Bug 1412690 - fennec release driver emails. r=aki

This patch adds the `release-notify-promote` and `release-notify-publish` kinds. It also genericizes all the notifications, and updates the kinds that use those notifications.

MozReview-Commit-ID: 9ymXKzthVF4

--HG--
extra : rebase_source : 6d7db48afa14e4124834321784da65e8d1e84c98

taskcluster/ci/release-bouncer-aliases/kind.yml

@@ -29,10 +29,28 @@ jobs:
                     mozilla-release: https://bounceradmin.mozilla.com/api
                     maple: https://bounceradmin.stage.allizom.org/api
                     default: http://localhost/api
-         notifications:
-            completed:
-               - releasetasks
-            failed:
-               - releasetasks
-            exception:
-               - releasetasks
+      notifications:
+         completed:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"
+         failed:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"
+         exception:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"

taskcluster/ci/release-bouncer-sub/kind.yml

@@ -23,10 +23,28 @@ jobs:
          routes:
             - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter
             - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter
-         notifications:
-            completed:
-               - releasetasks
-            failed:
-               - releasetasks
-            exception:
-               - releasetasks
+      notifications:
+         completed:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"
+         failed:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"
+         exception:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"

taskcluster/ci/release-mark-as-shipped/kind.yml

@@ -26,10 +26,28 @@ jobs:
          routes:
             - index.releases.v1.{branch}.latest.fennec.latest.mark_as_shipped
             - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.mark_as_shipped
-         notifications:
-            completed:
-               - releasetasks
-            failed:
-               - releasetasks
-            exception:
-               - releasetasks
+      notifications:
+         completed:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"
+         failed:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"
+         exception:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"

taskcluster/ci/release-notify-promote/kind.yml

@@ -0,0 +1,39 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - beetmover-checksums
+
+jobs:
+   fennec:
+      name: notify-release-drivers-promote
+      description: Sends email to release-drivers telling release was promoted.
+      run-on-projects: []
+      worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+      worker:
+         implementation: docker-worker
+         os: linux
+         docker-image: "ubuntu:16.10"
+         max-run-time: 600
+         command:
+            - /bin/bash
+            - -c
+            - echo "Dummy task"
+      notifications:
+         completed:
+            subject: "{config[params][project]} {release_config[version]} build{release_config[build_number]} is in the candidates directory"
+            message: "{config[params][project]} {release_config[version]} build{release_config[build_number]} is in the candidates directory"
+            ids:
+               by-project:
+                  maple:
+                     - "release-drivers-staging"
+                  try:
+                     #- "{task[tags][createdForUser]}"
+                  default:
+                     - "release-drivers"

taskcluster/ci/release-notify-publish/kind.yml

@@ -0,0 +1,39 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - push-apk
+
+jobs:
+   fennec:
+      name: notify-release-drivers-publish
+      description: Sends email to release-drivers telling release was published.
+      run-on-projects: []
+      worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+      worker:
+         implementation: docker-worker
+         os: linux
+         docker-image: "ubuntu:16.10"
+         max-run-time: 600
+         command:
+            - /bin/bash
+            - -c
+            - echo "Dummy task"
+      notifications:
+         completed:
+            subject: "{config[params][project]} {release_config[version]} build{release_config[build_number]} has been published to Google Play"
+            message: "{config[params][project]} {release_config[version]} build{release_config[build_number]} has been published to Google Play"
+            ids:
+               by-project:
+                  maple:
+                     - "release-drivers-staging"
+                  try:
+                     #- "{task[tags][createdForUser]}"
+                  default:
+                     - "release-drivers"

taskcluster/ci/release-uptake-monitoring/kind.yml

@@ -35,10 +35,28 @@ jobs:
                     mozilla-release: https://bounceradmin.mozilla.com/api
                     maple: https://bounceradmin.stage.allizom.org/api
                     default: http://localhost/api
-         notifications:
-            completed:
-               - releasetasks
-            failed:
-               - releasetasks
-            exception:
-               - releasetasks
+      notifications:
+         completed:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"
+         failed:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"
+         exception:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"

taskcluster/ci/release-version-bump/kind.yml

@@ -14,8 +14,8 @@ transforms:
 
 jobs:
    fennec:
-      name: fennec version bump
-      description: version bump
+      name: fennec-version-bump
+      description: Release Promotion version bump
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
       run:
@@ -26,10 +26,28 @@ jobs:
          routes:
             - index.releases.v1.{branch}.latest.fennec.latest.version_bump
             - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.version_bump
-         notifications:
-            completed:
-               - releasetasks
-            failed:
-               - releasetasks
-            exception:
-               - releasetasks
+      notifications:
+         completed:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"
+         failed:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"
+         exception:
+            by-project:
+               maple:
+                  - "release-drivers-staging"
+               try:
+                  #- "{task[tags][createdForUser]}"
+               default:
+                  - "release-drivers"

taskcluster/docs/kinds.rst

@@ -219,6 +219,14 @@ PushApk publishes Android packages onto Google Play Store. Jobs of this kind tak
 all the signed multi-locales (aka "multi") APKs for a given release and upload them
 all at once. They also depend on the breakpoint.
 
+release-notify-publish
+----------------------
+Notify when publishing a release.
+
+release-notify-promote
+----------------------
+Notify when promoting a release.
+
 release-bouncer-sub
 -------------------
 Submits bouncer updates for releases.

taskcluster/taskgraph/target_tasks.py

@@ -389,8 +389,11 @@ def target_tasks_candidates_fennec(full_task_graph, parameters):
             if task.kind not in ('balrog', 'push-apk', 'push-apk-breakpoint'):
                 if task.attributes.get('nightly'):
                     return True
-        if task.task['payload'].get('properties', {}).get('product') == 'fennec':
-            if task.kind in ('release-bouncer-sub', ):
+        if task.task['payload'].get('properties', {}).get('product') == 'fennec' or \
+           task.label.endswith('-fennec'):
+            if task.kind in ('release-bouncer-sub',
+                             'release-notify-promote',
+                             ):
                 return True
 
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(full_task_graph[l])]
@@ -406,11 +409,14 @@ def target_tasks_publish_fennec(full_task_graph, parameters):
         # Include candidates build tasks; these will be optimized out
         if task.label in filtered_for_candidates:
             return True
-        if task.task['payload'].get('properties', {}).get('product') == 'fennec':
+        # TODO: Include [beetmover] fennec mozilla-beta push to releases
+        if task.task['payload'].get('properties', {}).get('product') == 'fennec' or \
+           task.label.endswith('-fennec'):
             if task.kind in ('release-mark-as-shipped',
                              'release-bouncer-aliases',
                              'release-uptake-monitoring',
                              'release-version-bump',
+                             'release-notify-publish',
                              ):
                 return True
 

taskcluster/taskgraph/transforms/job/__init__.py

@@ -56,6 +56,7 @@ job_description_schema = Schema({
     Optional('scopes'): task_description_schema['scopes'],
     Optional('tags'): task_description_schema['tags'],
     Optional('extra'): task_description_schema['extra'],
+    Optional('notifications'): task_description_schema['notifications'],
     Optional('treeherder'): task_description_schema['treeherder'],
     Optional('index'): task_description_schema['index'],
     Optional('run-on-projects'): task_description_schema['run-on-projects'],

taskcluster/taskgraph/transforms/job/buildbot.py

@@ -17,7 +17,6 @@ from taskgraph.util.scriptworker import get_release_config
 from voluptuous import Optional, Required, Any
 
 from taskgraph.transforms.job import run_job_using
-from taskgraph.transforms.task import notification_schema
 
 
 buildbot_run_schema = Schema({
@@ -33,25 +32,9 @@ buildbot_run_schema = Schema({
     Optional('release-promotion'): bool,
     Optional('routes'): [basestring],
     Optional('properties'): {basestring: optionally_keyed_by('project', basestring)},
-
-    Optional('notifications'): {
-        Optional('completed'): Any(notification_schema, [basestring]),
-        Optional('failed'): Any(notification_schema, [basestring]),
-        Optional('artifact'): Any(notification_schema, [basestring]),
-        Optional('exception'): Any(notification_schema, [basestring]),
-    },
 })
 
 
-FULL_TASK_NAME = (
-    "[{task[payload][properties][product]} "
-    "{task[payload][properties][version]} "
-    "build{task[payload][properties][build_number]}/"
-    "{task[payload][sourcestamp][branch]}] "
-    "{task[metadata][name]} task"
-)
-
-
 def bb_release_worker(config, worker, run):
     # props
     release_props = get_release_config(config, force=True)
@@ -84,44 +67,6 @@ def bb_release_worker(config, worker, run):
             route = route.format(**repl_dict)
             worker['routes'].append(route)
 
-    notifications = run.get('notifications')
-    if notifications:
-
-        worker.setdefault('notifications', {})
-
-        completed = notifications.get('completed')
-        if completed:
-            if isinstance(completed, list):
-                worker['notifications']['task-completed'] = {
-                    "subject": "Completed: {}".format(FULL_TASK_NAME),
-                    "message": "{} has completed successfully! Yay!".format(FULL_TASK_NAME),
-                    "ids": completed,
-                }
-            else:
-                worker['notifications']['task-completed'] = completed
-
-        failed = notifications.get('failed')
-        if failed:
-            if isinstance(failed, list):
-                worker['notifications']['task-failed'] = {
-                    "subject": "Failed: {}".format(FULL_TASK_NAME),
-                    "message": "Uh-oh! {} failed.".format(FULL_TASK_NAME),
-                    "ids": failed,
-                }
-            else:
-                worker['notifications']['task-failed'] = failed
-
-        exception = notifications.get('exception')
-        if exception:
-            if isinstance(exception, list):
-                worker['notifications']['task-exception'] = {
-                    "subject": "Exception: {}".format(FULL_TASK_NAME),
-                    "message": "Uh-oh! {} resulted in an exception.".format(FULL_TASK_NAME),
-                    "ids": exception,
-                }
-            else:
-                worker['notifications']['task-exception'] = exception
-
 
 def bb_ci_worker(config, worker):
     worker['properties'].update({

taskcluster/taskgraph/transforms/task.py

@@ -23,7 +23,7 @@ from taskgraph.util.attributes import TRUNK_PROJECTS
 from taskgraph.util.hash import hash_path
 from taskgraph.util.treeherder import split_symbol
 from taskgraph.transforms.base import TransformSequence
-from taskgraph.util.schema import validate_schema, Schema, optionally_keyed_by
+from taskgraph.util.schema import validate_schema, Schema, optionally_keyed_by, resolve_keyed_by
 from taskgraph.util.scriptworker import get_release_config
 from voluptuous import Any, Required, Optional, Extra
 from taskgraph import GECKO
@@ -44,15 +44,25 @@ def _run_task_suffix():
 # shortcut for a string where task references are allowed
 taskref_or_string = Any(
     basestring,
-    {Required('task-reference'): basestring})
+    {Required('task-reference'): basestring},
+)
 
+notification_ids = optionally_keyed_by('project', Any(None, [basestring]))
 notification_schema = Schema({
     Required("subject"): basestring,
     Required("message"): basestring,
-    Required("ids"): [basestring],
+    Required("ids"): notification_ids,
 
 })
 
+FULL_TASK_NAME = (
+    "[{task[payload][properties][product]} "
+    "{task[payload][properties][version]} "
+    "build{task[payload][properties][build_number]}/"
+    "{task[payload][sourcestamp][branch]}] "
+    "{task[metadata][name]} task"
+)
+
 # A task description is a general description of a TaskCluster task
 task_description_schema = Schema({
     # the label for this task
@@ -210,6 +220,13 @@ task_description_schema = Schema({
     # Whether the job should use sccache compiler caching.
     Required('needs-sccache', default=False): bool,
 
+    # notifications
+    Optional('notifications'): {
+        Optional('completed'): Any(notification_schema, notification_ids),
+        Optional('failed'): Any(notification_schema, notification_ids),
+        Optional('exception'): Any(notification_schema, notification_ids),
+    },
+
     # information specific to the worker implementation that will run this task
     'worker': Any({
         Required('implementation'): Any('docker-worker', 'docker-engine'),
@@ -389,11 +406,6 @@ task_description_schema = Schema({
         },
         Optional('scopes'): [basestring],
         Optional('routes'): [basestring],
-        Optional('notifications'): {
-            Optional('task-completed'): notification_schema,
-            Optional('task-failed'): notification_schema,
-            Optional('task-exception'): notification_schema,
-        },
     }, {
         Required('implementation'): 'native-engine',
         Required('os'): Any('macosx', 'linux'),
@@ -1043,20 +1055,22 @@ def build_buildbot_bridge_payload(config, task, task_def):
     task_def['scopes'].extend(worker.get('scopes', []))
     task_def['routes'].extend(worker.get('routes', []))
 
-    notifications = worker.get('notifications')
-    if notifications:
-        task_def.setdefault('extra', {}).setdefault('notifications', {})
-        for k, v in notifications.items():
-            task_def['extra']['notifications'][k] = {
-                'subject': v['subject'].format(task=task_def),
-                'message': v['message'].format(task=task_def),
-                'ids': v['ids'],
-            }
-
 
 transforms = TransformSequence()
 
 
+@transforms.add
+def task_name_from_label(config, tasks):
+    for task in tasks:
+        if 'label' not in task:
+            if 'name' not in task:
+                raise Exception("task has neither a name nor a label")
+            task['label'] = '{}-{}'.format(config.kind, task['name'])
+        if task.get('name'):
+            del task['name']
+        yield task
+
+
 @transforms.add
 def validate(config, tasks):
     for task in tasks:
@@ -1312,6 +1326,48 @@ def build_task(config, tasks):
                 env = payload.setdefault('env', {})
                 env['MOZ_AUTOMATION'] = '1'
 
+        notifications = task.get('notifications')
+        if notifications:
+            task_def['extra'].setdefault('notifications', {})
+            for k, v in notifications.items():
+                if isinstance(v, dict) and len(v) == 1 and v.keys()[0].startswith('by-'):
+                    v = {'tmp': v}
+                    resolve_keyed_by(v, 'tmp', 'notifications', **config.params)
+                    v = v['tmp']
+                if isinstance(v, list):
+                    v = {'ids': v}
+                    if 'completed' == k:
+                        v.update({
+                            "subject": "Completed: {}".format(FULL_TASK_NAME),
+                            "message": "{} has completed successfully! Yay!".format(
+                                FULL_TASK_NAME),
+                        })
+                    elif k == 'failed':
+                        v.update({
+                            "subject": "Failed: {}".format(FULL_TASK_NAME),
+                            "message": "Uh-oh! {} failed.".format(FULL_TASK_NAME),
+                        })
+                    elif k == 'exception':
+                        v.update({
+                            "subject": "Exception: {}".format(FULL_TASK_NAME),
+                            "message": "Uh-oh! {} resulted in an exception.".format(
+                                FULL_TASK_NAME),
+                        })
+                else:
+                    resolve_keyed_by(v, 'ids', 'notifications', **config.params)
+                if v['ids'] is None:
+                    continue
+                notifications_kwargs = dict(
+                    task=task_def,
+                    config=config.__dict__,
+                    release_config=get_release_config(config, force=True),
+                )
+                task_def['extra']['notifications']['task-' + k] = {
+                    'subject': v['subject'].format(**notifications_kwargs),
+                    'message': v['message'].format(**notifications_kwargs),
+                    'ids': v['ids'],
+                }
+
         yield {
             'label': task['label'],
             'task': task_def,