diff --git a/taskcluster/ci/release-bouncer-aliases/kind.yml b/taskcluster/ci/release-bouncer-aliases/kind.yml index e037c143e9c0..3a277b25c2aa 100644 --- a/taskcluster/ci/release-bouncer-aliases/kind.yml +++ b/taskcluster/ci/release-bouncer-aliases/kind.yml @@ -29,10 +29,28 @@ jobs: mozilla-release: https://bounceradmin.mozilla.com/api maple: https://bounceradmin.stage.allizom.org/api default: http://localhost/api - notifications: - completed: - - releasetasks - failed: - - releasetasks - exception: - - releasetasks + notifications: + completed: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" + failed: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" + exception: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" diff --git a/taskcluster/ci/release-bouncer-sub/kind.yml b/taskcluster/ci/release-bouncer-sub/kind.yml index bc8abb167547..ecdbd5b72545 100644 --- a/taskcluster/ci/release-bouncer-sub/kind.yml +++ b/taskcluster/ci/release-bouncer-sub/kind.yml @@ -23,10 +23,28 @@ jobs: routes: - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter - notifications: - completed: - - releasetasks - failed: - - releasetasks - exception: - - releasetasks + notifications: + completed: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" + failed: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" + exception: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" diff --git a/taskcluster/ci/release-mark-as-shipped/kind.yml b/taskcluster/ci/release-mark-as-shipped/kind.yml index e040a1179f61..dbb4dce3b4e2 100644 --- a/taskcluster/ci/release-mark-as-shipped/kind.yml +++ b/taskcluster/ci/release-mark-as-shipped/kind.yml @@ -26,10 +26,28 @@ jobs: routes: - index.releases.v1.{branch}.latest.fennec.latest.mark_as_shipped - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.mark_as_shipped - notifications: - completed: - - releasetasks - failed: - - releasetasks - exception: - - releasetasks + notifications: + completed: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" + failed: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" + exception: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" diff --git a/taskcluster/ci/release-notify-promote/kind.yml b/taskcluster/ci/release-notify-promote/kind.yml new file mode 100644 index 000000000000..81e019d9aae8 --- /dev/null +++ b/taskcluster/ci/release-notify-promote/kind.yml @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.task:transforms + +kind-dependencies: + - beetmover-checksums + +jobs: + fennec: + name: notify-release-drivers-promote + description: Sends email to release-drivers telling release was promoted. + run-on-projects: [] + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + os: linux + docker-image: "ubuntu:16.10" + max-run-time: 600 + command: + - /bin/bash + - -c + - echo "Dummy task" + notifications: + completed: + subject: "{config[params][project]} {release_config[version]} build{release_config[build_number]} is in the candidates directory" + message: "{config[params][project]} {release_config[version]} build{release_config[build_number]} is in the candidates directory" + ids: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" diff --git a/taskcluster/ci/release-notify-publish/kind.yml b/taskcluster/ci/release-notify-publish/kind.yml new file mode 100644 index 000000000000..43877b37b15c --- /dev/null +++ b/taskcluster/ci/release-notify-publish/kind.yml @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.task:transforms + +kind-dependencies: + - push-apk + +jobs: + fennec: + name: notify-release-drivers-publish + description: Sends email to release-drivers telling release was published. + run-on-projects: [] + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + os: linux + docker-image: "ubuntu:16.10" + max-run-time: 600 + command: + - /bin/bash + - -c + - echo "Dummy task" + notifications: + completed: + subject: "{config[params][project]} {release_config[version]} build{release_config[build_number]} has been published to Google Play" + message: "{config[params][project]} {release_config[version]} build{release_config[build_number]} has been published to Google Play" + ids: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" diff --git a/taskcluster/ci/release-uptake-monitoring/kind.yml b/taskcluster/ci/release-uptake-monitoring/kind.yml index 3eaa4a7b0c03..43012c430c68 100644 --- a/taskcluster/ci/release-uptake-monitoring/kind.yml +++ b/taskcluster/ci/release-uptake-monitoring/kind.yml @@ -35,10 +35,28 @@ jobs: mozilla-release: https://bounceradmin.mozilla.com/api maple: https://bounceradmin.stage.allizom.org/api default: http://localhost/api - notifications: - completed: - - releasetasks - failed: - - releasetasks - exception: - - releasetasks + notifications: + completed: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" + failed: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" + exception: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" diff --git a/taskcluster/ci/release-version-bump/kind.yml b/taskcluster/ci/release-version-bump/kind.yml index f779eb1d2fc0..a5d9038f97fc 100644 --- a/taskcluster/ci/release-version-bump/kind.yml +++ b/taskcluster/ci/release-version-bump/kind.yml @@ -14,8 +14,8 @@ transforms: jobs: fennec: - name: fennec version bump - description: version bump + name: fennec-version-bump + description: Release Promotion version bump worker-type: buildbot-bridge/buildbot-bridge run-on-projects: [] run: @@ -26,10 +26,28 @@ jobs: routes: - index.releases.v1.{branch}.latest.fennec.latest.version_bump - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.version_bump - notifications: - completed: - - releasetasks - failed: - - releasetasks - exception: - - releasetasks + notifications: + completed: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" + failed: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" + exception: + by-project: + maple: + - "release-drivers-staging" + try: + #- "{task[tags][createdForUser]}" + default: + - "release-drivers" diff --git a/taskcluster/docs/kinds.rst b/taskcluster/docs/kinds.rst index b1f36e64e593..ce2b3fcf7b9a 100644 --- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -219,6 +219,14 @@ PushApk publishes Android packages onto Google Play Store. Jobs of this kind tak all the signed multi-locales (aka "multi") APKs for a given release and upload them all at once. They also depend on the breakpoint. +release-notify-publish +---------------------- +Notify when publishing a release. + +release-notify-promote +---------------------- +Notify when promoting a release. + release-bouncer-sub ------------------- Submits bouncer updates for releases. diff --git a/taskcluster/taskgraph/target_tasks.py b/taskcluster/taskgraph/target_tasks.py index 41989d53d593..ac07fd0a8a86 100644 --- a/taskcluster/taskgraph/target_tasks.py +++ b/taskcluster/taskgraph/target_tasks.py @@ -389,8 +389,11 @@ def target_tasks_candidates_fennec(full_task_graph, parameters): if task.kind not in ('balrog', 'push-apk', 'push-apk-breakpoint'): if task.attributes.get('nightly'): return True - if task.task['payload'].get('properties', {}).get('product') == 'fennec': - if task.kind in ('release-bouncer-sub', ): + if task.task['payload'].get('properties', {}).get('product') == 'fennec' or \ + task.label.endswith('-fennec'): + if task.kind in ('release-bouncer-sub', + 'release-notify-promote', + ): return True return [l for l, t in full_task_graph.tasks.iteritems() if filter(full_task_graph[l])] @@ -406,11 +409,14 @@ def target_tasks_publish_fennec(full_task_graph, parameters): # Include candidates build tasks; these will be optimized out if task.label in filtered_for_candidates: return True - if task.task['payload'].get('properties', {}).get('product') == 'fennec': + # TODO: Include [beetmover] fennec mozilla-beta push to releases + if task.task['payload'].get('properties', {}).get('product') == 'fennec' or \ + task.label.endswith('-fennec'): if task.kind in ('release-mark-as-shipped', 'release-bouncer-aliases', 'release-uptake-monitoring', 'release-version-bump', + 'release-notify-publish', ): return True diff --git a/taskcluster/taskgraph/transforms/job/__init__.py b/taskcluster/taskgraph/transforms/job/__init__.py index e3fbb8ff8fb7..4094f48d30ea 100644 --- a/taskcluster/taskgraph/transforms/job/__init__.py +++ b/taskcluster/taskgraph/transforms/job/__init__.py @@ -56,6 +56,7 @@ job_description_schema = Schema({ Optional('scopes'): task_description_schema['scopes'], Optional('tags'): task_description_schema['tags'], Optional('extra'): task_description_schema['extra'], + Optional('notifications'): task_description_schema['notifications'], Optional('treeherder'): task_description_schema['treeherder'], Optional('index'): task_description_schema['index'], Optional('run-on-projects'): task_description_schema['run-on-projects'], diff --git a/taskcluster/taskgraph/transforms/job/buildbot.py b/taskcluster/taskgraph/transforms/job/buildbot.py index e02427b563a0..403d0c9b85cd 100644 --- a/taskcluster/taskgraph/transforms/job/buildbot.py +++ b/taskcluster/taskgraph/transforms/job/buildbot.py @@ -17,7 +17,6 @@ from taskgraph.util.scriptworker import get_release_config from voluptuous import Optional, Required, Any from taskgraph.transforms.job import run_job_using -from taskgraph.transforms.task import notification_schema buildbot_run_schema = Schema({ @@ -33,25 +32,9 @@ buildbot_run_schema = Schema({ Optional('release-promotion'): bool, Optional('routes'): [basestring], Optional('properties'): {basestring: optionally_keyed_by('project', basestring)}, - - Optional('notifications'): { - Optional('completed'): Any(notification_schema, [basestring]), - Optional('failed'): Any(notification_schema, [basestring]), - Optional('artifact'): Any(notification_schema, [basestring]), - Optional('exception'): Any(notification_schema, [basestring]), - }, }) -FULL_TASK_NAME = ( - "[{task[payload][properties][product]} " - "{task[payload][properties][version]} " - "build{task[payload][properties][build_number]}/" - "{task[payload][sourcestamp][branch]}] " - "{task[metadata][name]} task" -) - - def bb_release_worker(config, worker, run): # props release_props = get_release_config(config, force=True) @@ -84,44 +67,6 @@ def bb_release_worker(config, worker, run): route = route.format(**repl_dict) worker['routes'].append(route) - notifications = run.get('notifications') - if notifications: - - worker.setdefault('notifications', {}) - - completed = notifications.get('completed') - if completed: - if isinstance(completed, list): - worker['notifications']['task-completed'] = { - "subject": "Completed: {}".format(FULL_TASK_NAME), - "message": "{} has completed successfully! Yay!".format(FULL_TASK_NAME), - "ids": completed, - } - else: - worker['notifications']['task-completed'] = completed - - failed = notifications.get('failed') - if failed: - if isinstance(failed, list): - worker['notifications']['task-failed'] = { - "subject": "Failed: {}".format(FULL_TASK_NAME), - "message": "Uh-oh! {} failed.".format(FULL_TASK_NAME), - "ids": failed, - } - else: - worker['notifications']['task-failed'] = failed - - exception = notifications.get('exception') - if exception: - if isinstance(exception, list): - worker['notifications']['task-exception'] = { - "subject": "Exception: {}".format(FULL_TASK_NAME), - "message": "Uh-oh! {} resulted in an exception.".format(FULL_TASK_NAME), - "ids": exception, - } - else: - worker['notifications']['task-exception'] = exception - def bb_ci_worker(config, worker): worker['properties'].update({ diff --git a/taskcluster/taskgraph/transforms/task.py b/taskcluster/taskgraph/transforms/task.py index 8a9f8f7dd8ea..96e2fde5a715 100755 --- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -23,7 +23,7 @@ from taskgraph.util.attributes import TRUNK_PROJECTS from taskgraph.util.hash import hash_path from taskgraph.util.treeherder import split_symbol from taskgraph.transforms.base import TransformSequence -from taskgraph.util.schema import validate_schema, Schema, optionally_keyed_by +from taskgraph.util.schema import validate_schema, Schema, optionally_keyed_by, resolve_keyed_by from taskgraph.util.scriptworker import get_release_config from voluptuous import Any, Required, Optional, Extra from taskgraph import GECKO @@ -44,15 +44,25 @@ def _run_task_suffix(): # shortcut for a string where task references are allowed taskref_or_string = Any( basestring, - {Required('task-reference'): basestring}) + {Required('task-reference'): basestring}, +) +notification_ids = optionally_keyed_by('project', Any(None, [basestring])) notification_schema = Schema({ Required("subject"): basestring, Required("message"): basestring, - Required("ids"): [basestring], + Required("ids"): notification_ids, }) +FULL_TASK_NAME = ( + "[{task[payload][properties][product]} " + "{task[payload][properties][version]} " + "build{task[payload][properties][build_number]}/" + "{task[payload][sourcestamp][branch]}] " + "{task[metadata][name]} task" +) + # A task description is a general description of a TaskCluster task task_description_schema = Schema({ # the label for this task @@ -210,6 +220,13 @@ task_description_schema = Schema({ # Whether the job should use sccache compiler caching. Required('needs-sccache', default=False): bool, + # notifications + Optional('notifications'): { + Optional('completed'): Any(notification_schema, notification_ids), + Optional('failed'): Any(notification_schema, notification_ids), + Optional('exception'): Any(notification_schema, notification_ids), + }, + # information specific to the worker implementation that will run this task 'worker': Any({ Required('implementation'): Any('docker-worker', 'docker-engine'), @@ -389,11 +406,6 @@ task_description_schema = Schema({ }, Optional('scopes'): [basestring], Optional('routes'): [basestring], - Optional('notifications'): { - Optional('task-completed'): notification_schema, - Optional('task-failed'): notification_schema, - Optional('task-exception'): notification_schema, - }, }, { Required('implementation'): 'native-engine', Required('os'): Any('macosx', 'linux'), @@ -1043,20 +1055,22 @@ def build_buildbot_bridge_payload(config, task, task_def): task_def['scopes'].extend(worker.get('scopes', [])) task_def['routes'].extend(worker.get('routes', [])) - notifications = worker.get('notifications') - if notifications: - task_def.setdefault('extra', {}).setdefault('notifications', {}) - for k, v in notifications.items(): - task_def['extra']['notifications'][k] = { - 'subject': v['subject'].format(task=task_def), - 'message': v['message'].format(task=task_def), - 'ids': v['ids'], - } - transforms = TransformSequence() +@transforms.add +def task_name_from_label(config, tasks): + for task in tasks: + if 'label' not in task: + if 'name' not in task: + raise Exception("task has neither a name nor a label") + task['label'] = '{}-{}'.format(config.kind, task['name']) + if task.get('name'): + del task['name'] + yield task + + @transforms.add def validate(config, tasks): for task in tasks: @@ -1312,6 +1326,48 @@ def build_task(config, tasks): env = payload.setdefault('env', {}) env['MOZ_AUTOMATION'] = '1' + notifications = task.get('notifications') + if notifications: + task_def['extra'].setdefault('notifications', {}) + for k, v in notifications.items(): + if isinstance(v, dict) and len(v) == 1 and v.keys()[0].startswith('by-'): + v = {'tmp': v} + resolve_keyed_by(v, 'tmp', 'notifications', **config.params) + v = v['tmp'] + if isinstance(v, list): + v = {'ids': v} + if 'completed' == k: + v.update({ + "subject": "Completed: {}".format(FULL_TASK_NAME), + "message": "{} has completed successfully! Yay!".format( + FULL_TASK_NAME), + }) + elif k == 'failed': + v.update({ + "subject": "Failed: {}".format(FULL_TASK_NAME), + "message": "Uh-oh! {} failed.".format(FULL_TASK_NAME), + }) + elif k == 'exception': + v.update({ + "subject": "Exception: {}".format(FULL_TASK_NAME), + "message": "Uh-oh! {} resulted in an exception.".format( + FULL_TASK_NAME), + }) + else: + resolve_keyed_by(v, 'ids', 'notifications', **config.params) + if v['ids'] is None: + continue + notifications_kwargs = dict( + task=task_def, + config=config.__dict__, + release_config=get_release_config(config, force=True), + ) + task_def['extra']['notifications']['task-' + k] = { + 'subject': v['subject'].format(**notifications_kwargs), + 'message': v['message'].format(**notifications_kwargs), + 'ids': v['ids'], + } + yield { 'label': task['label'], 'task': task_def,