mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-07 18:04:46 +00:00
95449daa6d
For a long time two copies of the 'taskgraph' module have existed in parallel. We've attempted to keep them in sync, but over time they have diverged and the maintenance burden has increased. In order to reduce this burden, we'd like to re-join the two code bases. The canonical repo will be the one that lives outside of mozilla-central, and this module will depend on it. Since they both have the same module name (taskgraph) we need to rename the version in mozilla-central to avoid collisions. Other consumers of 'taskgraph' (like mobile repos) have standardized on '<project>_taskgraph' as their module names. So replicating that here as well. Differential Revision: https://phabricator.services.mozilla.com/D127118
321 lines
17 KiB
YAML
321 lines
17 KiB
YAML
# yamllint disable rule:line-length
|
|
# This file is rendered via JSON-e by
|
|
# - hg-push - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/hg_push.py
|
|
# {
|
|
# tasks_for: 'hg-push',
|
|
# push: {owner, comment, pushlog_id, pushdate},
|
|
# repository: {url, project, level},
|
|
# now,
|
|
# as_slugid: // function
|
|
# ownTaskId: // taskId of the task that will be created
|
|
# }
|
|
#
|
|
# - cron tasks - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/cron/decision.py
|
|
# {
|
|
# tasks_for: 'cron',
|
|
# push: {revision, pushlog_id, pushdate, owner}
|
|
# repository: {url, project, level},
|
|
# cron: {task_id, job_name, job_symbol, quoted_args},
|
|
# now,
|
|
# ownTaskId: // taskId of the task that will be created
|
|
# }
|
|
#
|
|
# - action tasks - See:
|
|
# * taskcluster/gecko_taskgraph/actions/registry.py,
|
|
# * https://docs.taskcluster.net/docs/manual/using/actions/spec
|
|
# * ci-admin:ciadmin/generate/in_tree_actions.py
|
|
#
|
|
# The registry generates the hookPayload that appears in actions.json, and
|
|
# contains data from the decision task as well as JSON-e code to combine that
|
|
# with data supplied as part of the action spec. When the hook is fired, the
|
|
# hookPayload is rendered with JSON-e to produce a payload for the hook task
|
|
# template.
|
|
#
|
|
# The ci-admin code wraps the content of this file (.taskcluster.yml) with a
|
|
# JSON-e $let statement that produces the context described below, and
|
|
# installs that as the hook task template.
|
|
#
|
|
# {
|
|
# tasks_for: 'action',
|
|
# push: {owner, pushlog_id, revision},
|
|
# repository: {url, project, level},
|
|
# input,
|
|
# taskId, // targetted taskId
|
|
# taskGroupId, // targetted taskGroupId
|
|
# action: {name, title, description, taskGroupId, symbol, repo_scope, cb_name}
|
|
# ownTaskId: // taskId of the task that will be created
|
|
# clientId: // clientId that triggered this hook
|
|
# }
|
|
---
|
|
version: 1
|
|
tasks:
|
|
# NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering
|
|
# takes place.
|
|
- $if: 'tasks_for in ["hg-push", "action", "cron"]'
|
|
then:
|
|
$let:
|
|
# sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
|
|
ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
|
|
# ensure there's no trailing `/` on the repo URL
|
|
repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
|
|
# expire try earlier than other branches
|
|
expires:
|
|
$if: 'repository.project == "try"'
|
|
then: {$fromNow: '28 days'}
|
|
else: {$fromNow: '1 year'}
|
|
trustDomain: gecko
|
|
treeherder_link: '[Treeherder job](https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}&selectedTaskRun=${ownTaskId})'
|
|
in:
|
|
taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'}
|
|
taskGroupId:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
'${action.taskGroupId}'
|
|
else:
|
|
'${ownTaskId}' # same as taskId; this is how automation identifies a decision task
|
|
schedulerId: '${trustDomain}-level-${repository.level}'
|
|
|
|
created: {$fromNow: ''}
|
|
deadline: {$fromNow: '1 day'}
|
|
expires: {$eval: 'expires'}
|
|
metadata:
|
|
$merge:
|
|
- owner: "${ownerEmail}"
|
|
source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
|
|
- $if: 'tasks_for == "hg-push"'
|
|
then:
|
|
name: "Gecko Decision Task"
|
|
description: 'The task that creates all of the other tasks in the task graph (${treeherder_link})'
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
name: "Action: ${action.title}"
|
|
description: |
|
|
${action.description}
|
|
|
|
${treeherder_link}
|
|
|
|
Action triggered by clientID `${clientId}`
|
|
else:
|
|
name: "Decision Task for cron job ${cron.job_name}"
|
|
description: 'Created by a [cron task](https://firefox-ci-tc.services.mozilla.com/tasks/${cron.task_id}) (${treeherder_link})'
|
|
|
|
provisionerId: "${trustDomain}-${repository.level}"
|
|
workerType: "decision"
|
|
|
|
tags:
|
|
$if: 'tasks_for == "hg-push"'
|
|
then:
|
|
createdForUser: "${ownerEmail}"
|
|
kind: decision-task
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
createdForUser: '${ownerEmail}'
|
|
kind: 'action-callback'
|
|
else:
|
|
$if: 'tasks_for == "cron"'
|
|
then:
|
|
kind: cron-task
|
|
|
|
routes:
|
|
$flattenDeep:
|
|
- "tc-treeherder.v2.${repository.project}.${push.revision}"
|
|
- $if: 'tasks_for == "hg-push"'
|
|
then:
|
|
- "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision"
|
|
- "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
|
|
- "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
|
|
- "notify.email.${ownerEmail}.on-failed"
|
|
- "notify.email.${ownerEmail}.on-exception"
|
|
# Send a notification email if the push comes from try
|
|
- $if: 'repository.project == "try"'
|
|
then:
|
|
"notify.email.${ownerEmail}.on-completed"
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
- "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.actions.${ownTaskId}"
|
|
- "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}"
|
|
else: # cron
|
|
- "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"
|
|
- "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision-${cron.job_name}"
|
|
- "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision-${cron.job_name}"
|
|
# list each cron task on this revision, so actions can find them
|
|
- 'index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.cron.${ownTaskId}'
|
|
# BUG 1500166 Notify ciduty by email if a nightly hook fails
|
|
- $if: 'repository.project != "try"'
|
|
then:
|
|
- "notify.email.ciduty+failedcron@mozilla.com.on-failed"
|
|
- "notify.email.ciduty+exceptioncron@mozilla.com.on-exception"
|
|
- "notify.email.sheriffs+failedcron@mozilla.org.on-failed"
|
|
- "notify.email.sheriffs+exceptioncron@mozilla.org.on-exception"
|
|
|
|
scopes:
|
|
$if: 'tasks_for == "hg-push"'
|
|
then:
|
|
- 'assume:repo:${repoUrl[8:]}:branch:default'
|
|
- 'queue:route:notify.email.${ownerEmail}.*'
|
|
- 'in-tree:hook-action:project-${trustDomain}/in-tree-action-${repository.level}-*'
|
|
- 'index:insert-task:${trustDomain}.v2.${repository.project}.*'
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
# when all actions are hooks, we can calculate this directly rather than using a variable
|
|
- '${action.repo_scope}'
|
|
else:
|
|
- 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
|
|
|
|
dependencies: []
|
|
requires: all-completed
|
|
|
|
priority:
|
|
# Most times, there is plenty of worker capacity so everything runs
|
|
# quickly, but sometimes a storm of action tasks lands. Then we
|
|
# want, from highest to lowest:
|
|
# - cron tasks (time-sensitive) (low)
|
|
# - action tasks (avoid interfering with the other two) (very-low)
|
|
# - decision tasks (minimize user-visible delay) (lowest)
|
|
# SCM levels all use different workerTypes, so there is no need for priority
|
|
# between levels; "low" is the highest priority available at all levels, and
|
|
# nothing runs at any higher priority on these workerTypes.
|
|
$if: "tasks_for == 'cron'"
|
|
then: low
|
|
else:
|
|
$if: "tasks_for == 'action'"
|
|
then: very-low
|
|
else: lowest # tasks_for == 'hg-push'
|
|
retries:
|
|
$if: "tasks_for == 'hg-push'"
|
|
then: 0
|
|
else: 5
|
|
|
|
payload:
|
|
env:
|
|
# run-task uses these to check out the source; the inputs
|
|
# to `mach taskgraph decision` are all on the command line.
|
|
$merge:
|
|
- GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
|
|
GECKO_HEAD_REPOSITORY: '${repoUrl}'
|
|
GECKO_HEAD_REF: '${push.revision}'
|
|
GECKO_HEAD_REV: '${push.revision}'
|
|
HG_STORE_PATH: /builds/worker/checkouts/hg-store
|
|
TASKCLUSTER_CACHES: /builds/worker/checkouts
|
|
MOZ_AUTOMATION: '1'
|
|
# mach generates pyc files when reading `mach_commands.py`
|
|
# This causes cached_task digest generation to be random for
|
|
# some tasks. Disable bytecode generation to work around that.
|
|
PYTHONDONTWRITEBYTECODE: '1'
|
|
- $if: 'tasks_for == "action"'
|
|
then:
|
|
ACTION_TASK_GROUP_ID: '${action.taskGroupId}' # taskGroupId of the target task
|
|
ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded)
|
|
ACTION_INPUT: {$json: {$eval: 'input'}}
|
|
ACTION_CALLBACK: '${action.cb_name}'
|
|
|
|
cache:
|
|
"${trustDomain}-level-${repository.level}-checkouts-sparse-v2": /builds/worker/checkouts
|
|
|
|
features:
|
|
taskclusterProxy: true
|
|
chainOfTrust: true
|
|
|
|
# Note: This task is built server side without the context or tooling that
|
|
# exist in tree so we must hard code the hash
|
|
image: 'mozillareleases/gecko_decision:3.0.1@sha256:9964730ed951584e65032bd1d6a2b7ede6d880a8b7aefb72d0d7f82346d834e3'
|
|
|
|
maxRunTime: 3600
|
|
|
|
command:
|
|
- /builds/worker/bin/run-task
|
|
- '--gecko-checkout=/builds/worker/checkouts/gecko'
|
|
- '--gecko-sparse-profile=build/sparse-profiles/taskgraph'
|
|
- '--'
|
|
- bash
|
|
- -cx
|
|
- $let:
|
|
extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
|
|
in:
|
|
$if: 'tasks_for == "action"'
|
|
then: >
|
|
cd /builds/worker/checkouts/gecko &&
|
|
ln -s /builds/worker/artifacts artifacts &&
|
|
./mach --log-no-times taskgraph action-callback
|
|
else: >
|
|
cd /builds/worker/checkouts/gecko &&
|
|
ln -s /builds/worker/artifacts artifacts &&
|
|
./mach --log-no-times taskgraph decision
|
|
--pushlog-id='${push.pushlog_id}'
|
|
--pushdate='${push.pushdate}'
|
|
--project='${repository.project}'
|
|
--owner='${ownerEmail}'
|
|
--level='${repository.level}'
|
|
--tasks-for='${tasks_for}'
|
|
--repository-type=hg
|
|
--base-repository="$GECKO_BASE_REPOSITORY"
|
|
--head-repository="$GECKO_HEAD_REPOSITORY"
|
|
--head-ref="$GECKO_HEAD_REF"
|
|
--head-rev="$GECKO_HEAD_REV"
|
|
${extraArgs}
|
|
|
|
artifacts:
|
|
'public':
|
|
type: 'directory'
|
|
path: '/builds/worker/artifacts'
|
|
expires: {$eval: expires}
|
|
'public/docker-contexts':
|
|
type: 'directory'
|
|
path: '/builds/worker/checkouts/gecko/docker-contexts'
|
|
# This needs to be at least the deadline of the
|
|
# decision task + the docker-image task deadlines.
|
|
# It is set to a week to allow for some time for
|
|
# debugging, but they are not useful long-term.
|
|
expires: {$fromNow: '7 day'}
|
|
|
|
extra:
|
|
$merge:
|
|
- treeherder:
|
|
$merge:
|
|
- machine:
|
|
platform: gecko-decision
|
|
- $if: 'tasks_for == "hg-push"'
|
|
then:
|
|
symbol: D
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
groupName: 'action-callback'
|
|
groupSymbol: AC
|
|
symbol: "${action.symbol}"
|
|
else:
|
|
groupSymbol: cron
|
|
symbol: "${cron.job_symbol}"
|
|
- $if: 'tasks_for == "action"'
|
|
then:
|
|
parent: '${action.taskGroupId}'
|
|
action:
|
|
name: '${action.name}'
|
|
context:
|
|
taskGroupId: '${action.taskGroupId}'
|
|
taskId: {$eval: 'taskId'}
|
|
input: {$eval: 'input'}
|
|
clientId: {$eval: 'clientId'}
|
|
- $if: 'tasks_for == "cron"'
|
|
then:
|
|
cron: {$json: {$eval: 'cron'}}
|
|
- tasks_for: '${tasks_for}'
|
|
# Email for all pushes should link to treeherder
|
|
- $if: 'tasks_for == "hg-push"'
|
|
then:
|
|
notify:
|
|
email:
|
|
$merge:
|
|
- link:
|
|
text: "Treeherder Jobs"
|
|
href: "https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}"
|
|
# Email for try pushes should thank you for your revision
|
|
- $if: 'repository.project == "try"'
|
|
then:
|
|
subject: "Thank you for your try submission of ${push.revision}. It's the best!"
|
|
content: "Your try push has been submitted. It's the best! Use the link to view the status of your jobs."
|