mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-08 19:04:45 +00:00
c288f18538
We started using the "backstop" index added by bug 1660506 to determine whether a push should be a backstop based on a time interval. The problem is that this index gets added by an index-task that runs after the decision task has completed. Therefore, if two pushes land at roughly the same time (i.e, the second decision task starts before the first completes), then they can both determine themselves as backstops. This patch gets around the problem by inserting the "backstop" index as early as possible (immediately after resolving parameters), so the chances of this happening become very low. It's still theoretically possible that it could happen again, but we don't need this to be 100% perfect. As long as it is rare, it's good enough. Depends on D91191 Differential Revision: https://phabricator.services.mozilla.com/D91192
320 lines
17 KiB
YAML
320 lines
17 KiB
YAML
# yamllint disable rule:line-length
|
|
# This file is rendered via JSON-e by
|
|
# - hg-push - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/hg_push.py
|
|
# {
|
|
# tasks_for: 'hg-push',
|
|
# push: {owner, comment, pushlog_id, pushdate},
|
|
# repository: {url, project, level},
|
|
# now,
|
|
# as_slugid: // function
|
|
# ownTaskId: // taskId of the task that will be created
|
|
# }
|
|
#
|
|
# - cron tasks - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/cron/decision.py
|
|
# {
|
|
# tasks_for: 'cron',
|
|
# push: {revision, pushlog_id, pushdate, owner}
|
|
# repository: {url, project, level},
|
|
# cron: {task_id, job_name, job_symbol, quoted_args},
|
|
# now,
|
|
# ownTaskId: // taskId of the task that will be created
|
|
# }
|
|
#
|
|
# - action tasks - See:
|
|
# * taskcluster/taskgraph/actions/registry.py,
|
|
# * https://docs.taskcluster.net/docs/manual/using/actions/spec
|
|
# * ci-admin:ciadmin/generate/in_tree_actions.py
|
|
#
|
|
# The registry generates the hookPayload that appears in actions.json, and
|
|
# contains data from the decision task as well as JSON-e code to combine that
|
|
# with data supplied as part of the action spec. When the hook is fired, the
|
|
# hookPayload is rendered with JSON-e to produce a payload for the hook task
|
|
# template.
|
|
#
|
|
# The ci-admin code wraps the content of this file (.taskcluster.yml) with a
|
|
# JSON-e $let statement that produces the context described below, and
|
|
# installs that as the hook task template.
|
|
#
|
|
# {
|
|
# tasks_for: 'action',
|
|
# push: {owner, pushlog_id, revision},
|
|
# repository: {url, project, level},
|
|
# input,
|
|
# taskId, // targetted taskId
|
|
# taskGroupId, // targetted taskGroupId
|
|
# action: {name, title, description, taskGroupId, symbol, repo_scope, cb_name}
|
|
# ownTaskId: // taskId of the task that will be created
|
|
# clientId: // clientId that triggered this hook
|
|
# }
|
|
---
|
|
version: 1
|
|
tasks:
|
|
# NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering
|
|
# takes place.
|
|
- $if: 'tasks_for in ["hg-push", "action", "cron"]'
|
|
then:
|
|
$let:
|
|
# sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
|
|
ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
|
|
# ensure there's no trailing `/` on the repo URL
|
|
repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
|
|
# expire try earlier than other branches
|
|
expires:
|
|
$if: 'repository.project == "try"'
|
|
then: {$fromNow: '28 days'}
|
|
else: {$fromNow: '1 year'}
|
|
trustDomain: gecko
|
|
treeherder_link: '[Treeherder job](https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}&selectedTaskRun=${ownTaskId})'
|
|
in:
|
|
taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'}
|
|
taskGroupId:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
'${action.taskGroupId}'
|
|
else:
|
|
'${ownTaskId}' # same as taskId; this is how automation identifies a decision task
|
|
schedulerId: '${trustDomain}-level-${repository.level}'
|
|
|
|
created: {$fromNow: ''}
|
|
deadline: {$fromNow: '1 day'}
|
|
expires: {$eval: 'expires'}
|
|
metadata:
|
|
$merge:
|
|
- owner: "${ownerEmail}"
|
|
source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
|
|
- $if: 'tasks_for == "hg-push"'
|
|
then:
|
|
name: "Gecko Decision Task"
|
|
description: 'The task that creates all of the other tasks in the task graph (${treeherder_link})'
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
name: "Action: ${action.title}"
|
|
description: |
|
|
${action.description}
|
|
|
|
${treeherder_link}
|
|
|
|
Action triggered by clientID `${clientId}`
|
|
else:
|
|
name: "Decision Task for cron job ${cron.job_name}"
|
|
description: 'Created by a [cron task](https://firefox-ci-tc.services.mozilla.com/tasks/${cron.task_id}) (${treeherder_link})'
|
|
|
|
provisionerId: "${trustDomain}-${repository.level}"
|
|
workerType: "decision"
|
|
|
|
tags:
|
|
$if: 'tasks_for == "hg-push"'
|
|
then:
|
|
createdForUser: "${ownerEmail}"
|
|
kind: decision-task
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
createdForUser: '${ownerEmail}'
|
|
kind: 'action-callback'
|
|
else:
|
|
$if: 'tasks_for == "cron"'
|
|
then:
|
|
kind: cron-task
|
|
|
|
routes:
|
|
$flattenDeep:
|
|
- "tc-treeherder.v2.${repository.project}.${push.revision}"
|
|
- $if: 'tasks_for == "hg-push"'
|
|
then:
|
|
- "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision"
|
|
- "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
|
|
- "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
|
|
- "notify.email.${ownerEmail}.on-failed"
|
|
- "notify.email.${ownerEmail}.on-exception"
|
|
# Send a notification email if the push comes from try
|
|
- $if: 'repository.project == "try"'
|
|
then:
|
|
"notify.email.${ownerEmail}.on-completed"
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
- "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.actions.${ownTaskId}"
|
|
- "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}"
|
|
else: # cron
|
|
- "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"
|
|
- "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision-${cron.job_name}"
|
|
- "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision-${cron.job_name}"
|
|
# list each cron task on this revision, so actions can find them
|
|
- 'index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.cron.${ownTaskId}'
|
|
# BUG 1500166 Notify ciduty by email if a nightly hook fails
|
|
- $if: 'repository.project != "try"'
|
|
then:
|
|
- "notify.email.ciduty+failedcron@mozilla.com.on-failed"
|
|
- "notify.email.ciduty+exceptioncron@mozilla.com.on-exception"
|
|
- "notify.email.sheriffs+failedcron@mozilla.org.on-failed"
|
|
- "notify.email.sheriffs+exceptioncron@mozilla.org.on-exception"
|
|
|
|
scopes:
|
|
$if: 'tasks_for == "hg-push"'
|
|
then:
|
|
- 'assume:repo:${repoUrl[8:]}:branch:default'
|
|
- 'queue:route:notify.email.${ownerEmail}.*'
|
|
- 'in-tree:hook-action:project-${trustDomain}/in-tree-action-${repository.level}-*'
|
|
- 'index:insert-task:${trustDomain}.v2.${repository.project}.*'
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
# when all actions are hooks, we can calculate this directly rather than using a variable
|
|
- '${action.repo_scope}'
|
|
else:
|
|
- 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
|
|
|
|
dependencies: []
|
|
requires: all-completed
|
|
|
|
priority:
|
|
# Most times, there is plenty of worker capacity so everything runs
|
|
# quickly, but sometimes a storm of action tasks lands. Then we
|
|
# want, from highest to lowest:
|
|
# - cron tasks (time-sensitive) (low)
|
|
# - action tasks (avoid interfering with the other two) (very-low)
|
|
# - decision tasks (minimize user-visible delay) (lowest)
|
|
# SCM levels all use different workerTypes, so there is no need for priority
|
|
# between levels; "low" is the highest priority available at all levels, and
|
|
# nothing runs at any higher priority on these workerTypes.
|
|
$if: "tasks_for == 'cron'"
|
|
then: low
|
|
else:
|
|
$if: "tasks_for == 'action'"
|
|
then: very-low
|
|
else: lowest # tasks_for == 'hg-push'
|
|
retries:
|
|
$if: "tasks_for == 'hg-push'"
|
|
then: 0
|
|
else: 5
|
|
|
|
payload:
|
|
env:
|
|
# run-task uses these to check out the source; the inputs
|
|
# to `mach taskgraph decision` are all on the command line.
|
|
$merge:
|
|
- GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
|
|
GECKO_HEAD_REPOSITORY: '${repoUrl}'
|
|
GECKO_HEAD_REF: '${push.revision}'
|
|
GECKO_HEAD_REV: '${push.revision}'
|
|
HG_STORE_PATH: /builds/worker/checkouts/hg-store
|
|
TASKCLUSTER_CACHES: /builds/worker/checkouts
|
|
MOZ_AUTOMATION: '1'
|
|
# mach generates pyc files when reading `mach_commands.py`
|
|
# This causes cached_task digest generation to be random for
|
|
# some tasks. Disable bytecode generation to work around that.
|
|
PYTHONDONTWRITEBYTECODE: '1'
|
|
- $if: 'tasks_for == "action"'
|
|
then:
|
|
ACTION_TASK_GROUP_ID: '${action.taskGroupId}' # taskGroupId of the target task
|
|
ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded)
|
|
ACTION_INPUT: {$json: {$eval: 'input'}}
|
|
ACTION_CALLBACK: '${action.cb_name}'
|
|
|
|
cache:
|
|
"${trustDomain}-level-${repository.level}-checkouts-sparse-v2": /builds/worker/checkouts
|
|
|
|
features:
|
|
taskclusterProxy: true
|
|
chainOfTrust: true
|
|
|
|
# Note: This task is built server side without the context or tooling that
|
|
# exist in tree so we must hard code the hash
|
|
image: 'taskcluster/decision:2.2.0@sha256:0e9689e94605eb8395f5b49141a48148416b0d825f6f7be04c29642d1a85ee3d'
|
|
|
|
maxRunTime: 1800
|
|
|
|
command:
|
|
- /builds/worker/bin/run-task
|
|
- '--gecko-checkout=/builds/worker/checkouts/gecko'
|
|
- '--gecko-sparse-profile=build/sparse-profiles/taskgraph'
|
|
- '--'
|
|
- bash
|
|
- -cx
|
|
- $let:
|
|
extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
|
|
in:
|
|
$if: 'tasks_for == "action"'
|
|
then: >
|
|
cd /builds/worker/checkouts/gecko &&
|
|
ln -s /builds/worker/artifacts artifacts &&
|
|
./mach --log-no-times taskgraph action-callback
|
|
else: >
|
|
cd /builds/worker/checkouts/gecko &&
|
|
ln -s /builds/worker/artifacts artifacts &&
|
|
./mach --log-no-times taskgraph decision
|
|
--pushlog-id='${push.pushlog_id}'
|
|
--pushdate='${push.pushdate}'
|
|
--project='${repository.project}'
|
|
--owner='${ownerEmail}'
|
|
--level='${repository.level}'
|
|
--tasks-for='${tasks_for}'
|
|
--base-repository="$GECKO_BASE_REPOSITORY"
|
|
--head-repository="$GECKO_HEAD_REPOSITORY"
|
|
--head-ref="$GECKO_HEAD_REF"
|
|
--head-rev="$GECKO_HEAD_REV"
|
|
${extraArgs}
|
|
|
|
artifacts:
|
|
'public':
|
|
type: 'directory'
|
|
path: '/builds/worker/artifacts'
|
|
expires: {$eval: expires}
|
|
'public/docker-contexts':
|
|
type: 'directory'
|
|
path: '/builds/worker/checkouts/gecko/docker-contexts'
|
|
# This needs to be at least the deadline of the
|
|
# decision task + the docker-image task deadlines.
|
|
# It is set to a week to allow for some time for
|
|
# debugging, but they are not useful long-term.
|
|
expires: {$fromNow: '7 day'}
|
|
|
|
extra:
|
|
$merge:
|
|
- treeherder:
|
|
$merge:
|
|
- machine:
|
|
platform: gecko-decision
|
|
- $if: 'tasks_for == "hg-push"'
|
|
then:
|
|
symbol: D
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
groupName: 'action-callback'
|
|
groupSymbol: AC
|
|
symbol: "${action.symbol}"
|
|
else:
|
|
groupSymbol: cron
|
|
symbol: "${cron.job_symbol}"
|
|
- $if: 'tasks_for == "action"'
|
|
then:
|
|
parent: '${action.taskGroupId}'
|
|
action:
|
|
name: '${action.name}'
|
|
context:
|
|
taskGroupId: '${action.taskGroupId}'
|
|
taskId: {$eval: 'taskId'}
|
|
input: {$eval: 'input'}
|
|
clientId: {$eval: 'clientId'}
|
|
- $if: 'tasks_for == "cron"'
|
|
then:
|
|
cron: {$json: {$eval: 'cron'}}
|
|
- tasks_for: '${tasks_for}'
|
|
# Email for all pushes should link to treeherder
|
|
- $if: 'tasks_for == "hg-push"'
|
|
then:
|
|
notify:
|
|
email:
|
|
$merge:
|
|
- link:
|
|
text: "Treeherder Jobs"
|
|
href: "https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}"
|
|
# Email for try pushes should thank you for your revision
|
|
- $if: 'repository.project == "try"'
|
|
then:
|
|
subject: "Thank you for your try submission of ${push.revision}. It's the best!"
|
|
content: "Your try push has been submitted. It's the best! Use the link to view the status of your jobs."
|