mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-23 21:01:08 +00:00
766773ca7e
hg 4.8 (e.g. from the v3.0.1 decision image) seems to choke on cached repos created with 5.8 (from v4.0.0). Differential Revision: https://phabricator.services.mozilla.com/D142455
321 lines
17 KiB
YAML
321 lines
17 KiB
YAML
# yamllint disable rule:line-length
|
|
# This file is rendered via JSON-e by
|
|
# - hg-push - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/hg_push.py
|
|
# {
|
|
# tasks_for: 'hg-push',
|
|
# push: {owner, comment, pushlog_id, pushdate},
|
|
# repository: {url, project, level},
|
|
# now,
|
|
# as_slugid: // function
|
|
# ownTaskId: // taskId of the task that will be created
|
|
# }
|
|
#
|
|
# - cron tasks - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/cron/decision.py
|
|
# {
|
|
# tasks_for: 'cron',
|
|
# push: {revision, pushlog_id, pushdate, owner}
|
|
# repository: {url, project, level},
|
|
# cron: {task_id, job_name, job_symbol, quoted_args},
|
|
# now,
|
|
# ownTaskId: // taskId of the task that will be created
|
|
# }
|
|
#
|
|
# - action tasks - See:
|
|
# * taskcluster/gecko_taskgraph/actions/registry.py,
|
|
# * https://docs.taskcluster.net/docs/manual/using/actions/spec
|
|
# * ci-admin:ciadmin/generate/in_tree_actions.py
|
|
#
|
|
# The registry generates the hookPayload that appears in actions.json, and
|
|
# contains data from the decision task as well as JSON-e code to combine that
|
|
# with data supplied as part of the action spec. When the hook is fired, the
|
|
# hookPayload is rendered with JSON-e to produce a payload for the hook task
|
|
# template.
|
|
#
|
|
# The ci-admin code wraps the content of this file (.taskcluster.yml) with a
|
|
# JSON-e $let statement that produces the context described below, and
|
|
# installs that as the hook task template.
|
|
#
|
|
# {
|
|
# tasks_for: 'action',
|
|
# push: {owner, pushlog_id, revision},
|
|
# repository: {url, project, level},
|
|
# input,
|
|
# taskId, // targetted taskId
|
|
# taskGroupId, // targetted taskGroupId
|
|
# action: {name, title, description, taskGroupId, symbol, repo_scope, cb_name}
|
|
# ownTaskId: // taskId of the task that will be created
|
|
# clientId: // clientId that triggered this hook
|
|
# }
|
|
---
|
|
version: 1
|
|
tasks:
|
|
# NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering
|
|
# takes place.
|
|
- $if: 'tasks_for in ["hg-push", "action", "cron"]'
|
|
then:
|
|
$let:
|
|
# sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
|
|
ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
|
|
# ensure there's no trailing `/` on the repo URL
|
|
repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
|
|
# expire try earlier than other branches
|
|
expires:
|
|
$if: 'repository.project == "try"'
|
|
then: {$fromNow: '28 days'}
|
|
else: {$fromNow: '1 year'}
|
|
trustDomain: gecko
|
|
treeherder_link: '[Treeherder job](https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}&selectedTaskRun=${ownTaskId})'
|
|
in:
|
|
taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'}
|
|
taskGroupId:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
'${action.taskGroupId}'
|
|
else:
|
|
'${ownTaskId}' # same as taskId; this is how automation identifies a decision task
|
|
schedulerId: '${trustDomain}-level-${repository.level}'
|
|
|
|
created: {$fromNow: ''}
|
|
deadline: {$fromNow: '1 day'}
|
|
expires: {$eval: 'expires'}
|
|
metadata:
|
|
$merge:
|
|
- owner: "${ownerEmail}"
|
|
source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
|
|
- $if: 'tasks_for == "hg-push"'
|
|
then:
|
|
name: "Gecko Decision Task"
|
|
description: 'The task that creates all of the other tasks in the task graph (${treeherder_link})'
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
name: "Action: ${action.title}"
|
|
description: |
|
|
${action.description}
|
|
|
|
${treeherder_link}
|
|
|
|
Action triggered by clientID `${clientId}`
|
|
else:
|
|
name: "Decision Task for cron job ${cron.job_name}"
|
|
description: 'Created by a [cron task](https://firefox-ci-tc.services.mozilla.com/tasks/${cron.task_id}) (${treeherder_link})'
|
|
|
|
provisionerId: "${trustDomain}-${repository.level}"
|
|
workerType: "decision"
|
|
|
|
tags:
|
|
$if: 'tasks_for == "hg-push"'
|
|
then:
|
|
createdForUser: "${ownerEmail}"
|
|
kind: decision-task
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
createdForUser: '${ownerEmail}'
|
|
kind: 'action-callback'
|
|
else:
|
|
$if: 'tasks_for == "cron"'
|
|
then:
|
|
kind: cron-task
|
|
|
|
routes:
|
|
$flattenDeep:
|
|
- "tc-treeherder.v2.${repository.project}.${push.revision}"
|
|
- $if: 'tasks_for == "hg-push"'
|
|
then:
|
|
- "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision"
|
|
- "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
|
|
- "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
|
|
- "notify.email.${ownerEmail}.on-failed"
|
|
- "notify.email.${ownerEmail}.on-exception"
|
|
# Send a notification email if the push comes from try
|
|
- $if: 'repository.project == "try"'
|
|
then:
|
|
"notify.email.${ownerEmail}.on-completed"
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
- "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.actions.${ownTaskId}"
|
|
- "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}"
|
|
else: # cron
|
|
- "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"
|
|
- "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision-${cron.job_name}"
|
|
- "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision-${cron.job_name}"
|
|
# list each cron task on this revision, so actions can find them
|
|
- 'index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.cron.${ownTaskId}'
|
|
# BUG 1500166 Notify ciduty by email if a nightly hook fails
|
|
- $if: 'repository.project != "try"'
|
|
then:
|
|
- "notify.email.ciduty+failedcron@mozilla.com.on-failed"
|
|
- "notify.email.ciduty+exceptioncron@mozilla.com.on-exception"
|
|
- "notify.email.sheriffs+failedcron@mozilla.org.on-failed"
|
|
- "notify.email.sheriffs+exceptioncron@mozilla.org.on-exception"
|
|
|
|
scopes:
|
|
$if: 'tasks_for == "hg-push"'
|
|
then:
|
|
- 'assume:repo:${repoUrl[8:]}:branch:default'
|
|
- 'queue:route:notify.email.${ownerEmail}.*'
|
|
- 'in-tree:hook-action:project-${trustDomain}/in-tree-action-${repository.level}-*'
|
|
- 'index:insert-task:${trustDomain}.v2.${repository.project}.*'
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
# when all actions are hooks, we can calculate this directly rather than using a variable
|
|
- '${action.repo_scope}'
|
|
else:
|
|
- 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
|
|
|
|
dependencies: []
|
|
requires: all-completed
|
|
|
|
priority:
|
|
# Most times, there is plenty of worker capacity so everything runs
|
|
# quickly, but sometimes a storm of action tasks lands. Then we
|
|
# want, from highest to lowest:
|
|
# - cron tasks (time-sensitive) (low)
|
|
# - action tasks (avoid interfering with the other two) (very-low)
|
|
# - decision tasks (minimize user-visible delay) (lowest)
|
|
# SCM levels all use different workerTypes, so there is no need for priority
|
|
# between levels; "low" is the highest priority available at all levels, and
|
|
# nothing runs at any higher priority on these workerTypes.
|
|
$if: "tasks_for == 'cron'"
|
|
then: low
|
|
else:
|
|
$if: "tasks_for == 'action'"
|
|
then: very-low
|
|
else: lowest # tasks_for == 'hg-push'
|
|
retries:
|
|
$if: "tasks_for == 'hg-push'"
|
|
then: 0
|
|
else: 5
|
|
|
|
payload:
|
|
env:
|
|
# run-task uses these to check out the source; the inputs
|
|
# to `mach taskgraph decision` are all on the command line.
|
|
$merge:
|
|
- GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
|
|
GECKO_HEAD_REPOSITORY: '${repoUrl}'
|
|
GECKO_HEAD_REF: '${push.revision}'
|
|
GECKO_HEAD_REV: '${push.revision}'
|
|
HG_STORE_PATH: /builds/worker/checkouts/hg-store
|
|
TASKCLUSTER_CACHES: /builds/worker/checkouts
|
|
MOZ_AUTOMATION: '1'
|
|
# mach generates pyc files when reading `mach_commands.py`
|
|
# This causes cached_task digest generation to be random for
|
|
# some tasks. Disable bytecode generation to work around that.
|
|
PYTHONDONTWRITEBYTECODE: '1'
|
|
- $if: 'tasks_for == "action"'
|
|
then:
|
|
ACTION_TASK_GROUP_ID: '${action.taskGroupId}' # taskGroupId of the target task
|
|
ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded)
|
|
ACTION_INPUT: {$json: {$eval: 'input'}}
|
|
ACTION_CALLBACK: '${action.cb_name}'
|
|
|
|
cache:
|
|
"${trustDomain}-level-${repository.level}-checkouts-sparse-v3": /builds/worker/checkouts
|
|
|
|
features:
|
|
taskclusterProxy: true
|
|
chainOfTrust: true
|
|
|
|
# Note: This task is built server side without the context or tooling that
|
|
# exist in tree so we must hard code the hash
|
|
image: 'mozillareleases/gecko_decision:4.0.0@sha256:9f69fe08c28e3cb3cc296451f0a2735df6e25d0e3c877ea735ef1b7f0b345b06'
|
|
|
|
maxRunTime: 3600
|
|
|
|
command:
|
|
- /builds/worker/bin/run-task
|
|
- '--gecko-checkout=/builds/worker/checkouts/gecko'
|
|
- '--gecko-sparse-profile=build/sparse-profiles/taskgraph'
|
|
- '--'
|
|
- bash
|
|
- -cx
|
|
- $let:
|
|
extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
|
|
in:
|
|
$if: 'tasks_for == "action"'
|
|
then: >
|
|
cd /builds/worker/checkouts/gecko &&
|
|
ln -s /builds/worker/artifacts artifacts &&
|
|
./mach --log-no-times taskgraph action-callback
|
|
else: >
|
|
cd /builds/worker/checkouts/gecko &&
|
|
ln -s /builds/worker/artifacts artifacts &&
|
|
./mach --log-no-times taskgraph decision
|
|
--pushlog-id='${push.pushlog_id}'
|
|
--pushdate='${push.pushdate}'
|
|
--project='${repository.project}'
|
|
--owner='${ownerEmail}'
|
|
--level='${repository.level}'
|
|
--tasks-for='${tasks_for}'
|
|
--repository-type=hg
|
|
--base-repository="$GECKO_BASE_REPOSITORY"
|
|
--head-repository="$GECKO_HEAD_REPOSITORY"
|
|
--head-ref="$GECKO_HEAD_REF"
|
|
--head-rev="$GECKO_HEAD_REV"
|
|
${extraArgs}
|
|
|
|
artifacts:
|
|
'public':
|
|
type: 'directory'
|
|
path: '/builds/worker/artifacts'
|
|
expires: {$eval: expires}
|
|
'public/docker-contexts':
|
|
type: 'directory'
|
|
path: '/builds/worker/checkouts/gecko/docker-contexts'
|
|
# This needs to be at least the deadline of the
|
|
# decision task + the docker-image task deadlines.
|
|
# It is set to a week to allow for some time for
|
|
# debugging, but they are not useful long-term.
|
|
expires: {$fromNow: '7 day'}
|
|
|
|
extra:
|
|
$merge:
|
|
- treeherder:
|
|
$merge:
|
|
- machine:
|
|
platform: gecko-decision
|
|
- $if: 'tasks_for == "hg-push"'
|
|
then:
|
|
symbol: D
|
|
else:
|
|
$if: 'tasks_for == "action"'
|
|
then:
|
|
groupName: 'action-callback'
|
|
groupSymbol: AC
|
|
symbol: "${action.symbol}"
|
|
else:
|
|
groupSymbol: cron
|
|
symbol: "${cron.job_symbol}"
|
|
- $if: 'tasks_for == "action"'
|
|
then:
|
|
parent: '${action.taskGroupId}'
|
|
action:
|
|
name: '${action.name}'
|
|
context:
|
|
taskGroupId: '${action.taskGroupId}'
|
|
taskId: {$eval: 'taskId'}
|
|
input: {$eval: 'input'}
|
|
clientId: {$eval: 'clientId'}
|
|
- $if: 'tasks_for == "cron"'
|
|
then:
|
|
cron: {$json: {$eval: 'cron'}}
|
|
- tasks_for: '${tasks_for}'
|
|
# Email for all pushes should link to treeherder
|
|
- $if: 'tasks_for == "hg-push"'
|
|
then:
|
|
notify:
|
|
email:
|
|
$merge:
|
|
- link:
|
|
text: "Treeherder Jobs"
|
|
href: "https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}"
|
|
# Email for try pushes should thank you for your revision
|
|
- $if: 'repository.project == "try"'
|
|
then:
|
|
subject: "Thank you for your try submission of ${push.revision}. It's the best!"
|
|
content: "Your try push has been submitted. It's the best! Use the link to view the status of your jobs."
|