Bug 1290620 - Use run-task from decision task; r=dustin

This commit does a lot. But it's really not too difficult to comprehend
once you focus on the final state, which is basically the same as the
"lint" image and derived tasks.

Before, the "decision" image contained a "checkout-gecko" script and
"run-action" and "run-decision" scripts. The latter 2 invoked the first
script.

The "run-task" script basically does what the combination of these
scripts were doing before. So we switch to it.

While we're here, we also replaced the custom Mercurial installation in
this image with the shared install-mercurial.sh script. The
system-setup.sh script for the decision image is now short and sweet.

The YAML files for tasks using this image have been updated to use
"run-task." We no longer have to pass an environment variable to hold
command arguments. So we revert to putting these arguments inline in
the task's command. Dustin never liked passing the arguments as
environment variables, so it should make him happy ;) We add
--log-no-times because "run-task" prefixes its own timestamps on output
lines.

The path to the Gecko checkout has been changed from
/home/worker/workspace to /home/worker/checkouts to match changes made
in bug 1289643.

Finally, since "checkout-gecko-and-run" is no longer used, we delete it.

The Docker image version has been bumped accordingly.

MozReview-Commit-ID: HAGuCz6668Q

--HG--
extra : rebase_source : 9586873c21d67f8ce869b7ceee221e8fa8775539

.taskcluster.yml

@@ -70,36 +70,42 @@ tasks:
           GECKO_HEAD_REPOSITORY: '{{{url}}}'
           GECKO_HEAD_REF: '{{revision}}'
           GECKO_HEAD_REV: '{{revision}}'
-          # Arguments passed into `mach taskgraph decision`
-          # TODO use mozilla-unified for the base repository once the tc-vcs
-          # tar.gz archives are created or tc-vcs isn't being used.
-          DECISION_ARGS: >
-            --pushlog-id='{{pushlog_id}}'
-            --project='{{project}}'
-            --message='{{comment}}'
-            --owner='{{owner}}'
-            --level='{{level}}'
-            --base-repository='https://hg.mozilla.org/mozilla-central'
-            --head-repository='{{{url}}}'
-            --head-ref='{{revision}}'
-            --head-rev='{{revision}}'
-            --revision-hash='{{revision_hash}}'
 
         cache:
           level-{{level}}-hg-shared: /home/worker/hg-shared
-          level-{{level}}-gecko-decision: /home/worker/workspace
+          level-{{level}}-checkouts: /home/worker/checkouts
 
         features:
           taskclusterProxy: true
 
         # Note: This task is built server side without the context or tooling that
         # exist in tree so we must hard code the version
-        image: 'taskcluster/decision:0.1.2'
+        image: 'taskcluster/decision:0.1.3'
 
         maxRunTime: 1800
 
+        # TODO use mozilla-unified for the base repository once the tc-vcs
+        # tar.gz archives are created or tc-vcs isn't being used.
         command:
-          - /home/worker/bin/run-decision
+          - /home/worker/bin/run-task
+          - '--vcs-checkout=/home/worker/checkouts/gecko'
+          - '--'
+          - bash
+          - -cx
+          - >
+              cd /home/worker/checkouts/gecko &&
+              ln -s /home/worker/artifacts artifacts &&
+              ./mach --log-no-times taskgraph decision
+              --pushlog-id='{{pushlog_id}}'
+              --project='{{project}}'
+              --message='{{comment}}'
+              --owner='{{owner}}'
+              --level='{{level}}'
+              --base-repository='https://hg.mozilla.org/mozilla-central'
+              --head-repository='{{{url}}}'
+              --head-ref='{{revision}}'
+              --head-rev='{{revision}}'
+              --revision-hash='{{revision_hash}}'
 
         artifacts:
           'public':

taskcluster/taskgraph/action.yml

@@ -31,20 +31,17 @@ payload:
     GECKO_HEAD_REPOSITORY: '{{{head_repository}}}'
     GECKO_HEAD_REF: '{{head_ref}}'
     GECKO_HEAD_REV: '{{head_rev}}'
-    ACTION_ARGS: >
-      --decision-id='{{decision_task_id}}'
-      --task-labels='{{task_labels}}'
 
   cache:
     level-{{level}}-hg-shared: /home/worker/hg-shared
-    level-{{level}}-gecko-decision: /home/worker/workspace
+    level-{{level}}-checkouts: /home/worker/checkouts
 
   features:
     taskclusterProxy: true
 
   # Note: This task is built server side without the context or tooling that
   # exist in tree so we must hard code the version
-  image: 'taskcluster/decision:0.1.2'
+  image: 'taskcluster/decision:0.1.3'
 
   # Virtually no network or other potentially risky operations happen as part
   # of the task timeout aside from the initial clone. We intentionally have
@@ -53,7 +50,17 @@ payload:
   maxRunTime: 1800
 
   command:
-    - /home/worker/bin/run-action
+    - /home/worker/bin/run-task
+    - '--vcs-checkout=/home/worker/checkouts/gecko'
+    - '--'
+    - bash
+    - -cx
+    - >
+        cd /home/worker/checkouts/gecko &&
+        ln -s /home/worker/artifacts artifacts &&
+        ./mach --log-no-times taskgraph action-task
+        --decision-id='{{decision_task_id}}'
+        --task-label='{{task_labels}}'
 
   artifacts:
     'public':

testing/docker/decision/Dockerfile

@@ -8,10 +8,14 @@ RUN mkdir /home/worker/artifacts && chown worker:worker /home/worker/artifacts
 # %include testing/docker/recipes/tooltool.py
 ADD topsrcdir/testing/docker/recipes/tooltool.py /tmp/tooltool.py
 
+# %include testing/docker/recipes/install-mercurial.sh
+ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /tmp/install-mercurial.sh
+
 ADD system-setup.sh /tmp/system-setup.sh
 RUN bash /tmp/system-setup.sh
 
-ADD bin /home/worker/bin
+# %include testing/docker/recipes/run-task
+ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task
 
 ENV PATH /home/worker/bin:$PATH
 ENV SHELL /bin/bash

testing/docker/decision/VERSION

@@ -1 +1 @@
-0.1.2
+0.1.3

testing/docker/decision/bin/checkout-gecko

@@ -1,31 +0,0 @@
-#! /bin/bash -e
-
-set -ex
-
-# Ensure we have at least enough to check gecko out...
-test $GECKO_BASE_REPOSITORY
-
-DESTDIR=$1
-
-res=`curl --fail --retry 5 http://taskcluster/secrets/v1/secret/project/taskcluster/gecko/hgfingerprint`
-FP=`echo $res | jq -r .secret.content`
-
-if [[ ! "$FP" =~ ^[a-f0-9:]+$ ]]; then
-    echo
-    echo "[taskcluster:error] HG host fingerprint not properly loaded"
-    exit 1
-fi
-
-if [ -n "${GECKO_HEAD_REV}" ]; then
-  args="--revision ${GECKO_HEAD_REV}"
-else
-  args="--branch ${GECKO_HEAD_REF}"
-fi
-
-hg --config hostfingerprints.hg.mozilla.org=${FP} robustcheckout \
-  --sharebase /home/worker/hg-shared \
-  --purge \
-  ${args} \
-  --upstream ${GECKO_BASE_REPOSITORY} \
-  ${GECKO_HEAD_REPOSITORY} \
-  ${DESTDIR}

testing/docker/decision/bin/run-action

@@ -1,19 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-set -ex
-
-# The script starts executing as root. We need to change ownership
-# of the caches because they are initially owned by root:root.
-if [ $(id -u) = 0 ]; then
-    chown worker:worker /home/worker/hg-shared /home/worker/workspace
-
-    exec sudo -E -u worker /home/worker/bin/run-action "${@}"
-fi
-
-/home/worker/bin/checkout-gecko /home/worker/workspace/gecko
-cd /home/worker/workspace/gecko
-ln -s /home/worker/artifacts artifacts &&
-eval "./mach taskgraph action-task ${ACTION_ARGS}"

testing/docker/decision/bin/run-decision

@@ -1,19 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-set -ex
-
-# The script starts executing as root. We need to change ownership
-# of the caches because they are initially owned by root:root.
-if [ $(id -u) = 0 ]; then
-    chown worker:worker /home/worker/hg-shared /home/worker/workspace
-
-    exec sudo -E -u worker /home/worker/bin/run-decision "${@}"
-fi
-
-/home/worker/bin/checkout-gecko /home/worker/workspace/gecko
-cd /home/worker/workspace/gecko
-ln -s /home/worker/artifacts artifacts
-eval "./mach taskgraph decision ${DECISION_ARGS}"

testing/docker/decision/system-setup.sh

@@ -21,60 +21,8 @@ tooltool_fetch() {
     rm manifest.tt
 }
 
-# Install Mercurial from custom debs since distro packages tend to lag behind.
 cd $BUILD
-tooltool_fetch <<EOF
-[
-{
-    "size": 44878,
-    "digest": "7b1fc1217e0dcaeea852b0af2dc559b1aafb704fbee7e29cbec75af57bacb84910a7ec92b5c33f04ee98f23b3a57f1fa451173fe7c8a96f58faefe319dc7dde1",
-    "algorithm": "sha512",
-    "filename": "mercurial_3.8.4_amd64.deb"
-},
-{
-    "size": 1818422,
-    "digest": "b476e2612e7495a1c7c5adfd84511aa7479e26cc9070289513ec705fbfc4c61806ce2dbcceca0e63f2e80669be416f3467a3cebb522dcb8a6aeb62cdd3df82f2",
-    "algorithm": "sha512",
-    "filename": "mercurial-common_3.8.4_all.deb"
-}
-]
-EOF
-
-dpkg -i mercurial-common_3.8.4_all.deb mercurial_3.8.4_amd64.deb
-
-mkdir -p /usr/local/mercurial
-chown 755 /usr/local/mercurial
-cd /usr/local/mercurial
-tooltool_fetch <<'EOF'
-[
-{
-    "size": 11849,
-    "digest": "c88d9b8afd6649bd28bbacfa654ebefec8087a01d1662004aae088d485edeb03a92df1193d1310c0369d7721f475b974fcd4a911428ec65936f7e40cf1609c49",
-    "algorithm": "sha512",
-    "filename": "robustcheckout.py"
-}
-]
-EOF
-
-chmod 644 /usr/local/mercurial/robustcheckout.py
-
-# Install a global hgrc file with reasonable defaults.
-mkdir -p /etc/mercurial
-cat >/etc/mercurial/hgrc <<EOF
-# By default the progress bar starts after 3s and updates every 0.1s. We
-# change this so it shows and updates every 1.0s.
-[progress]
-delay = 1.0
-refresh = 1.0
-
-[web]
-cacerts = /etc/ssl/certs/ca-certificates.crt
-
-[extensions]
-robustcheckout = /usr/local/mercurial/robustcheckout.py
-EOF
-
-chmod 644 /etc/mercurial/hgrc
+. /tmp/install-mercurial.sh
 
 cd /
 rm -rf $BUILD

testing/docker/recipes/checkout-gecko-and-run

@@ -1,39 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-set -ex
-
-DEST=$1
-CHECKOUTS=`dirname ${DEST}`
-
-# The script starts executing as root. We need to change ownership
-# of the caches because they are initially owned by root:root. There
-# may not be a cache mount/directory on some instances. So create the
-# directory if missing.
-if [ $(id -u) = 0 ]; then
-    mkdir -p ${CHECKOUTS}
-    chown worker:worker /home/worker/hg-shared ${CHECKOUTS}
-
-    exec sudo -E -u worker ${0} "${@}"
-fi
-
-shift
-
-# We set the base repository to mozilla-central so tc-vcs doesn't get
-# confused. Switch to mozilla-unified because robustcheckout works best
-# with it.
-if [ "${GECKO_BASE_REPOSITORY}" = "https://hg.mozilla.org/mozilla-central" ]; then
-    GECKO_BASE_REPOSITORY=https://hg.mozilla.org/mozilla-unified
-fi
-
-/usr/bin/hg robustcheckout \
-    --sharebase /home/worker/hg-shared \
-    --purge \
-    --upstream ${GECKO_BASE_REPOSITORY} \
-    --revision ${GECKO_HEAD_REV} \
-    ${GECKO_HEAD_REPOSITORY} \
-    ${DEST}
-
-exec "${@}"