bug 1269355 - Enable sccache for taskcluster linux32/64 builds. r=dustin,gps

MozReview-Commit-ID: LGqpNDb7uHN

--HG--
extra : rebase_source : dfd3d0c7208178d7a44cd49a84cd1ead194e8095

build/mozconfig.cache

@@ -41,15 +41,20 @@ if test -z "$SCCACHE_DISABLE" -a -z "$no_sccache" -a -z "$MOZ_PGO_IS_SET" -a -z
         ;;
     esac
 fi
+fi
 
-# builds without buildprops (eg: taskcluster or non-buildbot) and without ccache env config and without sccache disabled:
-elif test -z "$CCACHE_DIR" -a -z "$SCCACHE_DISABLE" -a -z "$no_sccache" -a -z "$MOZ_PGO_IS_SET" -a -z "$MOZ_PGO"; then
+# builds where buildprops didn't have the data (eg: taskcluster or non-buildbot) and without sccache disabled:
+if test -z "$bucket" -a -z "$SCCACHE_DISABLE" -a -z "$no_sccache" -a -z "$MOZ_PGO_IS_SET" -a -z "$MOZ_PGO"; then
 
     # prevent rerun if az is set, or wget is not available
     if test -z "$availability_zone" -a -x "$(command -v wget)"; then
+        if test -n "${TASKCLUSTER_WORKER_GROUP}"; then
+            availability_zone="${TASKCLUSTER_WORKER_GROUP}"
+        else
             # timeout after 1 second, and don't retry (failure indicates instance is not in ec2 or network issue)
             # availability_zone is of the form <region><letter> where region is e.g. us-west-2, and az is us-west-2a
             availability_zone=$(wget -T 1 -t 1 -q -O - http://169.254.169.254/latest/meta-data/placement/availability-zone || true)
+        fi
         if test -z "$availability_zone" -o "$availability_zone" = "not-ec2"; then
             availability_zone=not-ec2
         else

taskcluster/scripts/builder/build-linux.sh

@@ -50,6 +50,11 @@ export MOZ_AUTOMATION_UPLOAD_SYMBOLS=0
 # Ensure that in tree libraries can be found
 export LIBRARY_PATH=$LIBRARY_PATH:$WORKSPACE/src/obj-firefox:$WORKSPACE/src/gcc/lib64
 
+if [[ -n ${USE_SCCACHE} ]]; then
+    # Point sccache at the Taskcluster proxy for AWS credentials.
+    export AWS_IAM_CREDENTIALS_URL="http://taskcluster/auth/v1/aws/s3/read-write/taskcluster-level-${MOZ_SCM_LEVEL}-sccache-${TASKCLUSTER_WORKER_GROUP%?}/?format=iam-role-compat"
+fi
+
 # test required parameters are supplied
 if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
 if [[ -z ${MOZHARNESS_CONFIG} ]]; then fail "MOZHARNESS_CONFIG is not set"; fi

taskcluster/taskgraph/transforms/build.py

@@ -19,6 +19,7 @@ def set_defaults(config, jobs):
     for job in jobs:
         job['treeherder'].setdefault('kind', 'build')
         job['treeherder'].setdefault('tier', 1)
+        job.setdefault('needs-sccache', True)
         if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
             job['worker'].setdefault('docker-image', {'in-tree': 'desktop-build'})
             job['worker']['chain-of-trust'] = True

taskcluster/taskgraph/transforms/job/__init__.py

@@ -53,6 +53,7 @@ job_description_schema = Schema({
     Optional('run-on-projects'): task_description_schema['run-on-projects'],
     Optional('coalesce-name'): task_description_schema['coalesce-name'],
     Optional('worker-type'): task_description_schema['worker-type'],
+    Optional('needs-sccache'): task_description_schema['needs-sccache'],
     Required('worker'): task_description_schema['worker'],
     Optional('when'): task_description_schema['when'],
 

taskcluster/taskgraph/transforms/task.py

@@ -140,6 +140,9 @@ task_description_schema = Schema({
     #  {level} -- the scm level of this push
     'worker-type': basestring,
 
+    # Whether the job should use sccache compiler caching.
+    Required('needs-sccache', default=False): bool,
+
     # information specific to the worker implementation that will run this task
     'worker': Any({
         Required('implementation'): Any('docker-worker', 'docker-engine'),
@@ -370,6 +373,14 @@ def build_docker_worker_payload(config, task, task_def):
     if worker.get('chain-of-trust'):
         features['chainOfTrust'] = True
 
+    if task.get('needs-sccache'):
+        features['taskclusterProxy'] = True
+        task_def['scopes'].append(
+            'assume:project:taskcluster:level-{level}-sccache-buckets'.format(
+                level=config.params['level'])
+        )
+        worker['env']['USE_SCCACHE'] = '1'
+
     capabilities = {}
 
     for lo in 'audio', 'video':
@@ -451,6 +462,8 @@ def build_generic_worker_payload(config, task, task_def):
         'osGroups': worker.get('os-groups', []),
     }
 
+    # needs-sccache is handled in mozharness_on_windows
+
     if 'retry-exit-status' in worker:
         raise Exception("retry-exit-status not supported in generic-worker")
 
@@ -472,6 +485,8 @@ def build_macosx_engine_payload(config, task, task_def):
         'artifacts': artifacts,
     }
 
+    if task.get('needs-sccache'):
+        raise Exception('needs-sccache not supported in macosx-engine')
 
 @payload_builder('buildbot-bridge')
 def build_buildbot_bridge_payload(config, task, task_def):