bug 1259832 - add a post-build task to upload generated source files. r=dustin

This change adds an upload-generated-sources task kind that runs after nightly
builds, fetches their `target.generated-files.tar.gz` artifact, and uploads
all the contained files to an S3 bucket. For actual nightly and release builds
on SCM level 3 trees, the S3 bucket is configured to be publicly accessible,
so that tools like Socorro will be able to fetch generated source files that
appear in crash reports, and debuggers will be able to fetch generated sources
when they show up while debugging Nightly or Release builds.

There are also level-2 and level-1 S3 buckets configured for builds happening
on trees of other levels such as try. They are not configured as publicly
accessible, but they exist so that these tasks can be tested in try.

MozReview-Commit-ID: Js1HRftbtep

--HG--
extra : rebase_source : 13d6b71200cda11938af3048cbeddce4edfc9c07

build/upload_generated_sources.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env/python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
+import gzip
+import io
+import logging
+from mozbuild.base import MozbuildObject
+from mozbuild.generated_sources import (
+    get_filename_with_digest,
+    get_s3_region_and_bucket,
+)
+import os
+from Queue import Queue
+import requests
+import sys
+import tarfile
+from threading import Event, Thread
+import time
+
+# Arbitrary, should probably measure this.
+NUM_WORKER_THREADS = 10
+log = logging.getLogger('upload-generated-sources')
+log.setLevel(logging.INFO)
+
+
+@contextmanager
+def timed():
+    '''
+    Yield a function that provides the elapsed time in seconds since this
+    function was called.
+    '''
+    start = time.time()
+    def elapsed():
+        return time.time() - start
+    yield elapsed
+
+
+def gzip_compress(data):
+    '''
+    Apply gzip compression to `data` and return the result as a `BytesIO`.
+    '''
+    b = io.BytesIO()
+    with gzip.GzipFile(fileobj=b, mode='w') as f:
+        f.write(data)
+    b.flush()
+    b.seek(0)
+    return b
+
+
+def upload_worker(queue, event, bucket, session_args):
+    '''
+    Get `(name, contents)` entries from `queue` and upload `contents`
+    to S3 with gzip compression using `name` as the key, prefixed with
+    the SHA-512 digest of `contents` as a hex string. If an exception occurs,
+    set `event`.
+    '''
+    try:
+        import boto3
+        session = boto3.session.Session(**session_args)
+        s3 = session.client('s3')
+        while True:
+            if event.is_set():
+                # Some other thread hit an exception.
+                return
+            (name, contents) = queue.get()
+            pathname = get_filename_with_digest(name, contents)
+            compressed = gzip_compress(contents)
+            extra_args = {
+                'ContentEncoding': 'gzip',
+                'ContentType': 'text/plain',
+            }
+            log.info('Uploading "{}" ({} bytes)'.format(pathname, len(compressed.getvalue())))
+            with timed() as elapsed:
+                s3.upload_fileobj(compressed, bucket, pathname, ExtraArgs=extra_args)
+                log.info('Finished uploading "{}" in {:0.3f}s'.format(pathname, elapsed()))
+            queue.task_done()
+    except Exception:
+        log.exception('Thread encountered exception:')
+        event.set()
+
+
+def do_work(artifact, region, bucket):
+    session_args = {'region_name': region}
+    session = requests.Session()
+    if 'TASK_ID' in os.environ:
+        level = os.environ.get('MOZ_SCM_LEVEL', '1')
+        secrets_url = 'http://taskcluster/secrets/v1/secret/project/releng/gecko/build/level-{}/gecko-generated-sources-upload'.format(level)
+        log.info('Using AWS credentials from the secrets service: "{}"'.format(secrets_url))
+        res = session.get(secrets_url)
+        res.raise_for_status()
+        secret = res.json()
+        session_args.update(
+            aws_access_key_id=secret['secret']['AWS_ACCESS_KEY_ID'],
+            aws_secret_access_key=secret['secret']['AWS_SECRET_ACCESS_KEY'],
+        )
+    else:
+        log.info('Trying to use your AWS credentials..')
+
+
+    # First, fetch the artifact containing the sources.
+    log.info('Fetching generated sources artifact: "{}"'.format(artifact))
+    with timed() as elapsed:
+        res = session.get(artifact)
+        log.info('Fetch HTTP status: {}, {} bytes downloaded in {:0.3f}s'.format(res.status_code, len(res.content), elapsed()))
+    res.raise_for_status()
+    # Create a queue and worker threads for uploading.
+    q = Queue()
+    event = Event()
+    log.info('Creating {} worker threads'.format(NUM_WORKER_THREADS))
+    for i in range(NUM_WORKER_THREADS):
+        t = Thread(target=upload_worker, args=(q, event, bucket, session_args))
+        t.daemon = True
+        t.start()
+    with tarfile.open(fileobj=io.BytesIO(res.content), mode='r|gz') as tar:
+        # Next, process each file.
+        for entry in tar:
+            if event.is_set():
+                break
+            log.info('Queueing "{}"'.format(entry.name))
+            q.put((entry.name, tar.extractfile(entry).read()))
+    # Wait until all uploads are finished.
+    # We don't use q.join() here because we want to also monitor event.
+    while q.unfinished_tasks:
+        if event.wait(0.1):
+            log.error('Worker thread encountered exception, exiting...')
+            break
+
+
+def main(argv):
+    logging.basicConfig(format='%(levelname)s - %(threadName)s - %(message)s')
+    parser = argparse.ArgumentParser(
+    description='Upload generated source files in ARTIFACT to BUCKET in S3.')
+    parser.add_argument('artifact',
+                        help='generated-sources artifact from build task')
+    args = parser.parse_args(argv)
+    region, bucket = get_s3_region_and_bucket()
+
+    config = MozbuildObject.from_environment()
+    config._activate_virtualenv()
+    config.virtualenv_manager.install_pip_package('boto3==1.4.4')
+
+    with timed() as elapsed:
+        do_work(region=region, bucket=bucket, artifact=args.artifact)
+        log.info('Finished in {:.03f}s'.format(elapsed()))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/generated_sources.py

@@ -2,6 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import hashlib
 import json
 import os
 
@@ -9,6 +10,23 @@ from mozpack.files import FileFinder
 import mozpack.path as mozpath
 
 
+def sha512_digest(data):
+    '''
+    Generate the SHA-512 digest of `data` and return it as a hex string.
+    '''
+    return hashlib.sha512(data).hexdigest()
+
+
+def get_filename_with_digest(name, contents):
+    '''
+    Return the filename that will be used to store the generated file
+    in the S3 bucket, consisting of the SHA-512 digest of `contents`
+    joined with the relative path `name`.
+    '''
+    digest = sha512_digest(contents)
+    return mozpath.join(digest, name)
+
+
 def get_generated_sources():
     '''
     Yield tuples of `(objdir-rel-path, file)` for generated source files
@@ -38,3 +56,18 @@ def get_generated_sources():
     finder = FileFinder(mozpath.join(buildconfig.topobjdir, base))
     for p, f in finder.find('**/*.rs'):
         yield mozpath.join(base, p), f
+
+
+def get_s3_region_and_bucket():
+    '''
+    Return a tuple of (region, bucket) giving the AWS region and S3
+    bucket to which generated sources should be uploaded.
+    '''
+    region = 'us-west-2'
+    level = os.environ.get('MOZ_SCM_LEVEL', '1')
+    bucket = {
+        '1': 'gecko-generated-sources-l1',
+        '2': 'gecko-generated-sources-l2',
+        '3': 'gecko-generated-sources',
+    }[level]
+    return (region, bucket)

taskcluster/ci/upload-generated-sources/kind.yml

@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.upload_generated_sources:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - build
+
+only-for-attributes:
+  - nightly
+
+job-template:
+  description: Upload generated source files from build
+  worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
+  treeherder:
+    symbol: Ugs
+    kind: build
+  worker:
+     docker-image: {in-tree: "lint"}
+     max-run-time: 600
+  run:
+    using: run-task
+    command: >
+            cd /home/worker/checkouts/gecko &&
+            ./mach python build/upload_generated_sources.py ${ARTIFACT_URL}
+  scopes:
+      - secrets:get:project/releng/gecko/build/level-{level}/gecko-generated-sources-upload

taskcluster/docs/kinds.rst

@@ -67,6 +67,11 @@ upload-symbols
 Upload-symbols tasks run after builds and upload the symbols files generated by
 build tasks to Socorro for later use in crash analysis.
 
+upload-generated-sources
+--------------
+
+Upload-generated-sources tasks run after builds and upload source files that were generated as part of the build process to an s3 bucket for later use in links from crash reports or when debugging shipped builds.
+
 valgrind
 --------
 

taskcluster/taskgraph/transforms/upload_generated_sources.py

@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the upload-generated-files task description template,
+  taskcluster/ci/upload-generated-sources/kind.yml
+into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.taskcluster import get_artifact_url
+
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def add_task_info(config, jobs):
+    for job in jobs:
+        dep_task = job['dependent-task']
+        del job['dependent-task']
+
+        # Add a dependency on the build task.
+        job['dependencies'] = {'build': dep_task.label}
+        # Label the job to match the build task it's uploading from.
+        job['label'] = dep_task.label.replace("build-", "upload-generated-sources-")
+        # Copy over some bits of treeherder metdata from the build task.
+        dep_th = dep_task.task['extra']['treeherder']
+        plat = '{}/{}'.format(dep_th['machine']['platform'], dep_task.attributes.get('build_type'))
+        job['treeherder']['platform'] = plat
+        job['treeherder']['tier'] = dep_th['tier']
+        # Add an environment variable pointing at the artifact from the build.
+        artifact_url = get_artifact_url('<build>',
+                                        'public/build/target.generated-files.tar.gz')
+        job['worker'].setdefault('env', {})['ARTIFACT_URL'] = {
+            'task-reference': artifact_url
+        }
+
+        yield job