merge mozilla-central to mozilla-inbound. r=merge a=merge

browser/app/blocklist.xml

@@ -3991,6 +3991,9 @@
     <certItem issuerName="MIGVMQswCQYDVQQGEwJVUzELMAkGA1UECBMCVVQxFzAVBgNVBAcTDlNhbHQgTGFrZSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxITAfBgNVBAsTGGh0dHA6Ly93d3cudXNlcnRydXN0LmNvbTEdMBsGA1UEAxMUVVROLVVTRVJGaXJzdC1PYmplY3Q=">
       <serialNumber>CMNfzETd7XxesS9FOUj9Mg==</serialNumber>
     </certItem>
+    <certItem issuerName="MGsxCzAJBgNVBAYTAlVTMQ0wCwYDVQQKEwRWSVNBMS8wLQYDVQQLEyZWaXNhIEludGVybmF0aW9uYWwgU2VydmljZSBBc3NvY2lhdGlvbjEcMBoGA1UEAxMTVmlzYSBlQ29tbWVyY2UgUm9vdA==">
+      <serialNumber>B2VhZAPxCDH3s9Mkbu3HfQ==</serialNumber>
+    </certItem>
     <certItem issuerName="MEUxCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBMaW1pdGVkMRswGQYDVQQDExJRdW9WYWRpcyBSb290IENBIDM=">
       <serialNumber>CLc=</serialNumber>
     </certItem>
@@ -4603,6 +4606,9 @@
     <certItem issuerName="MDMxCzAJBgNVBAYTAlBUMQ0wCwYDVQQKDARTQ0VFMRUwEwYDVQQDDAxFQ1JhaXpFc3RhZG8=">
       <serialNumber>cx0HrIEQg8JHWTP7DzOxSQ==</serialNumber>
     </certItem>
+    <certItem issuerName="MD8xJDAiBgNVBAoTG0RpZ2l0YWwgU2lnbmF0dXJlIFRydXN0IENvLjEXMBUGA1UEAxMORFNUIFJvb3QgQ0EgWDM=">
+      <serialNumber>ANUANvVYN7xqAISA9rvJPzQ=</serialNumber>
+    </certItem>
     <certItem issuerName="MGQxCzAJBgNVBAYTAmNoMREwDwYDVQQKEwhTd2lzc2NvbTElMCMGA1UECxMcRGlnaXRhbCBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczEbMBkGA1UEAxMSU3dpc3Njb20gUm9vdCBDQSAy">
       <serialNumber>GpO48aJ8GngtwECqZhm/xA==</serialNumber>
     </certItem>
@@ -4927,6 +4933,9 @@
     <certItem issuerName="MEwxIDAeBgNVBAsTF0dsb2JhbFNpZ24gUm9vdCBDQSAtIFIyMRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWdu">
       <serialNumber>BAAAAAABEAuMoRs=</serialNumber>
     </certItem>
+    <certItem issuerName="MD8xJDAiBgNVBAoTG0RpZ2l0YWwgU2lnbmF0dXJlIFRydXN0IENvLjEXMBUGA1UEAxMORFNUIFJvb3QgQ0EgWDM=">
+      <serialNumber>CgFBQQAAATjtdPY5AAAAAg==</serialNumber>
+    </certItem>
     <certItem issuerName="MGExCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMR0wGwYDVQQLExREb21haW4gVmFsaWRhdGVkIFNTTDEbMBkGA1UEAxMSR2VvVHJ1c3QgRFYgU1NMIENB">
       <serialNumber>CWhp</serialNumber>
     </certItem>

python/mozbuild/mozbuild/repackaging/mar.py

@@ -7,28 +7,39 @@ import sys
 import tempfile
 import shutil
 import zipfile
+import tarfile
 import subprocess
 import mozpack.path as mozpath
 from application_ini import get_application_ini_value
 from mozbuild.util import ensureParentDir
 
+
 def repackage_mar(topsrcdir, package, mar, output):
-    if not zipfile.is_zipfile(package):
-        raise Exception("Package file %s is not a valid .zip file." % package)
+    if not zipfile.is_zipfile(package) and not tarfile.is_tarfile(package):
+        raise Exception("Package file %s is not a valid .zip or .tar file." % package)
 
     ensureParentDir(output)
     tmpdir = tempfile.mkdtemp()
     try:
-        z = zipfile.ZipFile(package)
-        z.extractall(tmpdir)
-        filelist = z.namelist()
-        z.close()
+        if zipfile.is_zipfile(package):
+            z = zipfile.ZipFile(package)
+            z.extractall(tmpdir)
+            filelist = z.namelist()
+            z.close()
+        else:
+            z = tarfile.open(package)
+            z.extractall(tmpdir)
+            filelist = z.getnames()
+            z.close()
 
+        toplevel_dirs = set([mozpath.split(f)[0] for f in filelist])
+        excluded_stuff = set([' ', '.background', '.DS_Store', '.VolumeIcon.icns'])
+        toplevel_dirs = toplevel_dirs - excluded_stuff
         # Make sure the .zip file just contains a directory like 'firefox/' at
         # the top, and find out what it is called.
-        toplevel_dirs = set([mozpath.split(f)[0] for f in filelist])
         if len(toplevel_dirs) != 1:
-            raise Exception("Package file is expected to have a single top-level directory (eg: 'firefox'), not: %s" % toplevel_dirs)
+            raise Exception("Package file is expected to have a single top-level directory"
+                            "(eg: 'firefox'), not: %s" % toplevel_dirs)
         ffxdir = mozpath.join(tmpdir, toplevel_dirs.pop())
 
         make_full_update = mozpath.join(topsrcdir, 'tools/update-packaging/make_full_update.sh')

security/manager/ssl/StaticHPKPins.h

@@ -1161,4 +1161,4 @@ static const TransportSecurityPreload kPublicKeyPinningPreloadList[] = {
 
 static const int32_t kUnknownId = -1;
 
-static const PRTime kPreloadPKPinsExpirationTime = INT64_C(1507044980295000);
+static const PRTime kPreloadPKPinsExpirationTime = INT64_C(1507150328324000);

taskcluster/ci/balrog/kind.yml

@@ -12,7 +12,6 @@ kind-dependencies:
   - beetmover
   - beetmover-l10n
   - beetmover-repackage
-  - beetmover-repackage-l10n
 
 only-for-attributes:
   - nightly

taskcluster/ci/beetmover-repackage/kind.yml

@@ -5,11 +5,12 @@
 loader: taskgraph.loader.single_dep:loader
 
 transforms:
+   - taskgraph.transforms.beetmover_repackage_l10n:transforms
    - taskgraph.transforms.beetmover_repackage:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-  - repackage
+  - repackage-signing
 
 only-for-build-platforms:
   - macosx64-nightly/opt

taskcluster/ci/checksums-signing/kind.yml

@@ -12,7 +12,6 @@ kind-dependencies:
   - beetmover
   - beetmover-l10n
   - beetmover-repackage
-  - beetmover-repackage-l10n
 
 only-for-attributes:
   - nightly

taskcluster/ci/repackage-signing/kind.yml

@@ -5,11 +5,11 @@
 loader: taskgraph.loader.single_dep:loader
 
 transforms:
-   - taskgraph.transforms.beetmover_repackage_l10n:transforms
-   - taskgraph.transforms.beetmover_repackage:transforms
+   - taskgraph.transforms.repackage_signing:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
+  - repackage
   - repackage-l10n
 
 only-for-build-platforms:

taskcluster/docs/kinds.rst

@@ -195,16 +195,6 @@ Beetmover-repackage is beetmover but for tasks that need an intermediate step
 between signing and packaging, such as OSX. For more details see the definitions
 of the Beetmover kind above and the repackage kind below.
 
-beetmover-repackage-l10n
-------------------------
-Same as beetmover-repackage but for l10n.
-
-Beetmover L10n, takes specific artifacts, "Beets", and pushes them to a location outside
-of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the
-process determines the final location and a "pretty" name (versioned product name)
-This separate kind uses logic specific to localized artifacts, such as including
-the language in the final artifact names.
-
 checksums-signing
 -----------------
 Checksums-signing take as input the checksums file generated by beetmover tasks
@@ -239,3 +229,9 @@ and this task would package that up as an Apple Disk Image (.dmg)
 repackage-l10n
 --------------
 Repackage-L10n is a ```Repackage``` task split up to be suitable for use after l10n repacks.
+
+
+repackage-signing
+-----------------
+Repackage-signing take the repackaged installers (windows) and update packaging (with
+the signed internal bits) and signs them.

taskcluster/taskgraph/transforms/beetmover_repackage.py

@@ -47,24 +47,10 @@ _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
-_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US = [
-    "update/target.complete.mar",
-]
-# Until bug 1331141 is fixed, if you are adding any new artifacts here that
-# need to be transfered to S3, please be aware you also need to follow-up
-# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
-# See example in bug 1348286
 _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
     "target.langpack.xpi",
     "balrog_props.json",
 ]
-# Until bug 1331141 is fixed, if you are adding any new artifacts here that
-# need to be transfered to S3, please be aware you also need to follow-up
-# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
-# See example in bug 1348286
-_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N = [
-    "target.complete.mar",
-]
 
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
@@ -78,17 +64,17 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
-UPSTREAM_ARTIFACT_SIGNED_PATHS = {
-    'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US,
-    'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N,
+UPSTREAM_ARTIFACT_REPACKAGE_PATHS = {
+    'macosx64-nightly': ["target.dmg"],
+    'macosx64-nightly-l10n': ["target.dmg"],
 }
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
-UPSTREAM_ARTIFACT_REPACKAGE_PATHS = {
-    'macosx64-nightly': ["target.dmg"],
-    'macosx64-nightly-l10n': ["target.dmg"],
+UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS = {
+    'macosx64-nightly': ["target.complete.mar"],
+    'macosx64-nightly-l10n': ["target.complete.mar"],
 }
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
@@ -145,6 +131,7 @@ def make_task_description(config, jobs):
         treeherder.setdefault('tier', 1)
         treeherder.setdefault('kind', 'build')
         label = job.get('label', "beetmover-{}".format(dep_job.label))
+
         dependent_kind = str(dep_job.kind)
         dependencies = {dependent_kind: dep_job.label}
 
@@ -152,9 +139,10 @@ def make_task_description(config, jobs):
         # images and thus have two dependencies
         # change the signing_dependencies to be use the ones in
         docker_dependencies = {"docker-image":
-                               dep_job.dependencies["docker-image"]
+                               dep_job.dependencies['docker-image']
                                }
         dependencies.update(docker_dependencies)
+
         signing_name = "build-signing"
         if job.get('locale'):
             signing_name = "nightly-l10n-signing"
@@ -171,6 +159,13 @@ def make_task_description(config, jobs):
                               }
         dependencies.update(build_dependencies)
 
+        repackage_name = "repackage"
+        # repackage-l10n actually uses the repackage depname here
+        repackage_dependencies = {"repackage":
+                                  dep_job.dependencies[repackage_name]
+                                  }
+        dependencies.update(repackage_dependencies)
+
         attributes = {
             'nightly': dep_job.attributes.get('nightly', False),
             'signed': dep_job.attributes.get('signed', False),
@@ -198,13 +193,13 @@ def make_task_description(config, jobs):
         yield task
 
 
-def generate_upstream_artifacts(signing_task_ref, build_task_ref,
-                                repackage_task_ref, platform,
+def generate_upstream_artifacts(build_task_ref, repackage_task_ref,
+                                repackage_signing_task_ref, platform,
                                 locale=None):
 
-    signing_mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS
     build_mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS
     repackage_mapping = UPSTREAM_ARTIFACT_REPACKAGE_PATHS
+    repackage_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS
 
     artifact_prefix = 'public/build'
     if locale:
@@ -218,17 +213,17 @@ def generate_upstream_artifacts(signing_task_ref, build_task_ref,
                   for p in build_mapping[platform]],
         "locale": locale or "en-US",
         }, {
-        "taskId": {"task-reference": signing_task_ref},
-        "taskType": "signing",
-        "paths": ["{}/{}".format(artifact_prefix, p)
-                  for p in signing_mapping[platform]],
-        "locale": locale or "en-US",
-        }, {
         "taskId": {"task-reference": repackage_task_ref},
         "taskType": "repackage",
         "paths": ["{}/{}".format(artifact_prefix, p)
                   for p in repackage_mapping[platform]],
         "locale": locale or "en-US",
+        }, {
+        "taskId": {"task-reference": repackage_signing_task_ref},
+        "taskType": "repackage",
+        "paths": ["{}/{}".format(artifact_prefix, p)
+                  for p in repackage_signing_mapping[platform]],
+        "locale": locale or "en-US",
     }]
 
     return upstream_artifacts
@@ -237,29 +232,32 @@ def generate_upstream_artifacts(signing_task_ref, build_task_ref,
 @transforms.add
 def make_task_worker(config, jobs):
     for job in jobs:
-        valid_beetmover_job = (len(job["dependencies"]) == 4 and
+        valid_beetmover_job = (len(job["dependencies"]) == 5 and
                                any(['repackage' in j for j in job['dependencies']]))
         if not valid_beetmover_job:
-            raise NotImplementedError("Beetmover_repackage must have four dependencies.")
+            raise NotImplementedError("Beetmover_repackage must have five dependencies.")
 
         locale = job["attributes"].get("locale")
         platform = job["attributes"]["build_platform"]
         build_task = None
         repackage_task = None
-        signing_task = None
+        repackage_signing_task = None
         for dependency in job["dependencies"].keys():
-            if 'repackage' in dependency:
+            if 'repackage-signing' in dependency:
+                repackage_signing_task = dependency
+            elif 'repackage' in dependency:
                 repackage_task = dependency
             elif 'signing' in dependency:
-                signing_task = dependency
+                pass
             else:
                 build_task = "build"
 
-        signing_task_ref = "<" + str(signing_task) + ">"
         build_task_ref = "<" + str(build_task) + ">"
         repackage_task_ref = "<" + str(repackage_task) + ">"
+        repackage_signing_task_ref = "<" + str(repackage_signing_task) + ">"
         upstream_artifacts = generate_upstream_artifacts(
-            signing_task_ref, build_task_ref, repackage_task_ref, platform, locale
+            build_task_ref, repackage_task_ref,
+            repackage_signing_task_ref, platform, locale
         )
 
         worker = {'implementation': 'beetmover',

taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py

@@ -20,7 +20,8 @@ def make_beetmover_description(config, jobs):
 
         locale = dep_job.attributes.get('locale')
         if not locale:
-            return
+            yield job
+            continue
 
         label = job.get('label',
                         "beetmover-{}-{}".format(locale, dep_job.label))

taskcluster/taskgraph/transforms/build_signing.py

@@ -49,9 +49,6 @@ def make_signing_description(config, jobs):
                 {
                    'artifacts': ['public/build/target.dmg'],
                    'format': 'macapp',
-                }, {
-                   'artifacts': ['public/build/update/target.complete.mar'],
-                   'format': 'mar',
                 },
             ]
         else:

taskcluster/taskgraph/transforms/repackage.py

@@ -91,11 +91,23 @@ def make_job_description(config, jobs):
         treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
         treeherder.setdefault('tier', 1)
         treeherder.setdefault('kind', 'build')
+        build_task = None
         signing_task = None
         for dependency in dependencies.keys():
             if 'signing' in dependency:
                 signing_task = dependency
+            else:
+                build_task = dependency
+        if job.get('locale'):
+            # XXXCallek: todo: rewrite dependency finding
+            # Use string splice to strip out 'nightly-l10n-' .. '-<chunk>/opt'
+            # We need this additional dependency to support finding the mar binary
+            # Which is needed in order to generate a new complete.mar
+            dependencies['build'] = "build-{}/opt".format(
+                dependencies[build_task][13:dependencies[build_task].rfind('-')])
+            build_task = 'build'
         signing_task_ref = "<{}>".format(signing_task)
+        build_task_ref = "<{}>".format(build_task)
 
         attributes = {
             'nightly': dep_job.attributes.get('nightly', False),
@@ -109,6 +121,8 @@ def make_job_description(config, jobs):
 
         task_env = {}
         locale_output_path = ""
+        mar_prefix = 'https://queue.taskcluster.net/v1/task/' + \
+            '{}/artifacts/public/build/host/bin/'.format(build_task_ref)
         if attributes['build_platform'].startswith('macosx'):
             if job.get('locale'):
                 input_string = 'https://queue.taskcluster.net/v1/task/' + \
@@ -120,12 +134,17 @@ def make_job_description(config, jobs):
                     '{}/artifacts/public/build/target.tar.gz'.format(signing_task_ref)
             task_env.update(
                 SIGNED_INPUT={'task-reference': input_string},
+                UNSIGNED_MAR={'task-reference': "{}mar".format(mar_prefix)},
             )
             mozharness_config = ['repackage/osx_signed.py']
             output_files = [{
                 'type': 'file',
                 'path': '/home/worker/workspace/build/artifacts/target.dmg',
                 'name': 'public/build/{}target.dmg'.format(locale_output_path),
+            }, {
+                'type': 'file',
+                'path': '/home/worker/workspace/build/artifacts/target.complete.mar',
+                'name': 'public/build/{}target.complete.mar'.format(locale_output_path),
             }]
         else:
             raise Exception("Unexpected build platform for repackage")

taskcluster/taskgraph/transforms/repackage_signing.py

@@ -0,0 +1,98 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the repackage signing task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.scriptworker import get_signing_cert_scope
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Required, Optional
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+transforms = TransformSequence()
+
+repackage_signing_description_schema = Schema({
+    Required('dependent-task'): object,
+    Required('depname', default='repackage'): basestring,
+    Optional('label'): basestring,
+    Optional('treeherder'): task_description_schema['treeherder'],
+})
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        yield validate_schema(
+            repackage_signing_description_schema, job,
+            "In repackage-signing ({!r} kind) task for {!r}:".format(config.kind, label))
+
+
+@transforms.add
+def make_repackage_signing_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'tc-rs(N)')
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('tier', 1)
+        treeherder.setdefault('kind', 'build')
+
+        label = job.get('label', "repackage-signing-{}".format(dep_job.label))
+        dependencies = {"repackage": dep_job.label}
+
+        signing_dependencies = dep_job.dependencies
+        # This is so we get the build task etc in our dependencies to
+        # have better beetmover support.
+        dependencies.update(signing_dependencies)
+        attributes = {
+            'nightly': dep_job.attributes.get('nightly', False),
+            'build_platform': dep_job.attributes.get('build_platform'),
+            'build_type': dep_job.attributes.get('build_type'),
+        }
+        locale_str = ""
+        if dep_job.attributes.get('locale'):
+            treeherder['symbol'] = 'tc-rs({})'.format(dep_job.attributes.get('locale'))
+            attributes['locale'] = dep_job.attributes.get('locale')
+            locale_str = "{}/".format(dep_job.attributes.get('locale'))
+
+        scopes = [get_signing_cert_scope(config),
+                  "project:releng:signing:format:mar"]
+
+        upstream_artifacts = [{
+            "taskId": {"task-reference": "<repackage>"},
+            "taskType": "repackage",
+            "paths": [
+                "public/build/{}target.complete.mar".format(locale_str),
+            ],
+            "formats": ["mar"]
+        }]
+
+        task = {
+            'label': label,
+            'description': "Repackage signing {} ".format(
+                dep_job.task["metadata"]["description"]),
+            'worker-type': "scriptworker-prov-v1/signing-linux-v1",
+            'worker': {'implementation': 'scriptworker-signing',
+                       'upstream-artifacts': upstream_artifacts,
+                       'max-run-time': 3600},
+            'scopes': scopes,
+            'dependencies': dependencies,
+            'attributes': attributes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+        }
+
+        yield task

taskcluster/taskgraph/transforms/task.py

@@ -454,6 +454,7 @@ GROUP_NAMES = {
     'tc-BMR-L10n': 'Beetmover repackages for locales executed by Taskcluster',
     'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
     'tc-cs': 'Checksum signing executed by Taskcluster',
+    'tc-rs': 'Repackage signing executed by Taskcluster',
     'tc-BMcs': 'Beetmover checksums, executed by Taskcluster',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',

testing/mozharness/configs/repackage/osx_signed.py

@@ -2,16 +2,23 @@ import os
 
 config = {
     "input_home": "{abs_work_dir}/inputs",
+    "output_home": "{abs_work_dir}/artifacts",
     "src_mozconfig": "browser/config/mozconfigs/macosx64/repack",
 
     "download_config": {
         "target.tar.gz": os.environ.get("SIGNED_INPUT"),
+        "mar": os.environ.get("UNSIGNED_MAR"),
     },
 
     "repackage_config": [[
         "dmg",
         "-i", "{abs_work_dir}/inputs/target.tar.gz",
         "-o", "{output_home}/target.dmg"
+    ], [
+        "mar",
+        "-i", "{abs_work_dir}/inputs/target.tar.gz",
+        "--mar", "{abs_work_dir}/inputs/mar",
+        "-o", "{output_home}/target.complete.mar"
     ]],
 
     # ToolTool

testing/mozharness/scripts/repackage.py

@@ -37,6 +37,10 @@ class Repackage(BaseScript):
             if not status:
                 self.fatal("Unable to fetch signed input from %s" % url)
 
+            if 'mar' in path:
+                # Ensure mar is executable
+                self.chmod(os.path.join(input_home, path), 0755)
+
     def setup(self):
         self._run_tooltool()
         self._get_mozconfig()
@@ -46,13 +50,14 @@ class Repackage(BaseScript):
         if self.abs_dirs:
             return self.abs_dirs
         abs_dirs = super(Repackage, self).query_abs_dirs()
+        config = self.config
         for directory in abs_dirs:
             value = abs_dirs[directory]
             abs_dirs[directory] = value
         dirs = {}
         dirs['abs_tools_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tools')
         dirs['abs_mozilla_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'src')
-        dirs['output_home'] = os.path.join(abs_dirs['abs_work_dir'], 'artifacts')
+        dirs['output_home'] = config['output_home'].format(**abs_dirs)
         for key in dirs.keys():
             if key not in abs_dirs:
                 abs_dirs[key] = dirs[key]

widget/android/AndroidBridge.cpp

@@ -153,6 +153,7 @@ AndroidBridge::~AndroidBridge()
 }
 
 AndroidBridge::AndroidBridge()
+  : mUiTaskQueueLock("UiTaskQueue")
 {
     ALOG_BRIDGE("AndroidBridge::Init");
 
@@ -946,6 +947,109 @@ AndroidBridge::IsContentDocumentDisplayed(mozIDOMWindowProxy* aWindow)
     return layerClient->IsContentDocumentDisplayed();
 }
 
+class AndroidBridge::DelayedTask
+{
+    using TimeStamp = mozilla::TimeStamp;
+    using TimeDuration = mozilla::TimeDuration;
+
+public:
+    DelayedTask(already_AddRefed<nsIRunnable> aTask)
+        : mTask(aTask)
+        , mRunTime() // Null timestamp representing no delay.
+    {}
+
+    DelayedTask(already_AddRefed<nsIRunnable> aTask, int aDelayMs)
+        : mTask(aTask)
+        , mRunTime(TimeStamp::Now() + TimeDuration::FromMilliseconds(aDelayMs))
+    {}
+
+    bool IsEarlierThan(const DelayedTask& aOther) const
+    {
+        if (mRunTime) {
+            return aOther.mRunTime ? mRunTime < aOther.mRunTime : false;
+        }
+        // In the case of no delay, we're earlier if aOther has a delay.
+        // Otherwise, we're not earlier, to maintain task order.
+        return !!aOther.mRunTime;
+    }
+
+    int64_t MillisecondsToRunTime() const
+    {
+        if (mRunTime) {
+            return int64_t((mRunTime - TimeStamp::Now()).ToMilliseconds());
+        }
+        return 0;
+    }
+
+    already_AddRefed<nsIRunnable> TakeTask()
+    {
+        return mTask.forget();
+    }
+
+private:
+    nsCOMPtr<nsIRunnable> mTask;
+    const TimeStamp mRunTime;
+};
+
+
+void
+AndroidBridge::PostTaskToUiThread(already_AddRefed<nsIRunnable> aTask, int aDelayMs)
+{
+    // add the new task into the mUiTaskQueue, sorted with
+    // the earliest task first in the queue
+    size_t i;
+    DelayedTask newTask(aDelayMs ? DelayedTask(mozilla::Move(aTask), aDelayMs)
+                                 : DelayedTask(mozilla::Move(aTask)));
+
+    {
+        MutexAutoLock lock(mUiTaskQueueLock);
+
+        for (i = 0; i < mUiTaskQueue.Length(); i++) {
+            if (newTask.IsEarlierThan(mUiTaskQueue[i])) {
+                mUiTaskQueue.InsertElementAt(i, mozilla::Move(newTask));
+                break;
+            }
+        }
+
+        if (i == mUiTaskQueue.Length()) {
+            // We didn't insert the task, which means we should append it.
+            mUiTaskQueue.AppendElement(mozilla::Move(newTask));
+        }
+    }
+
+    if (i == 0) {
+        // if we're inserting it at the head of the queue, notify Java because
+        // we need to get a callback at an earlier time than the last scheduled
+        // callback
+        GeckoThread::RequestUiThreadCallback(int64_t(aDelayMs));
+    }
+}
+
+int64_t
+AndroidBridge::RunDelayedUiThreadTasks()
+{
+    MutexAutoLock lock(mUiTaskQueueLock);
+
+    while (!mUiTaskQueue.IsEmpty()) {
+        const int64_t timeLeft = mUiTaskQueue[0].MillisecondsToRunTime();
+        if (timeLeft > 0) {
+            // this task (and therefore all remaining tasks)
+            // have not yet reached their runtime. return the
+            // time left until we should be called again
+            return timeLeft;
+        }
+
+        // Retrieve task before unlocking/running.
+        nsCOMPtr<nsIRunnable> nextTask(mUiTaskQueue[0].TakeTask());
+        mUiTaskQueue.RemoveElementAt(0);
+
+        // Unlock to allow posting new tasks reentrantly.
+        MutexAutoUnlock unlock(mUiTaskQueueLock);
+        nextTask->Run();
+    }
+    return -1;
+}
+
 Object::LocalRef AndroidBridge::ChannelCreate(Object::Param stream) {
     JNIEnv* const env = GetEnvForThread();
     auto rv = Object::LocalRef::Adopt(env, env->CallStaticObjectMethod(

widget/android/AndroidBridge.h

@@ -224,6 +224,15 @@ protected:
     jni::Object::GlobalRef mMessageQueue;
     jfieldID mMessageQueueMessages;
     jmethodID mMessageQueueNext;
+
+private:
+    class DelayedTask;
+    nsTArray<DelayedTask> mUiTaskQueue;
+    mozilla::Mutex mUiTaskQueueLock;
+
+public:
+    void PostTaskToUiThread(already_AddRefed<nsIRunnable> aTask, int aDelayMs);
+    int64_t RunDelayedUiThreadTasks();
 };
 
 class AutoJNIClass {

widget/android/AndroidUiThread.cpp

@@ -3,15 +3,12 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+#include "AndroidBridge.h"
 #include "base/message_loop.h"
-#include "GeneratedJNIWrappers.h"
 #include "mozilla/Atomics.h"
-#include "mozilla/LinkedList.h"
 #include "mozilla/Monitor.h"
-#include "mozilla/Mutex.h"
 #include "mozilla/RefPtr.h"
 #include "mozilla/StaticPtr.h"
-#include "mozilla/TimeStamp.h"
 #include "nsThread.h"
 #include "nsThreadManager.h"
 #include "nsThreadUtils.h"
@@ -21,22 +18,17 @@ using namespace mozilla;
 namespace {
 
 class AndroidUiThread;
-class AndroidUiTask;
 
-StaticAutoPtr<LinkedList<AndroidUiTask> > sTaskQueue;
-StaticAutoPtr<mozilla::Mutex> sTaskQueueLock;
 StaticRefPtr<AndroidUiThread> sThread;
 static bool sThreadDestroyed;
 static MessageLoop* sMessageLoop;
 static Atomic<Monitor*> sMessageLoopAccessMonitor;
 
-void EnqueueTask(already_AddRefed<nsIRunnable> aTask, int aDelayMs);
-
 /*
  * The AndroidUiThread is derived from nsThread so that nsIRunnable objects that get
  * dispatched may be intercepted. Only nsIRunnable objects that need to be synchronously
  * executed are passed into the nsThread to be queued. All other nsIRunnable object
- * are immediately dispatched to the Android UI thread.
+ * are immediately dispatched to the Android UI thread via the AndroidBridge.
  * AndroidUiThread is derived from nsThread instead of being an nsIEventTarget
  * wrapper that contains an nsThread object because if nsIRunnable objects with a
  * delay were dispatch directly to an nsThread object, such as obtained from
@@ -56,6 +48,14 @@ public:
   nsresult Dispatch(already_AddRefed<nsIRunnable> aEvent, uint32_t aFlags) override;
   nsresult DelayedDispatch(already_AddRefed<nsIRunnable> aEvent, uint32_t aDelayMs) override;
 
+  static int64_t RunDelayedTasksIfValid() {
+    if (!AndroidBridge::Bridge() ||
+        sThreadDestroyed) {
+      return -1;
+    }
+    return AndroidBridge::Bridge()->RunDelayedUiThreadTasks();
+  }
+
 private:
   ~AndroidUiThread()
   {}
@@ -69,7 +69,7 @@ AndroidUiThread::Dispatch(already_AddRefed<nsIRunnable> aEvent, uint32_t aFlags)
   if (aFlags & NS_DISPATCH_SYNC) {
     return nsThread::Dispatch(Move(aEvent), aFlags);
   } else {
-    EnqueueTask(Move(aEvent), 0);
+    AndroidBridge::Bridge()->PostTaskToUiThread(Move(aEvent), 0);
     return NS_OK;
   }
 }
@@ -77,7 +77,7 @@ AndroidUiThread::Dispatch(already_AddRefed<nsIRunnable> aEvent, uint32_t aFlags)
 NS_IMETHODIMP
 AndroidUiThread::DelayedDispatch(already_AddRefed<nsIRunnable> aEvent, uint32_t aDelayMs)
 {
-  EnqueueTask(Move(aEvent), aDelayMs);
+  AndroidBridge::Bridge()->PostTaskToUiThread(Move(aEvent), aDelayMs);
   return NS_OK;
 }
 
@@ -105,7 +105,7 @@ NS_IMPL_ISUPPORTS(ThreadObserver, nsIThreadObserver)
 NS_IMETHODIMP
 ThreadObserver::OnDispatchedEvent(nsIThreadInternal *thread)
 {
-  EnqueueTask(NS_NewRunnableFunction("PumpEvents", &PumpEvents), 0);
+  AndroidBridge::Bridge()->PostTaskToUiThread(NS_NewRunnableFunction("PumpEvents", &PumpEvents), 0);
   return NS_OK;
 }
 
@@ -121,49 +121,6 @@ ThreadObserver::AfterProcessNextEvent(nsIThreadInternal *thread, bool eventWasPr
   return NS_OK;
 }
 
-class AndroidUiTask : public LinkedListElement<AndroidUiTask> {
-    using TimeStamp = mozilla::TimeStamp;
-    using TimeDuration = mozilla::TimeDuration;
-
-public:
-  AndroidUiTask(already_AddRefed<nsIRunnable> aTask)
-    : mTask(aTask)
-    , mRunTime() // Null timestamp representing no delay.
-  {}
-
-  AndroidUiTask(already_AddRefed<nsIRunnable> aTask, int aDelayMs)
-    : mTask(aTask)
-    , mRunTime(TimeStamp::Now() + TimeDuration::FromMilliseconds(aDelayMs))
-  {}
-
-  bool IsEarlierThan(const AndroidUiTask& aOther) const
-  {
-    if (mRunTime) {
-      return aOther.mRunTime ? mRunTime < aOther.mRunTime : false;
-    }
-    // In the case of no delay, we're earlier if aOther has a delay.
-    // Otherwise, we're not earlier, to maintain task order.
-    return !!aOther.mRunTime;
-  }
-
-  int64_t MillisecondsToRunTime() const
-  {
-    if (mRunTime) {
-      return int64_t((mRunTime - TimeStamp::Now()).ToMilliseconds());
-    }
-    return 0;
-  }
-
-  already_AddRefed<nsIRunnable> TakeTask()
-  {
-      return mTask.forget();
-  }
-
-private:
-  nsCOMPtr<nsIRunnable> mTask;
-  const TimeStamp mRunTime;
-};
-
 class CreateOnUiThread : public Runnable {
 public:
   CreateOnUiThread() : Runnable("CreateOnUiThread")
@@ -190,17 +147,7 @@ public:
   NS_IMETHOD Run() override {
     MOZ_ASSERT(!sThreadDestroyed);
     MOZ_ASSERT(sMessageLoopAccessMonitor);
-    MOZ_ASSERT(sTaskQueue);
     MonitorAutoLock lock(*sMessageLoopAccessMonitor);
-    sThreadDestroyed = true;
-
-    {
-      // Flush the queue
-      MutexAutoLock lock (*sTaskQueueLock);
-      while (AndroidUiTask* task = sTaskQueue->getFirst()) {
-        delete task;
-      }
-    }
 
     delete sMessageLoop;
     sMessageLoop = nullptr;
@@ -208,6 +155,7 @@ public:
     nsThreadManager::get().UnregisterCurrentThread(*sThread);
     sThread = nullptr;
     mDestroyed = true;
+    sThreadDestroyed = true;
     lock.NotifyAll();
     return NS_OK;
   }
@@ -225,47 +173,6 @@ private:
   bool mDestroyed;
 };
 
-void
-EnqueueTask(already_AddRefed<nsIRunnable> aTask, int aDelayMs)
-{
-
-  if (sThreadDestroyed) {
-    return;
-  }
-
-  // add the new task into the sTaskQueue, sorted with
-  // the earliest task first in the queue
-  AndroidUiTask* newTask = (aDelayMs ? new AndroidUiTask(mozilla::Move(aTask), aDelayMs)
-                                 : new AndroidUiTask(mozilla::Move(aTask)));
-
-  {
-    MOZ_ASSERT(sTaskQueue);
-    MOZ_ASSERT(sTaskQueueLock);
-    MutexAutoLock lock(*sTaskQueueLock);
-
-    AndroidUiTask* task = sTaskQueue->getFirst();
-
-    while (task) {
-      if (newTask->IsEarlierThan(*task)) {
-        task->setPrevious(newTask);
-        break;
-      }
-      task = task->getNext();
-    }
-
-    if (!newTask->isInList()) {
-      sTaskQueue->insertBack(newTask);
-    }
-  }
-
-  if (!newTask->getPrevious()) {
-    // if we're inserting it at the head of the queue, notify Java because
-    // we need to get a callback at an earlier time than the last scheduled
-    // callback
-    GeckoThread::RequestUiThreadCallback(int64_t(aDelayMs));
-  }
-}
-
 } // namespace
 
 namespace mozilla {
@@ -275,20 +182,20 @@ CreateAndroidUiThread()
 {
   MOZ_ASSERT(!sThread);
   MOZ_ASSERT(!sMessageLoopAccessMonitor);
-  sTaskQueue = new LinkedList<AndroidUiTask>();
-  sTaskQueueLock = new Mutex("AndroidUiThreadTaskQueueLock");
   sMessageLoopAccessMonitor = new Monitor("AndroidUiThreadMessageLoopAccessMonitor");
   sThreadDestroyed = false;
   RefPtr<CreateOnUiThread> runnable = new CreateOnUiThread;
-  EnqueueTask(do_AddRef(runnable), 0);
+  AndroidBridge::Bridge()->PostTaskToUiThread(do_AddRef(runnable), 0);
 }
 
 void
 DestroyAndroidUiThread()
 {
   MOZ_ASSERT(sThread);
+  // Insure the Android bridge has not already been deconstructed.
+  MOZ_ASSERT(AndroidBridge::Bridge() != nullptr);
   RefPtr<DestroyOnUiThread> runnable = new DestroyOnUiThread;
-  EnqueueTask(do_AddRef(runnable), 0);
+  AndroidBridge::Bridge()->PostTaskToUiThread(do_AddRef(runnable), 0);
   runnable->WaitForDestruction();
   delete sMessageLoopAccessMonitor;
   sMessageLoopAccessMonitor = nullptr;
@@ -324,38 +231,4 @@ GetAndroidUiThread()
   return sThread;
 }
 
-int64_t
-RunAndroidUiTasks()
-{
-  MutexAutoLock lock(*sTaskQueueLock);
-
-  if (sThreadDestroyed) {
-    return -1;
-  }
-
-  while (!sTaskQueue->isEmpty()) {
-    AndroidUiTask* task = sTaskQueue->getFirst();
-    const int64_t timeLeft = task->MillisecondsToRunTime();
-    if (timeLeft > 0) {
-      // this task (and therefore all remaining tasks)
-      // have not yet reached their runtime. return the
-      // time left until we should be called again
-      return timeLeft;
-    }
-
-    // Retrieve task before unlocking/running.
-    nsCOMPtr<nsIRunnable> runnable(task->TakeTask());
-    // LinkedListElements auto remove from list upon destruction
-    delete task;
-
-    // Unlock to allow posting new tasks reentrantly.
-    MutexAutoUnlock unlock(*sTaskQueueLock);
-    runnable->Run();
-    if (sThreadDestroyed) {
-      return -1;
-    }
-  }
-  return -1;
-}
-
 } // namespace mozilla

widget/android/AndroidUiThread.h

@@ -15,7 +15,6 @@ namespace mozilla {
 
 void CreateAndroidUiThread();
 void DestroyAndroidUiThread();
-int64_t RunAndroidUiTasks();
 
 MessageLoop* GetAndroidUiThreadMessageLoop();
 RefPtr<nsThread> GetAndroidUiThread();

widget/android/nsAppShell.cpp

@@ -242,7 +242,7 @@ public:
 
     static int64_t RunUiThreadCallback()
     {
-        return RunAndroidUiTasks();
+        return AndroidUiThread::RunDelayedTasksIfValid();
     }
 };
 

widget/android/nsWindow.cpp

@@ -422,13 +422,9 @@ public:
         };
 
         NativePanZoomController::GlobalRef npzc = mNPZC;
-        RefPtr<nsThread> uiThread = GetAndroidUiThread();
-        if (!uiThread) {
-            return;
-        }
-        uiThread->Dispatch(NewRunnableFunction(
+        AndroidBridge::Bridge()->PostTaskToUiThread(NewRunnableFunction(
                 static_cast<void(*)(const NPZCRef&)>(callDestroy),
-                mozilla::Move(npzc)), nsIThread::DISPATCH_NORMAL);
+                mozilla::Move(npzc)), 0);
     }
 
 public: