Bug 1402015 - Submit re-compressed MAR metadata to Balrog r=sfraser a=release DONTBUILD

MozReview-Commit-ID: DuJxPBErAhg

--HG--
extra : rebase_source : a84be0640891a779dd36830b60c9c07da0462372
extra : amend_source : 3afe179bfe6bf226e246f08fec57e05cfe330ccb

taskcluster/docker/funsize-balrog-submitter/Dockerfile

@@ -26,7 +26,8 @@ COPY scripts/* /home/worker/bin/
 RUN mkdir /home/worker/keys
 COPY *.pubkey /home/worker/keys/
 COPY runme.sh /runme.sh
-RUN chmod 755 /home/worker/bin/* /runme.sh
+COPY submit_complete.sh /submit_complete.sh
+RUN chmod 755 /home/worker/bin/* /runme.sh /submit_complete.sh
 
 ENV           HOME          /home/worker
 ENV           SHELL         /bin/bash

taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter-complete.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+from __future__ import absolute_import, print_function
+
+import site
+import os
+import logging
+import argparse
+import json
+
+site.addsitedir("/home/worker/tools/lib/python")
+
+from balrog.submitter.cli import ReleaseSubmitterV4
+from util.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--manifest", required=True)
+    parser.add_argument("-a", "--api-root", required=True,
+                        help="Balrog API root")
+    parser.add_argument("-v", "--verbose", action="store_const",
+                        dest="loglevel", const=logging.DEBUG,
+                        default=logging.INFO)
+    parser.add_argument("--product", help="Override product name from application.ini")
+    args = parser.parse_args()
+    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
+                        level=args.loglevel)
+    logging.getLogger("requests").setLevel(logging.WARNING)
+    logging.getLogger("boto").setLevel(logging.WARNING)
+
+    balrog_username = os.environ.get("BALROG_USERNAME")
+    balrog_password = os.environ.get("BALROG_PASSWORD")
+    suffix = os.environ.get("BALROG_BLOB_SUFFIX")
+    if not balrog_username and not balrog_password:
+        raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment "
+                           "variables should be set")
+    if not suffix:
+        raise RuntimeError("BALROG_BLOB_SUFFIX environment variable should be set")
+
+    manifest = json.load(open(args.manifest))
+    auth = (balrog_username, balrog_password)
+
+    for e in manifest:
+        complete_info = [{
+            "hash": e["hash"],
+            "size": e["size"],
+        }]
+
+        submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth,
+                                       suffix=suffix)
+        productName = args.product or e["appName"]
+        retry(lambda: submitter.run(
+            platform=e["platform"], productName=productName,
+            version=e["toVersion"],
+            build_number=e["toBuildNumber"],
+            appVersion=e["version"], extVersion=e["version"],
+            buildID=e["to_buildid"], locale=e["locale"],
+            hashFunction='sha512', completeInfo=complete_info),
+            attempts=30, sleeptime=10, max_sleeptime=60, jitter=3,
+        )
+
+
+if __name__ == '__main__':
+    main()

taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py

@@ -133,6 +133,8 @@ def main():
     if not balrog_username and not balrog_password:
         raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment "
                            "variables should be set")
+    # blob suffix used for releases only
+    suffix = os.environ.get("BALROG_BLOB_SUFFIX")
 
     s3_bucket = os.environ.get("S3_BUCKET")
     aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID")
@@ -169,8 +171,11 @@ def main():
             partial_info[0]["previousVersion"] = e["previousVersion"]
             partial_info[0]["previousBuildNumber"] = e["previousBuildNumber"]
             submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth,
-                                           dummy=args.dummy)
+                                           dummy=args.dummy, suffix=suffix)
             productName = args.product or e["appName"]
+            if suffix:
+                log.warning("Not submitting complete info")
+                complete_info = None
             retry(lambda: submitter.run(
                 platform=e["platform"], productName=productName,
                 version=e["toVersion"],

taskcluster/docker/funsize-balrog-submitter/submit_complete.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -xe
+
+test $PARENT_TASK_ARTIFACTS_URL_PREFIX
+test $BALROG_API_ROOT
+# BALROG_BLOB_SUFFIX is used by the script implicitly to avoid possible CLI
+# issues with suffixes starting with "-"
+test $BALROG_BLOB_SUFFIX
+
+
+ARTIFACTS_DIR="/home/worker/artifacts"
+mkdir -p "$ARTIFACTS_DIR"
+
+curl --location --retry 10 --retry-delay 10 -o "$ARTIFACTS_DIR/manifest.json" \
+    "$PARENT_TASK_ARTIFACTS_URL_PREFIX/manifest.json"
+
+cat "$ARTIFACTS_DIR/manifest.json"
+python /home/worker/bin/funsize-balrog-submitter-complete.py  \
+    --manifest "$ARTIFACTS_DIR/manifest.json" \
+    -a "$BALROG_API_ROOT" \
+    --verbose \
+    $EXTRA_BALROG_SUBMITTER_PARAMS

taskcluster/docker/funsize-update-generator/scripts/recompress.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 from __future__ import absolute_import, print_function
 
+import ConfigParser
 import argparse
 import functools
 import hashlib
@@ -29,6 +30,26 @@ ALLOWED_URL_PREFIXES = [
 ]
 
 
+def find_file(directory, filename):
+    log.debug("Searching for %s in %s", filename, directory)
+    for root, dirs, files in os.walk(directory):
+        if filename in files:
+            f = os.path.join(root, filename)
+            log.debug("Found %s", f)
+            return f
+
+
+def get_option(directory, filename, section, option):
+    log.debug("Exctracting [%s]: %s from %s/**/%s", section, option, directory,
+              filename)
+    f = find_file(directory, filename)
+    config = ConfigParser.ConfigParser()
+    config.read(f)
+    rv = config.get(section, option)
+    log.debug("Found %s", rv)
+    return rv
+
+
 def verify_signature(mar, certs):
     log.info("Checking %s signature", mar)
     with open(mar, 'rb') as mar_fh:
@@ -195,6 +216,16 @@ def main():
         mar_data = {
             "file_to_sign": output_filename,
             "hash": get_hash(dest),
+            "appName": get_option(unpack_dir, filename="application.ini",
+                                  section="App", option="Name"),
+            "version": get_option(unpack_dir, filename="application.ini",
+                                  section="App", option="Version"),
+            "to_buildid": get_option(unpack_dir, filename="application.ini",
+                                     section="App", option="BuildID"),
+            "toVersion": e["toVersion"],
+            "toBuildNumber": e["toBuildNumber"],
+            "platform": e["platform"],
+            "locale": e["locale"],
         }
         shutil.copy(dest, os.path.join(args.artifacts_dir, output_filename))
         work_env.cleanup()

testing/mozharness/configs/releases/dev_updates_firefox_beta.py

@@ -11,8 +11,8 @@ config = {
     "vcs_share_base": "/builds/hg-shared",
     # TODO: use real repo
     "push_dest": "ssh://hg.mozilla.org/users/raliiev_mozilla.com/tools",
-    # date repo used for staging beta
-    "shipped-locales-url": "https://hg.mozilla.org/projects/date/raw-file/{revision}/browser/locales/shipped-locales",
+    # jamun repo used for staging beta
+    "shipped-locales-url": "https://hg.mozilla.org/projects/jamun/raw-file/{revision}/browser/locales/shipped-locales",
     "ignore_no_changes": True,
     "ssh_user": "ffxbld",
     "ssh_key": "~/.ssh/ffxbld_rsa",
@@ -20,19 +20,24 @@ config = {
     "archive_prefix": "https://ftp.stage.mozaws.net/pub",
     "previous_archive_prefix": "https://archive.mozilla.org/pub",
     "download_domain": "download.mozilla.org",
-    "balrog_url": "http://ec2-54-241-39-23.us-west-1.compute.amazonaws.com",
+    "balrog_url": "http://54.90.211.22:9090",
     "balrog_username": "balrog-stage-ffxbld",
     "update_channels": {
         "beta-dev": {
             "version_regex": r"^(\d+\.\d+(b\d+)?)$",
             "requires_mirrors": True,
-            # TODO - when we use a real repo, rename this file # s/MozDate/MozBeta-dev/
-            "patcher_config": "mozDate-branch-patcher2.cfg",
+            # TODO - when we use a real repo, rename this file # s/MozJamun/MozBeta-dev/
+            "patcher_config": "mozJamun-branch-patcher2.cfg",
             "update_verify_channel": "beta-dev-localtest",
             "mar_channel_ids": [],
             "channel_names": ["beta-dev", "beta-dev-localtest", "beta-dev-cdntest"],
             "rules_to_update": ["firefox-beta-dev-cdntest", "firefox-beta-dev-localtest"],
             "publish_rules": ["firefox-beta"],
+            "bz2_blob_suffix": "-bz2",
+            "bz2_rules_to_update": ["firefox-beta-cdntest-bz2", "firefox-beta-localtest-bz2"],
+            "bz2_publish_rules": [652],
+            "complete_mar_filename_pattern": '%s-%s.bz2.complete.mar',
+            "complete_mar_bouncer_product_pattern": '%s-%s-complete-bz2',
         }
     },
     "balrog_use_dummy_suffix": False,

testing/mozharness/configs/releases/dev_updates_firefox_release.py

@@ -20,7 +20,7 @@ config = {
     "archive_prefix": "https://ftp.stage.mozaws.net/pub",
     "previous_archive_prefix": "https://archive.mozilla.org/pub",
     "download_domain": "download.mozilla.org",
-    "balrog_url": "http://ec2-54-241-39-23.us-west-1.compute.amazonaws.com",
+    "balrog_url": "http://54.90.211.22:9090",
     "balrog_username": "balrog-stage-ffxbld",
     "update_channels": {
         "beta-dev": {
@@ -35,6 +35,11 @@ config = {
             "rules_to_update": ["firefox-beta-dev-cdntest", "firefox-beta-dev-localtest"],
             "publish_rules": ["firefox-beta"],
             "schedule_asap": True,
+            "bz2_blob_suffix": "-bz2",
+            "bz2_rules_to_update": ["firefox-release-cdntest-bz2", "firefox-release-localtest-bz2"],
+            "bz2_publish_rules": [624],
+            "complete_mar_filename_pattern": '%s-%s.bz2.complete.mar',
+            "complete_mar_bouncer_product_pattern": '%s-%s-complete-bz2',
         },
         "release-dev": {
             "version_regex": r"^\d+\.\d+(\.\d+)?$",
@@ -45,6 +50,11 @@ config = {
             "channel_names": ["release-dev", "release-dev-localtest", "release-dev-cdntest"],
             "rules_to_update": ["firefox-release-dev-cdntest", "firefox-release-dev-localtest"],
             "publish_rules": ["firefox-release"],
+            "bz2_blob_suffix": "-bz2",
+            "bz2_rules_to_update": ["firefox-release-cdntest-bz2", "firefox-release-localtest-bz2"],
+            "bz2_publish_rules": [624],
+            "complete_mar_filename_pattern": '%s-%s.bz2.complete.mar',
+            "complete_mar_bouncer_product_pattern": '%s-%s-complete-bz2',
         },
     },
     "balrog_use_dummy_suffix": False,

testing/mozharness/configs/releases/updates_firefox_release.py

@@ -42,6 +42,11 @@ config = {
             "channel_names": ["release", "release-localtest", "release-cdntest"],
             "rules_to_update": ["firefox-release-cdntest", "firefox-release-localtest"],
             "publish_rules": [145],
+            "bz2_blob_suffix": "-bz2",
+            "bz2_rules_to_update": ["firefox-release-cdntest-bz2", "firefox-release-localtest-bz2"],
+            "bz2_publish_rules": [624],
+            "complete_mar_filename_pattern": '%s-%s.bz2.complete.mar',
+            "complete_mar_bouncer_product_pattern": '%s-%s-complete-bz2',
         },
     },
     "balrog_use_dummy_suffix": False,

testing/mozharness/scripts/release/publish_balrog.py

@@ -55,7 +55,8 @@ class PublishBalrog(MercurialScript, BuildbotMixin):
         # TODO: version and appVersion should come from repo
         props = self.buildbot_config["properties"]
         for prop in ['product', 'version', 'build_number', 'channels',
-                     'balrog_api_root', 'schedule_at', 'background_rate']:
+                     'balrog_api_root', 'schedule_at', 'background_rate',
+                     'publish_bz2_blob']:
             if props.get(prop):
                 self.info("Overriding %s with %s" % (prop, props[prop]))
                 self.config[prop] = props.get(prop)
@@ -89,6 +90,10 @@ class PublishBalrog(MercurialScript, BuildbotMixin):
     def submit_to_balrog(self):
         for _, channel_config in self.query_channel_configs():
             self._submit_to_balrog(channel_config)
+        if 'publish_bz2_blob' in self.config and \
+                self.config['publish_bz2_blob']:
+            for _, channel_config in self.query_channel_configs():
+                self._submit_to_balrog_bz2(channel_config)
 
     def _submit_to_balrog(self, channel_config):
         dirs = self.query_abs_dirs()
@@ -123,6 +128,44 @@ class PublishBalrog(MercurialScript, BuildbotMixin):
         self.retry(lambda: self.run_command(cmd, halt_on_failure=True),
                    error_level=FATAL)
 
+    def _submit_to_balrog_bz2(self, channel_config):
+        dirs = self.query_abs_dirs()
+        # Use env varialbe instead of command line to avoid issues with blob
+        # names starting with "-", e.g. "-bz2"
+        env = {"BALROG_BLOB_SUFFIX": channel_config["bz2_blob_suffix"]}
+        auth = os.path.join(os.getcwd(), self.config['credentials_file'])
+        cmd = [
+            sys.executable,
+            os.path.join(dirs["abs_tools_dir"],
+                         "scripts/build-promotion/balrog-release-shipper.py")]
+        cmd.extend([
+            "--api-root", self.config["balrog_api_root"],
+            "--credentials-file", auth,
+            "--username", self.config["balrog_username"],
+            "--version", self.config["version"],
+            "--product", self.config["product"],
+            "--build-number", str(self.config["build_number"]),
+            "--suffix", channel_config["bz2_blob_suffix"],
+            "--verbose",
+        ])
+        for r in channel_config["bz2_publish_rules"]:
+            cmd.extend(["--rules", str(r)])
+        if channel_config.get("schedule_asap"):
+            # RC releases going to the beta channel have no ETA set for the
+            # RC-to-beta push. The corresponding task is scheduled after we
+            # resolve the push-to-beta human decision task, so we can schedule
+            # it ASAP plus some additional 30m to avoid retry() to fail.
+            schedule_at = datetime.utcnow() + timedelta(minutes=30)
+            cmd.extend(["--schedule-at", schedule_at.isoformat()])
+        elif self.config.get("schedule_at"):
+            cmd.extend(["--schedule-at", self.config["schedule_at"]])
+        if self.config.get("background_rate"):
+            cmd.extend(["--background-rate", str(self.config["background_rate"])])
+
+        self.retry(lambda: self.run_command(cmd, halt_on_failure=True, env=env),
+                   error_level=FATAL)
+
+
 
 # __main__ {{{1
 if __name__ == '__main__':

testing/mozharness/scripts/release/updates.py

@@ -94,7 +94,8 @@ class UpdatesBumper(MercurialScript, BuildbotMixin,
         # TODO: version and appVersion should come from repo
         props = self.buildbot_config["properties"]
         for prop in ['product', 'version', 'build_number', 'revision',
-                     'appVersion', 'balrog_api_root', "channels"]:
+                     'appVersion', 'balrog_api_root', "channels",
+                     'generate_bz2_blob']:
             if props.get(prop):
                 self.info("Overriding %s with %s" % (prop, props[prop]))
                 self.config[prop] = props.get(prop)
@@ -269,6 +270,10 @@ class UpdatesBumper(MercurialScript, BuildbotMixin,
     def submit_to_balrog(self):
         for _, channel_config in self.query_channel_configs():
             self._submit_to_balrog(channel_config)
+        if 'generate_bz2_blob' in self.config and \
+                self.config['generate_bz2_blob']:
+            for _, channel_config in self.query_channel_configs():
+                self._submit_to_balrog_bz2(channel_config)
 
     def _submit_to_balrog(self, channel_config):
         dirs = self.query_abs_dirs()
@@ -306,6 +311,60 @@ class UpdatesBumper(MercurialScript, BuildbotMixin,
 
         self.retry(lambda: self.run_command(cmd, halt_on_failure=True))
 
+    def _submit_to_balrog_bz2(self, channel_config):
+        if "bz2_blob_suffix" not in channel_config:
+            self.info("No need to generate BZ2 blob")
+            return
+
+        dirs = self.query_abs_dirs()
+        # Use env varialbe instead of command line to avoid issues with blob
+        # names starting with "-", e.g. "-bz2"
+        env = {"BALROG_BLOB_SUFFIX": channel_config["bz2_blob_suffix"]}
+        auth = os.path.join(os.getcwd(), self.config['credentials_file'])
+        cmd = [
+            sys.executable,
+            os.path.join(dirs["abs_tools_dir"],
+                         "scripts/build-promotion/balrog-release-pusher.py")]
+        cmd.extend([
+            "--api-root", self.config["balrog_api_root"],
+            "--download-domain", self.config["download_domain"],
+            "--archive-domain", self.config["archive_domain"],
+            "--credentials-file", auth,
+            "--product", self.config["product"],
+            "--version", self.config["version"],
+            "--build-number", str(self.config["build_number"]),
+            "--app-version", self.config["appVersion"],
+            "--username", self.config["balrog_username"],
+            "--complete-mar-filename-pattern",
+            channel_config["complete_mar_filename_pattern"],
+            "--complete-mar-bouncer-product-pattern",
+            channel_config["complete_mar_bouncer_product_pattern"],
+            "--verbose",
+        ])
+
+        for v, build_number in self.query_matching_partials(channel_config):
+            if v < "56.0":
+                self.info("Adding %s to partials" % v)
+                partial = "{version}build{build_number}".format(
+                    version=v, build_number=build_number)
+                cmd.extend(["--partial-update", partial])
+            else:
+                self.info("Not adding %s to partials" % v)
+
+        for c in channel_config["channel_names"]:
+            cmd.extend(["--channel", c])
+        for r in channel_config["bz2_rules_to_update"]:
+            cmd.extend(["--rule-to-update", r])
+        for p in self.config["platforms"]:
+            cmd.extend(["--platform", p])
+        if channel_config["requires_mirrors"]:
+            cmd.append("--requires-mirrors")
+        if self.config["balrog_use_dummy_suffix"]:
+            cmd.append("--dummy")
+
+        self.retry(lambda: self.run_command(cmd, halt_on_failure=True, env=env))
+
+
 # __main__ {{{1
 if __name__ == '__main__':
     UpdatesBumper().run_and_exit()