diff --git a/taskcluster/docker/funsize-balrog-submitter/Dockerfile b/taskcluster/docker/funsize-balrog-submitter/Dockerfile index 43eeadaa3877..9ae503987d38 100644 --- a/taskcluster/docker/funsize-balrog-submitter/Dockerfile +++ b/taskcluster/docker/funsize-balrog-submitter/Dockerfile @@ -26,7 +26,8 @@ COPY scripts/* /home/worker/bin/ RUN mkdir /home/worker/keys COPY *.pubkey /home/worker/keys/ COPY runme.sh /runme.sh -RUN chmod 755 /home/worker/bin/* /runme.sh +COPY submit_complete.sh /submit_complete.sh +RUN chmod 755 /home/worker/bin/* /runme.sh /submit_complete.sh ENV HOME /home/worker ENV SHELL /bin/bash diff --git a/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter-complete.py b/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter-complete.py new file mode 100644 index 000000000000..c30fb1468fb0 --- /dev/null +++ b/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter-complete.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +from __future__ import absolute_import, print_function + +import site +import os +import logging +import argparse +import json + +site.addsitedir("/home/worker/tools/lib/python") + +from balrog.submitter.cli import ReleaseSubmitterV4 +from util.retry import retry + +log = logging.getLogger(__name__) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", required=True) + parser.add_argument("-a", "--api-root", required=True, + help="Balrog API root") + parser.add_argument("-v", "--verbose", action="store_const", + dest="loglevel", const=logging.DEBUG, + default=logging.INFO) + parser.add_argument("--product", help="Override product name from application.ini") + args = parser.parse_args() + logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", + level=args.loglevel) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("boto").setLevel(logging.WARNING) + + balrog_username = os.environ.get("BALROG_USERNAME") + balrog_password = os.environ.get("BALROG_PASSWORD") + suffix = os.environ.get("BALROG_BLOB_SUFFIX") + if not balrog_username and not balrog_password: + raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment " + "variables should be set") + if not suffix: + raise RuntimeError("BALROG_BLOB_SUFFIX environment variable should be set") + + manifest = json.load(open(args.manifest)) + auth = (balrog_username, balrog_password) + + for e in manifest: + complete_info = [{ + "hash": e["hash"], + "size": e["size"], + }] + + submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth, + suffix=suffix) + productName = args.product or e["appName"] + retry(lambda: submitter.run( + platform=e["platform"], productName=productName, + version=e["toVersion"], + build_number=e["toBuildNumber"], + appVersion=e["version"], extVersion=e["version"], + buildID=e["to_buildid"], locale=e["locale"], + hashFunction='sha512', completeInfo=complete_info), + attempts=30, sleeptime=10, max_sleeptime=60, jitter=3, + ) + + +if __name__ == '__main__': + main() diff --git a/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py b/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py index 833da9146ce9..7eecaf0db8c5 100644 --- a/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py +++ b/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py @@ -133,6 +133,8 @@ def main(): if not balrog_username and not balrog_password: raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment " "variables should be set") + # blob suffix used for releases only + suffix = os.environ.get("BALROG_BLOB_SUFFIX") s3_bucket = os.environ.get("S3_BUCKET") aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID") @@ -169,8 +171,11 @@ def main(): partial_info[0]["previousVersion"] = e["previousVersion"] partial_info[0]["previousBuildNumber"] = e["previousBuildNumber"] submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth, - dummy=args.dummy) + dummy=args.dummy, suffix=suffix) productName = args.product or e["appName"] + if suffix: + log.warning("Not submitting complete info") + complete_info = None retry(lambda: submitter.run( platform=e["platform"], productName=productName, version=e["toVersion"], diff --git a/taskcluster/docker/funsize-balrog-submitter/submit_complete.sh b/taskcluster/docker/funsize-balrog-submitter/submit_complete.sh new file mode 100644 index 000000000000..b18b1da3b400 --- /dev/null +++ b/taskcluster/docker/funsize-balrog-submitter/submit_complete.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -xe + +test $PARENT_TASK_ARTIFACTS_URL_PREFIX +test $BALROG_API_ROOT +# BALROG_BLOB_SUFFIX is used by the script implicitly to avoid possible CLI +# issues with suffixes starting with "-" +test $BALROG_BLOB_SUFFIX + + +ARTIFACTS_DIR="/home/worker/artifacts" +mkdir -p "$ARTIFACTS_DIR" + +curl --location --retry 10 --retry-delay 10 -o "$ARTIFACTS_DIR/manifest.json" \ + "$PARENT_TASK_ARTIFACTS_URL_PREFIX/manifest.json" + +cat "$ARTIFACTS_DIR/manifest.json" +python /home/worker/bin/funsize-balrog-submitter-complete.py \ + --manifest "$ARTIFACTS_DIR/manifest.json" \ + -a "$BALROG_API_ROOT" \ + --verbose \ + $EXTRA_BALROG_SUBMITTER_PARAMS diff --git a/taskcluster/docker/funsize-update-generator/scripts/recompress.py b/taskcluster/docker/funsize-update-generator/scripts/recompress.py index 91ed26688b36..5c0e538ee020 100755 --- a/taskcluster/docker/funsize-update-generator/scripts/recompress.py +++ b/taskcluster/docker/funsize-update-generator/scripts/recompress.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from __future__ import absolute_import, print_function +import ConfigParser import argparse import functools import hashlib @@ -29,6 +30,26 @@ ALLOWED_URL_PREFIXES = [ ] +def find_file(directory, filename): + log.debug("Searching for %s in %s", filename, directory) + for root, dirs, files in os.walk(directory): + if filename in files: + f = os.path.join(root, filename) + log.debug("Found %s", f) + return f + + +def get_option(directory, filename, section, option): + log.debug("Exctracting [%s]: %s from %s/**/%s", section, option, directory, + filename) + f = find_file(directory, filename) + config = ConfigParser.ConfigParser() + config.read(f) + rv = config.get(section, option) + log.debug("Found %s", rv) + return rv + + def verify_signature(mar, certs): log.info("Checking %s signature", mar) with open(mar, 'rb') as mar_fh: @@ -195,6 +216,16 @@ def main(): mar_data = { "file_to_sign": output_filename, "hash": get_hash(dest), + "appName": get_option(unpack_dir, filename="application.ini", + section="App", option="Name"), + "version": get_option(unpack_dir, filename="application.ini", + section="App", option="Version"), + "to_buildid": get_option(unpack_dir, filename="application.ini", + section="App", option="BuildID"), + "toVersion": e["toVersion"], + "toBuildNumber": e["toBuildNumber"], + "platform": e["platform"], + "locale": e["locale"], } shutil.copy(dest, os.path.join(args.artifacts_dir, output_filename)) work_env.cleanup() diff --git a/testing/mozharness/configs/releases/dev_updates_firefox_beta.py b/testing/mozharness/configs/releases/dev_updates_firefox_beta.py index 40b87c57b421..4248f2e729af 100644 --- a/testing/mozharness/configs/releases/dev_updates_firefox_beta.py +++ b/testing/mozharness/configs/releases/dev_updates_firefox_beta.py @@ -11,8 +11,8 @@ config = { "vcs_share_base": "/builds/hg-shared", # TODO: use real repo "push_dest": "ssh://hg.mozilla.org/users/raliiev_mozilla.com/tools", - # date repo used for staging beta - "shipped-locales-url": "https://hg.mozilla.org/projects/date/raw-file/{revision}/browser/locales/shipped-locales", + # jamun repo used for staging beta + "shipped-locales-url": "https://hg.mozilla.org/projects/jamun/raw-file/{revision}/browser/locales/shipped-locales", "ignore_no_changes": True, "ssh_user": "ffxbld", "ssh_key": "~/.ssh/ffxbld_rsa", @@ -20,19 +20,24 @@ config = { "archive_prefix": "https://ftp.stage.mozaws.net/pub", "previous_archive_prefix": "https://archive.mozilla.org/pub", "download_domain": "download.mozilla.org", - "balrog_url": "http://ec2-54-241-39-23.us-west-1.compute.amazonaws.com", + "balrog_url": "http://54.90.211.22:9090", "balrog_username": "balrog-stage-ffxbld", "update_channels": { "beta-dev": { "version_regex": r"^(\d+\.\d+(b\d+)?)$", "requires_mirrors": True, - # TODO - when we use a real repo, rename this file # s/MozDate/MozBeta-dev/ - "patcher_config": "mozDate-branch-patcher2.cfg", + # TODO - when we use a real repo, rename this file # s/MozJamun/MozBeta-dev/ + "patcher_config": "mozJamun-branch-patcher2.cfg", "update_verify_channel": "beta-dev-localtest", "mar_channel_ids": [], "channel_names": ["beta-dev", "beta-dev-localtest", "beta-dev-cdntest"], "rules_to_update": ["firefox-beta-dev-cdntest", "firefox-beta-dev-localtest"], "publish_rules": ["firefox-beta"], + "bz2_blob_suffix": "-bz2", + "bz2_rules_to_update": ["firefox-beta-cdntest-bz2", "firefox-beta-localtest-bz2"], + "bz2_publish_rules": [652], + "complete_mar_filename_pattern": '%s-%s.bz2.complete.mar', + "complete_mar_bouncer_product_pattern": '%s-%s-complete-bz2', } }, "balrog_use_dummy_suffix": False, diff --git a/testing/mozharness/configs/releases/dev_updates_firefox_release.py b/testing/mozharness/configs/releases/dev_updates_firefox_release.py index afc23d256151..06a867670fe5 100644 --- a/testing/mozharness/configs/releases/dev_updates_firefox_release.py +++ b/testing/mozharness/configs/releases/dev_updates_firefox_release.py @@ -20,7 +20,7 @@ config = { "archive_prefix": "https://ftp.stage.mozaws.net/pub", "previous_archive_prefix": "https://archive.mozilla.org/pub", "download_domain": "download.mozilla.org", - "balrog_url": "http://ec2-54-241-39-23.us-west-1.compute.amazonaws.com", + "balrog_url": "http://54.90.211.22:9090", "balrog_username": "balrog-stage-ffxbld", "update_channels": { "beta-dev": { @@ -35,6 +35,11 @@ config = { "rules_to_update": ["firefox-beta-dev-cdntest", "firefox-beta-dev-localtest"], "publish_rules": ["firefox-beta"], "schedule_asap": True, + "bz2_blob_suffix": "-bz2", + "bz2_rules_to_update": ["firefox-release-cdntest-bz2", "firefox-release-localtest-bz2"], + "bz2_publish_rules": [624], + "complete_mar_filename_pattern": '%s-%s.bz2.complete.mar', + "complete_mar_bouncer_product_pattern": '%s-%s-complete-bz2', }, "release-dev": { "version_regex": r"^\d+\.\d+(\.\d+)?$", @@ -45,6 +50,11 @@ config = { "channel_names": ["release-dev", "release-dev-localtest", "release-dev-cdntest"], "rules_to_update": ["firefox-release-dev-cdntest", "firefox-release-dev-localtest"], "publish_rules": ["firefox-release"], + "bz2_blob_suffix": "-bz2", + "bz2_rules_to_update": ["firefox-release-cdntest-bz2", "firefox-release-localtest-bz2"], + "bz2_publish_rules": [624], + "complete_mar_filename_pattern": '%s-%s.bz2.complete.mar', + "complete_mar_bouncer_product_pattern": '%s-%s-complete-bz2', }, }, "balrog_use_dummy_suffix": False, diff --git a/testing/mozharness/configs/releases/updates_firefox_release.py b/testing/mozharness/configs/releases/updates_firefox_release.py index d0e1701ea068..9430a0df38f4 100644 --- a/testing/mozharness/configs/releases/updates_firefox_release.py +++ b/testing/mozharness/configs/releases/updates_firefox_release.py @@ -42,6 +42,11 @@ config = { "channel_names": ["release", "release-localtest", "release-cdntest"], "rules_to_update": ["firefox-release-cdntest", "firefox-release-localtest"], "publish_rules": [145], + "bz2_blob_suffix": "-bz2", + "bz2_rules_to_update": ["firefox-release-cdntest-bz2", "firefox-release-localtest-bz2"], + "bz2_publish_rules": [624], + "complete_mar_filename_pattern": '%s-%s.bz2.complete.mar', + "complete_mar_bouncer_product_pattern": '%s-%s-complete-bz2', }, }, "balrog_use_dummy_suffix": False, diff --git a/testing/mozharness/scripts/release/publish_balrog.py b/testing/mozharness/scripts/release/publish_balrog.py index fa32b2c31ce0..25695f5f4dc7 100644 --- a/testing/mozharness/scripts/release/publish_balrog.py +++ b/testing/mozharness/scripts/release/publish_balrog.py @@ -55,7 +55,8 @@ class PublishBalrog(MercurialScript, BuildbotMixin): # TODO: version and appVersion should come from repo props = self.buildbot_config["properties"] for prop in ['product', 'version', 'build_number', 'channels', - 'balrog_api_root', 'schedule_at', 'background_rate']: + 'balrog_api_root', 'schedule_at', 'background_rate', + 'publish_bz2_blob']: if props.get(prop): self.info("Overriding %s with %s" % (prop, props[prop])) self.config[prop] = props.get(prop) @@ -89,6 +90,10 @@ class PublishBalrog(MercurialScript, BuildbotMixin): def submit_to_balrog(self): for _, channel_config in self.query_channel_configs(): self._submit_to_balrog(channel_config) + if 'publish_bz2_blob' in self.config and \ + self.config['publish_bz2_blob']: + for _, channel_config in self.query_channel_configs(): + self._submit_to_balrog_bz2(channel_config) def _submit_to_balrog(self, channel_config): dirs = self.query_abs_dirs() @@ -123,6 +128,44 @@ class PublishBalrog(MercurialScript, BuildbotMixin): self.retry(lambda: self.run_command(cmd, halt_on_failure=True), error_level=FATAL) + def _submit_to_balrog_bz2(self, channel_config): + dirs = self.query_abs_dirs() + # Use env varialbe instead of command line to avoid issues with blob + # names starting with "-", e.g. "-bz2" + env = {"BALROG_BLOB_SUFFIX": channel_config["bz2_blob_suffix"]} + auth = os.path.join(os.getcwd(), self.config['credentials_file']) + cmd = [ + sys.executable, + os.path.join(dirs["abs_tools_dir"], + "scripts/build-promotion/balrog-release-shipper.py")] + cmd.extend([ + "--api-root", self.config["balrog_api_root"], + "--credentials-file", auth, + "--username", self.config["balrog_username"], + "--version", self.config["version"], + "--product", self.config["product"], + "--build-number", str(self.config["build_number"]), + "--suffix", channel_config["bz2_blob_suffix"], + "--verbose", + ]) + for r in channel_config["bz2_publish_rules"]: + cmd.extend(["--rules", str(r)]) + if channel_config.get("schedule_asap"): + # RC releases going to the beta channel have no ETA set for the + # RC-to-beta push. The corresponding task is scheduled after we + # resolve the push-to-beta human decision task, so we can schedule + # it ASAP plus some additional 30m to avoid retry() to fail. + schedule_at = datetime.utcnow() + timedelta(minutes=30) + cmd.extend(["--schedule-at", schedule_at.isoformat()]) + elif self.config.get("schedule_at"): + cmd.extend(["--schedule-at", self.config["schedule_at"]]) + if self.config.get("background_rate"): + cmd.extend(["--background-rate", str(self.config["background_rate"])]) + + self.retry(lambda: self.run_command(cmd, halt_on_failure=True, env=env), + error_level=FATAL) + + # __main__ {{{1 if __name__ == '__main__': diff --git a/testing/mozharness/scripts/release/updates.py b/testing/mozharness/scripts/release/updates.py index 47d8c7fdaab3..9676db3315a2 100644 --- a/testing/mozharness/scripts/release/updates.py +++ b/testing/mozharness/scripts/release/updates.py @@ -94,7 +94,8 @@ class UpdatesBumper(MercurialScript, BuildbotMixin, # TODO: version and appVersion should come from repo props = self.buildbot_config["properties"] for prop in ['product', 'version', 'build_number', 'revision', - 'appVersion', 'balrog_api_root', "channels"]: + 'appVersion', 'balrog_api_root', "channels", + 'generate_bz2_blob']: if props.get(prop): self.info("Overriding %s with %s" % (prop, props[prop])) self.config[prop] = props.get(prop) @@ -269,6 +270,10 @@ class UpdatesBumper(MercurialScript, BuildbotMixin, def submit_to_balrog(self): for _, channel_config in self.query_channel_configs(): self._submit_to_balrog(channel_config) + if 'generate_bz2_blob' in self.config and \ + self.config['generate_bz2_blob']: + for _, channel_config in self.query_channel_configs(): + self._submit_to_balrog_bz2(channel_config) def _submit_to_balrog(self, channel_config): dirs = self.query_abs_dirs() @@ -306,6 +311,60 @@ class UpdatesBumper(MercurialScript, BuildbotMixin, self.retry(lambda: self.run_command(cmd, halt_on_failure=True)) + def _submit_to_balrog_bz2(self, channel_config): + if "bz2_blob_suffix" not in channel_config: + self.info("No need to generate BZ2 blob") + return + + dirs = self.query_abs_dirs() + # Use env varialbe instead of command line to avoid issues with blob + # names starting with "-", e.g. "-bz2" + env = {"BALROG_BLOB_SUFFIX": channel_config["bz2_blob_suffix"]} + auth = os.path.join(os.getcwd(), self.config['credentials_file']) + cmd = [ + sys.executable, + os.path.join(dirs["abs_tools_dir"], + "scripts/build-promotion/balrog-release-pusher.py")] + cmd.extend([ + "--api-root", self.config["balrog_api_root"], + "--download-domain", self.config["download_domain"], + "--archive-domain", self.config["archive_domain"], + "--credentials-file", auth, + "--product", self.config["product"], + "--version", self.config["version"], + "--build-number", str(self.config["build_number"]), + "--app-version", self.config["appVersion"], + "--username", self.config["balrog_username"], + "--complete-mar-filename-pattern", + channel_config["complete_mar_filename_pattern"], + "--complete-mar-bouncer-product-pattern", + channel_config["complete_mar_bouncer_product_pattern"], + "--verbose", + ]) + + for v, build_number in self.query_matching_partials(channel_config): + if v < "56.0": + self.info("Adding %s to partials" % v) + partial = "{version}build{build_number}".format( + version=v, build_number=build_number) + cmd.extend(["--partial-update", partial]) + else: + self.info("Not adding %s to partials" % v) + + for c in channel_config["channel_names"]: + cmd.extend(["--channel", c]) + for r in channel_config["bz2_rules_to_update"]: + cmd.extend(["--rule-to-update", r]) + for p in self.config["platforms"]: + cmd.extend(["--platform", p]) + if channel_config["requires_mirrors"]: + cmd.append("--requires-mirrors") + if self.config["balrog_use_dummy_suffix"]: + cmd.append("--dummy") + + self.retry(lambda: self.run_command(cmd, halt_on_failure=True, env=env)) + + # __main__ {{{1 if __name__ == '__main__': UpdatesBumper().run_and_exit()