mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-26 14:22:01 +00:00
Bug 1225905 - create a mozharness script that manages each beet mover task logic, NPOTB DONTBUILD r=rail
--HG-- extra : commitid : L1hpWyOgKxa extra : rebase_source : ef585199e5819894d251ecb11ca7150eb3b1efcc
This commit is contained in:
Jordan Lund
parent
cedf0dc1a3
commit
e89c66f355
84
testing/mozharness/configs/beetmover/en_us.yml.tmpl
Normal file
84
testing/mozharness/configs/beetmover/en_us.yml.tmpl
Normal file
@ -0,0 +1,84 @@
|
|||||||
|
---
|
||||||
|
metadata:
|
||||||
|
name: "Beet Mover Manifest"
|
||||||
|
description: "Maps artifact locations to s3 key names for the en-US locale"
|
||||||
|
owner: "release@mozilla.com"
|
||||||
|
|
||||||
|
mapping:
|
||||||
|
{% for locale in locales %}
|
||||||
|
# common deliverables
|
||||||
|
{{ locale }}:
|
||||||
|
complete_mar:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.complete.mar
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/update/{{ platform }}/{{ locale }}/firefox-{{ version }}.complete.mar
|
||||||
|
checksum:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.checksums
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.checksums
|
||||||
|
checksum_sig:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.checksums.asc
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.checksums.asc
|
||||||
|
buildinfo:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.json
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.json
|
||||||
|
mozinfo:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.mozinfo.json
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.mozinfo.json
|
||||||
|
socorroinfo:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.txt
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.txt
|
||||||
|
|
||||||
|
{% if platform == "win32" %}
|
||||||
|
full_installer:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.installer.exe
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/Firefox Setup {{ version }}.exe
|
||||||
|
stub_installer:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.installer-stub.exe
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/Firefox Setup Stub {{ version }}.exe
|
||||||
|
package:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.zip
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.zip
|
||||||
|
symbols:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.crashreporter-symbols.zip
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% if platform == "win64" %}
|
||||||
|
full_installer:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.installer.exe
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/Firefox Setup {{ version }}.exe
|
||||||
|
package:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.zip
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.zip
|
||||||
|
symbols:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.crashreporter-symbols.zip
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% if platform == "linux-i686" %}
|
||||||
|
package:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.tar.bz2
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.tar.bz2
|
||||||
|
symbols:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.crashreporter-symbols.zip
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% if platform == "linux-x86_64" %}
|
||||||
|
package:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.tar.bz2
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.tar.bz2
|
||||||
|
symbols:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.crashreporter-symbols.zip
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% if platform == "mac" %}
|
||||||
|
package:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.dmg
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/Firefox {{ version }}.dmg
|
||||||
|
symbols:
|
||||||
|
artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
|
||||||
|
s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/Firefox {{ version }}.crashreporter-symbols.zip
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% endfor %}
|
||||||
240
testing/mozharness/scripts/release/beet_mover.py
Executable file
240
testing/mozharness/scripts/release/beet_mover.py
Executable file
@ -0,0 +1,240 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
# ***** BEGIN LICENSE BLOCK *****
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||||
|
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
# ***** END LICENSE BLOCK *****
|
||||||
|
"""beet_mover.py.
|
||||||
|
|
||||||
|
downloads artifacts and uploads them to s3
|
||||||
|
"""
|
||||||
|
import hashlib
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import pprint
|
||||||
|
|
||||||
|
sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
|
||||||
|
from mozharness.base.log import FATAL
|
||||||
|
from mozharness.base.python import VirtualenvMixin
|
||||||
|
from mozharness.base.script import BaseScript
|
||||||
|
|
||||||
|
|
||||||
|
def get_hash(content, hash_type="md5"):
|
||||||
|
h = hashlib.new(hash_type)
|
||||||
|
h.update(content)
|
||||||
|
return h.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def get_aws_auth():
|
||||||
|
"""
|
||||||
|
retrieves aws creds and deletes them from os.environ if present.
|
||||||
|
"""
|
||||||
|
aws_key_id = os.environ.get("AWS_ACCESS_KEY_ID")
|
||||||
|
aws_secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY")
|
||||||
|
|
||||||
|
if aws_key_id and aws_secret_key:
|
||||||
|
del os.environ['AWS_ACCESS_KEY_ID']
|
||||||
|
del os.environ['AWS_SECRET_ACCESS_KEY']
|
||||||
|
else:
|
||||||
|
exit("could not determine aws credentials from os environment")
|
||||||
|
|
||||||
|
return aws_key_id, aws_secret_key
|
||||||
|
|
||||||
|
|
||||||
|
CONFIG_OPTIONS = [
|
||||||
|
[["--template"], {
|
||||||
|
"dest": "template",
|
||||||
|
"help": "Specify jinja2 template file",
|
||||||
|
}],
|
||||||
|
[['--locale', ], {
|
||||||
|
"action": "extend",
|
||||||
|
"dest": "locales",
|
||||||
|
"type": "string",
|
||||||
|
"help": "Specify the locale(s) to upload."}],
|
||||||
|
[["--platform"], {
|
||||||
|
"dest": "platform",
|
||||||
|
"help": "Specify the platform of the build",
|
||||||
|
}],
|
||||||
|
[["--version"], {
|
||||||
|
"dest": "version",
|
||||||
|
"help": "full release version based on gecko and tag/stage identifier. e.g. '44.0b1'"
|
||||||
|
}],
|
||||||
|
[["--app-version"], {
|
||||||
|
"dest": "app_version",
|
||||||
|
"help": "numbered version based on gecko. e.g. '44.0'"
|
||||||
|
}],
|
||||||
|
[["--build-num"], {
|
||||||
|
"dest": "build_num",
|
||||||
|
"help": "the release build identifier"
|
||||||
|
}],
|
||||||
|
[["--taskid"], {
|
||||||
|
"dest": "taskid",
|
||||||
|
"help": "taskcluster task id to download artifacts from",
|
||||||
|
}],
|
||||||
|
[["--production"], {
|
||||||
|
"dest": "production",
|
||||||
|
"default": False,
|
||||||
|
"help": "taskcluster task id to download artifacts from",
|
||||||
|
}],
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class BeetMover(BaseScript, VirtualenvMixin, object):
|
||||||
|
def __init__(self, aws_creds):
|
||||||
|
beetmover_kwargs = {
|
||||||
|
'config_options': CONFIG_OPTIONS,
|
||||||
|
'all_actions': [
|
||||||
|
# 'clobber',
|
||||||
|
'create-virtualenv',
|
||||||
|
'activate-virtualenv',
|
||||||
|
'generate-candidates-manifest',
|
||||||
|
'verify-bits', # beets
|
||||||
|
'upload-bits', # beets
|
||||||
|
],
|
||||||
|
'require_config_file': False,
|
||||||
|
# Default configuration
|
||||||
|
'config': {
|
||||||
|
# base index url where to find taskcluster artifact based on taskid
|
||||||
|
# TODO - find out if we need to support taskcluster run number other than 0.
|
||||||
|
# e.g. maybe we could end up with artifacts in > 'run 0' in a re-trigger situation?
|
||||||
|
"artifact_base_url": 'https://queue.taskcluster.net/v1/task/{taskid}/runs/0/artifacts/public/build',
|
||||||
|
"virtualenv_modules": [
|
||||||
|
"boto",
|
||||||
|
"PyYAML",
|
||||||
|
"Jinja2",
|
||||||
|
],
|
||||||
|
"virtualenv_path": "venv",
|
||||||
|
'buckets': {
|
||||||
|
'development': "mozilla-releng-beet-mover-dev",
|
||||||
|
'production': "mozilla-releng-beet-mover",
|
||||||
|
},
|
||||||
|
'product': 'firefox',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
super(BeetMover, self).__init__(**beetmover_kwargs)
|
||||||
|
|
||||||
|
c = self.config
|
||||||
|
self.manifest = {}
|
||||||
|
# assigned in _post_create_virtualenv
|
||||||
|
self.virtualenv_imports = None
|
||||||
|
self.bucket = c['buckets']['production'] if c['production'] else c['buckets']['development']
|
||||||
|
self.aws_key_id, self.aws_secret_key = aws_creds
|
||||||
|
|
||||||
|
def activate_virtualenv(self):
|
||||||
|
"""
|
||||||
|
activates virtualenv and adds module imports to a instance wide namespace.
|
||||||
|
|
||||||
|
creating and activating a virtualenv onto the currently executing python interpreter is a
|
||||||
|
bit black magic. Rather than having import statements added in various places within the
|
||||||
|
script, we import them here immediately after we activate the newly created virtualenv
|
||||||
|
"""
|
||||||
|
VirtualenvMixin.activate_virtualenv(self)
|
||||||
|
|
||||||
|
import boto
|
||||||
|
import yaml
|
||||||
|
import jinja2
|
||||||
|
self.virtualenv_imports = {
|
||||||
|
'boto': boto,
|
||||||
|
'yaml': yaml,
|
||||||
|
'jinja2': jinja2,
|
||||||
|
}
|
||||||
|
self.log("activated virtualenv with the modules: {}".format(str(self.virtualenv_imports)))
|
||||||
|
|
||||||
|
def generate_candidates_manifest(self):
|
||||||
|
"""
|
||||||
|
generates and outputs a manifest that maps expected Taskcluster artifact names
|
||||||
|
to release deliverable names
|
||||||
|
"""
|
||||||
|
self.log('generating manifest from {}...'.format(self.config['template']))
|
||||||
|
template_dir, template_file = os.path.split(os.path.abspath(self.config['template']))
|
||||||
|
jinja2 = self.virtualenv_imports['jinja2']
|
||||||
|
yaml = self.virtualenv_imports['yaml']
|
||||||
|
|
||||||
|
jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir),
|
||||||
|
undefined=jinja2.StrictUndefined)
|
||||||
|
template = jinja_env.get_template(template_file)
|
||||||
|
template_vars = {
|
||||||
|
"platform": self.config['platform'],
|
||||||
|
"locales": self.config['locales'],
|
||||||
|
"version": self.config['version'],
|
||||||
|
"app_version": self.config['app_version'],
|
||||||
|
"build_num": self.config['build_num'],
|
||||||
|
# mirror current release folder structure
|
||||||
|
"s3_prefix": 'pub/{}/candidates'.format(self.config['product']),
|
||||||
|
"artifact_base_url": self.config['artifact_base_url'].format(
|
||||||
|
taskid=self.config['taskid']
|
||||||
|
)
|
||||||
|
}
|
||||||
|
self.manifest = yaml.safe_load(template.render(**template_vars))
|
||||||
|
|
||||||
|
self.log("manifest generated:")
|
||||||
|
self.log(pprint.pformat(self.manifest['mapping']))
|
||||||
|
|
||||||
|
def verify_bits(self):
|
||||||
|
"""
|
||||||
|
inspects each artifact and verifies that they were created by trustworthy tasks
|
||||||
|
"""
|
||||||
|
# TODO
|
||||||
|
self.log('skipping verification. unimplemented...')
|
||||||
|
|
||||||
|
def upload_bits(self):
|
||||||
|
"""
|
||||||
|
downloads and uploads list of artifacts to s3 candidates dir based on a given manifest
|
||||||
|
"""
|
||||||
|
self.log('downloading and uploading artifacts to s3...')
|
||||||
|
|
||||||
|
# connect to s3
|
||||||
|
boto = self.virtualenv_imports['boto']
|
||||||
|
conn = boto.connect_s3(self.aws_key_id, self.aws_secret_key)
|
||||||
|
bucket = conn.get_bucket(self.bucket)
|
||||||
|
|
||||||
|
for locale in self.manifest['mapping']:
|
||||||
|
for deliverable in self.manifest['mapping'][locale]:
|
||||||
|
self.log("uploading '{}' deliverable for '{}' locale".format(deliverable, locale))
|
||||||
|
self.upload_bit(
|
||||||
|
source=self.manifest['mapping'][locale][deliverable]['artifact'],
|
||||||
|
s3_key=self.manifest['mapping'][locale][deliverable]['s3_key'],
|
||||||
|
bucket=bucket,
|
||||||
|
)
|
||||||
|
self.log('Success!')
|
||||||
|
|
||||||
|
def upload_bit(self, source, s3_key, bucket):
|
||||||
|
# TODO - do we want to mirror/upload to more than one region?
|
||||||
|
dirs = self.query_abs_dirs()
|
||||||
|
boto = self.virtualenv_imports['boto']
|
||||||
|
|
||||||
|
# download locally
|
||||||
|
file_name = self.retry(self.download_file,
|
||||||
|
args=[source],
|
||||||
|
kwargs={'parent_dir': dirs['abs_work_dir']},
|
||||||
|
error_level=FATAL)
|
||||||
|
|
||||||
|
self.info('uploading to s3 with key: {}'.format(s3_key))
|
||||||
|
key = boto.s3.key.Key(bucket) # create new key
|
||||||
|
key.key = s3_key # set key name
|
||||||
|
|
||||||
|
self.info("Checking if `{}` already exists".format(s3_key))
|
||||||
|
key = bucket.get_key(s3_key)
|
||||||
|
if not key:
|
||||||
|
self.info("Uploading to `{}`".format(s3_key))
|
||||||
|
key = bucket.new_key(s3_key)
|
||||||
|
|
||||||
|
# set key value
|
||||||
|
self.retry(key.set_contents_from_filename, args=[file_name], error_level=FATAL),
|
||||||
|
|
||||||
|
# key.make_public() may lead to race conditions, because
|
||||||
|
# it doesn't pass version_id, so it may not set permissions
|
||||||
|
bucket.set_canned_acl(acl_str='public-read', key_name=s3_key,
|
||||||
|
version_id=key.version_id)
|
||||||
|
else:
|
||||||
|
if not get_hash(key.get_contents_as_string()) == get_hash(open(file_name).read()):
|
||||||
|
# for now, let's halt. If necessary, we can revisit this and allow for overwrites
|
||||||
|
# to the same buildnum release with different bits
|
||||||
|
self.fatal("`{}` already exists with different checksum.".format(s3_key))
|
||||||
|
self.log("`{}` has the same MD5 checksum, not uploading".format(s3_key))
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
beet_mover = BeetMover(get_aws_auth())
|
||||||
|
beet_mover.run_and_exit()
|
||||||
Loading…
Reference in New Issue
Block a user