mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-17 23:35:34 +00:00
Bug 1492033 - Create partner repacks during esr60 releases r=tomprince
Supports partner builds on esr60 by re-arranging how we specify the manifest repositories, which are different for esr60 compared to beta+release. The defaults for enabling partners and EME-free now depend on the partner-urls definition in taskcluster/ci/config.yml. This patch also adds support for partner & EMEfree repacks in try staging releases, for all the various release branches. It makes release_level available early in the decision task (for the release promotion action) where a Parameters object is not available. Signing worker type is no longer hard-coded, and we use level-1 secrets for querying Github (via bug 1513375). Differential Revision: https://phabricator.services.mozilla.com/D15185 --HG-- extra : moz-landing-system : lando
This commit is contained in:
Nick Thomas
parent
c5132c335e
commit
463964570d
@ -217,11 +217,28 @@ scriptworker:
|
||||
'scriptworker-prov-v1/balrog-dev':
|
||||
- 'project:releng:balrog:server:dep'
|
||||
|
||||
|
||||
partner:
|
||||
release:
|
||||
release-partner-repack: git@github.com:mozilla-partners/repack-manifests.git
|
||||
release-eme-free-repack: git@github.com:mozilla-partners/mozilla-EME-free-manifest
|
||||
staging:
|
||||
release-partner-repack: git@github.com:mozilla-releng/staging-repack-manifests.git
|
||||
release-eme-free-repack: git@github.com:mozilla-releng/staging-repack-manifests.git
|
||||
partner-urls:
|
||||
release-partner-repack:
|
||||
by-release-product:
|
||||
default: null
|
||||
firefox:
|
||||
by-release-type:
|
||||
default: null
|
||||
beta|release.*:
|
||||
by-release-level:
|
||||
production: 'git@github.com:mozilla-partners/repack-manifests.git'
|
||||
staging: 'git@github.com:moz-releng-automation-stage/repack-manifests.git'
|
||||
esr60:
|
||||
by-release-level:
|
||||
production: 'git@github.com:mozilla-partners/esr-repack-manifests.git'
|
||||
staging: 'git@github.com:moz-releng-automation-stage/esr-repack-manifests.git'
|
||||
release-eme-free-repack:
|
||||
by-release-product:
|
||||
default: null
|
||||
firefox:
|
||||
by-release-type:
|
||||
default: null
|
||||
beta|release.*:
|
||||
by-release-level:
|
||||
production: 'git@github.com:mozilla-partners/mozilla-EME-free-manifest.git'
|
||||
staging: 'git@github.com:moz-releng-automation-stage/mozilla-EME-free-manifest.git'
|
||||
|
||||
@ -16,8 +16,6 @@ from taskgraph.util.hg import find_hg_revision_push_info
|
||||
from taskgraph.util.taskcluster import get_artifact
|
||||
from taskgraph.util.partials import populate_release_history
|
||||
from taskgraph.util.partners import (
|
||||
EMEFREE_BRANCHES,
|
||||
PARTNER_BRANCHES,
|
||||
fix_partner_config,
|
||||
get_partner_config_by_url,
|
||||
get_partner_url_config,
|
||||
@ -39,7 +37,8 @@ def is_release_promotion_available(parameters):
|
||||
def get_partner_config(partner_url_config, github_token):
|
||||
partner_config = {}
|
||||
for kind, url in partner_url_config.items():
|
||||
partner_config[kind] = get_partner_config_by_url(url, kind, github_token)
|
||||
if url:
|
||||
partner_config[kind] = get_partner_config_by_url(url, kind, github_token)
|
||||
return partner_config
|
||||
|
||||
|
||||
@ -191,8 +190,7 @@ def get_flavors(graph_config, param):
|
||||
},
|
||||
'release_enable_partners': {
|
||||
'type': 'boolean',
|
||||
'default': False,
|
||||
'description': ('Toggle for creating partner repacks'),
|
||||
'description': 'Toggle for creating partner repacks',
|
||||
},
|
||||
'release_partner_build_number': {
|
||||
'type': 'integer',
|
||||
@ -212,14 +210,13 @@ def get_flavors(graph_config, param):
|
||||
},
|
||||
'release_partner_config': {
|
||||
'type': 'object',
|
||||
'description': ('Partner configuration to use for partner repacks.'),
|
||||
'description': 'Partner configuration to use for partner repacks.',
|
||||
'properties': {},
|
||||
'additionalProperties': True,
|
||||
},
|
||||
'release_enable_emefree': {
|
||||
'type': 'boolean',
|
||||
'default': False,
|
||||
'description': ('Toggle for creating EME-free repacks'),
|
||||
'description': 'Toggle for creating EME-free repacks',
|
||||
},
|
||||
'required_signoffs': {
|
||||
'type': 'array',
|
||||
@ -276,14 +273,6 @@ def release_promotion_action(parameters, graph_config, input, task_group_id, tas
|
||||
do_not_optimize = input.get(
|
||||
'do_not_optimize', promotion_config.get('do-not-optimize', [])
|
||||
)
|
||||
release_enable_partners = input.get(
|
||||
'release_enable_partners',
|
||||
parameters['project'] in PARTNER_BRANCHES and product in ('firefox',)
|
||||
)
|
||||
release_enable_emefree = input.get(
|
||||
'release_enable_emefree',
|
||||
parameters['project'] in EMEFREE_BRANCHES and product in ('firefox',)
|
||||
)
|
||||
|
||||
# make parameters read-write
|
||||
parameters = dict(parameters)
|
||||
@ -322,28 +311,36 @@ def release_promotion_action(parameters, graph_config, input, task_group_id, tas
|
||||
if promotion_config.get('is-rc'):
|
||||
parameters['release_type'] += '-rc'
|
||||
parameters['release_eta'] = input.get('release_eta', '')
|
||||
parameters['release_enable_partners'] = release_enable_partners
|
||||
parameters['release_partners'] = input.get('release_partners')
|
||||
parameters['release_enable_emefree'] = release_enable_emefree
|
||||
parameters['release_product'] = product
|
||||
# When doing staging releases on try, we still want to re-use tasks from
|
||||
# previous graphs.
|
||||
parameters['optimize_target_tasks'] = True
|
||||
|
||||
# Partner/EMEfree are enabled by default when get_partner_url_config() returns a non-null url
|
||||
# The action input may override by sending False. It's an error to send True with no url found
|
||||
partner_url_config = get_partner_url_config(parameters, graph_config)
|
||||
release_enable_partners = partner_url_config['release-partner-repack'] is not None
|
||||
release_enable_emefree = partner_url_config['release-eme-free-repack'] is not None
|
||||
if input.get('release_enable_partners') is False:
|
||||
release_enable_partners = False
|
||||
elif input.get('release_enable_partners') is True and not release_enable_partners:
|
||||
raise Exception("Can't enable partner repacks when no config url found")
|
||||
if input.get('release_enable_emefree') is False:
|
||||
release_enable_emefree = False
|
||||
elif input.get('release_enable_emefree') is True and not release_enable_emefree:
|
||||
raise Exception("Can't enable EMEfree when no config url found")
|
||||
parameters['release_enable_partners'] = release_enable_partners
|
||||
parameters['release_enable_emefree'] = release_enable_emefree
|
||||
|
||||
partner_config = input.get('release_partner_config')
|
||||
if not partner_config and (release_enable_emefree or release_enable_partners):
|
||||
partner_url_config = get_partner_url_config(
|
||||
parameters, graph_config, enable_emefree=release_enable_emefree,
|
||||
enable_partners=release_enable_partners
|
||||
)
|
||||
github_token = get_token(parameters)
|
||||
partner_config = get_partner_config(partner_url_config, github_token)
|
||||
|
||||
if input.get('release_partner_build_number'):
|
||||
parameters['release_partner_build_number'] = input['release_partner_build_number']
|
||||
|
||||
if partner_config:
|
||||
parameters['release_partner_config'] = fix_partner_config(partner_config)
|
||||
parameters['release_partners'] = input.get('release_partners')
|
||||
if input.get('release_partner_build_number'):
|
||||
parameters['release_partner_build_number'] = input['release_partner_build_number']
|
||||
|
||||
if input['version']:
|
||||
parameters['version'] = input['version']
|
||||
|
||||
@ -10,8 +10,8 @@ import attr
|
||||
import yaml
|
||||
from mozpack import path
|
||||
|
||||
from .util.schema import validate_schema, Schema
|
||||
from voluptuous import Required, Optional
|
||||
from .util.schema import validate_schema, Schema, optionally_keyed_by
|
||||
from voluptuous import Required, Optional, Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@ -58,11 +58,13 @@ graph_config_schema = Schema({
|
||||
# Mapping of scriptworker types to scopes they accept
|
||||
Required('worker-types'): {basestring: [basestring]}
|
||||
},
|
||||
Required('partner'): {
|
||||
# Release config for partner repacks
|
||||
Required('release'): {basestring: basestring},
|
||||
# Staging config for partner repacks
|
||||
Required('staging'): {basestring: basestring},
|
||||
Required('partner-urls'): {
|
||||
Required('release-partner-repack'):
|
||||
optionally_keyed_by('release-product', 'release-level', 'release-type',
|
||||
Any(basestring, None)),
|
||||
Required('release-eme-free-repack'):
|
||||
optionally_keyed_by('release-product', 'release-level', 'release-type',
|
||||
Any(basestring, None)),
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@ -16,7 +16,7 @@ from mozbuild.util import ReadOnlyDict, memoize
|
||||
from mozversioncontrol import get_repository_object
|
||||
|
||||
from . import GECKO
|
||||
from .util.attributes import RELEASE_PROJECTS
|
||||
from .util.attributes import release_level
|
||||
|
||||
|
||||
class ParameterMismatch(Exception):
|
||||
@ -183,7 +183,7 @@ class Parameters(ReadOnlyDict):
|
||||
|
||||
:return basestring: One of "production" or "staging".
|
||||
"""
|
||||
return 'production' if self['project'] in RELEASE_PROJECTS else 'staging'
|
||||
return release_level(self['project'])
|
||||
|
||||
|
||||
def load_parameters_file(filename, strict=True, overrides=None):
|
||||
|
||||
@ -14,6 +14,7 @@ from taskgraph.util.partners import check_if_partners_enabled
|
||||
from taskgraph.util.scriptworker import (
|
||||
add_scope_prefix,
|
||||
get_signing_cert_scope_per_platform,
|
||||
get_worker_type_for_scope,
|
||||
)
|
||||
from taskgraph.util.taskcluster import get_artifact_path
|
||||
from taskgraph.transforms.task import task_description_schema
|
||||
@ -109,8 +110,7 @@ def make_repackage_signing_description(config, jobs):
|
||||
task = {
|
||||
'label': label,
|
||||
'description': description,
|
||||
# 'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
|
||||
'worker-type': 'scriptworker-prov-v1/signing-linux-v1',
|
||||
'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
|
||||
'worker': {'implementation': 'scriptworker-signing',
|
||||
'upstream-artifacts': upstream_artifacts,
|
||||
'max-run-time': 3600},
|
||||
|
||||
@ -138,3 +138,12 @@ def sorted_unique_list(*args):
|
||||
"""Join one or more lists, and return a sorted list of unique members"""
|
||||
combined = set().union(*args)
|
||||
return sorted(combined)
|
||||
|
||||
|
||||
def release_level(project):
|
||||
"""
|
||||
Whether this is a staging release or not.
|
||||
|
||||
:return basestring: One of "production" or "staging".
|
||||
"""
|
||||
return 'production' if project in RELEASE_PROJECTS else 'staging'
|
||||
|
||||
@ -8,6 +8,8 @@ from redo import retry
|
||||
import requests
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from taskgraph.util.attributes import release_level
|
||||
from taskgraph.util.schema import resolve_keyed_by
|
||||
|
||||
# Suppress chatty requests logging
|
||||
logging.getLogger("requests").setLevel(logging.WARNING)
|
||||
@ -15,20 +17,6 @@ logging.getLogger("requests").setLevel(logging.WARNING)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
GITHUB_API_ENDPOINT = "https://api.github.com/graphql"
|
||||
PARTNER_BRANCHES = {
|
||||
'mozilla-beta': 'release',
|
||||
'mozilla-release': 'release',
|
||||
'maple': 'release',
|
||||
'birch': 'release',
|
||||
'jamun': 'release',
|
||||
}
|
||||
EMEFREE_BRANCHES = {
|
||||
'mozilla-beta': 'release',
|
||||
'mozilla-release': 'release',
|
||||
'maple': 'release',
|
||||
'birch': 'release',
|
||||
'jamun': 'release',
|
||||
}
|
||||
|
||||
"""
|
||||
LOGIN_QUERY, MANIFEST_QUERY, and REPACK_CFG_QUERY are all written to the Github v4 API,
|
||||
@ -75,7 +63,7 @@ MANIFEST_QUERY = """query {
|
||||
# Returns the contents of desktop/*/repack.cfg for a partner repository
|
||||
REPACK_CFG_QUERY = """query{
|
||||
repository(owner:"%(owner)s", name:"%(repo)s") {
|
||||
object(expression: "master:desktop/"){
|
||||
object(expression: "%(revision)s:desktop/"){
|
||||
... on Tree {
|
||||
entries {
|
||||
name
|
||||
@ -151,8 +139,6 @@ LOCALES_FILE = os.path.join(
|
||||
partner_configs = {}
|
||||
|
||||
|
||||
# TODO - grant private repo access to P.A.T.
|
||||
# TODO - add level-3 as well, cleanup as of level
|
||||
def get_token(params):
|
||||
""" We use a Personal Access Token from Github to lookup partner config. No extra scopes are
|
||||
needed on the token to read public repositories, but need the 'repo' scope to see private
|
||||
@ -161,8 +147,7 @@ def get_token(params):
|
||||
"""
|
||||
|
||||
# The 'usual' method - via taskClusterProxy for decision tasks
|
||||
# TODO use {level}? Or allow the token to level 1 and remove level from the path?
|
||||
url = "{secret_root}/project/releng/gecko/build/level-2/partner-github-api".format(
|
||||
url = "{secret_root}/project/releng/gecko/build/level-{level}/partner-github-api".format(
|
||||
secret_root=TASKCLUSTER_PROXY_SECRET_ROOT, **params
|
||||
)
|
||||
try:
|
||||
@ -226,22 +211,27 @@ def get_partners(manifestRepo, token):
|
||||
name = child.attrib['name']
|
||||
url = child.attrib['fetch']
|
||||
remotes[name] = url
|
||||
log.debug('Added remote %s from %s', name, url)
|
||||
log.debug('Added remote %s at %s', name, url)
|
||||
elif child.tag == 'project':
|
||||
# we don't need to check any code repos
|
||||
if 'scripts' in child.attrib['path']:
|
||||
continue
|
||||
partner_url = "%s%s" % (remotes[child.attrib['remote']],
|
||||
child.attrib['name'])
|
||||
owner, _ = get_repo_params(remotes[child.attrib['remote']] + '_')
|
||||
partner_url = {
|
||||
'owner': owner,
|
||||
'repo': child.attrib['name'],
|
||||
'revision': child.attrib['revision'],
|
||||
}
|
||||
partners[child.attrib['name']] = partner_url
|
||||
log.debug("Added partner %s" % partner_url)
|
||||
log.debug("Added partner %s at revision %s" % (partner_url['repo'],
|
||||
partner_url['revision']))
|
||||
return partners
|
||||
|
||||
|
||||
def parse_config(data):
|
||||
""" Parse a single repack.cfg file into a python dictionary.
|
||||
data is contents of the file, in "foo=bar\nbaz=buzz" style. We do some translation on
|
||||
locales and platforms data, otherewise passthrough
|
||||
locales and platforms data, otherwise passthrough
|
||||
"""
|
||||
ALLOWED_KEYS = ('locales', 'upload_to_candidates', 'platforms')
|
||||
config = {'platforms': []}
|
||||
@ -266,8 +256,7 @@ def parse_config(data):
|
||||
def get_repack_configs(repackRepo, token):
|
||||
""" For a partner repository, retrieve all the repack.cfg files and parse them into a dict """
|
||||
log.debug("Querying for configs in %s", repackRepo)
|
||||
owner, repo = get_repo_params(repackRepo)
|
||||
query = REPACK_CFG_QUERY % {'owner': owner, 'repo': repo}
|
||||
query = REPACK_CFG_QUERY % repackRepo
|
||||
raw_configs = query_api(query, token)
|
||||
raw_configs = raw_configs['data']['repository']['object']['entries']
|
||||
|
||||
@ -291,6 +280,8 @@ def get_partner_config_by_url(manifest_url, kind, token, partner_subset=None):
|
||||
Supports caching data by kind to avoid repeated requests, relying on the related kinds for
|
||||
partner repacking, signing, repackage, repackage signing all having the same kind prefix.
|
||||
"""
|
||||
if not manifest_url:
|
||||
raise RuntimeError('Manifest url for {} not defined'.format(kind))
|
||||
if kind not in partner_configs:
|
||||
log.info('Looking up data for %s from %s', kind, manifest_url)
|
||||
check_login(token)
|
||||
@ -397,15 +388,15 @@ def locales_per_build_platform(build_platform, locales):
|
||||
return [locale for locale in locales if locale not in exclude]
|
||||
|
||||
|
||||
def get_partner_url_config(parameters, graph_config, enable_emefree=True, enable_partners=True):
|
||||
partner_url_config = {}
|
||||
project = parameters['project']
|
||||
if enable_emefree:
|
||||
alias = EMEFREE_BRANCHES[project]
|
||||
partner_url_config['release-eme-free-repack'] = \
|
||||
graph_config['partner'][alias]['release-eme-free-repack']
|
||||
if enable_partners:
|
||||
alias = PARTNER_BRANCHES[project]
|
||||
partner_url_config['release-partner-repack'] = \
|
||||
graph_config['partner'][alias]['release-partner-repack']
|
||||
def get_partner_url_config(parameters, graph_config):
|
||||
partner_url_config = deepcopy(graph_config['partner-urls'])
|
||||
substitutions = {
|
||||
'release-product': parameters['release_product'],
|
||||
'release-level': release_level(parameters['project']),
|
||||
'release-type': parameters["release_type"]
|
||||
}
|
||||
resolve_keyed_by(partner_url_config, 'release-eme-free-repack', 'eme-free manifest_url',
|
||||
**substitutions)
|
||||
resolve_keyed_by(partner_url_config, 'release-partner-repack', 'partner manifest url',
|
||||
**substitutions)
|
||||
return partner_url_config
|
||||
|
||||
Loading…
Reference in New Issue
Block a user