mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-21 09:15:35 +00:00
Bug 1491371 - Pin comm branches to mozilla revs without .taskcluster.yml. r=tomprince,dustin
comm-task-env runs before run-task and updates the environment with GECKO_* variables that are defined in a file at the root of the comm repository, ".gecko_rev.yml". run-task needs these variables to be set to find the correct mozilla repository to check out for a particular TB build. The current pinning method of updating ".taskcluster.yml" with the mozilla repository and revision to pin tois no longer supported. Differential Revision: https://phabricator.services.mozilla.com/D16783 --HG-- extra : moz-landing-system : lando
This commit is contained in:
parent
07abfb743a
commit
d7db4546d1
199
taskcluster/scripts/comm-task-env
Executable file
199
taskcluster/scripts/comm-task-env
Executable file
@ -0,0 +1,199 @@
|
||||
#!/usr/bin/python3 -u
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Thunderbird build environment prep for run-task,
|
||||
for use with comm-central derived repositories.
|
||||
|
||||
This script is meant to run prior to run-task on repositories like
|
||||
comm-central that need to check out a copy of a mozilla repository
|
||||
in order to build.
|
||||
See bug 1491371 for background on why this is necessary.
|
||||
|
||||
A project will have a file named ".gecko_rev.yml" in it's root. See the
|
||||
constant "GECKO_REV_CONF" if you want to change that. To download it, the
|
||||
script uses the project repository URL and the revision number.
|
||||
Those are defined in the environment variables:
|
||||
COMM_HEAD_REPOSITORY
|
||||
COMM_HEAD_REV
|
||||
|
||||
.gecko_rev.yml has a structure like (for comm-central):
|
||||
```
|
||||
GECKO_BASE_REPOSITORY: https://hg.mozilla.org/mozilla-unified
|
||||
GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/mozilla-central
|
||||
GECKO_HEAD_REF: default
|
||||
```
|
||||
or for branches:
|
||||
```
|
||||
GECKO_BASE_REPOSITORY: https://hg.mozilla.org/mozilla-unified
|
||||
GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/releases/mozilla-beta
|
||||
GECKO_HEAD_REF: THUNDERBIRD_60_VERBRANCH
|
||||
GECKO_HEAD_REV: 6a830d12f15493a70b1192022c9985eba2139910
|
||||
|
||||
Note about GECKO_HEAD_REV and GECKO_HEAD_REF:
|
||||
GECKO_HEAD_REF is a branch name or "default".
|
||||
GECKO_HEAD_REV is a revision hash.
|
||||
```
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pprint import pformat
|
||||
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
import yaml
|
||||
|
||||
if sys.version_info[0:2] < (3, 5):
|
||||
print('run-task-wrapper requires Python 3.5+')
|
||||
sys.exit(1)
|
||||
|
||||
GECKO_REV_CONF = ".gecko_rev.yml"
|
||||
DEBUG = bool(os.environ.get("RTW_DEBUG", False))
|
||||
|
||||
|
||||
def print_message(msg, prefix=__file__, level=""):
|
||||
"""
|
||||
Print messages.
|
||||
:param object msg: message to print, usually a string, but not always
|
||||
:param str prefix: message prefix
|
||||
:param str level: message level (DEBUG, ERROR, INFO)
|
||||
"""
|
||||
if not isinstance(msg, str):
|
||||
msg = pformat(msg)
|
||||
now = datetime.utcnow().isoformat()
|
||||
# slice microseconds to 3 decimals.
|
||||
now = now[:-3] if now[-7:-6] == '.' else now
|
||||
if level:
|
||||
sys.stdout.write('[{prefix} {now}Z] {level}: {msg}\n'.format(
|
||||
prefix=prefix, now=now, level=level, msg=msg))
|
||||
else:
|
||||
sys.stdout.write('[{prefix} {now}Z] {msg}\n'.format(
|
||||
prefix=prefix, now=now, msg=msg))
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def error_exit(msg):
|
||||
"""Print the error message and exit with error."""
|
||||
print_message(msg, level="ERROR")
|
||||
if DEBUG:
|
||||
raise Exception(msg)
|
||||
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def print_debug(msg):
|
||||
"""Prints a message with DEBUG prefix if DEBUG is enabled
|
||||
with the environment variable "RTW_DEBUG".
|
||||
"""
|
||||
if DEBUG:
|
||||
print_message(msg, level="DEBUG")
|
||||
|
||||
|
||||
def check_environ():
|
||||
"""Check that the necessary environment variables to find the
|
||||
comm- repository are defined. (Set in .taskcluster.yml)
|
||||
:return: tuple(str, str)
|
||||
"""
|
||||
print_debug("Checking environment variables...")
|
||||
project_head_repo = os.environ.get("COMM_HEAD_REPOSITORY", None)
|
||||
project_head_rev = os.environ.get("COMM_HEAD_REV", None)
|
||||
|
||||
if project_head_repo is None or project_head_rev is None:
|
||||
error_exit("Environment NOT Ok:\n\tHead: {}\n\tRev: {}\n").format(
|
||||
project_head_repo, project_head_rev)
|
||||
|
||||
print_debug("Environment Ok:\n\tHead: {}\n\tRev: {}\n".format(
|
||||
project_head_repo, project_head_rev))
|
||||
return project_head_repo, project_head_rev
|
||||
|
||||
|
||||
def download_url(url, retry=1):
|
||||
"""Downloads the given URL. Naively retries (when asked) upon failure
|
||||
:param url: str
|
||||
:param retry: int
|
||||
:return: str
|
||||
"""
|
||||
# Use 1-based counting for display and calculation purposes.
|
||||
for i in range(1, retry+1):
|
||||
try:
|
||||
print_message('Fetching {}. Attempt {} of {}.'.format(
|
||||
url, i, retry))
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
data = response.read().decode("utf-8")
|
||||
return data
|
||||
except (urllib.error.URLError, socket.timeout) as exc:
|
||||
print_message('Unable to retrieve {}'.format(url))
|
||||
if isinstance(exc, urllib.error.URLError):
|
||||
print_message(exc.reason)
|
||||
else: # socket.timeout
|
||||
print_message('Connection timed out.')
|
||||
|
||||
if i < retry: # No more retries
|
||||
wait_time = i * 5 # fail #1: sleep 5s. #2, sleep 10s
|
||||
print_message('Retrying in {} seconds.'.format(wait_time))
|
||||
time.sleep(wait_time)
|
||||
|
||||
error_exit('No more retry attempts! Aborting.')
|
||||
|
||||
|
||||
def fetch_gecko_conf(project_head_repo, project_revision):
|
||||
"""Downloads .gecko_rev.yml from the project repository
|
||||
:param project_head_repo: str
|
||||
:param project_revision: str
|
||||
:return: dict
|
||||
"""
|
||||
gecko_conf_url = '/'.join(
|
||||
[project_head_repo, 'raw-file', project_revision, GECKO_REV_CONF])
|
||||
|
||||
gecko_conf_yml = download_url(gecko_conf_url, retry=5)
|
||||
|
||||
try:
|
||||
gecko_conf = yaml.safe_load(gecko_conf_yml)
|
||||
return gecko_conf
|
||||
except yaml.YAMLError as exc:
|
||||
err_txt = ["Error processing Gecko YAML configuration."]
|
||||
if hasattr(exc, "problem_mark"):
|
||||
mark = exc.problem_mark # pylint: disable=no-member
|
||||
err_txt.append("Error position: line {}, column {}".format(
|
||||
mark.line + 1, mark.column + 1))
|
||||
error_exit('\n'.join(err_txt))
|
||||
|
||||
|
||||
def update_environment(gecko_conf):
|
||||
"""Adds the new variables defined in gecko_conf to the
|
||||
running environment.
|
||||
:param gecko_conf: dict
|
||||
"""
|
||||
print_message("Updating environment with:")
|
||||
print_message(gecko_conf)
|
||||
os.environ.update(gecko_conf)
|
||||
|
||||
print_debug("New environment:")
|
||||
print_debug(os.environ)
|
||||
|
||||
|
||||
def exec_run_task(args):
|
||||
"""Executes run-task with a modified environment."""
|
||||
print_message("Executing: {}".format(pformat(args)))
|
||||
os.execv(args[0], args[1:])
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function."""
|
||||
args = sys.argv[1:] # Remaining args starting with run-task
|
||||
|
||||
project_head_repo, project_revision = check_environ()
|
||||
gecko_conf = fetch_gecko_conf(project_head_repo, project_revision)
|
||||
update_environment(gecko_conf)
|
||||
exec_run_task(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
Loading…
Reference in New Issue
Block a user