Merge mozilla-central to autoland. CLOSED TREE

This commit is contained in:
Csoregi Natalia 2019-11-11 14:34:22 +02:00
commit 6db9f61fb6
37 changed files with 39 additions and 131 deletions

View File

@ -27,7 +27,6 @@ mk_export_correct_style VC_PATH
if ! which mozmake 2>/dev/null; then
export PATH="$PATH:$SOURCE/.."
if ! which mozmake 2>/dev/null; then
TT_SERVER=${TT_SERVER:-https://tooltool.mozilla-releng.net/}
( cd $SOURCE/..; $SOURCE/mach artifact toolchain -v --tooltool-manifest $SOURCE/browser/config/tooltool-manifests/${platform:-win32}/releng.manifest --tooltool-url $TT_SERVER --retry 4${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}})
( cd $SOURCE/..; $SOURCE/mach artifact toolchain -v --tooltool-manifest $SOURCE/browser/config/tooltool-manifests/${platform:-win32}/releng.manifest --retry 4${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}})
fi
fi

View File

@ -1279,14 +1279,17 @@ def main(argv, _skip_logging=False):
(options_obj, args) = parser.parse_args(argv[1:])
# default the options list if not provided
if not options_obj.base_url:
options_obj.base_url = ['https://tooltool.mozilla-releng.net/']
tooltool_host = os.environ.get('TOOLTOOL_HOST', 'tooltool.mozilla-releng.net')
taskcluster_proxy_url = os.environ.get('TASKCLUSTER_PROXY_URL')
if taskcluster_proxy_url:
tooltool_url = '{}/{}'.format(taskcluster_proxy_url, tooltool_host)
else:
tooltool_url = 'https://{}'.format(tooltool_host)
# ensure all URLs have a trailing slash
def add_slash(url):
return url if url.endswith('/') else (url + '/')
options_obj.base_url = [add_slash(u) for u in options_obj.base_url]
options_obj.base_url = [add_slash(tooltool_url)]
# expand ~ in --authentication-file
if options_obj.auth_file:

View File

@ -165,8 +165,6 @@ class PackageFrontend(MachCommandBase):
help='Explicit tooltool manifest to process')
@CommandArgument('--authentication-file', metavar='FILE',
help='Use the RelengAPI token found in the given file to authenticate')
@CommandArgument('--tooltool-url', metavar='URL',
help='Use the given url as tooltool server')
@CommandArgument('--no-unpack', action='store_true',
help='Do not unpack any downloaded file')
@CommandArgument('--retry', type=int, default=4,
@ -179,7 +177,7 @@ class PackageFrontend(MachCommandBase):
def artifact_toolchain(self, verbose=False, cache_dir=None,
skip_cache=False, from_build=(),
tooltool_manifest=None, authentication_file=None,
tooltool_url=None, no_unpack=False, retry=None,
no_unpack=False, retry=None,
artifact_manifest=None, files=()):
'''Download, cache and install pre-built toolchains.
'''
@ -189,7 +187,6 @@ class PackageFrontend(MachCommandBase):
open_manifest,
unpack_file,
)
from requests.adapters import HTTPAdapter
import redo
import requests
@ -213,26 +210,16 @@ class PackageFrontend(MachCommandBase):
if not cache_dir:
cache_dir = os.path.join(self._mach_context.state_dir, 'toolchains')
tooltool_url = (tooltool_url or
'https://tooltool.mozilla-releng.net').rstrip('/')
tooltool_host = os.environ.get('TOOLTOOL_HOST', 'tooltool.mozilla-releng.net')
taskcluster_proxy_url = os.environ.get('TASKCLUSTER_PROXY_URL')
if taskcluster_proxy_url:
tooltool_url = '{}/{}'.format(taskcluster_proxy_url, tooltool_host)
else:
tooltool_url = 'https://{}'.format(tooltool_host)
cache = ArtifactCache(cache_dir=cache_dir, log=self.log,
skip_cache=skip_cache)
if authentication_file:
with open(authentication_file, 'rb') as f:
token = f.read().strip()
class TooltoolAuthenticator(HTTPAdapter):
def send(self, request, *args, **kwargs):
request.headers['Authorization'] = \
'Bearer {}'.format(token)
return super(TooltoolAuthenticator, self).send(
request, *args, **kwargs)
cache._download_manager.session.mount(
tooltool_url, TooltoolAuthenticator())
class DownloadRecord(FileRecord):
def __init__(self, url, *args, **kwargs):
super(DownloadRecord, self).__init__(*args, **kwargs)

View File

@ -7,9 +7,6 @@ job-defaults:
using: mozharness
use-caches: false
tooltool-downloads: internal
extra-config:
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
win32/debug:
description: "Win32 Debug"

View File

@ -149,8 +149,6 @@ jobs:
tooltool-downloads: internal
extra-config:
stage_platform: win32
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
fetches:
toolchain:
- win64-clang-cl
@ -183,8 +181,6 @@ jobs:
tooltool-downloads: internal
extra-config:
stage_platform: win64
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
fetches:
toolchain:
- win64-clang-cl

View File

@ -120,9 +120,6 @@ jobs:
- builds/taskcluster_base_win64.py
- builds/taskcluster_sub_win64/searchfox_debug.py
tooltool-downloads: internal
extra-config:
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
fetches:
toolchain:
- win64-clang-cl

View File

@ -8,6 +8,8 @@ job-defaults:
docker-image: {in-tree: debian7-amd64-build}
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/jsshell.manifest"
run:
tooltool-downloads: public
sm-package-linux64/opt:
description: "Spidermonkey source package and test"

View File

@ -8,6 +8,8 @@ job-defaults:
max-run-time: 36000
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
run:
tooltool-downloads: internal
sm-plain-win64/debug:
description: "Spidermonkey Plain win64 debug"

View File

@ -84,8 +84,6 @@ jobs:
tooltool-downloads: internal
extra-config:
mozconfig_variant: debug
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
fetches:
toolchain:
- win64-clang-cl

View File

@ -2,7 +2,6 @@
set -x
TOOLTOOL_SERVER=${TOOLTOOL_SERVER:-https://tooltool.mozilla-releng.net/}
SPIDERMONKEY_VARIANT=${SPIDERMONKEY_VARIANT:-plain}
UPLOAD_DIR=${UPLOAD_DIR:-$HOME/artifacts/}
WORK=${WORK:-$HOME/workspace}
@ -57,7 +56,7 @@ fi
# manifests.
BROWSER_PLATFORM=$PLATFORM_OS$BITS
(cd $TOOLTOOL_CHECKOUT && ${SRCDIR}/mach artifact toolchain${TOOLTOOL_MANIFEST:+ -v $TOOLTOOL_AUTH_FLAGS --tooltool-url $TOOLTOOL_SERVER --tooltool-manifest $SRCDIR/$TOOLTOOL_MANIFEST}${TOOLTOOL_CACHE:+ --cache-dir $TOOLTOOL_CACHE}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}})
(cd $TOOLTOOL_CHECKOUT && ${SRCDIR}/mach artifact toolchain${TOOLTOOL_MANIFEST:+ -v $TOOLTOOL_AUTH_FLAGS --tooltool-manifest $SRCDIR/$TOOLTOOL_MANIFEST}${TOOLTOOL_CACHE:+ --cache-dir $TOOLTOOL_CACHE}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}})
) || exit 1 # end of set -e scope

View File

@ -18,11 +18,6 @@ if [ -e "$TOOLTOOL_AUTH_FILE" ]; then
TOOLTOOL_DL_FLAGS="${TOOLTOOL_DL_FLAGS=} --authentication-file=$TOOLTOOL_AUTH_FILE"
fi
if [ -n "$TASKCLUSTER_PROXY_URL" ]; then
# When the worker has the relengapi proxy setup, use it.
TOOLTOOL_DL_FLAGS="${TOOLTOOL_DL_FLAGS=} --tooltool-url=${TASKCLUSTER_PROXY_URL}/tooltool.mozilla-releng.net/"
fi
if [ -n "$UPLOAD_DIR" ]; then
TOOLTOOL_DL_FLAGS="${TOOLTOOL_DL_FLAGS=} --artifact-manifest $UPLOAD_DIR/toolchains.json"
fi

View File

@ -17,7 +17,6 @@ from taskgraph.transforms.job import (
from taskgraph.transforms.job.common import (
docker_worker_add_artifacts,
generic_worker_add_artifacts,
add_tooltool,
)
sm_run_schema = Schema({
@ -30,6 +29,12 @@ sm_run_schema = Schema({
# Base work directory used to set up the task.
Required('workdir'): basestring,
Required('tooltool-downloads'): Any(
False,
'public',
'internal',
),
})
@ -52,7 +57,6 @@ def docker_worker_spidermonkey(config, job, taskdesc):
})
docker_worker_add_artifacts(config, job, taskdesc)
add_tooltool(config, job, taskdesc)
env = worker.setdefault('env', {})
env.update({

View File

@ -385,6 +385,7 @@ def make_job_description(config, jobs):
if job.get('extra'):
job_description['extra'] = job['extra']
job_description['run']['tooltool-downloads'] = job['tooltool']
if job['worker-type'] == "b-win2012":
job_description['worker'] = {
'os': 'windows',
@ -398,7 +399,6 @@ def make_job_description(config, jobs):
'max-run-time': job['run-time'],
'chain-of-trust': True,
}
job_description['run']['tooltool-downloads'] = job['tooltool']
job_description['run']['need-xvfb'] = True
if job.get('docker-image'):

View File

@ -43,7 +43,6 @@ config = {
'run-tests',
],
"tooltool_cache": os.environ.get("TOOLTOOL_CACHE"),
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/'],
"hostutils_manifest_path": "testing/config/tooltool-manifests/linux64/hostutils.manifest",
"avds_dir": "/builds/worker/workspace/build/.android",
# "log_format": "%(levelname)8s - %(message)s",

View File

@ -22,7 +22,6 @@ config = {
# from android_common.py
"download_tooltool": True,
"minidump_stackwalk_path": "linux64-minidump_stackwalk",
"tooltool_servers": ['https://tooltool.mozilla-releng.net/'],
"minidump_tooltool_manifest_path": "config/tooltool-manifests/linux64/releng.manifest",
"xpcshell_extra": "--remoteTestRoot=/data/local/tests",
}

View File

@ -5,7 +5,6 @@
config = {
"tooltool_manifest_path": "testing/config/tooltool-manifests/androidx86_7_0/releng.manifest",
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/'],
"emulator_manifest": """
[
{

View File

@ -3,9 +3,6 @@
config = {
"taskcluster": {
# use the relengapi proxy to talk to tooltool
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/'],
"tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
'upload_env': {
'UPLOAD_PATH': '/builds/worker/artifacts',
},

View File

@ -32,6 +32,4 @@ config = {
'PATH': ('{MOZ_FETCHES_DIR}/clang/bin/:%(PATH)s'
.format(MOZ_FETCHES_DIR=os.environ['MOZ_FETCHES_DIR'])),
},
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/'],
"tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
}

View File

@ -15,7 +15,6 @@ config = {
"tooltool_cache": "/builds/tooltool_cache",
"download_tooltool": True,
"minidump_stackwalk_path": "linux64-minidump_stackwalk",
"tooltool_servers": ['https://tooltool.mozilla-releng.net/'],
"minidump_tooltool_manifest_path": "config/tooltool-manifests/linux64/releng.manifest",
"hostutils_manifest_path": "testing/config/tooltool-manifests/linux64/hostutils.manifest",
}

View File

@ -6,7 +6,6 @@ config = {
"locale": os.environ.get("LOCALE"),
# ToolTool
"tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
'run_configure': False,

View File

@ -6,7 +6,6 @@ config = {
"locale": os.environ.get("LOCALE"),
# ToolTool
"tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
'run_configure': False,

View File

@ -6,6 +6,5 @@ config = {
"repack_id": os.environ.get("REPACK_ID"),
# ToolTool
"tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
}

View File

@ -6,6 +6,5 @@ config = {
"locale": os.environ.get("LOCALE"),
# ToolTool
"tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
}

View File

@ -5,7 +5,6 @@ platform = "win32"
config = {
"repack_id": os.environ.get("REPACK_ID"),
'tooltool_url': 'https://tooltool.mozilla-releng.net/',
'run_configure': False,
'env': {

View File

@ -5,7 +5,6 @@ platform = "win64"
config = {
"repack_id": os.environ.get("REPACK_ID"),
'tooltool_url': 'https://tooltool.mozilla-releng.net/',
'run_configure': False,
'env': {

View File

@ -15,7 +15,6 @@ config = {
"manifest": "mobile/android/config/tooltool-manifests/android/releng.manifest",
"output_dir": "%(abs_work_dir)s/src",
},
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/'],
"upload_env": {
'UPLOAD_PATH': '/builds/worker/artifacts/',

View File

@ -13,8 +13,6 @@ config = {
'UPLOAD_PATH': '/builds/worker/artifacts/',
},
"tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
"vcs_share_base": "/builds/hg-shared",
}

View File

@ -15,7 +15,6 @@ config = {
'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
},
"tooltool_url": 'https://tooltool.mozilla-releng.net/',
'tooltool_manifest_src': "browser/config/tooltool-manifests/win32/releng.manifest",
# use mozmake?

View File

@ -15,7 +15,6 @@ config = {
'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
},
"tooltool_url": 'https://tooltool.mozilla-releng.net/',
'tooltool_manifest_src': "browser/config/tooltool-manifests/win64/releng.manifest",
# use mozmake?

View File

@ -25,5 +25,4 @@ config = {
"minidump_stackwalk_path": "linux64-minidump_stackwalk",
"per_test_category": "web-platform",
"tooltool_cache": os.environ.get("TOOLTOOL_CACHE"),
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/'],
}

View File

@ -1278,15 +1278,18 @@ def main(argv, _skip_logging=False):
(options_obj, args) = parser.parse_args(argv[1:])
# default the options list if not provided
if not options_obj.base_url:
options_obj.base_url = ['https://tooltool.mozilla-releng.net/']
tooltool_host = os.environ.get('TOOLTOOL_HOST', 'tooltool.mozilla-releng.net')
taskcluster_proxy_url = os.environ.get('TASKCLUSTER_PROXY_URL')
if taskcluster_proxy_url:
tooltool_url = '{}/{}'.format(taskcluster_proxy_url, tooltool_host)
else:
tooltool_url = 'https://{}'.format(tooltool_host)
# ensure all URLs have a trailing slash
def add_slash(url):
return url if url.endswith('/') else (url + '/')
options_obj.base_url = [add_slash(u) for u in options_obj.base_url]
options_obj.base_url = [add_slash(tooltool_url)]
# expand ~ in --authentication-file
if options_obj.auth_file:
options_obj.auth_file = os.path.expanduser(options_obj.auth_file)

View File

@ -596,33 +596,6 @@ items from that key's value."
self.info("Both --dump-config and --dump-config-hierarchy don't "
"actually run any actions.")
def _assert_cfg_valid_for_action(self, dependencies, action):
""" assert dependency keys are in config for given action.
Takes a list of dependencies and ensures that each have an
assoctiated key in the config. Displays error messages as
appropriate.
"""
# TODO add type and value checking, not just keys
# TODO solution should adhere to: bug 699343
# TODO add this to BaseScript when the above is done
# for now, let's just use this as a way to save typing...
c = self.config
undetermined_keys = []
err_template = "The key '%s' could not be determined \
and is needed for the action '%s'. Please add this to your config \
or run without that action (ie: --no-{action})"
for dep in dependencies:
if dep not in c:
undetermined_keys.append(dep)
if undetermined_keys:
fatal_msgs = [err_template % (key, action)
for key in undetermined_keys]
self.fatal("".join(fatal_msgs))
# otherwise:
return # all good
def _query_build_prop_from_app_ini(self, prop, app_ini_path=None):
dirs = self.query_abs_dirs()
print_conf_setting_path = os.path.join(dirs['abs_src_dir'],
@ -781,10 +754,6 @@ or run without that action (ie: --no-{action})"
env = self.query_build_env()
env.update(self.query_mach_build_env())
self._assert_cfg_valid_for_action(
['tooltool_url'],
'build'
)
c = self.config
dirs = self.query_abs_dirs()
toolchains = os.environ.get('MOZ_TOOLCHAINS')
@ -807,8 +776,6 @@ or run without that action (ie: --no-{action})"
cmd.extend([
'--tooltool-manifest',
os.path.join(dirs['abs_src_dir'], manifest_src),
'--tooltool-url',
c['tooltool_url'],
])
auth_file = self._get_tooltool_auth_file()
if auth_file:

View File

@ -10,10 +10,6 @@ TooltoolErrorList = PythonErrorList + [{
}]
TOOLTOOL_SERVERS = [
'https://tooltool.mozilla-releng.net/',
]
_here = os.path.abspath(os.path.dirname(__file__))
_external_tools_path = os.path.normpath(os.path.join(_here, '..', '..',
'external_tools'))
@ -21,8 +17,8 @@ _external_tools_path = os.path.normpath(os.path.join(_here, '..', '..',
class TooltoolMixin(object):
"""Mixin class for handling tooltool manifests.
To use a tooltool server other than the Mozilla server, override
config['tooltool_servers']. To specify a different authentication
To use a tooltool server other than the Mozilla server, set
TOOLTOOL_HOST in the environment. To specify a different authentication
file than that used in releng automation,override
config['tooltool_authentication_file']; set it to None to not pass
any authentication information (OK for public files)
@ -67,17 +63,6 @@ class TooltoolMixin(object):
os.path.join(_external_tools_path, 'tooltool.py'),
]
# get the tooltool servers from configuration
default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS)
# add slashes (bug 1155630)
def add_slash(url):
return url if url.endswith('/') else (url + '/')
default_urls = [add_slash(u) for u in default_urls]
for url in default_urls:
cmd.extend(['--tooltool-url' if self.topsrcdir else '--url', url])
# handle authentication file, if given
auth_file = self._get_auth_file()
if auth_file and os.path.exists(auth_file):

View File

@ -465,8 +465,6 @@ class DesktopSingleLocale(LocalesMixin, AutomationMixin,
cmd.extend([
'--tooltool-manifest',
os.path.join(dirs['abs_mozilla_dir'], manifest_src),
'--tooltool-url',
config['tooltool_url'],
])
auth_file = self._get_tooltool_auth_file()
if auth_file and os.path.exists(auth_file):

View File

@ -52,7 +52,6 @@ class FxDesktopBuild(BuildScript, TryToolsMixin, object):
# jobs have a minimal `hg pull`.
"clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
"repo_base": "https://hg.mozilla.org",
'tooltool_url': 'https://tooltool.mozilla-releng.net/',
"graph_selector": "/server/collect.cgi",
# only used for make uploadsymbols
'old_packages': [

View File

@ -120,8 +120,6 @@ class Repackage(BaseScript):
cmd.extend([
'--tooltool-manifest',
os.path.join(dirs['abs_mozilla_dir'], manifest_src),
'--tooltool-url',
config['tooltool_url'],
])
auth_file = self._get_tooltool_auth_file()
if auth_file:

View File

@ -56,10 +56,10 @@ tar cvz --exclude=eslint-plugin-mozilla --exclude=eslint-plugin-spidermonkey-js
echo "Adding eslint.tar.gz to tooltool..."
rm tools/lint/eslint/manifest.tt
./python/mozbuild/mozbuild/action/tooltool.py add --visibility public --unpack eslint.tar.gz --url="https://tooltool.mozilla-releng.net/"
./python/mozbuild/mozbuild/action/tooltool.py add --visibility public --unpack eslint.tar.gz
echo "Uploading eslint.tar.gz to tooltool..."
./python/mozbuild/mozbuild/action/tooltool.py upload --authentication-file=~/.tooltool-token --message "node_modules folder update for tools/lint/eslint" --url="https://tooltool.mozilla-releng.net/"
./python/mozbuild/mozbuild/action/tooltool.py upload --authentication-file=~/.tooltool-token --message "node_modules folder update for tools/lint/eslint"
echo "Cleaning up..."
mv manifest.tt tools/lint/eslint/manifest.tt