Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-08 10:44:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Backed out 3 changesets (bug 1636271) for breaking the Gecko Decision Task. CLOSED TREE

Browse Source 
Backed out changeset 4b0f13fcf941 (bug 1636271)
Backed out changeset 4250f49877ba (bug 1636271)
Backed out changeset d3f93bd6b1f9 (bug 1636271)
This commit is contained in:
Narcis Beleuzu 2020-07-22 23:40:32 +03:00
parent f197b3c60e
commit ef290a86e8
17 changed files with 83 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
taskcluster/docs/parameters.rst
View File

@ -135,10 +135,6 @@ syntax or reading a project-specific configuration file).
Optimization
------------
``optimize_strategies``
A python path of the form ``<module>:<object>`` containing a dictionary of
optimization strategies to use, overwriting the defaults.
``optimize_target_tasks``
If true, then target tasks are eligible for optimization.

5
taskcluster/taskgraph/actions/cancel_all.py
View File

@ -12,7 +12,7 @@ import os
import requests
from taskgraph.util.taskcluster import (
list_task_group_incomplete_task_ids,
list_task_group_incomplete_tasks,
cancel_task,
CONCURRENCY,
)
@ -48,8 +48,7 @@ def cancel_all_action(parameters, graph_config, input, task_group_id, task_id):
raise
own_task_id = os.environ.get('TASK_ID', '')
to_cancel = [t for t in list_task_group_incomplete_task_ids(task_group_id) if t != own_task_id]
to_cancel = [t for t in list_task_group_incomplete_tasks(task_group_id) if t != own_task_id]
logger.info("Cancelling {} tasks".format(len(to_cancel)))
with futures.ThreadPoolExecutor(CONCURRENCY) as e:
cancel_futs = [e.submit(do_cancel_task, t) for t in to_cancel]

7
taskcluster/taskgraph/decision.py
View File

@ -72,10 +72,6 @@ PER_PROJECT_PARAMETERS = {
'target_tasks_method': 'graphics_tasks',
},
'autoland': {
'optimize_strategies': 'taskgraph.optimize:project.autoland',
},
'mozilla-central': {
'target_tasks_method': 'default',
'release_type': 'nightly',
@ -322,6 +318,7 @@ def get_decision_parameters(graph_config, options):
parameters['filters'] = [
'target_tasks_method',
]
parameters['optimize_target_tasks'] = True
parameters['existing_tasks'] = {}
parameters['do_not_optimize'] = []
parameters['build_number'] = 1
@ -330,8 +327,6 @@ def get_decision_parameters(graph_config, options):
parameters['message'] = try_syntax_from_message(commit_message)
parameters['hg_branch'] = get_hg_revision_branch(GECKO, revision=parameters['head_rev'])
parameters['next_version'] = None
parameters['optimize_strategies'] = None
parameters['optimize_target_tasks'] = True
parameters['phabricator_diff'] = None
parameters['release_type'] = ''
parameters['release_eta'] = ''

2
taskcluster/taskgraph/generator.py
View File

@ -346,7 +346,7 @@ class TaskGraphGenerator(object):
# this is used for testing experimental optimization strategies
strategies = os.environ.get('TASKGRAPH_OPTIMIZE_STRATEGIES',
parameters.get('optimize_strategies'))
parameters['try_task_config'].get('optimize-strategies'))
if strategies:
strategies = find_object(strategies)

32
taskcluster/taskgraph/optimize/__init__.py
View File

@ -397,12 +397,11 @@ class Alias(CompositeStrategy):
def split_bugbug_arg(arg):
"""Split args for bugbug based strategies.
Many bugbug based optimizations require passing an empty dict by reference
to communicate to downstream strategies. This function passes the provided
arg to the first strategy and an empty dict to second (bugbug based)
strategy.
"""
The bugbug optimization strageies require passing an dict as
scratch space for communicating with downstream stratgies.
This function pass the provided argument to the first strategy,
and a fresh dictionary to the second stratgey.
"""
return (arg, {})
@ -418,25 +417,16 @@ register_strategy('build-optimized', args=(
'backstop',
))(All)
register_strategy('build-fuzzing', args=('push-interval-10',))(Alias)
register_strategy('test', args=('skip-unless-schedules',))(Alias)
register_strategy('test-inclusive', args=('skip-unless-schedules',))(Alias)
# Strategy overrides used to tweak the default strategies. These are referenced
# by the `optimize_strategies` parameter.
class project(object):
"""Strategies that should be applied per-project."""
autoland = {
'test': All(
register_strategy('test', args=(
Any('skip-unless-schedules', 'bugbug-reduced-fallback', split_args=split_bugbug_arg),
'backstop',
),
}
"""Strategy overrides that apply to autoland."""
))(All)
register_strategy('test-inclusive', args=('skip-unless-schedules',))(Alias)
register_strategy('test-try', args=('skip-unless-schedules',))(Alias)
# Strategy overrides used by |mach try| and/or shadow-scheduler tasks.
class experimental(object):
"""Experimental strategies either under development or used as benchmarks.

2
taskcluster/taskgraph/parameters.py
View File

@ -79,7 +79,6 @@ base_schema = Schema({
Required('message'): text_type,
Required('moz_build_date'): text_type,
Required('next_version'): Any(None, text_type),
Required('optimize_strategies'): Any(None, text_type),
Required('optimize_target_tasks'): bool,
Required('owner'): text_type,
Required('phabricator_diff'): Any(None, text_type),
@ -162,7 +161,6 @@ class Parameters(ReadOnlyDict):
'message': '',
'moz_build_date': six.ensure_text(now.strftime("%Y%m%d%H%M%S")),
'next_version': None,
'optimize_strategies': None,
'optimize_target_tasks': True,
'owner': 'nobody@mozilla.com',
'phabricator_diff': None,

1
taskcluster/taskgraph/test/test_parameters.py
View File

@ -31,7 +31,6 @@ class TestParameters(unittest.TestCase):
'message': 'message',
'moz_build_date': 'moz_build_date',
'next_version': 'next_version',
'optimize_strategies': None,
'optimize_target_tasks': False,
'owner': 'owner',
'phabricator_diff': 'phabricator_diff',

3
taskcluster/taskgraph/transforms/tests.py
View File

@ -1817,6 +1817,9 @@ def make_job_description(config, tasks):
jobdesc['when'] = task['when']
elif 'optimization' in task:
jobdesc['optimization'] = task['optimization']
# Pushes generated by `mach try auto` should use the non-try optimizations.
elif config.params.is_try() and config.params['try_mode'] != 'try_auto':
jobdesc['optimization'] = {'test-try': schedules}
elif set(schedules) & set(INCLUSIVE_COMPONENTS):
jobdesc['optimization'] = {'test-inclusive': schedules}
else:

1
taskcluster/taskgraph/util/schema.py
View File

@ -225,6 +225,7 @@ OptimizationSchema = voluptuous.Any(
# optimize strategy aliases for the test kind
{'test': list(schedules.ALL_COMPONENTS)},
{'test-inclusive': list(schedules.ALL_COMPONENTS)},
{'test-try': list(schedules.ALL_COMPONENTS)},
)
# shortcut for a string where task references are allowed

16
taskcluster/taskgraph/util/taskcluster.py
View File

@ -306,23 +306,17 @@ def send_email(address, subject, content, link, use_proxy=False):
})
def list_task_group_tasks(task_group_id):
"""Generate the tasks in a task group"""
def list_task_group_incomplete_tasks(task_group_id):
"""Generate the incomplete tasks in a task group"""
params = {}
while True:
url = liburls.api(get_root_url(False), 'queue', 'v1',
'task-group/{}/list'.format(task_group_id))
resp = _do_request(url, force_get=True, params=params).json()
for task in resp['tasks']:
yield task
for task in [t['status'] for t in resp['tasks']]:
if task['state'] in ['running', 'pending', 'unscheduled']:
yield task['taskId']
if resp.get('continuationToken'):
params = {'continuationToken': resp.get('continuationToken')}
else:
break
def list_task_group_incomplete_task_ids(task_group_id):
states = ('running', 'pending', 'unscheduled')
for task in [t['status'] for t in list_task_group_tasks(task_group_id)]:
if task['state'] in states:
yield task['taskId']

1
taskcluster/test/test_mach_try_auto.py
View File

@ -64,6 +64,7 @@ def test_only_important_manifests(params, full_task_graph, filter_tasks):
assert unimportant == []
@pytest.mark.xfail(reason="Bug 1641131 - shippable builds still showing up due to dependencies")
def test_no_shippable_builds(tgg, filter_tasks):
optimized_task_graph = tgg.optimized_task_graph
# We can still sometimes get macosx64-shippable builds with |mach try

37
tools/tryselect/selectors/auto.py
View File

@ -4,14 +4,11 @@
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.util.python_path import find_object
from ..cli import BaseTryParser
from ..push import push_to_try
TRY_AUTO_PARAMETERS = {
'optimize_strategies': 'taskgraph.optimize:tryselect.bugbug_debug_disperse',
'optimize_target_tasks': True,
'target_tasks_method': 'try_auto',
'test_manifest_loader': 'bugbug',
@ -28,43 +25,23 @@ class AutoParser(BaseTryParser):
"env",
"chemspill-prio",
"disable-pgo",
"strategy",
"worker-overrides",
]
arguments = [
[['--strategy'],
{'default': None,
'help': 'Override the default optimization strategy. Valid values '
'are the experimental strategies defined at the bottom of '
'`taskcluster/taskgraph/optimize/__init__.py`.'
}],
]
def validate(self, args):
super(AutoParser, self).validate(args)
if args.strategy:
if ':' not in args.strategy:
args.strategy = "taskgraph.optimize:tryselect.{}".format(args.strategy)
try:
obj = find_object(args.strategy)
except (ImportError, AttributeError):
self.error("invalid module path '{}'".format(args.strategy))
if not isinstance(obj, dict):
self.error("object at '{}' must be a dict".format(args.strategy))
def run(message='{msg}', push=True, closed_tree=False, strategy=None, try_config=None, **ignored):
def run(message='{msg}', push=True, closed_tree=False, try_config=None, **ignored):
msg = message.format(msg='Tasks automatically selected.')
# XXX Remove once an intelligent scheduling algorithm is running on
# autoland by default. This ensures `mach try auto` doesn't run SETA.
try_config.setdefault('optimize-strategies',
'taskgraph.optimize:tryselect.bugbug_debug_disperse')
params = TRY_AUTO_PARAMETERS.copy()
if try_config:
params['try_task_config'] = try_config
if strategy:
params['optimize_strategies'] = strategy
task_config = {
'version': 2,
'parameters': params,

33
tools/tryselect/task_config.py
View File

@ -20,6 +20,7 @@ from textwrap import dedent
import mozpack.path as mozpath
from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
from taskgraph.util.python_path import find_object
from .tasks import resolve_tests_by_suite
@ -313,6 +314,37 @@ class GeckoProfile(TryConfig):
}
class OptimizeStrategies(TryConfig):
arguments = [
[['--strategy'],
{'default': None,
'help': 'Override the default optimization strategy. Valid values '
'are the experimental strategies defined at the bottom of '
'`taskcluster/taskgraph/optimize/__init__.py`.'
}],
]
def try_config(self, strategy, **kwargs):
if strategy:
if ':' not in strategy:
strategy = "taskgraph.optimize:tryselect.{}".format(strategy)
try:
obj = find_object(strategy)
except (ImportError, AttributeError):
print("error: invalid module path '{}'".format(strategy))
sys.exit(1)
if not isinstance(obj, dict):
print("error: object at '{}' must be a dict".format(strategy))
sys.exit(1)
return {
'optimize-strategies': strategy,
}
class Browsertime(TryConfig):
arguments = [
[['--browsertime'],
@ -429,5 +461,6 @@ all_task_configs = {
'pernosco': Pernosco,
'rebuild': Rebuild,
'routes': Routes,
'strategy': OptimizeStrategies,
'worker-overrides': WorkerOverrides,
}

1
tools/tryselect/test/python.ini
View File

@ -2,7 +2,6 @@
subsuite=try
[test_again.py]
[test_auto.py]
[test_chooser.py]
requirements = tools/tryselect/selectors/chooser/requirements.txt
[test_fuzzy.py]

32
tools/tryselect/test/test_auto.py
View File

@ -1,32 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import mozunit
import pytest
from tryselect.selectors.auto import AutoParser
def test_strategy_validation():
parser = AutoParser()
args = parser.parse_args(["--strategy", "seta"])
assert args.strategy == "taskgraph.optimize:tryselect.seta"
args = parser.parse_args(["--strategy", "taskgraph.optimize:experimental.seta"])
assert args.strategy == "taskgraph.optimize:experimental.seta"
with pytest.raises(SystemExit):
parser.parse_args(["--strategy", "taskgraph.optimize:tryselect"])
with pytest.raises(SystemExit):
parser.parse_args(["--strategy", "foo"])
with pytest.raises(SystemExit):
parser.parse_args(["--strategy", "foo:bar"])
if __name__ == "__main__":
mozunit.main()

15
tools/tryselect/test/test_auto.t
View File

@ -12,12 +12,13 @@ Test auto selector
Calculated try_task_config.json:
{
"parameters": {
"optimize_strategies": "taskgraph.optimize:tryselect.bugbug_debug_disperse",
"optimize_target_tasks": true,
"target_tasks_method": "try_auto",
"test_manifest_loader": "bugbug",
"try_mode": "try_auto",
"try_task_config": {}
"try_task_config": {
"optimize-strategies": "taskgraph.optimize:tryselect.bugbug_debug_disperse"
}
},
"version": 2
}
@ -31,12 +32,13 @@ Test auto selector
Calculated try_task_config.json:
{
"parameters": {
"optimize_strategies": "taskgraph.optimize:tryselect.bugbug_debug_disperse",
"optimize_target_tasks": true,
"target_tasks_method": "try_auto",
"test_manifest_loader": "bugbug",
"try_mode": "try_auto",
"try_task_config": {}
"try_task_config": {
"optimize-strategies": "taskgraph.optimize:tryselect.bugbug_debug_disperse"
}
},
"version": 2
}
@ -49,12 +51,13 @@ Test auto selector
Calculated try_task_config.json:
{
"parameters": {
"optimize_strategies": "taskgraph.optimize:tryselect.bugbug_debug_disperse",
"optimize_target_tasks": true,
"target_tasks_method": "try_auto",
"test_manifest_loader": "bugbug",
"try_mode": "try_auto",
"try_task_config": {}
"try_task_config": {
"optimize-strategies": "taskgraph.optimize:tryselect.bugbug_debug_disperse"
}
},
"version": 2
}

8
tools/tryselect/test/test_task_configs.py
View File

@ -45,6 +45,14 @@ TASK_CONFIG_TESTS = {
(['--rebuild', '1'], SystemExit),
(['--rebuild', '21'], SystemExit),
],
'strategy': [
([], None),
(['--strategy', 'seta'], {'optimize-strategies': 'taskgraph.optimize:tryselect.seta'}),
(['--strategy', 'taskgraph.optimize:experimental.seta'], {'optimize-strategies': 'taskgraph.optimize:experimental.seta'}), # noqa
(['--strategy', 'taskgraph.optimize:tryselect'], SystemExit),
(['--strategy', 'foo'], SystemExit),
(['--strategy', 'foo:bar'], SystemExit),
],
'worker-overrides': [
([], None),
(['--worker-override', 'alias=worker/pool'],