Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-23 12:51:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
d43a8490ef
gecko-dev/testing/skipfails.py

1883 lines
76 KiB
Python
Raw Blame History

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import gzip
import io
import json
import logging
import os
import os.path
import pprint
import re
import sys
import tempfile
import urllib.parse
from copy import deepcopy
from pathlib import Path
from statistics import median
from xmlrpc.client import Fault
from yaml import load
try:
from yaml import CLoader as Loader
except ImportError:
from yaml import Loader
import bugzilla
import mozci.push
import requests
from manifestparser import ManifestParser
from manifestparser.toml import add_skip_if, alphabetize_toml_str, sort_paths
from mozci.task import TestTask
from mozci.util.taskcluster import get_task
from taskcluster.exceptions import TaskclusterRestFailure
TASK_LOG = "live_backing.log"
TASK_ARTIFACT = "public/logs/" + TASK_LOG
ATTACHMENT_DESCRIPTION = "Compressed " + TASK_ARTIFACT + " for task "
ATTACHMENT_REGEX = (
r".*Created attachment ([0-9]+)\n.*"
+ ATTACHMENT_DESCRIPTION
+ "([A-Za-z0-9_-]+)\n.*"
)
BUGZILLA_AUTHENTICATION_HELP = "Must create a Bugzilla API key per https://github.com/mozilla/mozci-tools/blob/main/citools/test_triage_bug_filer.py"
MS_PER_MINUTE = 60 * 1000 # ms per minute
DEBUG_THRESHOLD = 40 * MS_PER_MINUTE # 40 minutes in ms
OPT_THRESHOLD = 20 * MS_PER_MINUTE # 20 minutes in ms
ANYJS = "anyjs"
CC = "classification"
DEF = "DEFAULT"
DIFFERENCE = "difference"
DURATIONS = "durations"
EQEQ = "=="
ERROR = "error"
FAIL = "FAIL"
FAILED_RUNS = "runs_failed"
FAILURE_RATIO = 0.4 # more than this fraction of failures will disable
INTERMITTENT_RATIO_REFTEST = 0.4 # reftest low frequency intermittent
FAILURE_RATIO_REFTEST = 0.8 # disable ratio for reftest (high freq intermittent)
GROUP = "group"
KIND = "kind"
LINENO = "lineno"
LL = "label"
MEDIAN_DURATION = "duration_median"
MINIMUM_RUNS = 3 # mininum number of runs to consider success/failure
MOCK_BUG_DEFAULTS = {"blocks": [], "comments": []}
MOCK_TASK_DEFAULTS = {"extra": {}, "failure_types": {}, "results": []}
MOCK_TASK_INITS = ["results"]
MODIFIERS = "modifiers"
NOTEQ = "!="
OPT = "opt"
PASS = "PASS"
PIXELS = "pixels"
PP = "path"
QUERY = "query"
RR = "result"
RUNS = "runs"
STATUS = "status"
SUBTEST = "subtest"
SUBTEST_REGEX = (
r"image comparison, max difference: ([0-9]+), number of differing pixels: ([0-9]+)"
)
SUM_BY_LABEL = "sum_by_label"
TEST = "test"
TEST_TYPES = [EQEQ, NOTEQ]
TOTAL_DURATION = "duration_total"
TOTAL_RUNS = "runs_total"
WP = "testing/web-platform/"
WPT0 = WP + "tests/infrastructure"
WPT_META0 = WP + "tests/infrastructure/metadata"
WPT_META0_CLASSIC = WP + "meta/infrastructure"
WPT1 = WP + "tests"
WPT_META1 = WPT1.replace("tests", "meta")
WPT2 = WP + "mozilla/tests"
WPT_META2 = WPT2.replace("tests", "meta")
WPT_MOZILLA = "/_mozilla"
class Mock(object):
def __init__(self, data, defaults={}, inits=[]):
self._data = data
self._defaults = defaults
for name in inits:
values = self._data.get(name, []) # assume type is an array
values = [Mock(value, defaults, inits) for value in values]
self._data[name] = values
def __getattr__(self, name):
if name in self._data:
return self._data[name]
if name in self._defaults:
return self._defaults[name]
return ""
class Classification(object):
"Classification of the failure (not the task result)"
DISABLE_INTERMITTENT = "disable_intermittent" # reftest [40%, 80%)
DISABLE_FAILURE = "disable_failure" # reftest (80%,100%] failure
DISABLE_MANIFEST = "disable_manifest" # crash found
DISABLE_RECOMMENDED = "disable_recommended" # disable first failing path
DISABLE_TOO_LONG = "disable_too_long" # runtime threshold exceeded
INTERMITTENT = "intermittent"
SECONDARY = "secondary" # secondary failing path
SUCCESS = "success" # path always succeeds
UNKNOWN = "unknown"
class Kind(object):
"Kind of manifest"
LIST = "list"
TOML = "toml"
UNKNOWN = "unknown"
WPT = "wpt"
class Skipfails(object):
"mach manifest skip-fails implementation: Update manifests to skip failing tests"
REPO = "repo"
REVISION = "revision"
TREEHERDER = "treeherder.mozilla.org"
BUGZILLA_SERVER_DEFAULT = "bugzilla.allizom.org"
def __init__(
self,
command_context=None,
try_url="",
verbose=True,
bugzilla=None,
dry_run=False,
turbo=False,
implicit_vars=False,
):
self.command_context = command_context
if self.command_context is not None:
self.topsrcdir = self.command_context.topsrcdir
else:
self.topsrcdir = Path(__file__).parent.parent
self.topsrcdir = os.path.normpath(self.topsrcdir)
if isinstance(try_url, list) and len(try_url) == 1:
self.try_url = try_url[0]
else:
self.try_url = try_url
self.dry_run = dry_run
self.implicit_vars = implicit_vars
self.verbose = verbose
self.turbo = turbo
if bugzilla is not None:
self.bugzilla = bugzilla
elif "BUGZILLA" in os.environ:
self.bugzilla = os.environ["BUGZILLA"]
else:
self.bugzilla = Skipfails.BUGZILLA_SERVER_DEFAULT
if self.bugzilla == "disable":
self.bugzilla = None # Bug filing disabled
self.component = "skip-fails"
self._bzapi = None
self._attach_rx = None
self.variants = {}
self.tasks = {}
self.pp = None
self.headers = {} # for Treeherder requests
self.headers["Accept"] = "application/json"
self.headers["User-Agent"] = "treeherder-pyclient"
self.jobs_url = "https://treeherder.mozilla.org/api/jobs/"
self.push_ids = {}
self.job_ids = {}
self.extras = {}
self.bugs = [] # preloaded bugs, currently not an updated cache
self.error_summary = {}
self._subtest_rx = None
self.lmp = None
self.failure_types = None
def _initialize_bzapi(self):
"""Lazily initializes the Bugzilla API (returns True on success)"""
if self._bzapi is None and self.bugzilla is not None:
self._bzapi = bugzilla.Bugzilla(self.bugzilla)
self._attach_rx = re.compile(ATTACHMENT_REGEX, flags=re.M)
return self._bzapi is not None
def pprint(self, obj):
if self.pp is None:
self.pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
self.pp.pprint(obj)
sys.stderr.flush()
def error(self, e):
if self.command_context is not None:
self.command_context.log(
logging.ERROR, self.component, {ERROR: str(e)}, "ERROR: {error}"
)
else:
print(f"ERROR: {e}", file=sys.stderr, flush=True)
def warning(self, e):
if self.command_context is not None:
self.command_context.log(
logging.WARNING, self.component, {ERROR: str(e)}, "WARNING: {error}"
)
else:
print(f"WARNING: {e}", file=sys.stderr, flush=True)
def info(self, e):
if self.command_context is not None:
self.command_context.log(
logging.INFO, self.component, {ERROR: str(e)}, "INFO: {error}"
)
else:
print(f"INFO: {e}", file=sys.stderr, flush=True)
def vinfo(self, e):
if self.verbose:
self.info(e)
def full_path(self, filename):
"""Returns full path for the relative filename"""
return os.path.join(self.topsrcdir, os.path.normpath(filename))
def isdir(self, filename):
"""Returns True if filename is a directory"""
return os.path.isdir(self.full_path(filename))
def exists(self, filename):
"""Returns True if filename exists"""
return os.path.exists(self.full_path(filename))
def run(
self,
meta_bug_id=None,
save_tasks=None,
use_tasks=None,
save_failures=None,
use_failures=None,
max_failures=-1,
):
"Run skip-fails on try_url, return True on success"
try_url = self.try_url
revision, repo = self.get_revision(try_url)
if use_tasks is not None:
tasks = self.read_tasks(use_tasks)
self.vinfo(f"use tasks: {use_tasks}")
self.failure_types = None # do NOT cache failure_types
else:
tasks = self.get_tasks(revision, repo)
self.failure_types = {} # cache failure_types
if use_failures is not None:
failures = self.read_failures(use_failures)
self.vinfo(f"use failures: {use_failures}")
else:
failures = self.get_failures(tasks)
if save_failures is not None:
self.write_json(save_failures, failures)
self.vinfo(f"save failures: {save_failures}")
if save_tasks is not None:
self.write_tasks(save_tasks, tasks)
self.vinfo(f"save tasks: {save_tasks}")
num_failures = 0
self.vinfo(
f"skip-fails assumes implicit-vars for reftest: {self.implicit_vars}"
)
for manifest in failures:
kind = failures[manifest][KIND]
for label in failures[manifest][LL]:
for path in failures[manifest][LL][label][PP]:
classification = failures[manifest][LL][label][PP][path][CC]
if classification.startswith("disable_") or (
self.turbo and classification == Classification.SECONDARY
):
anyjs = {} # anyjs alternate basename = False
differences = []
pixels = []
status = FAIL
lineno = failures[manifest][LL][label][PP][path].get(LINENO, 0)
for task_id in failures[manifest][LL][label][PP][path][RUNS]:
if kind == Kind.TOML:
break # just use the first task_id
elif kind == Kind.LIST:
difference = failures[manifest][LL][label][PP][path][
RUNS
][task_id].get(DIFFERENCE, 0)
if difference > 0:
differences.append(difference)
pixel = failures[manifest][LL][label][PP][path][RUNS][
task_id
].get(PIXELS, 0)
if pixel > 0:
pixels.append(pixel)
status = failures[manifest][LL][label][PP][path][RUNS][
task_id
].get(STATUS, FAIL)
elif kind == Kind.WPT:
filename = os.path.basename(path)
anyjs[filename] = False
if (
QUERY
in failures[manifest][LL][label][PP][path][RUNS][
task_id
]
):
query = failures[manifest][LL][label][PP][path][
RUNS
][task_id][QUERY]
anyjs[filename + query] = False
else:
query = None
if (
ANYJS
in failures[manifest][LL][label][PP][path][RUNS][
task_id
]
):
any_filename = os.path.basename(
failures[manifest][LL][label][PP][path][RUNS][
task_id
][ANYJS]
)
anyjs[any_filename] = False
if query is not None:
anyjs[any_filename + query] = False
self.skip_failure(
manifest,
kind,
path,
anyjs,
differences,
pixels,
lineno,
status,
label,
classification,
task_id,
try_url,
revision,
repo,
meta_bug_id,
)
num_failures += 1
if max_failures >= 0 and num_failures >= max_failures:
self.warning(
f"max_failures={max_failures} threshold reached: stopping."
)
return True
return True
def get_revision(self, url):
parsed = urllib.parse.urlparse(url)
if parsed.scheme != "https":
raise ValueError("try_url scheme not https")
if parsed.netloc != Skipfails.TREEHERDER:
raise ValueError(f"try_url server not {Skipfails.TREEHERDER}")
if len(parsed.query) == 0:
raise ValueError("try_url query missing")
query = urllib.parse.parse_qs(parsed.query)
if Skipfails.REVISION not in query:
raise ValueError("try_url query missing revision")
revision = query[Skipfails.REVISION][0]
if Skipfails.REPO in query:
repo = query[Skipfails.REPO][0]
else:
repo = "try"
self.vinfo(f"considering {repo} revision={revision}")
return revision, repo
def get_tasks(self, revision, repo):
push = mozci.push.Push(revision, repo)
return push.tasks
def get_failures(self, tasks):
"""
find failures and create structure comprised of runs by path:
result:
* False (failed)
* True (passed)
classification: Classification
* unknown (default) < 3 runs
* intermittent (not enough failures)
* disable_recommended (enough repeated failures) >3 runs >= 4
* disable_manifest (disable DEFAULT if no other failures)
* secondary (not first failure in group)
* success
"""
ff = {}
manifest_paths = {}
manifest_ = {
KIND: Kind.UNKNOWN,
LL: {},
}
label_ = {
DURATIONS: {},
MEDIAN_DURATION: 0,
OPT: False,
PP: {},
SUM_BY_LABEL: {}, # All sums implicitly zero
TOTAL_DURATION: 0,
}
path_ = {
CC: Classification.UNKNOWN,
FAILED_RUNS: 0,
RUNS: {},
TOTAL_RUNS: 0,
}
run_ = {
RR: False,
}
for task in tasks: # add explicit failures
try:
if len(task.results) == 0:
continue # ignore aborted tasks
failure_types = task.failure_types # call magic property once
if self.failure_types is not None:
self.failure_types[task.id] = failure_types
self.vinfo(f"Getting failure_types from task: {task.id}")
for manifest in failure_types:
mm = manifest
ll = task.label
kind = Kind.UNKNOWN
if mm.endswith(".ini"):
self.warning(
f"cannot analyze skip-fails on INI manifests: {mm}"
)
continue
elif mm.endswith(".list"):
kind = Kind.LIST
elif mm.endswith(".toml"):
kind = Kind.TOML
else:
kind = Kind.WPT
path, mm, _query, _anyjs = self.wpt_paths(mm)
if path is None: # not WPT
self.warning(
f"cannot analyze skip-fails on unknown manifest type: {manifest}"
)
continue
if kind != Kind.WPT:
if mm not in manifest_paths:
manifest_paths[mm] = []
if mm not in ff:
ff[mm] = deepcopy(manifest_)
ff[mm][KIND] = kind
if ll not in ff[mm][LL]:
ff[mm][LL][ll] = deepcopy(label_)
for path_type in failure_types[manifest]:
path, _type = path_type
query = None
anyjs = None
allpaths = []
if kind == Kind.WPT:
path, mmpath, query, anyjs = self.wpt_paths(path)
if path is None:
self.warning(
f"non existant failure path: {path_type[0]}"
)
break
allpaths = [path]
mm = os.path.dirname(mmpath)
if mm not in manifest_paths:
manifest_paths[mm] = []
if mm not in ff:
ff[mm] = deepcopy(manifest_)
ff[mm][KIND] = kind
if ll not in ff[mm][LL]:
ff[mm][LL][ll] = deepcopy(label_)
elif kind == Kind.LIST:
words = path.split()
if len(words) != 3 or words[1] not in TEST_TYPES:
self.warning(f"reftest type not supported: {path}")
continue
allpaths = self.get_allpaths(task.id, mm, path)
elif kind == Kind.TOML:
if path == mm:
path = DEF # refers to the manifest itself
allpaths = [path]
for path in allpaths:
if path in manifest_paths[mm]:
continue # duplicate path for this task
manifest_paths[mm].append(path)
self.vinfo(
f"Getting failure info in manifest: {mm}, path: {path}"
)
if path not in ff[mm][LL][ll][PP]:
ff[mm][LL][ll][PP][path] = deepcopy(path_)
if task.id not in ff[mm][LL][ll][PP][path][RUNS]:
ff[mm][LL][ll][PP][path][RUNS][task.id] = deepcopy(run_)
ff[mm][LL][ll][PP][path][RUNS][task.id][RR] = False
if query is not None:
ff[mm][LL][ll][PP][path][RUNS][task.id][QUERY] = query
if anyjs is not None:
ff[mm][LL][ll][PP][path][RUNS][task.id][ANYJS] = anyjs
ff[mm][LL][ll][PP][path][TOTAL_RUNS] += 1
ff[mm][LL][ll][PP][path][FAILED_RUNS] += 1
if kind == Kind.LIST:
(
lineno,
difference,
pixels,
status,
) = self.get_lineno_difference_pixels_status(
task.id, mm, path
)
if lineno > 0:
ff[mm][LL][ll][PP][path][LINENO] = lineno
else:
self.vinfo(f"ERROR no lineno for {path}")
if status != FAIL:
ff[mm][LL][ll][PP][path][RUNS][task.id][
STATUS
] = status
if status == FAIL and difference == 0 and pixels == 0:
# intermittent, not error
ff[mm][LL][ll][PP][path][RUNS][task.id][RR] = True
ff[mm][LL][ll][PP][path][FAILED_RUNS] -= 1
elif difference > 0:
ff[mm][LL][ll][PP][path][RUNS][task.id][
DIFFERENCE
] = difference
if pixels > 0:
ff[mm][LL][ll][PP][path][RUNS][task.id][
PIXELS
] = pixels
except AttributeError:
pass # self.warning(f"unknown attribute in task (#1): {ae}")
for task in tasks: # add results
try:
if len(task.results) == 0:
continue # ignore aborted tasks
self.vinfo(f"Getting results from task: {task.id}")
for result in task.results:
mm = result.group
ll = task.label
kind = Kind.UNKNOWN
if mm.endswith(".ini"):
self.warning(
f"cannot analyze skip-fails on INI manifests: {mm}"
)
continue
elif mm.endswith(".list"):
kind = Kind.LIST
elif mm.endswith(".toml"):
kind = Kind.TOML
else:
kind = Kind.WPT
path, mm, _query, _anyjs = self.wpt_paths(mm)
if path is None: # not WPT
self.warning(
f"cannot analyze skip-fails on unknown manifest type: {result.group}"
)
continue
if mm not in manifest_paths:
continue
if mm not in ff:
ff[mm] = deepcopy(manifest_)
if ll not in ff[mm][LL]:
ff[mm][LL][ll] = deepcopy(label_)
if task.id not in ff[mm][LL][ll][DURATIONS]:
# duration may be None !!!
ff[mm][LL][ll][DURATIONS][task.id] = result.duration or 0
if ff[mm][LL][ll][OPT] is None:
ff[mm][LL][ll][OPT] = self.get_opt_for_task(task.id)
for path in manifest_paths[mm]: # all known paths
self.vinfo(f"Getting result for manifest: {mm}, path: {path}")
if path not in ff[mm][LL][ll][PP]:
ff[mm][LL][ll][PP][path] = deepcopy(path_)
if task.id not in ff[mm][LL][ll][PP][path][RUNS]:
ff[mm][LL][ll][PP][path][RUNS][task.id] = deepcopy(run_)
ff[mm][LL][ll][PP][path][RUNS][task.id][RR] = result.ok
ff[mm][LL][ll][PP][path][TOTAL_RUNS] += 1
if not result.ok:
ff[mm][LL][ll][PP][path][FAILED_RUNS] += 1
if kind == Kind.LIST:
(
lineno,
difference,
pixels,
status,
) = self.get_lineno_difference_pixels_status(
task.id, mm, path
)
if lineno > 0:
ff[mm][LL][ll][PP][path][LINENO] = lineno
else:
self.vinfo(f"ERROR no lineno for {path}")
if status != FAIL:
ff[mm][LL][ll][PP][path][RUNS][task.id][
STATUS
] = status
if (
status == FAIL
and difference == 0
and pixels == 0
and not result.ok
):
# intermittent, not error
ff[mm][LL][ll][PP][path][RUNS][task.id][RR] = True
ff[mm][LL][ll][PP][path][FAILED_RUNS] -= 1
if difference > 0:
ff[mm][LL][ll][PP][path][RUNS][task.id][
DIFFERENCE
] = difference
if pixels > 0:
ff[mm][LL][ll][PP][path][RUNS][task.id][
PIXELS
] = pixels
except AttributeError:
pass # self.warning(f"unknown attribute in task (#2): {ae}")
for mm in ff: # determine classifications
kind = ff[mm][KIND]
for label in ff[mm][LL]:
ll = label
opt = ff[mm][LL][ll][OPT]
durations = [] # summarize durations
for task_id in ff[mm][LL][ll][DURATIONS]:
duration = ff[mm][LL][ll][DURATIONS][task_id]
durations.append(duration)
if len(durations) > 0:
total_duration = sum(durations)
median_duration = median(durations)
ff[mm][LL][ll][TOTAL_DURATION] = total_duration
ff[mm][LL][ll][MEDIAN_DURATION] = median_duration
if (opt and median_duration > OPT_THRESHOLD) or (
(not opt) and median_duration > DEBUG_THRESHOLD
):
if kind == Kind.TOML:
paths = [DEF]
else:
paths = ff[mm][LL][ll][PP].keys()
for path in paths:
if path not in ff[mm][LL][ll][PP]:
ff[mm][LL][ll][PP][path] = deepcopy(path_)
if task_id not in ff[mm][LL][ll][PP][path][RUNS]:
ff[mm][LL][ll][PP][path][RUNS][task.id] = deepcopy(run_)
ff[mm][LL][ll][PP][path][RUNS][task.id][RR] = False
ff[mm][LL][ll][PP][path][TOTAL_RUNS] += 1
ff[mm][LL][ll][PP][path][FAILED_RUNS] += 1
ff[mm][LL][ll][PP][path][
CC
] = Classification.DISABLE_TOO_LONG
primary = True # we have not seen the first failure
for path in sort_paths(ff[mm][LL][ll][PP]):
classification = ff[mm][LL][ll][PP][path][CC]
if classification == Classification.UNKNOWN:
failed_runs = ff[mm][LL][ll][PP][path][FAILED_RUNS]
total_runs = ff[mm][LL][ll][PP][path][TOTAL_RUNS]
status = FAIL # default status, only one run could be PASS
for task_id in ff[mm][LL][ll][PP][path][RUNS]:
status = ff[mm][LL][ll][PP][path][RUNS][task_id].get(
STATUS, status
)
if kind == Kind.LIST:
failure_ratio = INTERMITTENT_RATIO_REFTEST
else:
failure_ratio = FAILURE_RATIO
if total_runs >= MINIMUM_RUNS:
if failed_runs / total_runs < failure_ratio:
if failed_runs == 0:
classification = Classification.SUCCESS
else:
classification = Classification.INTERMITTENT
elif kind == Kind.LIST:
if failed_runs / total_runs < FAILURE_RATIO_REFTEST:
classification = Classification.DISABLE_INTERMITTENT
else:
classification = Classification.DISABLE_FAILURE
elif primary:
if path == DEF:
classification = Classification.DISABLE_MANIFEST
else:
classification = Classification.DISABLE_RECOMMENDED
primary = False
else:
classification = Classification.SECONDARY
ff[mm][LL][ll][PP][path][CC] = classification
if classification not in ff[mm][LL][ll][SUM_BY_LABEL]:
ff[mm][LL][ll][SUM_BY_LABEL][classification] = 0
ff[mm][LL][ll][SUM_BY_LABEL][classification] += 1
return ff
def _get_os_version(self, os, platform):
"""Return the os_version given the label platform string"""
i = platform.find(os)
j = i + len(os)
yy = platform[j : j + 2]
mm = platform[j + 2 : j + 4]
return yy + "." + mm
def get_bug_by_id(self, id):
"""Get bug by bug id"""
bug = None
for b in self.bugs:
if b.id == id:
bug = b
break
if bug is None and self._initialize_bzapi():
bug = self._bzapi.getbug(id)
return bug
def get_bugs_by_summary(self, summary):
"""Get bug by bug summary"""
bugs = []
for b in self.bugs:
if b.summary == summary:
bugs.append(b)
if len(bugs) > 0:
return bugs
if self._initialize_bzapi():
query = self._bzapi.build_query(short_desc=summary)
query["include_fields"] = [
"id",
"product",
"component",
"status",
"resolution",
"summary",
"blocks",
]
bugs = self._bzapi.query(query)
return bugs
def create_bug(
self,
summary="Bug short description",
description="Bug description",
product="Testing",
component="General",
version="unspecified",
bugtype="task",
):
"""Create a bug"""
bug = None
if self._initialize_bzapi():
if not self._bzapi.logged_in:
self.error(
"Must create a Bugzilla API key per https://github.com/mozilla/mozci-tools/blob/main/citools/test_triage_bug_filer.py"
)
raise PermissionError(f"Not authenticated for Bugzilla {self.bugzilla}")
createinfo = self._bzapi.build_createbug(
product=product,
component=component,
summary=summary,
version=version,
description=description,
)
createinfo["type"] = bugtype
bug = self._bzapi.createbug(createinfo)
return bug
def add_bug_comment(self, id, comment, meta_bug_id=None):
"""Add a comment to an existing bug"""
if self._initialize_bzapi():
if not self._bzapi.logged_in:
self.error(BUGZILLA_AUTHENTICATION_HELP)
raise PermissionError("Not authenticated for Bugzilla")
if meta_bug_id is not None:
blocks_add = [meta_bug_id]
else:
blocks_add = None
updateinfo = self._bzapi.build_update(
comment=comment, blocks_add=blocks_add
)
self._bzapi.update_bugs([id], updateinfo)
def skip_failure(
self,
manifest,
kind,
path,
anyjs,
differences,
pixels,
lineno,
status,
label,
classification,
task_id,
try_url,
revision,
repo,
meta_bug_id=None,
):
"""
Skip a failure (for TOML, WPT and REFTEST manifests)
For wpt anyjs is a dictionary mapping from alternate basename to
a boolean (indicating if the basename has been handled in the manifest)
"""
self.vinfo(f"\n\n===== Skip failure in manifest: {manifest} =====")
self.vinfo(f" path: {path}")
if task_id is None:
skip_if = "true"
else:
skip_if = self.task_to_skip_if(task_id, kind)
if skip_if is None:
self.warning(
f"Unable to calculate skip-if condition from manifest={manifest} from failure label={label}"
)
return
if kind == Kind.TOML:
filename = DEF
elif kind == Kind.WPT:
_path, manifest, _query, _anyjs = self.wpt_paths(path)
filename = os.path.basename(path)
elif kind == Kind.LIST:
filename = path
if status == PASS:
self.info(f"Unexpected status: {status}")
if status == PASS or classification == Classification.DISABLE_INTERMITTENT:
zero = True # refest lower ranges should include zero
else:
zero = False
bug_reference = ""
if classification == Classification.DISABLE_MANIFEST:
comment = "Disabled entire manifest due to crash result"
elif classification == Classification.DISABLE_TOO_LONG:
comment = "Disabled entire manifest due to excessive run time"
else:
if kind == Kind.TOML:
filename = self.get_filename_in_manifest(manifest, path)
comment = f'Disabled test due to failures in test file: "{filename}"'
if classification == Classification.SECONDARY:
comment += " (secondary)"
if kind != Kind.WPT:
bug_reference = " (secondary)"
if kind != Kind.LIST:
self.vinfo(f"filename: {filename}")
if kind == Kind.WPT and len(anyjs) > 1:
comment += "\nAdditional WPT wildcard paths:"
for p in sorted(anyjs.keys()):
if p != filename:
comment += f'\n "{p}"'
platform, testname = self.label_to_platform_testname(label)
if platform is not None:
comment += "\nCommand line to reproduce (experimental):\n"
comment += f" \"mach try fuzzy -q '{platform}' {testname}\""
comment += f"\nTry URL = {try_url}"
comment += f"\nrevision = {revision}"
comment += f"\nrepo = {repo}"
comment += f"\nlabel = {label}"
if task_id is not None:
comment += f"\ntask_id = {task_id}"
if kind != Kind.LIST:
push_id = self.get_push_id(revision, repo)
if push_id is not None:
comment += f"\npush_id = {push_id}"
job_id = self.get_job_id(push_id, task_id)
if job_id is not None:
comment += f"\njob_id = {job_id}"
(
suggestions_url,
line_number,
line,
log_url,
) = self.get_bug_suggestions(repo, job_id, path, anyjs)
if log_url is not None:
comment += f"\nBug suggestions: {suggestions_url}"
comment += f"\nSpecifically see at line {line_number} in the attached log: {log_url}"
comment += f'\n\n "{line}"\n'
bug_summary = f"MANIFEST {manifest}"
attachments = {}
bugid = "TBD"
if self.bugzilla is None:
self.vinfo("Bugzilla has been disabled: no bugs created or updated")
else:
bugs = self.get_bugs_by_summary(bug_summary)
if len(bugs) == 0:
description = (
f"This bug covers excluded failing tests in the MANIFEST {manifest}"
)
description += "\n(generated by `mach manifest skip-fails`)"
product, component = self.get_file_info(path)
if self.dry_run:
self.warning(
f'Dry-run NOT creating bug: {product}::{component} "{bug_summary}"'
)
else:
bug = self.create_bug(bug_summary, description, product, component)
bugid = bug.id
self.vinfo(
f'Created Bug {bugid} {product}::{component} : "{bug_summary}"'
)
elif len(bugs) == 1:
bugid = bugs[0].id
product = bugs[0].product
component = bugs[0].component
self.vinfo(f'Found Bug {bugid} {product}::{component} "{bug_summary}"')
if meta_bug_id is not None:
if meta_bug_id in bugs[0].blocks:
self.vinfo(
f" Bug {bugid} already blocks meta bug {meta_bug_id}"
)
meta_bug_id = None # no need to add again
comments = bugs[0].getcomments()
for i in range(len(comments)):
text = comments[i]["text"]
m = self._attach_rx.findall(text)
if len(m) == 1:
a_task_id = m[0][1]
attachments[a_task_id] = m[0][0]
if a_task_id == task_id:
self.vinfo(
f" Bug {bugid} already has the compressed log attached for this task"
)
else:
self.error(f'More than one bug found for summary: "{bug_summary}"')
return
bug_reference = f"Bug {bugid}" + bug_reference
extra = self.get_extra(task_id)
json.dumps(extra)
if kind == Kind.LIST:
comment += (
f"\nfuzzy-if condition on line {lineno}: {skip_if} # {bug_reference}"
)
else:
comment += f"\nskip-if condition: {skip_if} # {bug_reference}"
manifest_path = self.full_path(manifest)
manifest_str = ""
additional_comment = ""
if kind == Kind.WPT:
if os.path.exists(manifest_path):
manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
else:
# ensure parent directories exist
os.makedirs(os.path.dirname(manifest_path), exist_ok=True)
manifest_str, additional_comment = self.wpt_add_skip_if(
manifest_str, anyjs, skip_if, bug_reference
)
elif kind == Kind.TOML:
mp = ManifestParser(use_toml=True, document=True)
mp.read(manifest_path)
document = mp.source_documents[manifest_path]
additional_comment = add_skip_if(document, filename, skip_if, bug_reference)
manifest_str = alphabetize_toml_str(document)
elif kind == Kind.LIST:
if lineno == 0:
self.error(
f"cannot determine line to edit in manifest: {manifest_path}"
)
# elif skip_if.find("winWidget") >= 0 and skip_if.find("!is64Bit") >= 0:
# self.error(
# "Skipping failures for Windows 32-bit are temporarily disabled"
# )
elif not os.path.exists(manifest_path):
self.error(f"manifest does not exist: {manifest_path}")
else:
manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
manifest_str, additional_comment = self.reftest_add_fuzzy_if(
manifest_str,
filename,
skip_if,
differences,
pixels,
lineno,
zero,
bug_reference,
)
if not manifest_str and additional_comment:
self.warning(additional_comment)
if additional_comment:
comment += "\n" + additional_comment
if len(manifest_str) > 0:
fp = io.open(manifest_path, "w", encoding="utf-8", newline="\n")
fp.write(manifest_str)
fp.close()
self.info(f'Edited ["{filename}"] in manifest: "{manifest}"')
if kind != Kind.LIST:
self.info(f'added skip-if condition: "{skip_if}" # {bug_reference}')
if self.dry_run:
self.info(f"Dry-run NOT adding comment to Bug {bugid}:\n{comment}")
self.info(
f'Dry-run NOT editing ["{filename}"] in manifest: "{manifest}"'
)
self.info(f'would add skip-if condition: "{skip_if}" # {bug_reference}')
if task_id is not None and task_id not in attachments:
self.info("would add compressed log for this task")
return
elif self.bugzilla is None:
self.warning(f"NOT adding comment to Bug {bugid}:\n{comment}")
else:
self.add_bug_comment(bugid, comment, meta_bug_id)
self.info(f"Added comment to Bug {bugid}:\n{comment}")
if meta_bug_id is not None:
self.info(f" Bug {bugid} blocks meta Bug: {meta_bug_id}")
if task_id is not None and task_id not in attachments:
self.add_attachment_log_for_task(bugid, task_id)
self.info("Added compressed log for this task")
else:
self.error(f'Error editing ["{filename}"] in manifest: "{manifest}"')
def get_variants(self):
"""Get mozinfo for each test variants"""
if len(self.variants) == 0:
variants_file = "taskcluster/kinds/test/variants.yml"
variants_path = self.full_path(variants_file)
fp = io.open(variants_path, "r", encoding="utf-8")
raw_variants = load(fp, Loader=Loader)
fp.close()
for k, v in raw_variants.items():
mozinfo = k
if "mozinfo" in v:
mozinfo = v["mozinfo"]
self.variants[k] = mozinfo
return self.variants
def get_task_details(self, task_id):
"""Download details for task task_id"""
if task_id in self.tasks: # if cached
task = self.tasks[task_id]
else:
self.vinfo(f"get_task_details for task: {task_id}")
try:
task = get_task(task_id)
except TaskclusterRestFailure:
self.warning(f"Task {task_id} no longer exists.")
return None
self.tasks[task_id] = task
return task
def get_extra(self, task_id):
"""Calculate extra for task task_id"""
if task_id in self.extras: # if cached
extra = self.extras[task_id]
else:
self.get_variants()
task = self.get_task_details(task_id) or {}
arch = None
bits = None
build = None
build_types = []
display = None
os = None
os_version = None
runtimes = []
test_setting = task.get("extra", {}).get("test-setting", {})
platform = test_setting.get("platform", {})
platform_os = platform.get("os", {})
opt = False
debug = False
if "name" in platform_os:
os = platform_os["name"]
if os == "windows":
os = "win"
if os == "macosx":
os = "mac"
if "version" in platform_os:
os_version = platform_os["version"]
if len(os_version) == 4:
os_version = os_version[0:2] + "." + os_version[2:4]
if "build" in platform_os:
build = platform_os["build"]
if "arch" in platform:
arch = platform["arch"]
if arch == "x86" or arch.find("32") >= 0:
bits = "32"
arch = "x86"
else:
bits = "64"
if arch != "aarch64" and arch != "ppc":
arch = "x86_64"
if "display" in platform:
display = platform["display"]
if "runtime" in test_setting:
for k in test_setting["runtime"]:
if k == "no-fission" and test_setting["runtime"][k]:
runtimes.append("no-fission")
elif k in self.variants: # draw-snapshot -> snapshot
runtimes.append(self.variants[k]) # adds mozinfo
if "build" in test_setting:
tbuild = test_setting["build"]
for k in tbuild:
if k == "type":
if tbuild[k] == "opt":
opt = True
elif tbuild[k] == "debug":
debug = True
build_types.append(tbuild[k])
else:
build_types.append(k)
unknown = None
extra = {
"arch": arch or unknown,
"bits": bits or unknown,
"build": build or unknown,
"build_types": build_types,
"debug": debug,
"display": display or unknown,
"opt": opt,
"os": os or unknown,
"os_version": os_version or unknown,
"runtimes": runtimes,
}
self.extras[task_id] = extra
return extra
def get_opt_for_task(self, task_id):
extra = self.get_extra(task_id)
return extra["opt"]
def task_to_skip_if(self, task_id, kind):
"""Calculate the skip-if condition for failing task task_id"""
if kind == Kind.WPT:
qq = '"'
aa = " and "
nn = "not "
elif kind == Kind.LIST:
qq = "'"
aa = "&&"
nn = "!"
else:
qq = "'"
aa = " && "
nn = "!"
eq = " == "
arch = "processor"
version = "os_version"
extra = self.get_extra(task_id)
skip_if = None
os = extra.get("os")
if os is not None:
if kind == Kind.LIST:
if os == "linux":
skip_if = "gtkWidget"
elif os == "win":
skip_if = "winWidget"
elif os == "mac":
skip_if = "cocoaWidget"
elif os == "android":
skip_if = "Android"
else:
self.error(f"cannot calculate skip-if for unknown OS: '{os}'")
return None
elif extra.get("os_version") is not None:
if (
extra.get("build") is not None
and os == "win"
and extra["os_version"] == "11"
and extra["build"] == "2009"
):
skip_if = "win11_2009" # mozinfo.py:137
else:
skip_if = "os" + eq + qq + os + qq
skip_if += aa + version + eq + qq + extra["os_version"] + qq
if kind != Kind.LIST and extra.get("arch") is not None:
skip_if += aa + arch + eq + qq + extra["arch"] + qq
# since we always give arch/processor, bits are not required
# if extra["bits"] is not None:
# skip_if += aa + "bits" + eq + extra["bits"]
debug = extra.get("debug", False)
runtimes = extra.get("runtimes", [])
fission = "no-fission" not in runtimes
snapshot = "snapshot" in runtimes
swgl = "swgl" in runtimes
build_types = extra.get("build_types", [])
asan = "asan" in build_types
ccov = "ccov" in build_types
tsan = "tsan" in build_types
optimized = (not debug) and (not ccov) and (not asan) and (not tsan)
skip_if += aa
if kind == Kind.LIST:
if optimized:
skip_if += "optimized"
elif debug:
skip_if += "isDebugBuild"
elif ccov:
skip_if += "isCoverageBuild"
elif asan:
skip_if += "AddressSanitizer"
elif tsan:
skip_if += "ThreadSanitizer"
# See implicit VARIANT_DEFAULTS in
# https://searchfox.org/mozilla-central/source/layout/tools/reftest/manifest.sys.mjs#30
if not self.implicit_vars and fission:
skip_if += aa + "fission"
elif not fission: # implicit default: fission
skip_if += aa + nn + "fission"
if extra.get("bits") is not None:
if extra["bits"] == "32":
skip_if += aa + nn + "is64Bit" # override implicit is64Bit
elif not self.implicit_vars and os == "winWidget":
skip_if += aa + "is64Bit"
if not self.implicit_vars and not swgl:
skip_if += aa + nn + "swgl"
elif swgl: # implicit default: !swgl
skip_if += aa + "swgl"
if os == "gtkWidget":
if not self.implicit_vars and not snapshot:
skip_if += aa + nn + "useDrawSnapshot"
elif snapshot: # implicit default: !useDrawSnapshot
skip_if += aa + "useDrawSnapshot"
else:
if not debug:
skip_if += nn
skip_if += "debug"
if extra.get("display") is not None:
skip_if += aa + "display" + eq + qq + extra["display"] + qq
for runtime in extra.get("runtimes", []):
skip_if += aa + runtime
for build_type in extra.get("build_types", []):
# note: lite will not evaluate on non-android platforms
if build_type not in ["debug", "lite", "opt", "shippable"]:
skip_if += aa + build_type
return skip_if
def get_file_info(self, path, product="Testing", component="General"):
"""
Get bugzilla product and component for the path.
Provide defaults (in case command_context is not defined
or there isn't file info available).
"""
if path != DEF and self.command_context is not None:
reader = self.command_context.mozbuild_reader(config_mode="empty")
info = reader.files_info([path])
cp = info[path]["BUG_COMPONENT"]
product = cp.product
component = cp.component
return product, component
def get_filename_in_manifest(self, manifest, path):
"""return relative filename for path in manifest"""
filename = os.path.basename(path)
if filename == DEF:
return filename
manifest_dir = os.path.dirname(manifest)
i = 0
j = min(len(manifest_dir), len(path))
while i < j and manifest_dir[i] == path[i]:
i += 1
if i < len(manifest_dir):
for _ in range(manifest_dir.count("/", i) + 1):
filename = "../" + filename
elif i < len(path):
filename = path[i + 1 :]
return filename
def get_push_id(self, revision, repo):
"""Return the push_id for revision and repo (or None)"""
self.vinfo(f"Retrieving push_id for {repo} revision: {revision} ...")
if revision in self.push_ids: # if cached
push_id = self.push_ids[revision]
else:
push_id = None
push_url = f"https://treeherder.mozilla.org/api/project/{repo}/push/"
params = {}
params["full"] = "true"
params["count"] = 10
params["revision"] = revision
r = requests.get(push_url, headers=self.headers, params=params)
if r.status_code != 200:
self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
else:
response = r.json()
if "results" in response:
results = response["results"]
if len(results) > 0:
r0 = results[0]
if "id" in r0:
push_id = r0["id"]
self.push_ids[revision] = push_id
return push_id
def get_job_id(self, push_id, task_id):
"""Return the job_id for push_id, task_id (or None)"""
self.vinfo(f"Retrieving job_id for push_id: {push_id}, task_id: {task_id} ...")
k = f"{push_id}:{task_id}"
if k in self.job_ids: # if cached
job_id = self.job_ids[k]
else:
job_id = None
params = {}
params["push_id"] = push_id
r = requests.get(self.jobs_url, headers=self.headers, params=params)
if r.status_code != 200:
self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
else:
response = r.json()
if "results" in response:
results = response["results"]
if len(results) > 0:
for result in results:
if len(result) > 14:
if result[14] == task_id:
job_id = result[1]
break
self.job_ids[k] = job_id
return job_id
def get_bug_suggestions(self, repo, job_id, path, anyjs=None):
"""
Return the (suggestions_url, line_number, line, log_url)
for the given repo and job_id
"""
self.vinfo(
f"Retrieving bug_suggestions for {repo} job_id: {job_id}, path: {path} ..."
)
suggestions_url = f"https://treeherder.mozilla.org/api/project/{repo}/jobs/{job_id}/bug_suggestions/"
line_number = None
line = None
log_url = None
r = requests.get(suggestions_url, headers=self.headers)
if r.status_code != 200:
self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
else:
if anyjs is not None:
pathdir = os.path.dirname(path) + "/"
paths = [pathdir + f for f in anyjs.keys()]
else:
paths = [path]
response = r.json()
if len(response) > 0:
for sugg in response:
for p in paths:
path_end = sugg.get("path_end", None)
# handles WPT short paths
if path_end is not None and p.endswith(path_end):
line_number = sugg["line_number"] + 1
line = sugg["search"]
log_url = f"https://treeherder.mozilla.org/logviewer?repo={repo}&job_id={job_id}&lineNumber={line_number}"
break
rv = (suggestions_url, line_number, line, log_url)
return rv
def read_json(self, filename):
"""read data as JSON from filename"""
fp = io.open(filename, "r", encoding="utf-8")
data = json.load(fp)
fp.close()
return data
def read_tasks(self, filename):
"""read tasks as JSON from filename"""
if not os.path.exists(filename):
msg = f"use-tasks JSON file does not exist: {filename}"
raise OSError(2, msg, filename)
tasks = self.read_json(filename)
tasks = [Mock(task, MOCK_TASK_DEFAULTS, MOCK_TASK_INITS) for task in tasks]
for task in tasks:
if len(task.extra) > 0: # pre-warm cache for extra information
self.extras[task.id] = task.extra
return tasks
def read_failures(self, filename):
"""read failures as JSON from filename"""
if not os.path.exists(filename):
msg = f"use-failures JSON file does not exist: {filename}"
raise OSError(2, msg, filename)
failures = self.read_json(filename)
return failures
def read_bugs(self, filename):
"""read bugs as JSON from filename"""
if not os.path.exists(filename):
msg = f"bugs JSON file does not exist: {filename}"
raise OSError(2, msg, filename)
bugs = self.read_json(filename)
bugs = [Mock(bug, MOCK_BUG_DEFAULTS) for bug in bugs]
return bugs
def write_json(self, filename, data):
"""saves data as JSON to filename"""
fp = io.open(filename, "w", encoding="utf-8")
json.dump(data, fp, indent=2, sort_keys=True)
fp.close()
def write_tasks(self, save_tasks, tasks):
"""saves tasks as JSON to save_tasks"""
jtasks = []
for task in tasks:
if not isinstance(task, TestTask):
continue
jtask = {}
jtask["id"] = task.id
jtask["label"] = task.label
jtask["duration"] = task.duration
jtask["result"] = task.result
jtask["state"] = task.state
jtask["extra"] = self.get_extra(task.id)
jtags = {}
for k, v in task.tags.items():
if k == "createdForUser":
jtags[k] = "ci@mozilla.com"
else:
jtags[k] = v
jtask["tags"] = jtags
jtask["tier"] = task.tier
jtask["results"] = [
{"group": r.group, "ok": r.ok, "duration": r.duration}
for r in task.results
]
jtask["errors"] = None # Bug with task.errors property??
jft = {}
if self.failure_types is not None and task.id in self.failure_types:
failure_types = self.failure_types[task.id] # use cache
else:
failure_types = task.failure_types
for k in failure_types:
jft[k] = [[f[0], f[1].value] for f in task.failure_types[k]]
jtask["failure_types"] = jft
jtasks.append(jtask)
self.write_json(save_tasks, jtasks)
def label_to_platform_testname(self, label):
"""convert from label to platform, testname for mach command line"""
platform = None
testname = None
platform_details = label.split("/")
if len(platform_details) == 2:
platform, details = platform_details
words = details.split("-")
if len(words) > 2:
platform += "/" + words.pop(0) # opt or debug
try:
_chunk = int(words[-1])
words.pop()
except ValueError:
pass
words.pop() # remove test suffix
testname = "-".join(words)
else:
platform = None
return platform, testname
def add_attachment_log_for_task(self, bugid, task_id):
"""Adds compressed log for this task to bugid"""
log_url = f"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{task_id}/artifacts/public/logs/live_backing.log"
r = requests.get(log_url, headers=self.headers)
if r.status_code != 200:
self.error(f"Unable to get log for task: {task_id}")
return
attach_fp = tempfile.NamedTemporaryFile()
fp = gzip.open(attach_fp, "wb")
fp.write(r.text.encode("utf-8"))
fp.close()
if self._initialize_bzapi():
description = ATTACHMENT_DESCRIPTION + task_id
file_name = TASK_LOG + ".gz"
comment = "Added compressed log"
content_type = "application/gzip"
try:
self._bzapi.attachfile(
[bugid],
attach_fp.name,
description,
file_name=file_name,
comment=comment,
content_type=content_type,
is_private=False,
)
except Fault:
pass # Fault expected: Failed to fetch key 9372091 from network storage: The specified key does not exist.
def get_wpt_path_meta(self, shortpath):
if shortpath.startswith(WPT0):
path = shortpath
meta = shortpath.replace(WPT0, WPT_META0, 1)
elif shortpath.startswith(WPT1):
path = shortpath
meta = shortpath.replace(WPT1, WPT_META1, 1)
elif shortpath.startswith(WPT2):
path = shortpath
meta = shortpath.replace(WPT2, WPT_META2, 1)
elif shortpath.startswith(WPT_MOZILLA):
shortpath = shortpath[len(WPT_MOZILLA) :]
path = WPT2 + shortpath
meta = WPT_META2 + shortpath
else:
path = WPT1 + shortpath
meta = WPT_META1 + shortpath
return (path, meta)
def wpt_paths(self, shortpath):
"""
Analyzes the WPT short path for a test and returns
(path, manifest, query, anyjs) where
path is the relative path to the test file
manifest is the relative path to the file metadata
query is the test file query paramters (or None)
anyjs is the html test file as reported by mozci (or None)
"""
query = None
anyjs = None
i = shortpath.find("?")
if i > 0:
query = shortpath[i:]
shortpath = shortpath[0:i]
path, manifest = self.get_wpt_path_meta(shortpath)
failure_type = not self.isdir(path)
if failure_type:
i = path.find(".any.")
if i > 0:
anyjs = path # orig path
manifest = manifest.replace(path[i:], ".any.js")
path = path[0:i] + ".any.js"
else:
i = path.find(".window.")
if i > 0:
anyjs = path # orig path
manifest = manifest.replace(path[i:], ".window.js")
path = path[0:i] + ".window.js"
else:
i = path.find(".worker.")
if i > 0:
anyjs = path # orig path
manifest = manifest.replace(path[i:], ".worker.js")
path = path[0:i] + ".worker.js"
manifest += ".ini"
manifest_classic = ""
if manifest.startswith(WPT_META0):
manifest_classic = manifest.replace(WPT_META0, WPT_META0_CLASSIC, 1)
if self.exists(manifest_classic):
if self.exists(manifest):
self.warning(
f"Both classic {manifest_classic} and metadata {manifest} manifests exist"
)
else:
self.warning(
f"Using the classic {manifest_classic} manifest as the metadata manifest {manifest} does not exist"
)
manifest = manifest_classic
if not self.exists(path):
return (None, None, None, None)
return (path, manifest, query, anyjs)
def wpt_add_skip_if(self, manifest_str, anyjs, skip_if, bug_reference):
"""
Edits a WPT manifest string to add disabled condition
anyjs is a dictionary mapping from filename and any alternate basenames to
a boolean (indicating if the file has been handled in the manifest).
Returns additional_comment (if any)
"""
additional_comment = ""
disabled_key = False
disabled = " disabled:"
condition_start = " if "
condition = condition_start + skip_if + ": " + bug_reference
lines = manifest_str.splitlines()
section = None # name of the section
i = 0
n = len(lines)
while i < n:
line = lines[i]
if line.startswith("["):
if section is not None and not anyjs[section]: # not yet handled
if not disabled_key:
lines.insert(i, disabled)
i += 1
lines.insert(i, condition)
lines.insert(i + 1, "") # blank line after condition
i += 2
n += 2
anyjs[section] = True
section = line[1:-1]
if section in anyjs and not anyjs[section]:
disabled_key = False
else:
section = None # ignore section we are not interested in
elif section is not None:
if line == disabled:
disabled_key = True
elif line.startswith(" ["):
if i > 0 and i - 1 < n and lines[i - 1] == "":
del lines[i - 1]
i -= 1
n -= 1
if not disabled_key:
lines.insert(i, disabled)
i += 1
n += 1
lines.insert(i, condition)
lines.insert(i + 1, "") # blank line after condition
i += 2
n += 2
anyjs[section] = True
section = None
elif line.startswith(" ") and not line.startswith(" "):
if disabled_key: # insert condition above new key
lines.insert(i, condition)
i += 1
n += 1
anyjs[section] = True
section = None
disabled_key = False
elif line.startswith(" "):
if disabled_key and line == condition:
anyjs[section] = True # condition already present
section = None
i += 1
if section is not None and not anyjs[section]: # not yet handled
if i > 0 and i - 1 < n and lines[i - 1] == "":
del lines[i - 1]
if not disabled_key:
lines.append(disabled)
i += 1
n += 1
lines.append(condition)
lines.append("") # blank line after condition
i += 2
n += 2
anyjs[section] = True
for section in anyjs:
if not anyjs[section]:
if i > 0 and i - 1 < n and lines[i - 1] != "":
lines.append("") # blank line before condition
i += 1
n += 1
lines.append("[" + section + "]")
lines.append(disabled)
lines.append(condition)
lines.append("") # blank line after condition
i += 4
n += 4
manifest_str = "\n".join(lines) + "\n"
return manifest_str, additional_comment
def reftest_add_fuzzy_if(
self,
manifest_str,
filename,
fuzzy_if,
differences,
pixels,
lineno,
zero,
bug_reference,
):
"""
Edits a reftest manifest string to add disabled condition
"""
if self.lmp is None:
from parse_reftest import ListManifestParser
self.lmp = ListManifestParser(
self.implicit_vars, self.verbose, self.error, self.warning, self.info
)
manifest_str, additional_comment = self.lmp.reftest_add_fuzzy_if(
manifest_str,
filename,
fuzzy_if,
differences,
pixels,
lineno,
zero,
bug_reference,
)
return manifest_str, additional_comment
def get_lineno_difference_pixels_status(self, task_id, manifest, allmods):
"""
Returns
- lineno in manifest
- image comparison, max *difference*
- number of differing *pixels*
- status (PASS or FAIL)
as cached from reftest_errorsummary.log for a task
"""
manifest_obj = self.error_summary.get(manifest, {})
allmods_obj = manifest_obj.get(allmods, {})
lineno = allmods_obj.get(LINENO, 0)
runs_obj = allmods_obj.get(RUNS, {})
task_obj = runs_obj.get(task_id, {})
difference = task_obj.get(DIFFERENCE, 0)
pixels = task_obj.get(PIXELS, 0)
status = task_obj.get(STATUS, FAIL)
return lineno, difference, pixels, status
def reftest_find_lineno(self, manifest, modifiers, allmods):
"""
Return the line number with modifiers in manifest (else 0)
"""
lineno = 0
mods = []
prefs = []
for i in range(len(modifiers)):
if modifiers[i].find("pref(") >= 0 or modifiers[i].find("skip-if(") >= 0:
prefs.append(modifiers[i])
else:
mods.append(modifiers[i])
m = len(mods)
manifest_str = io.open(manifest, "r", encoding="utf-8").read()
lines = manifest_str.splitlines()
defaults = []
found = False
alt_lineno = 0
for linenum in range(len(lines)):
line = lines[linenum]
if len(line) > 0 and line[0] == "#":
continue
comment_start = line.find(" #") # MUST NOT match anchors!
if comment_start > 0:
line = line[0:comment_start].strip()
words = line.split()
n = len(words)
if n > 1 and words[0] == "defaults":
defaults = words[1:].copy()
continue
line_defaults = defaults.copy()
i = 0
while i < n:
if words[i].find("pref(") >= 0 or words[i].find("skip-if(") >= 0:
line_defaults.append(words[i])
del words[i]
n -= 1
else:
i += 1
if (len(prefs) == 0 or prefs == line_defaults) and words == mods:
found = True
lineno = linenum + 1
break
elif m > 2 and n > 2:
if words[-3:] == mods[-3:]:
alt_lineno = linenum + 1
else:
bwords = [os.path.basename(f) for f in words[-2:]]
bmods = [os.path.basename(f) for f in mods[-2:]]
if bwords == bmods:
alt_lineno = linenum + 1
if not found:
if alt_lineno > 0:
lineno = alt_lineno
self.warning(
f"manifest '{manifest}' found lineno: {lineno}, but it does not contain all the prefs from modifiers,\nSEARCH: {allmods}\nFOUND : {lines[alt_lineno - 1]}"
)
else:
lineno = 0
self.error(
f"manifest '{manifest}' does not contain line with modifiers: {allmods}"
)
return lineno
def get_allpaths(self, task_id, manifest, path):
"""
Looks up the reftest_errorsummary.log for a task
and caches the details in self.error_summary by
task_id, manifest, allmods
where allmods is the concatenation of all modifiers
and the details include
- image comparison, max *difference*
- number of differing *pixels*
- status: unexpected PASS or FAIL
The list iof unique modifiers (allmods) for the given path are returned
"""
allpaths = []
words = path.split()
if len(words) != 3 or words[1] not in TEST_TYPES:
self.warning(
f"reftest_errorsummary.log for task: {task_id} has unsupported test type '{path}'"
)
return allpaths
if manifest in self.error_summary:
for allmods in self.error_summary[manifest]:
if self.error_summary[manifest][allmods][
TEST
] == path and task_id in self.error_summary[manifest][allmods].get(
RUNS, {}
):
allpaths.append(allmods)
if len(allpaths) > 0:
return allpaths # cached (including self tests)
error_url = f"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{task_id}/artifacts/public/test_info/reftest_errorsummary.log"
self.vinfo(f"Requesting reftest_errorsummary.log for task: {task_id}")
r = requests.get(error_url, headers=self.headers)
if r.status_code != 200:
self.error(f"Unable to get reftest_errorsummary.log for task: {task_id}")
return allpaths
for line in r.text.encode("utf-8").splitlines():
summary = json.loads(line)
group = summary.get(GROUP, "")
if not group or not os.path.exists(group): # not error line
continue
test = summary.get(TEST, None)
if test is None:
continue
if not MODIFIERS in summary:
self.warning(
f"reftest_errorsummary.log for task: {task_id} does not have modifiers for '{test}'"
)
continue
words = test.split()
if len(words) != 3 or words[1] not in TEST_TYPES:
self.warning(
f"reftest_errorsummary.log for task: {task_id} has unsupported test '{test}'"
)
continue
status = summary.get(STATUS, "")
if status not in [FAIL, PASS]:
self.warning(
f"reftest_errorsummary.log for task: {task_id} has unknown status: {status} for '{test}'"
)
continue
error = summary.get(SUBTEST, "")
mods = summary[MODIFIERS]
allmods = " ".join(mods)
if group not in self.error_summary:
self.error_summary[group] = {}
if allmods not in self.error_summary[group]:
self.error_summary[group][allmods] = {}
self.error_summary[group][allmods][TEST] = test
lineno = self.error_summary[group][allmods].get(LINENO, 0)
if lineno == 0:
lineno = self.reftest_find_lineno(group, mods, allmods)
if lineno > 0:
self.error_summary[group][allmods][LINENO] = lineno
if RUNS not in self.error_summary[group][allmods]:
self.error_summary[group][allmods][RUNS] = {}
if task_id not in self.error_summary[group][allmods][RUNS]:
self.error_summary[group][allmods][RUNS][task_id] = {}
self.error_summary[group][allmods][RUNS][task_id][ERROR] = error
if self._subtest_rx is None:
self._subtest_rx = re.compile(SUBTEST_REGEX)
m = self._subtest_rx.findall(error)
if len(m) == 1:
difference = int(m[0][0])
pixels = int(m[0][1])
else:
difference = 0
pixels = 0
if difference > 0:
self.error_summary[group][allmods][RUNS][task_id][
DIFFERENCE
] = difference
if pixels > 0:
self.error_summary[group][allmods][RUNS][task_id][PIXELS] = pixels
if status != FAIL:
self.error_summary[group][allmods][RUNS][task_id][STATUS] = status
if test == path:
allpaths.append(allmods)
return allpaths
Reference in New Issue View Git Blame Copy Permalink