Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-17 23:35:34 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1587080 - Part 1: Add performance documentation verification code. r=ahal,perftest-reviewers,alexandru.irimovici,octavian_negru

Browse Source 
This patch adds the performance documentation (perfdocs) verification code under `tools/lint/perfdocs`. This tool currently validates `perfdocs` folders found within the `testing` folder to ensure all performance tests have documentation (it only does this for raptor at the moment). See `tools/lint/docs/perfdocs.rst` for more details.

Differential Revision: https://phabricator.services.mozilla.com/D53647

--HG--
extra : moz-landing-system : lando
This commit is contained in:
alexandru.ionescu 2019-12-10 17:04:35 +00:00
parent 69c53e713c
commit 9f6b4ddb80
8 changed files with 752 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
tools/lint/perfdocs/__init__.py Normal file
View File

@ -0,0 +1,22 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
from perfdocs import perfdocs
from mozlint.util import pip
here = os.path.abspath(os.path.dirname(__file__))
PERFDOCS_REQUIREMENTS_PATH = os.path.join(here, 'requirements.txt')
def setup(root, **lintargs):
if not pip.reinstall_program(PERFDOCS_REQUIREMENTS_PATH):
print("Cannot install requirements.")
return 1
def lint(paths, config, logger, fix=None, **lintargs):
return perfdocs.run_perfdocs(
config, logger=logger, paths=paths, verify=True
)

107
tools/lint/perfdocs/framework_gatherers.py Normal file
View File

@ -0,0 +1,107 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import
import os
import re
from perfdocs.utils import read_yaml
from manifestparser import TestManifest
'''
This file is for framework specific gatherers since manifests
might be parsed differently in each of them. The gatherers
must implement the FrameworkGatherer class.
'''
class FrameworkGatherer(object):
'''
Abstract class for framework gatherers.
'''
def __init__(self, yaml_path, workspace_dir):
'''
Generic initialization for a framework gatherer.
'''
self.workspace_dir = workspace_dir
self._yaml_path = yaml_path
self._suite_list = {}
self._manifest_path = ''
self._manifest = None
def get_manifest_path(self):
'''
Returns the path to the manifest based on the
manifest entry in the frameworks YAML configuration
file.
:return str: Path to the manifest.
'''
if self._manifest_path:
return self._manifest_path
yaml_content = read_yaml(self._yaml_path)
self._manifest_path = os.path.join(
self.workspace_dir, yaml_content["manifest"]
)
return self._manifest_path
def get_suite_list(self):
'''
Each framework gatherer must return a dictionary with
the following structure. Note that the test names must
be relative paths so that issues can be correctly issued
by the reviewbot.
:return dict: A dictionary with the following structure: {
"suite_name": [
'testing/raptor/test1',
'testing/raptor/test2'
]
}
'''
raise NotImplementedError
class RaptorGatherer(FrameworkGatherer):
'''
Gatherer for the Raptor framework.
'''
def get_suite_list(self):
'''
Returns a dictionary containing a mapping from suites
to the tests they contain.
:return dict: A dictionary with the following structure: {
"suite_name": [
'testing/raptor/test1',
'testing/raptor/test2'
]
}
'''
if self._suite_list:
return self._suite_list
manifest_path = self.get_manifest_path()
# Get the tests from the manifest
test_manifest = TestManifest([manifest_path], strict=False)
test_list = test_manifest.active_tests(exists=False, disabled=False)
# Parse the tests into the expected dictionary
for test in test_list:
# Get the top-level suite
s = os.path.basename(test["here"])
if s not in self._suite_list:
self._suite_list[s] = []
# Get the individual test
fpath = re.sub(".*testing", "testing", test['manifest'])
if fpath not in self._suite_list[s]:
self._suite_list[s].append(fpath)
return self._suite_list

124
tools/lint/perfdocs/gatherer.py Normal file
View File

@ -0,0 +1,124 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import
import os
import re
from perfdocs.logger import PerfDocLogger
from perfdocs.utils import read_yaml
from perfdocs.framework_gatherers import RaptorGatherer
logger = PerfDocLogger()
# TODO: Implement decorator/searcher to find the classes.
frameworks = {
"raptor": RaptorGatherer,
}
class Gatherer(object):
'''
Gatherer produces the tree of the perfdoc's entries found
and can obtain manifest-based test lists. Used by the Verifier.
'''
def __init__(self, root_dir, workspace_dir):
'''
Initialzie the Gatherer.
:param str root_dir: Path to the testing directory.
:param str workspace_dir: Path to the gecko checkout.
'''
self.root_dir = root_dir
self.workspace_dir = workspace_dir
self._perfdocs_tree = []
self._test_list = []
@property
def perfdocs_tree(self):
'''
Returns the perfdocs_tree, and computes it
if it doesn't exist.
:return dict: The perfdocs tree containing all
framework perfdoc entries. See `fetch_perfdocs_tree`
for information on the data strcture.
'''
if self._perfdocs_tree:
return self.perfdocs_tree
else:
self.fetch_perfdocs_tree()
return self._perfdocs_tree
def fetch_perfdocs_tree(self):
'''
Creates the perfdocs tree with the following structure:
[
{
"path": Path to the perfdocs directory.
"yml": Name of the configuration YAML file.
"rst": Name of the RST file.
}, ...
]
This method doesn't return anything. The result can be found in
the perfdocs_tree attribute.
'''
yml_match = re.compile('^config.y(a)?ml$')
rst_match = re.compile('^index.rst$')
for dirpath, dirname, files in os.walk(self.root_dir):
# Walk through the testing directory tree
if dirpath.endswith('/perfdocs'):
matched = {"path": dirpath, "yml": "", "rst": ""}
for file in files:
# Add the yml/rst file to its key if re finds the searched file
if re.search(yml_match, file):
matched["yml"] = re.search(yml_match, file).string
if re.search(rst_match, file):
matched["rst"] = re.search(rst_match, file).string
# Append to structdocs if all the searched files were found
if all(matched.values()):
self._perfdocs_tree.append(matched)
logger.log("Found {} perfdocs directories in {}"
.format(len(self._perfdocs_tree), self.root_dir))
def get_test_list(self, sdt_entry):
'''
Use a perfdocs_tree entry to find the test list for
the framework that was found.
:return: A framework info dictionary with fields: {
'yml_path': Path to YAML,
'yml_content': Content of YAML,
'name': Name of framework,
'test_list': Test list found for the framework
}
'''
# If it was computed before, return it
yaml_path = os.path.join(sdt_entry["path"], sdt_entry['yml'])
for entry in self._test_list:
if entry['yml_path'] == yaml_path:
return entry
# Set up framework entry with meta data
yaml_content = read_yaml(yaml_path)
framework = {
'yml_content': yaml_content,
'yml_path': yaml_path,
'name': yaml_content["name"],
}
# Get and then store the frameworks tests
framework_gatherer = frameworks[framework["name"]](
framework["yml_path"],
self.workspace_dir
)
framework["test_list"] = framework_gatherer.get_suite_list()
self._test_list.append(framework)
return framework

73
tools/lint/perfdocs/logger.py Normal file
View File

@ -0,0 +1,73 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import
import re
class PerfDocLogger(object):
'''
Logger for the PerfDoc tooling. Handles the warnings by outputting
them into through the StructuredLogger provided by lint.
'''
PATHS = []
LOGGER = None
def __init__(self):
'''Initializes the PerfDocLogger.'''
# Set up class attributes for all logger instances
if not PerfDocLogger.LOGGER:
raise Exception(
"Missing linting LOGGER instance for PerfDocLogger initialization"
)
if not PerfDocLogger.PATHS:
raise Exception(
"Missing PATHS for PerfDocLogger initialization"
)
self.logger = PerfDocLogger.LOGGER
def log(self, msg):
'''
Log a message.
:param str msg: Message to log.
'''
self.logger.info(msg)
def warning(self, msg, files):
'''
Logs a validation warning message. The warning message is
used as the error message that is output in the reviewbot.
:param str msg: Message to log, it's also used as the error message
for the issue that is output by the reviewbot.
:param list/str files: The file(s) that this warning is about.
'''
if type(files) != list:
files = [files]
# Add a reviewbot error for each file that is given
for file in files:
# Get a relative path (reviewbot can't handle absolute paths)
# TODO: Expand to outside of the testing directory
fpath = re.sub(".*testing", "testing", file)
# Filter out any issues that do not relate to the paths
# that are being linted
for path in PerfDocLogger.PATHS:
if path not in file:
continue
# Output error entry
self.logger.lint_error(
message=msg,
lineno=0,
column=None,
path=fpath,
linter='perfdocs',
rule="Flawless performance docs."
)
break

73
tools/lint/perfdocs/perfdocs.py Normal file
View File

@ -0,0 +1,73 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function
import os
import re
def run_perfdocs(config, logger=None, paths=None, verify=True, generate=False):
'''
Build up performance testing documentation dynamically by combining
text data from YAML files that reside in `perfdoc` folders
across the `testing` directory. Each directory is expected to have
an `index.rst` file along with `config.yml` YAMLs defining what needs
to be added to the documentation.
The YAML must also define the name of the "framework" that should be
used in the main index.rst for the performance testing documentation.
The testing documentation list will be ordered alphabetically once
it's produced (to avoid unwanted shifts because of unordered dicts
and path searching).
Note that the suite name headings will be given the H4 (---) style so it
is suggested that you use H3 (===) style as the heading for your
test section. H5 will be used be used for individual tests within each
suite.
Usage for verification: ./mach lint -l perfdocs
Usage for generation: Not Implemented
Currently, doc generation is not implemented - only validation.
For validation, see the Verifier class for a description of how
it works.
The run will fail if the valid result from validate_tree is not
False, implying some warning/problem was logged.
:param dict config: The configuration given by mozlint.
:param StructuredLogger logger: The StructuredLogger instance to be used to
output the linting warnings/errors.
:param list paths: The paths that are being tested. Used to filter
out errors from files outside of these paths.
:param bool verify: If true, the verification will be performed.
:param bool generate: If true, the docs will be generated.
'''
from perfdocs.logger import PerfDocLogger
top_dir = os.environ.get('WORKSPACE', None)
if not top_dir:
floc = os.path.abspath(__file__)
top_dir = floc.split('tools')[0]
PerfDocLogger.LOGGER = logger
# Convert all the paths to relative ones
rel_paths = [re.sub(".*testing", "testing", path) for path in paths]
PerfDocLogger.PATHS = rel_paths
# TODO: Expand search to entire tree rather than just the testing directory
testing_dir = os.path.join(top_dir, 'testing')
if not os.path.exists(testing_dir):
raise Exception("Cannot locate testing directory at %s" % testing_dir)
# Run either the verifier or generator
if generate:
raise NotImplementedError
if verify:
from perfdocs.verifier import Verifier
verifier = Verifier(testing_dir, top_dir)
verifier.validate_tree()

8
tools/lint/perfdocs/requirements.txt Normal file
View File

@ -0,0 +1,8 @@
jsonschema==3.1.1 --hash=sha256:94c0a13b4a0616458b42529091624e66700a17f847453e52279e35509a5b7631
importlib-metadata==0.23 --hash=sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af
attrs==17.4.0 --hash=sha256:a17a9573a6f475c99b551c0e0a812707ddda1ec9653bed04c13841404ed6f450
setuptools==41.6.0 --hash=sha256:3e8e8505e563631e7cb110d9ad82d135ee866b8146d5efe06e42be07a72db20a
pyrsistent==0.15.5 --hash=sha256:eb6545dbeb1aa69ab1fb4809bfbf5a8705e44d92ef8fc7c2361682a47c46c778
zipp==0.6.0 --hash=sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335
six==1.13.0 --hash=sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd
more-itertools==7.2.0 --hash=sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4

38
tools/lint/perfdocs/utils.py Normal file
View File

@ -0,0 +1,38 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import
import yaml
from perfdocs.logger import PerfDocLogger
logger = PerfDocLogger()
def read_file(path):
'''Opens a file and returns its contents.
:param str path: Path to the file.
:return list: List containing the lines in the file.
'''
with open(path, 'r') as f:
return f.readlines()
def read_yaml(yaml_path):
'''
Opens a YAML file and returns the contents.
:param str yaml_path: Path to the YAML to open.
:return dict: Dictionary containing the YAML content.
'''
contents = {}
try:
with open(yaml_path, 'r') as f:
contents = yaml.safe_load(f)
except Exception as e:
logger.warning(
"Error opening file {}: {}".format(yaml_path, str(e)), yaml_path
)
return contents

307
tools/lint/perfdocs/verifier.py Normal file
View File

@ -0,0 +1,307 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import
import jsonschema
import os
import re
from perfdocs.logger import PerfDocLogger
from perfdocs.utils import read_file, read_yaml
from perfdocs.gatherer import Gatherer
logger = PerfDocLogger()
'''
Schema for the config.yml file.
Expecting a YAML file with a format such as this:
name: raptor
manifest testing/raptor/raptor/raptor.ini
suites:
desktop:
description: "Desktop tests."
tests:
raptor-tp6: "Raptor TP6 tests."
mobile:
description: "Mobile tests"
benchmarks:
description: "Benchmark tests."
tests:
wasm: "All wasm tests."
'''
CONFIG_SCHEMA = {
"type": "object",
"properties": {
"name": {"type": "string"},
"manifest": {"type": "string"},
"suites": {
"type": "object",
"properties": {
"suite_name": {
"type": "object",
"properties": {
"tests": {
"type": "object",
"properties": {
"test_name": {"type": "string"},
}
},
"description": {"type": "string"},
},
"required": [
"description"
]
}
}
}
},
"required": [
"name",
"manifest",
"suites"
]
}
class Verifier(object):
'''
Verifier is used for validating the perfdocs folders/tree. In the future,
the generator will make use of this class to obtain a validated set of
descriptions that can be used to build up a document.
'''
def __init__(self, root_dir, workspace_dir):
'''
Initialize the Verifier.
:param str root_dir: Path to the 'testing' directory.
:param str workspace_dir: Path to the top-level checkout directory.
'''
self.workspace_dir = workspace_dir
self._gatherer = Gatherer(root_dir, workspace_dir)
def validate_descriptions(self, framework_info):
'''
Cross-validate the tests found in the manifests and the YAML
test definitions. This function doesn't return a valid flag. Instead,
the StructDocLogger.VALIDATION_LOG is used to determine validity.
The validation proceeds as follows:
1. Check that all tests/suites in the YAML exist in the manifests.
- At the same time, build a list of global descriptions which
define descriptions for groupings of tests.
2. Check that all tests/suites found in the manifests exist in the YAML.
- For missing tests, check if a global description for them exists.
As the validation is completed, errors are output into the validation log
for any issues that are found.
:param dict framework_info: Contains information about the framework. See
`Gatherer.get_test_list` for information about its structure.
'''
yaml_content = framework_info['yml_content']
# Check for any bad test/suite names in the yaml config file
global_descriptions = {}
for suite, ytests in yaml_content['suites'].items():
# Find the suite, then check against the tests within it
if framework_info["test_list"].get(suite):
global_descriptions[suite] = []
if not ytests.get("tests"):
# It's possible a suite entry has no tests
continue
# Suite found - now check if any tests in YAML
# definitions doesn't exist
ytests = ytests['tests']
for mnf_pth in ytests:
foundtest = False
for t in framework_info["test_list"][suite]:
tb = os.path.basename(t)
tb = re.sub("\..*", "", tb)
if mnf_pth == tb:
# Found an exact match for the mnf_pth
foundtest = True
break
if mnf_pth in tb:
# Found a 'fuzzy' match for the mnf_pth
# i.e. 'wasm' could exist for all raptor wasm tests
global_descriptions[suite].append(mnf_pth)
foundtest = True
break
if not foundtest:
logger.warning(
"Could not find an existing test for {} - bad test name?".format(
mnf_pth
),
framework_info["yml_path"]
)
else:
logger.warning(
"Could not find an existing suite for {} - bad suite name?".format(suite),
framework_info["yml_path"]
)
# Check for any missing tests/suites
for suite, manifest_paths in framework_info["test_list"].items():
if not yaml_content["suites"].get(suite):
# Description doesn't exist for the suite
logger.warning(
"Missing suite description for {}".format(suite),
yaml_content['manifest']
)
continue
# If only a description is provided for the suite, assume
# that this is a suite-wide description and don't check for
# it's tests
stests = yaml_content['suites'][suite].get('tests', None)
if not stests:
continue
tests_found = 0
missing_tests = []
test_to_manifest = {}
for mnf_pth in manifest_paths:
tb = os.path.basename(mnf_pth)
tb = re.sub("\..*", "", tb)
if stests.get(tb) or stests.get(mnf_pth):
# Test description exists, continue with the next test
tests_found += 1
continue
test_to_manifest[tb] = mnf_pth
missing_tests.append(tb)
# Check if global test descriptions exist (i.e.
# ones that cover all of tp6) for the missing tests
new_mtests = []
for mt in missing_tests:
found = False
for mnf_pth in global_descriptions[suite]:
if mnf_pth in mt:
# Global test exists for this missing test
found = True
break
if not found:
new_mtests.append(mt)
if len(new_mtests):
# Output an error for each manifest with a missing
# test description
for mnf_pth in new_mtests:
logger.warning(
"Could not find a test description for {}".format(mnf_pth),
test_to_manifest[mnf_pth]
)
continue
def validate_yaml(self, yaml_path):
'''
Validate that the YAML file has all the fields that are
required and parse the descriptions into strings in case
some are give as relative file paths.
:param str yaml_path: Path to the YAML to validate.
:return bool: True/False => Passed/Failed Validation
'''
def _get_description(desc):
'''
Recompute the description in case it's a file.
'''
desc_path = os.path.join(self.workspace_dir, desc)
if os.path.exists(desc_path):
with open(desc_path, 'r') as f:
desc = f.readlines()
return desc
def _parse_descriptions(content):
for suite, sinfo in content.items():
desc = sinfo['description']
sinfo['description'] = _get_description(desc)
# It's possible that the suite has no tests and
# only a description. If they exist, then parse them.
if 'tests' in sinfo:
for test, desc in sinfo['tests'].items():
sinfo['tests'][test] = _get_description(desc)
valid = False
yaml_content = read_yaml(yaml_path)
try:
jsonschema.validate(instance=yaml_content, schema=CONFIG_SCHEMA)
_parse_descriptions(yaml_content['suites'])
valid = True
except Exception as e:
logger.warning(
"YAML ValidationError: {}".format(str(e)), yaml_path
)
return valid
def validate_rst_content(self, rst_path):
'''
Validate that the index file given has a {documentation} entry
so that the documentation can be inserted there.
:param str rst_path: Path to the RST file.
:return bool: True/False => Passed/Failed Validation
'''
rst_content = read_file(rst_path)
# Check for a {documentation} entry in some line,
# if we can't find one, then the validation fails.
valid = False
docs_match = re.compile('.*{documentation}.*')
for line in rst_content:
if docs_match.search(line):
valid = True
break
if not valid:
logger.warning(
"Cannot find a '{documentation}' entry in the given index file",
rst_path
)
return valid
def _check_framework_descriptions(self, item):
'''
Helper method for validating descriptions
'''
framework_info = self._gatherer.get_test_list(item)
self.validate_descriptions(framework_info)
def validate_tree(self):
'''
Validate the `perfdocs` directory that was found.
Returns True if it is good, false otherwise.
:return bool: True/False => Passed/Failed Validation
'''
found_good = 0
# For each framework, check their files and validate descriptions
for matched in self._gatherer.perfdocs_tree:
# Get the paths to the YAML and RST for this framework
matched_yml = os.path.join(matched['path'], matched['yml'])
matched_rst = os.path.join(matched['path'], matched['rst'])
_valid_files = {
"yml": self.validate_yaml(matched_yml),
"rst": self.validate_rst_content(matched_rst)
}
if not all(_valid_files.values()):
# Don't check the descriptions if the YAML or RST is bad
logger.log("Bad perfdocs directory found in {}".format(matched['path']))
continue
found_good += 1
self._check_framework_descriptions(matched)
if not found_good:
raise Exception("No valid perfdocs directories found")