mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-20 00:35:44 +00:00
Bug 1675534 - Update glean-parser to v1.29.0. r=janerik
This version bump is required by the Rust glean-core version bump to v33.1.2. Differential Revision: https://phabricator.services.mozilla.com/D96305
This commit is contained in:
parent
52ea4ea9d9
commit
d201cffd9b
@ -62,14 +62,14 @@ commands:
|
||||
jobs:
|
||||
build-36:
|
||||
docker:
|
||||
- image: circleci/python:3.6.9
|
||||
- image: circleci/python:3.6.12
|
||||
steps:
|
||||
- test-start
|
||||
- test-python-version
|
||||
|
||||
build-36-min:
|
||||
docker:
|
||||
- image: circleci/python:3.6.9
|
||||
- image: circleci/python:3.6.12
|
||||
steps:
|
||||
- test-start
|
||||
- test-min-requirements
|
||||
@ -77,7 +77,7 @@ jobs:
|
||||
|
||||
build-37:
|
||||
docker:
|
||||
- image: circleci/python:3.7.5
|
||||
- image: circleci/python:3.7.9
|
||||
steps:
|
||||
- test-start
|
||||
- test-python-version
|
||||
@ -92,19 +92,26 @@ jobs:
|
||||
|
||||
build-38:
|
||||
docker:
|
||||
- image: circleci/python:3.8.0
|
||||
- image: circleci/python:3.8.5
|
||||
steps:
|
||||
- test-start
|
||||
- test-python-version
|
||||
|
||||
build-38-min:
|
||||
docker:
|
||||
- image: circleci/python:3.8.0
|
||||
- image: circleci/python:3.8.5
|
||||
steps:
|
||||
- test-start
|
||||
- test-min-requirements
|
||||
- test-python-version
|
||||
|
||||
build-39:
|
||||
docker:
|
||||
- image: circleci/python:3.9.0rc1
|
||||
steps:
|
||||
- test-start
|
||||
- test-python-version
|
||||
|
||||
docs-deploy:
|
||||
docker:
|
||||
- image: node:8.10.0
|
||||
@ -174,6 +181,10 @@ workflows:
|
||||
filters:
|
||||
tags:
|
||||
only: /.*/
|
||||
- build-39:
|
||||
filters:
|
||||
tags:
|
||||
only: /.*/
|
||||
- docs-deploy:
|
||||
requires:
|
||||
- build-37
|
||||
|
@ -141,7 +141,7 @@ Get a clean main branch with all of the changes from `upstream`::
|
||||
|
||||
- Make sure all your changes are committed.
|
||||
|
||||
- Push the changes upstream::
|
||||
- Push the changes upstream. (Normally pushing directly without review is frowned upon, but the `main` branch is protected from force pushes and release tagging requires the same permissions as pushing to `main`)::
|
||||
|
||||
$ git push upstream main
|
||||
|
||||
@ -149,8 +149,12 @@ Get a clean main branch with all of the changes from `upstream`::
|
||||
|
||||
- Make the release on GitHub using [this link](https://github.com/mozilla/glean_parser/releases/new)
|
||||
|
||||
- Enter the new version in the form `vX.Y.Z`.
|
||||
- Both the tag and the release title should be in the form `vX.Y.Z`.
|
||||
|
||||
- Copy and paste the relevant part of the `HISTORY.rst` file into the description.
|
||||
|
||||
- Tagging the release will trigger a CI workflow which will build the distribution of `glean_parser` and publish it to PyPI.
|
||||
|
||||
The continuous integration system will then automatically deploy to PyPI.
|
||||
|
||||
See also the [instructions for updating the version of `glean_parser` used by the Glean SDK](https://mozilla.github.io/glean/book/dev/upgrading-glean-parser.html).
|
||||
|
12
third_party/python/glean_parser/HISTORY.rst
vendored
12
third_party/python/glean_parser/HISTORY.rst
vendored
@ -5,6 +5,18 @@ History
|
||||
Unreleased
|
||||
----------
|
||||
|
||||
1.29.0 (2020-10-07)
|
||||
-------------------
|
||||
|
||||
* **Breaking change:** `glean_parser` will now return an error code when any of the input files do not exist (unless the `--allow-missing-files` flag is passed).
|
||||
* Generated code now includes a comment next to each metric containing the name of the metric in its original `snake_case` form.
|
||||
* When metrics don't provide a `unit` parameter, it is not included in the output (as provided by probe-scraper).
|
||||
|
||||
1.28.6 (2020-09-24)
|
||||
-------------------
|
||||
|
||||
* BUGFIX: Ensure Kotlin arguments are deterministically ordered
|
||||
|
||||
1.28.5 (2020-09-14)
|
||||
-------------------
|
||||
|
||||
|
4
third_party/python/glean_parser/Makefile
vendored
4
third_party/python/glean_parser/Makefile
vendored
@ -36,9 +36,7 @@ clean-test: ## remove test and coverage artifacts
|
||||
|
||||
lint: ## check style with flake8
|
||||
python3 -m flake8 glean_parser tests
|
||||
if python3 --version | grep 'Python 3\.[678]\..*'; then \
|
||||
python3 -m black --check glean_parser tests setup.py; \
|
||||
fi
|
||||
python3 -m black --check glean_parser tests setup.py
|
||||
python3 -m yamllint glean_parser tests
|
||||
python3 -m mypy glean_parser
|
||||
|
||||
|
15
third_party/python/glean_parser/PKG-INFO
vendored
15
third_party/python/glean_parser/PKG-INFO
vendored
@ -1,6 +1,6 @@
|
||||
Metadata-Version: 1.1
|
||||
Name: glean_parser
|
||||
Version: 1.28.5
|
||||
Version: 1.29.0
|
||||
Summary: Parser tools for Mozilla's Glean telemetry
|
||||
Home-page: https://github.com/mozilla/glean_parser
|
||||
Author: Michael Droettboom
|
||||
@ -69,6 +69,18 @@ Description: ============
|
||||
Unreleased
|
||||
----------
|
||||
|
||||
1.29.0 (2020-10-07)
|
||||
-------------------
|
||||
|
||||
* **Breaking change:** `glean_parser` will now return an error code when any of the input files do not exist (unless the `--allow-missing-files` flag is passed).
|
||||
* Generated code now includes a comment next to each metric containing the name of the metric in its original `snake_case` form.
|
||||
* When metrics don't provide a `unit` parameter, it is not included in the output (as provided by probe-scraper).
|
||||
|
||||
1.28.6 (2020-09-24)
|
||||
-------------------
|
||||
|
||||
* BUGFIX: Ensure Kotlin arguments are deterministically ordered
|
||||
|
||||
1.28.5 (2020-09-14)
|
||||
-------------------
|
||||
|
||||
@ -475,3 +487,4 @@ Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Programming Language :: Python :: 3.6
|
||||
Classifier: Programming Language :: Python :: 3.7
|
||||
Classifier: Programming Language :: Python :: 3.8
|
||||
Classifier: Programming Language :: Python :: 3.9
|
||||
|
@ -53,7 +53,12 @@ from . import validate_ping
|
||||
"Should only be set when building the Glean library itself."
|
||||
),
|
||||
)
|
||||
def translate(input, format, output, option, allow_reserved):
|
||||
@click.option(
|
||||
"--allow-missing-files",
|
||||
is_flag=True,
|
||||
help=("Do not treat missing input files as an error."),
|
||||
)
|
||||
def translate(input, format, output, option, allow_reserved, allow_missing_files):
|
||||
"""
|
||||
Translate metrics.yaml and pings.yaml files to other formats.
|
||||
"""
|
||||
@ -68,7 +73,10 @@ def translate(input, format, output, option, allow_reserved):
|
||||
format,
|
||||
Path(output),
|
||||
option_dict,
|
||||
{"allow_reserved": allow_reserved},
|
||||
{
|
||||
"allow_reserved": allow_reserved,
|
||||
"allow_missing_files": allow_missing_files,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
@ -112,11 +120,24 @@ def check(schema):
|
||||
"Should only be set when building the Glean library itself."
|
||||
),
|
||||
)
|
||||
def glinter(input, allow_reserved):
|
||||
@click.option(
|
||||
"--allow-missing-files",
|
||||
is_flag=True,
|
||||
help=("Do not treat missing input files as an error."),
|
||||
)
|
||||
def glinter(input, allow_reserved, allow_missing_files):
|
||||
"""
|
||||
Runs a linter over the metrics.
|
||||
"""
|
||||
sys.exit(lint.glinter([Path(x) for x in input], {"allow_reserved": allow_reserved}))
|
||||
sys.exit(
|
||||
lint.glinter(
|
||||
[Path(x) for x in input],
|
||||
{
|
||||
"allow_reserved": allow_reserved,
|
||||
"allow_missing_files": allow_missing_files,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@click.group()
|
||||
@ -131,5 +152,18 @@ main.add_command(check)
|
||||
main.add_command(glinter)
|
||||
|
||||
|
||||
def main_wrapper(args=None):
|
||||
"""
|
||||
A simple wrapper around click's `main` to display the glean_parser version
|
||||
when there is an error.
|
||||
"""
|
||||
try:
|
||||
main(args=args)
|
||||
except SystemExit as e:
|
||||
if e.code != 0:
|
||||
print(f"ERROR running glean_parser v{glean_parser.__version__}")
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main()) # pragma: no cover
|
||||
main_wrapper() # pragma: no cover
|
||||
|
@ -11,7 +11,7 @@ Outputter to generate C# code for metrics.
|
||||
import enum
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Union # noqa
|
||||
from typing import Any, Dict, List, Optional, Union # noqa
|
||||
|
||||
from . import metrics
|
||||
from . import pings
|
||||
@ -104,7 +104,7 @@ def class_name(obj_type: str) -> str:
|
||||
|
||||
|
||||
def output_csharp(
|
||||
objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {}
|
||||
objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""
|
||||
Given a tree of objects, output C# code to `output_dir`.
|
||||
@ -120,6 +120,9 @@ def output_csharp(
|
||||
This is where glean objects will be imported from in the generated
|
||||
code.
|
||||
"""
|
||||
if options is None:
|
||||
options = {}
|
||||
|
||||
template = util.get_jinja2_template(
|
||||
"csharp.jinja2",
|
||||
filters=(
|
||||
|
@ -12,7 +12,7 @@ from collections import OrderedDict
|
||||
import enum
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Union # noqa
|
||||
from typing import Any, Dict, List, Optional, Union # noqa
|
||||
|
||||
from . import metrics
|
||||
from . import pings
|
||||
@ -102,7 +102,7 @@ def class_name(obj_type: str) -> str:
|
||||
|
||||
|
||||
def output_gecko_lookup(
|
||||
objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {}
|
||||
objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""
|
||||
Given a tree of objects, generate a Kotlin map between Gecko histograms and
|
||||
@ -119,6 +119,9 @@ def output_gecko_lookup(
|
||||
This is where glean objects will be imported from in the generated
|
||||
code.
|
||||
"""
|
||||
if options is None:
|
||||
options = {}
|
||||
|
||||
template = util.get_jinja2_template(
|
||||
"kotlin.geckoview.jinja2",
|
||||
filters=(
|
||||
@ -197,7 +200,7 @@ def output_gecko_lookup(
|
||||
|
||||
|
||||
def output_kotlin(
|
||||
objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {}
|
||||
objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""
|
||||
Given a tree of objects, output Kotlin code to `output_dir`.
|
||||
@ -213,6 +216,9 @@ def output_kotlin(
|
||||
This is where glean objects will be imported from in the generated
|
||||
code.
|
||||
"""
|
||||
if options is None:
|
||||
options = {}
|
||||
|
||||
template = util.get_jinja2_template(
|
||||
"kotlin.jinja2",
|
||||
filters=(
|
||||
|
@ -7,7 +7,16 @@ import enum
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Callable, Dict, Generator, List, Iterable, Tuple, Union # noqa
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Dict,
|
||||
Generator,
|
||||
List,
|
||||
Iterable,
|
||||
Optional,
|
||||
Tuple,
|
||||
) # noqa
|
||||
|
||||
|
||||
from . import metrics
|
||||
@ -96,7 +105,7 @@ def check_common_prefix(
|
||||
|
||||
|
||||
def check_unit_in_name(
|
||||
metric: metrics.Metric, parser_config: Dict[str, Any] = {}
|
||||
metric: metrics.Metric, parser_config: Dict[str, Any]
|
||||
) -> LintGenerator:
|
||||
"""
|
||||
The metric name ends in a unit.
|
||||
@ -189,7 +198,7 @@ def check_category_generic(
|
||||
|
||||
|
||||
def check_bug_number(
|
||||
metric: metrics.Metric, parser_config: Dict[str, Any] = {}
|
||||
metric: metrics.Metric, parser_config: Dict[str, Any]
|
||||
) -> LintGenerator:
|
||||
number_bugs = [str(bug) for bug in metric.bugs if isinstance(bug, int)]
|
||||
|
||||
@ -202,7 +211,7 @@ def check_bug_number(
|
||||
|
||||
|
||||
def check_valid_in_baseline(
|
||||
metric: metrics.Metric, parser_config: Dict[str, Any] = {}
|
||||
metric: metrics.Metric, parser_config: Dict[str, Any]
|
||||
) -> LintGenerator:
|
||||
allow_reserved = parser_config.get("allow_reserved", False)
|
||||
|
||||
@ -214,7 +223,7 @@ def check_valid_in_baseline(
|
||||
|
||||
|
||||
def check_misspelled_pings(
|
||||
metric: metrics.Metric, parser_config: Dict[str, Any] = {}
|
||||
metric: metrics.Metric, parser_config: Dict[str, Any]
|
||||
) -> LintGenerator:
|
||||
for ping in metric.send_in_pings:
|
||||
for builtin in pings.RESERVED_PING_NAMES:
|
||||
@ -224,7 +233,7 @@ def check_misspelled_pings(
|
||||
|
||||
|
||||
def check_user_lifetime_expiration(
|
||||
metric: metrics.Metric, parser_config: Dict[str, Any] = {}
|
||||
metric: metrics.Metric, parser_config: Dict[str, Any]
|
||||
) -> LintGenerator:
|
||||
|
||||
if metric.lifetime == metrics.Lifetime.user and metric.expires != "never":
|
||||
@ -236,7 +245,7 @@ def check_user_lifetime_expiration(
|
||||
|
||||
|
||||
def check_expired_date(
|
||||
metric: metrics.Metric, parser_config: Dict[str, Any] = {}
|
||||
metric: metrics.Metric, parser_config: Dict[str, Any]
|
||||
) -> LintGenerator:
|
||||
try:
|
||||
metric.validate_expires()
|
||||
@ -245,7 +254,7 @@ def check_expired_date(
|
||||
|
||||
|
||||
def check_expired_metric(
|
||||
metric: metrics.Metric, parser_config: Dict[str, Any] = {}
|
||||
metric: metrics.Metric, parser_config: Dict[str, Any]
|
||||
) -> LintGenerator:
|
||||
if metric.is_expired():
|
||||
yield ("Metric has expired. Please consider removing it.")
|
||||
@ -291,7 +300,9 @@ class GlinterNit:
|
||||
|
||||
|
||||
def lint_metrics(
|
||||
objs: metrics.ObjectTree, parser_config: Dict[str, Any] = {}, file=sys.stderr
|
||||
objs: metrics.ObjectTree,
|
||||
parser_config: Optional[Dict[str, Any]] = None,
|
||||
file=sys.stderr,
|
||||
) -> List[GlinterNit]:
|
||||
"""
|
||||
Performs glinter checks on a set of metrics objects.
|
||||
@ -300,6 +311,9 @@ def lint_metrics(
|
||||
:param file: The stream to write errors to.
|
||||
:returns: List of nits.
|
||||
"""
|
||||
if parser_config is None:
|
||||
parser_config = {}
|
||||
|
||||
nits: List[GlinterNit] = []
|
||||
for (category_name, category) in sorted(list(objs.items())):
|
||||
if category_name == "pings":
|
||||
@ -322,7 +336,7 @@ def lint_metrics(
|
||||
for msg in cat_check_func(category_name, category_metrics.values())
|
||||
)
|
||||
|
||||
for (metric_name, metric) in sorted(list(category_metrics.items())):
|
||||
for (_metric_name, metric) in sorted(list(category_metrics.items())):
|
||||
for (check_name, (check_func, check_type)) in INDIVIDUAL_CHECKS.items():
|
||||
new_nits = list(check_func(metric, parser_config))
|
||||
if len(new_nits):
|
||||
@ -354,7 +368,11 @@ def lint_metrics(
|
||||
return nits
|
||||
|
||||
|
||||
def lint_yaml_files(input_filepaths: Iterable[Path], file=sys.stderr) -> List:
|
||||
def lint_yaml_files(
|
||||
input_filepaths: Iterable[Path],
|
||||
file=sys.stderr,
|
||||
parser_config: Dict[str, Any] = None,
|
||||
) -> List:
|
||||
"""
|
||||
Performs glinter YAML lint on a set of files.
|
||||
|
||||
@ -363,10 +381,16 @@ def lint_yaml_files(input_filepaths: Iterable[Path], file=sys.stderr) -> List:
|
||||
:returns: List of nits.
|
||||
"""
|
||||
|
||||
if parser_config is None:
|
||||
parser_config = {}
|
||||
|
||||
# Generic type since the actual type comes from yamllint, which we don't
|
||||
# control.
|
||||
nits: List = []
|
||||
for path in input_filepaths:
|
||||
if not path.is_file() and parser_config.get("allow_missing_files", False):
|
||||
continue
|
||||
|
||||
# yamllint needs both the file content and the path.
|
||||
file_content = None
|
||||
with path.open("r", encoding="utf-8") as fd:
|
||||
@ -386,7 +410,9 @@ def lint_yaml_files(input_filepaths: Iterable[Path], file=sys.stderr) -> List:
|
||||
|
||||
|
||||
def glinter(
|
||||
input_filepaths: Iterable[Path], parser_config: Dict[str, Any] = {}, file=sys.stderr
|
||||
input_filepaths: Iterable[Path],
|
||||
parser_config: Optional[Dict[str, Any]] = None,
|
||||
file=sys.stderr,
|
||||
) -> int:
|
||||
"""
|
||||
Commandline helper for glinter.
|
||||
@ -397,7 +423,10 @@ def glinter(
|
||||
:param file: The stream to write the errors to.
|
||||
:return: Non-zero if there were any glinter errors.
|
||||
"""
|
||||
if lint_yaml_files(input_filepaths, file=file):
|
||||
if parser_config is None:
|
||||
parser_config = {}
|
||||
|
||||
if lint_yaml_files(input_filepaths, file=file, parser_config=parser_config):
|
||||
return 1
|
||||
|
||||
objs = parser.parse_objects(input_filepaths, parser_config)
|
||||
|
@ -42,7 +42,9 @@ def extra_info(obj: Union[metrics.Metric, pings.Ping]) -> List[Tuple[str, str]]:
|
||||
return extra_info
|
||||
|
||||
|
||||
def ping_desc(ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}) -> str:
|
||||
def ping_desc(
|
||||
ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None
|
||||
) -> str:
|
||||
"""
|
||||
Return a text description of the ping. If a custom_pings_cache
|
||||
is available, look in there for non-reserved ping names description.
|
||||
@ -56,7 +58,7 @@ def ping_desc(ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}) ->
|
||||
)
|
||||
elif ping_name == "all-pings":
|
||||
desc = "These metrics are sent in every ping."
|
||||
elif ping_name in custom_pings_cache:
|
||||
elif custom_pings_cache is not None and ping_name in custom_pings_cache:
|
||||
desc = custom_pings_cache[ping_name].description
|
||||
|
||||
return desc
|
||||
@ -87,8 +89,10 @@ def ping_docs(ping_name: str) -> str:
|
||||
return f"https://mozilla.github.io/glean/book/user/pings/{ping_name}.html"
|
||||
|
||||
|
||||
def if_empty(ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}) -> bool:
|
||||
if ping_name in custom_pings_cache:
|
||||
def if_empty(
|
||||
ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None
|
||||
) -> bool:
|
||||
if custom_pings_cache is not None and ping_name in custom_pings_cache:
|
||||
return custom_pings_cache[ping_name].send_if_empty
|
||||
else:
|
||||
return False
|
||||
@ -109,27 +113,27 @@ def ping_reasons(
|
||||
|
||||
|
||||
def ping_data_reviews(
|
||||
ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}
|
||||
ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None
|
||||
) -> Optional[List[str]]:
|
||||
if ping_name in custom_pings_cache:
|
||||
if custom_pings_cache is not None and ping_name in custom_pings_cache:
|
||||
return custom_pings_cache[ping_name].data_reviews
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def ping_bugs(
|
||||
ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}
|
||||
ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None
|
||||
) -> Optional[List[str]]:
|
||||
if ping_name in custom_pings_cache:
|
||||
if custom_pings_cache is not None and ping_name in custom_pings_cache:
|
||||
return custom_pings_cache[ping_name].bugs
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def ping_include_client_id(
|
||||
ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}
|
||||
ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None
|
||||
) -> bool:
|
||||
if ping_name in custom_pings_cache:
|
||||
if custom_pings_cache is not None and ping_name in custom_pings_cache:
|
||||
return custom_pings_cache[ping_name].include_client_id
|
||||
else:
|
||||
return False
|
||||
@ -145,7 +149,7 @@ def data_sensitivity_numbers(
|
||||
|
||||
|
||||
def output_markdown(
|
||||
objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {}
|
||||
objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""
|
||||
Given a tree of objects, output Markdown docs to `output_dir`.
|
||||
@ -159,6 +163,8 @@ def output_markdown(
|
||||
:param options: options dictionary, with the following optional key:
|
||||
- `project_title`: The projects title.
|
||||
"""
|
||||
if options is None:
|
||||
options = {}
|
||||
|
||||
# Build a dictionary that associates pings with their metrics.
|
||||
#
|
||||
@ -177,7 +183,7 @@ def output_markdown(
|
||||
# This also builds a dictionary of custom pings, if available.
|
||||
custom_pings_cache: Dict[str, pings.Ping] = defaultdict()
|
||||
metrics_by_pings: Dict[str, List[metrics.Metric]] = defaultdict(list)
|
||||
for category_key, category_val in objs.items():
|
||||
for _category_key, category_val in objs.items():
|
||||
for obj in category_val.values():
|
||||
# Filter out custom pings. We will need them for extracting
|
||||
# the description
|
||||
|
@ -52,7 +52,7 @@ class Metric:
|
||||
disabled: bool = False,
|
||||
lifetime: str = "ping",
|
||||
send_in_pings: Optional[List[str]] = None,
|
||||
unit: str = "",
|
||||
unit: Optional[str] = None,
|
||||
gecko_datapoint: str = "",
|
||||
no_lint: Optional[List[str]] = None,
|
||||
data_sensitivity: Optional[List[str]] = None,
|
||||
@ -78,7 +78,8 @@ class Metric:
|
||||
if send_in_pings is None:
|
||||
send_in_pings = ["default"]
|
||||
self.send_in_pings = send_in_pings
|
||||
self.unit = unit
|
||||
if unit is not None:
|
||||
self.unit = unit
|
||||
self.gecko_datapoint = gecko_datapoint
|
||||
if no_lint is None:
|
||||
no_lint = []
|
||||
@ -120,7 +121,7 @@ class Metric:
|
||||
category: str,
|
||||
name: str,
|
||||
metric_info: Dict[str, util.JSONType],
|
||||
config: Dict[str, Any] = {},
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
validated: bool = False,
|
||||
):
|
||||
"""
|
||||
@ -136,6 +137,9 @@ class Metric:
|
||||
jsonschema validation
|
||||
:return: A new Metric instance.
|
||||
"""
|
||||
if config is None:
|
||||
config = {}
|
||||
|
||||
metric_type = metric_info["type"]
|
||||
if not isinstance(metric_type, str):
|
||||
raise TypeError(f"Unknown metric type {metric_type}")
|
||||
|
@ -53,13 +53,21 @@ def _update_validator(validator):
|
||||
|
||||
|
||||
def _load_file(
|
||||
filepath: Path,
|
||||
filepath: Path, parser_config: Dict[str, Any]
|
||||
) -> Generator[str, None, Tuple[Dict[str, util.JSONType], Optional[str]]]:
|
||||
"""
|
||||
Load a metrics.yaml or pings.yaml format file.
|
||||
|
||||
If the `filepath` does not exist, raises `FileNotFoundError`, unless
|
||||
`parser_config["allow_missing_files"]` is `True`.
|
||||
"""
|
||||
try:
|
||||
content = util.load_yaml_or_json(filepath, ordered_dict=True)
|
||||
except FileNotFoundError:
|
||||
if not parser_config.get("allow_missing_files", False):
|
||||
raise
|
||||
else:
|
||||
return {}, None
|
||||
except Exception as e:
|
||||
yield util.format_error(filepath, "", textwrap.fill(str(e)))
|
||||
return {}, None
|
||||
@ -313,7 +321,7 @@ def _preprocess_objects(objs: ObjectTree, config: Dict[str, Any]) -> ObjectTree:
|
||||
|
||||
@util.keep_value
|
||||
def parse_objects(
|
||||
filepaths: Iterable[Path], config: Dict[str, Any] = {}
|
||||
filepaths: Iterable[Path], config: Optional[Dict[str, Any]] = None
|
||||
) -> Generator[str, None, ObjectTree]:
|
||||
"""
|
||||
Parse one or more metrics.yaml and/or pings.yaml files, returning a tree of
|
||||
@ -342,12 +350,17 @@ def parse_objects(
|
||||
This is useful when you want to retain the original "disabled"
|
||||
value from the `metrics.yaml`, rather than having it overridden when
|
||||
the metric expires.
|
||||
- `allow_missing_files`: Do not raise a `FileNotFoundError` if any of
|
||||
the input `filepaths` do not exist.
|
||||
"""
|
||||
if config is None:
|
||||
config = {}
|
||||
|
||||
all_objects: ObjectTree = OrderedDict()
|
||||
sources: Dict[Any, Path] = {}
|
||||
filepaths = util.ensure_list(filepaths)
|
||||
for filepath in filepaths:
|
||||
content, filetype = yield from _load_file(filepath)
|
||||
content, filetype = yield from _load_file(filepath, config)
|
||||
if filetype == "metrics":
|
||||
yield from _instantiate_metrics(
|
||||
all_objects, sources, content, filepath, config
|
||||
|
@ -73,3 +73,9 @@ class Ping:
|
||||
d = self.__dict__.copy()
|
||||
del d["name"]
|
||||
return d
|
||||
|
||||
def identifier(self) -> str:
|
||||
"""
|
||||
Used for the "generated from ..." comment in the output.
|
||||
"""
|
||||
return self.name
|
||||
|
@ -11,7 +11,7 @@ Outputter to generate Swift code for metrics.
|
||||
import enum
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Union
|
||||
from typing import Any, Dict, Optional, Union
|
||||
|
||||
from . import metrics
|
||||
from . import pings
|
||||
@ -117,7 +117,7 @@ class Category:
|
||||
|
||||
|
||||
def output_swift(
|
||||
objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {}
|
||||
objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""
|
||||
Given a tree of objects, output Swift code to `output_dir`.
|
||||
@ -130,6 +130,9 @@ def output_swift(
|
||||
- glean_namespace: The namespace to import Glean from
|
||||
- allow_reserved: When True, this is a Glean-internal build
|
||||
"""
|
||||
if options is None:
|
||||
options = {}
|
||||
|
||||
template = util.get_jinja2_template(
|
||||
"swift.jinja2",
|
||||
filters=(
|
||||
|
@ -11,7 +11,7 @@ Jinja2 template is not. Please file bugs! #}
|
||||
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
{% macro obj_declaration(obj, suffix='', access='', lazy=False) %}
|
||||
{{ access }} {% if lazy %} Lazy<{{ obj|type_name }}>{%- else %} {{ obj|type_name }}{% endif %} {{ obj.name|camelize }}{{ suffix }}
|
||||
{%- if lazy %} = new Lazy<{{ obj|type_name }}>(() => {%- else %} ={% endif %}
|
||||
{%- if lazy %} = new Lazy<{{ obj|type_name }}>(() => {%- else %} = // generated from {{ obj.identifier() }}{% endif %}
|
||||
|
||||
new {{ obj|type_name }}(
|
||||
{% for arg_name in extra_args if obj[arg_name] is defined %}
|
||||
@ -57,7 +57,7 @@ namespace {{ namespace }}
|
||||
{% for obj in objs.values() %}
|
||||
{% if obj.labeled %}
|
||||
{{ obj_declaration(obj, 'Label', 'private ') }}
|
||||
private readonly Lazy<LabeledMetricType<{{ obj|type_name }}>> {{ obj.name|camelize }}Lazy = new Lazy<LabeledMetricType<{{ obj|type_name }}>>(() => new LabeledMetricType<{{ obj|type_name }}>(
|
||||
private readonly Lazy<LabeledMetricType<{{ obj|type_name }}>> {{ obj.name|camelize }}Lazy = new Lazy<LabeledMetricType<{{ obj|type_name }}>>(() => new LabeledMetricType<{{ obj|type_name }}>( // generated from {{ obj.identifier() }}
|
||||
category: {{ obj.category|csharp }},
|
||||
name: {{ obj.name|csharp }},
|
||||
submetric: {{ category_name|Camelize }}.{{ obj.name|camelize }}Label,
|
||||
@ -82,7 +82,7 @@ namespace {{ namespace }}
|
||||
/// <summary>
|
||||
/// {{ obj.description|wordwrap() | replace('\n', '\n /// ') }}
|
||||
/// </summary>
|
||||
internal {{ obj|type_name }} {{ obj.name|camelize }} => {{ obj.name|camelize }}Lazy.Value;
|
||||
internal {{ obj|type_name }} {{ obj.name|camelize }} => {{ obj.name|camelize }}Lazy.Value; // generated from {{ obj.identifier() }}
|
||||
|
||||
{% else %}
|
||||
{# Finally handle pings. #}
|
||||
|
@ -13,7 +13,7 @@ Jinja2 template is not. Please file bugs! #}
|
||||
{% if (access != "private ") -%}
|
||||
@get:JvmName("{{ obj.name|camelize }}{{ suffix }}")
|
||||
{% endif -%}
|
||||
{{ access }}val {{ obj.name|camelize }}{{ suffix }}: {{ obj|type_name }}{% if lazy %} by lazy { {%- else %} ={% endif %}
|
||||
{{ access }}val {{ obj.name|camelize }}{{ suffix }}: {{ obj|type_name }}{% if lazy %} by lazy { {%- else %} ={% endif %} // generated from {{ obj.identifier() }}
|
||||
|
||||
{{ obj|type_name }}(
|
||||
{% for arg_name in extra_args if obj[arg_name] is defined %}
|
||||
@ -60,7 +60,7 @@ internal object {{ category_name|Camelize }} {
|
||||
/**
|
||||
* {{ obj.description|wordwrap() | replace('\n', '\n * ') }}
|
||||
*/
|
||||
val {{ obj.name|camelize }}: LabeledMetricType<{{ obj|type_name }}> by lazy {
|
||||
val {{ obj.name|camelize }}: LabeledMetricType<{{ obj|type_name }}> by lazy { // generated from {{ obj.identifier() }}
|
||||
LabeledMetricType(
|
||||
category = {{ obj.category|kotlin }},
|
||||
name = {{ obj.name|kotlin }},
|
||||
|
@ -8,7 +8,7 @@ Jinja2 template is not. Please file bugs! #}
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
{% macro obj_declaration(obj, suffix='', access='') %}
|
||||
{{ access }}static let {{ obj.name|camelize|variable_name }}{{ suffix }} = {{ obj|type_name }}(
|
||||
{{ access }}static let {{ obj.name|camelize|variable_name }}{{ suffix }} = {{ obj|type_name }}( // generated from {{ obj.identifier() }}
|
||||
{% for arg_name in extra_args if obj[arg_name] is defined %}
|
||||
{{ arg_name|camelize }}: {{ obj[arg_name]|swift }}{{ "," if not loop.last }}
|
||||
{% endfor %}
|
||||
@ -86,7 +86,7 @@ extension {{ namespace }} {
|
||||
{% if obj.labeled %}
|
||||
{{ obj_declaration(obj, 'Label', 'private ') }}
|
||||
/// {{ obj.description|wordwrap() | replace('\n', '\n /// ') }}
|
||||
static let {{ obj.name|camelize|variable_name }} = try! LabeledMetricType<{{ obj|type_name }}>(
|
||||
static let {{ obj.name|camelize|variable_name }} = try! LabeledMetricType<{{ obj|type_name }}>( // generated from {{ obj.identifier() }}
|
||||
category: {{ obj.category|swift }},
|
||||
name: {{ obj.name|swift }},
|
||||
sendInPings: {{ obj.send_in_pings|swift }},
|
||||
|
@ -12,7 +12,7 @@ from pathlib import Path
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from typing import Any, Callable, Dict, Iterable, List
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||
|
||||
from . import lint
|
||||
from . import parser
|
||||
@ -40,8 +40,11 @@ class Outputter:
|
||||
def __init__(
|
||||
self,
|
||||
output_func: Callable[[metrics.ObjectTree, Path, Dict[str, Any]], None],
|
||||
clear_patterns: List[str] = [],
|
||||
clear_patterns: Optional[List[str]] = None,
|
||||
):
|
||||
if clear_patterns is None:
|
||||
clear_patterns = []
|
||||
|
||||
self.output_func = output_func
|
||||
self.clear_patterns = clear_patterns
|
||||
|
||||
@ -49,7 +52,7 @@ class Outputter:
|
||||
OUTPUTTERS = {
|
||||
"csharp": Outputter(csharp.output_csharp, ["*.cs"]),
|
||||
"kotlin": Outputter(kotlin.output_kotlin, ["*.kt"]),
|
||||
"markdown": Outputter(markdown.output_markdown),
|
||||
"markdown": Outputter(markdown.output_markdown, []),
|
||||
"swift": Outputter(swift.output_swift, ["*.swift"]),
|
||||
}
|
||||
|
||||
@ -58,9 +61,9 @@ def translate_metrics(
|
||||
input_filepaths: Iterable[Path],
|
||||
output_dir: Path,
|
||||
translation_func: Callable[[metrics.ObjectTree, Path, Dict[str, Any]], None],
|
||||
clear_patterns: List[str] = [],
|
||||
options: Dict[str, Any] = {},
|
||||
parser_config: Dict[str, Any] = {},
|
||||
clear_patterns: Optional[List[str]] = None,
|
||||
options: Optional[Dict[str, Any]] = None,
|
||||
parser_config: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Translate the files in `input_filepaths` by running the metrics through a
|
||||
@ -85,6 +88,15 @@ def translate_metrics(
|
||||
:param parser_config: A dictionary of options that change parsing behavior.
|
||||
See `parser.parse_metrics` for more info.
|
||||
"""
|
||||
if clear_patterns is None:
|
||||
clear_patterns = []
|
||||
|
||||
if options is None:
|
||||
options = {}
|
||||
|
||||
if parser_config is None:
|
||||
parser_config = {}
|
||||
|
||||
input_filepaths = util.ensure_list(input_filepaths)
|
||||
|
||||
if lint.glinter(input_filepaths, parser_config):
|
||||
@ -128,8 +140,8 @@ def translate(
|
||||
input_filepaths: Iterable[Path],
|
||||
output_format: str,
|
||||
output_dir: Path,
|
||||
options: Dict[str, Any] = {},
|
||||
parser_config: Dict[str, Any] = {},
|
||||
options: Optional[Dict[str, Any]] = None,
|
||||
parser_config: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Translate the files in `input_filepaths` to the given `output_format` and
|
||||
@ -143,6 +155,12 @@ def translate(
|
||||
:param parser_config: A dictionary of options that change parsing behavior.
|
||||
See `parser.parse_metrics` for more info.
|
||||
"""
|
||||
if options is None:
|
||||
options = {}
|
||||
|
||||
if parser_config is None:
|
||||
parser_config = {}
|
||||
|
||||
format_desc = OUTPUTTERS.get(output_format, None)
|
||||
|
||||
if format_desc is None:
|
||||
|
@ -114,14 +114,12 @@ def load_yaml_or_json(path: Path, ordered_dict: bool = False):
|
||||
:param path: `pathlib.Path` object
|
||||
:rtype object: The tree of objects as a result of parsing the file.
|
||||
:raises ValueError: The file is neither a .json, .yml or .yaml file.
|
||||
:raises FileNotFoundError: The file does not exist.
|
||||
"""
|
||||
# If in py.test, support bits of literal JSON/YAML content
|
||||
if TESTING_MODE and isinstance(path, dict):
|
||||
return path
|
||||
|
||||
if not path.is_file():
|
||||
return {}
|
||||
|
||||
if path.suffix == ".json":
|
||||
with path.open("r", encoding="utf-8") as fd:
|
||||
return json.load(fd)
|
||||
@ -434,5 +432,7 @@ extra_ping_args = [
|
||||
]
|
||||
|
||||
|
||||
# Names of parameters to pass to both metric and ping constructors.
|
||||
extra_args = list(set(extra_metric_args) | set(extra_ping_args))
|
||||
# Names of parameters to pass to both metric and ping constructors (no duplicates).
|
||||
extra_args = extra_metric_args + [
|
||||
v for v in extra_ping_args if v not in extra_metric_args
|
||||
]
|
||||
|
@ -1,13 +1,14 @@
|
||||
black==20.8b1
|
||||
coverage==5.2.1
|
||||
flake8==3.8.3
|
||||
coverage==5.3
|
||||
flake8==3.8.4
|
||||
flake8-bugbear==20.1.4
|
||||
m2r==0.2.1
|
||||
mypy==0.782
|
||||
pip
|
||||
pytest-runner==5.2
|
||||
pytest==6.0.1
|
||||
pytest==6.1.1
|
||||
Sphinx==3.2.1
|
||||
twine==3.2.0
|
||||
watchdog==0.10.3
|
||||
wheel
|
||||
yamllint==1.24.2
|
||||
yamllint==1.25.0
|
||||
|
3
third_party/python/glean_parser/setup.py
vendored
3
third_party/python/glean_parser/setup.py
vendored
@ -51,11 +51,12 @@ setup(
|
||||
"Programming Language :: Python :: 3.6",
|
||||
"Programming Language :: Python :: 3.7",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
],
|
||||
description="Parser tools for Mozilla's Glean telemetry",
|
||||
entry_points={
|
||||
"console_scripts": [
|
||||
"glean_parser=glean_parser.__main__:main",
|
||||
"glean_parser=glean_parser.__main__:main_wrapper",
|
||||
],
|
||||
},
|
||||
install_requires=requirements,
|
||||
|
2
third_party/python/requirements.in
vendored
2
third_party/python/requirements.in
vendored
@ -26,7 +26,7 @@ ecdsa==0.15
|
||||
esprima==4.0.1
|
||||
fluent.migrate==0.10
|
||||
fluent.syntax==0.18.1
|
||||
glean_parser==1.28.5
|
||||
glean_parser==1.29.0
|
||||
jsmin==2.1.0
|
||||
json-e==2.7.0
|
||||
mozilla-version==0.3.4
|
||||
|
6
third_party/python/requirements.txt
vendored
6
third_party/python/requirements.txt
vendored
@ -87,9 +87,9 @@ fluent.syntax==0.18.1 \
|
||||
--hash=sha256:0e63679fa4f1b3042565220a5127b4bab842424f07d6a13c12299e3b3835486a \
|
||||
--hash=sha256:3a55f5e605d1b029a65cc8b6492c86ec4608e15447e73db1495de11fd46c104f \
|
||||
# via -r requirements-mach-vendor-python.in, compare-locales, fluent.migrate
|
||||
glean_parser==1.28.5 \
|
||||
--hash=sha256:29ac33298898e0fd607163b704d68f598c1d118c5056852246d621ec26f973bb \
|
||||
--hash=sha256:330e045fd8410f661e8e4a67edc8ab4a125996381e5519c40ca98b34b9dc5ec8 \
|
||||
glean_parser==1.29.0 \
|
||||
--hash=sha256:7cf1b02ef87fad57bf0f6b9711a98c1fd8f89c9df702245d16c09bf1b042a255 \
|
||||
--hash=sha256:df7436e164148594176ec55f7d7c3c5c944daca67c3cc30428514628625b214b \
|
||||
# via -r requirements-mach-vendor-python.in
|
||||
jinja2==2.11.2 \
|
||||
--hash=sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0 \
|
||||
|
Loading…
Reference in New Issue
Block a user