gecko-dev/dom/base/usecounters.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import re


def read_conf(conf_filename):
    # Can't read/write from a single StringIO, so make a new one for reading.
    stream = open(conf_filename, "r")

    def parse_counters(stream):
        for line_num, full_line in enumerate(stream):
            line = full_line.rstrip("\n")
            if not line or line.startswith("//"):
                # empty line or comment
                continue
            m = re.match(r"method ([A-Za-z0-9]+)\.([A-Za-z0-9]+)$", line)
            if m:
                interface_name, method_name = m.groups()
                yield {
                    "type": "method",
                    "interface_name": interface_name,
                    "method_name": method_name,
                }
                continue
            m = re.match(r"attribute ([A-Za-z0-9]+)\.([A-Za-z0-9]+)$", line)
            if m:
                interface_name, attribute_name = m.groups()
                yield {
                    "type": "attribute",
                    "interface_name": interface_name,
                    "attribute_name": attribute_name,
                }
                continue
            m = re.match(r"custom ([A-Za-z0-9_]+) (.*)$", line)
            if m:
                name, desc = m.groups()
                yield {"type": "custom", "name": name, "desc": desc}
                continue
            raise ValueError(
                "error parsing %s at line %d" % (conf_filename, line_num + 1)
            )

    return parse_counters(stream)


YAML_HEADER = """\
# This file is AUTOGENERATED by usecounters.py. DO NOT EDIT.
# (instead, re-run ./mach gen-use-counter-metrics)

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

---
$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0
$tags:
  - 'Core :: DOM: Core & HTML'

"""

BASE_METRICS = """\
use.counter:
  content_documents_destroyed:
    type: counter
    description: >
      A count of how many content documents were destroyed.
      Used to turn document use counters' counts into rates.
      Excludes documents for which we do not count use counters
      (See `Document::ShouldIncludeInTelemetry`).
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1204994
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1569672
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1845779
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1852098
    data_reviews:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1569672
    notification_emails:
      - dom-core@mozilla.com
      - emilio@mozilla.com
    expires: never
    send_in_pings:
      - use-counters

  top_level_content_documents_destroyed:
    type: counter
    description: >
      A count of how many "pages" were destroyed.
      Used to turn page use counters' counts into rates.
      Excludes pages that contain only documents for which we do not count use
      counters (See `Document::ShouldIncludeInTelemetry`).
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1204994
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1569672
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1845779
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1852098
    data_reviews:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1569672
    notification_emails:
      - dom-core@mozilla.com
      - emilio@mozilla.com
    expires: never
    send_in_pings:
      - use-counters

  dedicated_workers_destroyed:
    type: counter
    description: >
      A count of how many `Dedicated`-kind workers were destroyed.
      Used to turn dedicated worker use counters' counts into rates.
      Excludes chrome workers.
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1202706
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1852098
    data_reviews:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1202706
    notification_emails:
      - dom-core@mozilla.com
      - emilio@mozilla.com
    expires: never
    send_in_pings:
      - use-counters

  shared_workers_destroyed:
    type: counter
    description: >
      A count of how many `Shared`-kind workers were destroyed.
      Used to turn shared worker use counters' counts into rates.
      Excludes chrome workers.
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1202706
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1852098
    data_reviews:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1202706
    notification_emails:
      - dom-core@mozilla.com
      - emilio@mozilla.com
    expires: never
    send_in_pings:
      - use-counters

  service_workers_destroyed:
    type: counter
    description: >
      A count of how many `Service`-kind workers were destroyed.
      Used to turn service worker use counters' counts into rates.
      Excludes chrome workers.
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1202706
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1852098
    data_reviews:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1202706
    notification_emails:
      - dom-core@mozilla.com
      - emilio@mozilla.com
    expires: never
    send_in_pings:
      - use-counters

"""

USE_COUNTER_TEMPLATE = """\
  {name}:
    type: counter
    description: >
      {desc}
      Compare against `{denominator}`
      to calculate the rate.
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1852098
    data_reviews:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1852098
    notification_emails:
      - dom-core@mozilla.com
      - emilio@mozilla.com
    expires: never
    send_in_pings:
      - use-counters

"""


def gen_use_counter_metrics():
    """
    Finds use counters in:
      * dom/base/UseCounters.conf
      * dom/base/UseCountersWorker.conf
      * dom/base/nsDeprecatedOperationsList.h
      * !/layout/style/ServoCSSPropList.py
      * servo/components/style/properties/counted_unknown_properties.py
    and overwrites the Glean metrics definition file
    `dom/base/use_counter_metrics.yaml` with definitions for each use counter found.

    IF YOU CHANGE THIS FUNCTION:
      * You should probably add your bug's number to USE_COUNTER_TEMPLATE, above.

    Returns 0 on success.
    """
    (
        page,
        doc,
        dedicated,
        shared,
        service,
        ops_page,
        ops_doc,
        css_page,
        css_doc,
    ) = parse_use_counters()
    import os

    import buildconfig
    from mozbuild.util import FileAvoidWrite

    yaml_path = os.path.join(
        buildconfig.topsrcdir, "dom", "base", "use_counter_metrics.yaml"
    )
    with FileAvoidWrite(yaml_path) as f:
        f.write(YAML_HEADER)
        f.write(BASE_METRICS)

        total = (
            len(page)
            + len(doc)
            + len(dedicated)
            + len(shared)
            + len(service)
            + len(ops_page)
            + len(ops_doc)
            + len(css_page)
            + len(css_doc)
        )
        f.write(f"# Total of {total} use counter metrics (excludes denominators).\n")
        f.write(f"# Total of {len(page)} 'page' use counters.\n")
        f.write("use.counter.page:\n")
        for [_, name, desc] in page:
            f.write(
                USE_COUNTER_TEMPLATE.format(
                    name=name,
                    desc=desc,
                    denominator="use.counter.top_level_content_documents_destroyed",
                )
            )

        f.write(f"# Total of {len(doc)} 'document' use counters.\n")
        f.write("use.counter.doc:\n")
        for [_, name, desc] in doc:
            f.write(
                USE_COUNTER_TEMPLATE.format(
                    name=name,
                    desc=desc,
                    denominator="use.counter.content_documents_destroyed",
                )
            )

        f.write(f"# Total of {len(dedicated)} 'dedicated worker' use counters.\n")
        f.write("use.counter.worker.dedicated:\n")
        for [_, name, desc] in dedicated:
            f.write(
                USE_COUNTER_TEMPLATE.format(
                    name=name,
                    desc=desc,
                    denominator="use.counter.dedicated_workers_destroyed",
                )
            )

        f.write(f"# Total of {len(shared)} 'shared worker' use counters.\n")
        f.write("use.counter.worker.shared:\n")
        for [_, name, desc] in shared:
            f.write(
                USE_COUNTER_TEMPLATE.format(
                    name=name,
                    desc=desc,
                    denominator="use.counter.shared_workers_destroyed",
                )
            )

        f.write(f"# Total of {len(service)} 'service worker' use counters.\n")
        f.write("use.counter.worker.service:\n")
        for [_, name, desc] in service:
            f.write(
                USE_COUNTER_TEMPLATE.format(
                    name=name,
                    desc=desc,
                    denominator="use.counter.service_workers_destroyed",
                )
            )

        f.write(
            f"# Total of {len(ops_page)} 'deprecated operations (page)' use counters.\n"
        )
        f.write("use.counter.deprecated_ops.page:\n")
        for [_, name, desc] in ops_page:
            f.write(
                USE_COUNTER_TEMPLATE.format(
                    name=name,
                    desc=desc,
                    denominator="use.counter.top_level_content_documents_destroyed",
                )
            )

        f.write(
            f"# Total of {len(ops_doc)} 'deprecated operations (document)' use counters.\n"
        )
        f.write("use.counter.deprecated_ops.doc:\n")
        for [_, name, desc] in ops_doc:
            f.write(
                USE_COUNTER_TEMPLATE.format(
                    name=name,
                    desc=desc,
                    denominator="use.counter.content_documents_destroyed",
                )
            )

        f.write(f"# Total of {len(css_page)} 'CSS (page)' use counters.\n")
        f.write("use.counter.css.page:\n")
        for [_, name, desc] in css_page:
            f.write(
                USE_COUNTER_TEMPLATE.format(
                    name=name,
                    desc=desc,
                    denominator="use.counter.top_level_content_documents_destroyed",
                )
            )

        f.write(f"# Total of {len(css_doc)} 'CSS (document)' use counters.\n")
        f.write("use.counter.css.doc:\n")
        for [_, name, desc] in css_doc:
            f.write(
                USE_COUNTER_TEMPLATE.format(
                    name=name,
                    desc=desc,
                    denominator="use.counter.content_documents_destroyed",
                )
            )

    return 0


def parse_use_counters():
    """
    Finds use counters in:
      * dom/base/UseCounters.conf
      * dom/base/UseCountersWorker.conf
      * dom/base/nsDeprecatedOperationsList.h
      * !/layout/style/ServoCSSPropList.py
      * servo/components/style/properties/counted_unknown_properties.py
    and returns them as a tuple of lists of tuples of the form:
    (page, doc, dedicated, shared, service, ops_page, ops_doc, css_page, css_doc)
    where each of the items is a List<Tuple<enum_name, glean_name, description>>
    where `enum_name` is the name of the enum variant from UseCounter.h
    (like `eUseCounter_custom_CustomizedBuiltin`), and
    where `glean_name` is the name conjugated to Glean metric name safety.
    """

    # Note, this function contains a duplication of enum naming logic from UseCounter.h.
    # If you change the enum name format, you'll need to do it twice.

    # There are 3 kinds of Use Counters in conf files: method, attribute, custom.
    # `method` and `attribute` are presumed label-safe and are taken as-is.
    # `custom` can be any case, so are coerced to snake_case.
    import os

    import buildconfig

    uc_path = os.path.join(buildconfig.topsrcdir, "dom", "base", "UseCounters.conf")
    page = []
    doc = []
    for counter in read_conf(uc_path):
        if counter["type"] == "method":
            enum_name = (
                f"eUseCounter_{counter['interface_name']}_{counter['method_name']}"
            )
            glean_name = f"{counter['interface_name']}_{counter['method_name']}".lower()
            method = f"called {counter['interface_name']}.{counter['method_name']}"
            page.append((enum_name, glean_name, f"Whether a page called {method}."))
            doc.append((enum_name, glean_name, f"Whether a document called {method}."))
        elif counter["type"] == "attribute":
            enum_root = (
                f"eUseCounter_{counter['interface_name']}_{counter['attribute_name']}"
            )
            name = f"{counter['interface_name']}_{counter['attribute_name']}".lower()
            attr = f"{counter['interface_name']}.{counter['attribute_name']}"
            page.append(
                (f"{enum_root}_getter", f"{name}_getter", f"Whether a page got {attr}.")
            )
            page.append(
                (f"{enum_root}_setter", f"{name}_setter", f"Whether a page set {attr}.")
            )
            doc.append(
                (
                    f"{enum_root}_getter",
                    f"{name}_getter",
                    f"Whether a document got {attr}.",
                )
            )
            doc.append(
                (
                    f"{enum_root}_setter",
                    f"{name}_setter",
                    f"Whether a document set {attr}.",
                )
            )
        elif counter["type"] == "custom":
            enum_name = f"eUseCounter_custom_{counter['name']}"
            page.append(
                (
                    enum_name,
                    to_snake_case(counter["name"]),
                    f"Whether a page {counter['desc']}.",
                )
            )
            doc.append(
                (
                    enum_name,
                    to_snake_case(counter["name"]),
                    f"Whether a document {counter['desc']}.",
                )
            )
        else:
            print(f"Found unexpected use counter type {counter['type']}. Returning 1.")
            return 1

    worker_uc_path = os.path.join(
        buildconfig.topsrcdir, "dom", "base", "UseCountersWorker.conf"
    )
    dedicated = []
    shared = []
    service = []
    for counter in read_conf(worker_uc_path):
        if counter["type"] == "method":
            enum_name = f"{counter['interface_name']}_{counter['method_name']}"
            name = f"{counter['interface_name']}_{counter['method_name']}".lower()
            method = f"called {counter['interface_name']}.{counter['method_name']}"
            dedicated.append(
                (enum_name, name, f"Whether a dedicated worker called {method}.")
            )
            shared.append(
                (enum_name, name, f"Whether a shared worker called {method}.")
            )
            service.append(
                (enum_name, name, f"Whether a service worker called {method}.")
            )
        elif counter["type"] == "attribute":
            enum_root = f"{counter['interface_name']}_{counter['attribute_name']}"
            name = f"{counter['interface_name']}_{counter['attribute_name']}".lower()
            attr = f"{counter['interface_name']}.{counter['attribute_name']}"
            dedicated.append(
                (
                    f"{enum_root}_getter",
                    f"{name}_getter",
                    f"Whether a dedicated worker got {attr}.",
                )
            )
            dedicated.append(
                (
                    f"{enum_root}_setter",
                    f"{name}_setter",
                    f"Whether a dedicated worker set {attr}.",
                )
            )
            shared.append(
                (
                    f"{enum_root}_getter",
                    f"{name}_getter",
                    f"Whether a shared worker got {attr}.",
                )
            )
            shared.append(
                (
                    f"{enum_root}_setter",
                    f"{name}_setter",
                    f"Whether a shared worker set {attr}.",
                )
            )
            service.append(
                (
                    f"{enum_root}_getter",
                    f"{name}_getter",
                    f"Whether a service worker got {attr}.",
                )
            )
            service.append(
                (
                    f"{enum_root}_setter",
                    f"{name}_setter",
                    f"Whether a service worker set {attr}.",
                )
            )
        elif counter["type"] == "custom":
            enum_name = f"Custom_{counter['name']}"
            dedicated.append(
                (
                    enum_name,
                    to_snake_case(counter["name"]),
                    f"Whether a dedicated worker {counter['desc']}.",
                )
            )
            shared.append(
                (
                    enum_name,
                    to_snake_case(counter["name"]),
                    f"Whether a shared worker {counter['desc']}.",
                )
            )
            service.append(
                (
                    enum_name,
                    to_snake_case(counter["name"]),
                    f"Whether a service worker {counter['desc']}.",
                )
            )
        else:
            print(
                f"Found unexpected worker use counter type {counter['type']}. Returning 1."
            )
            return 1

    # nsDeprecatedOperationsList.h parsing is adapted from parse_histograms.py.
    operation_list_path = os.path.join(
        buildconfig.topsrcdir, "dom", "base", "nsDeprecatedOperationList.h"
    )
    operation_regex = re.compile("^DEPRECATED_OPERATION\\(([^)]+)\\)")
    ops_page = []
    ops_doc = []
    with open(operation_list_path) as f:
        for line in f:
            match = operation_regex.search(line)
            if not match:
                # No macro, probably whitespace or comment.
                continue

            op = match.group(1)
            op_name = to_snake_case(op)
            enum_name = f"eUseCounter_{op}"
            ops_page.append((enum_name, op_name, f"Whether a page used {op}."))
            ops_doc.append((enum_name, op_name, f"Whether a document used {op}."))

    # Theoretically, we could do this without a completed build
    # (ie, without the generated ServoCSSPropList.py) by sourcing direct from
    # servo/components/style/properties/data.py:PropertiesData(engine=gecko).
    #
    # ...but parse_histograms.py doesn't do this the hard way. Should we?

    import runpy

    proplist_path = os.path.join(
        buildconfig.topobjdir, "layout", "style", "ServoCSSPropList.py"
    )
    css_properties = runpy.run_path(proplist_path)["data"]
    css_page = []
    css_doc = []
    for prop in css_properties.values():
        # We prefix `prop_name` with `css_` to avoid colliding with C++ keywords
        # like `float`.
        prop_name = "css_" + to_snake_case(prop.name)

        # Dependency keywords: CSS_PROP_PUBLIC_OR_PRIVATE, GenerateServoCSSPropList.py.
        method = "Float" if prop.method == "CssFloat" else prop.method
        # Dependency keywords: CSS_PROP_DOMPROP_PREFIXED, GenerateServoCSSPropList.py.
        if method.startswith("Moz") and prop.type() != "alias":
            method = method[3:]  # remove the moz prefix

        enum_name = f"eUseCounter_property_{method}"
        css_page.append(
            (enum_name, prop_name, f"Whether a page used the CSS property {prop.name}.")
        )
        css_doc.append(
            (
                enum_name,
                prop_name,
                f"Whether a document used the CSS property {prop.name}.",
            )
        )

    # Counted unknown properties: AKA - stuff that doesn't exist, but we want
    # to count uses of anyway.
    # We _might_ decide to implement these in the future, though, so we just add
    # them to the css_page, css_doc lists directly for continuity.
    # (We do give them a different description, though)

    import sys

    sys.path.append(os.path.join(buildconfig.topsrcdir, "layout", "style"))
    from GenerateCountedUnknownProperties import to_camel_case

    unknown_proplist_path = os.path.join(
        buildconfig.topsrcdir,
        "servo",
        "components",
        "style",
        "properties",
        "counted_unknown_properties.py",
    )
    unknown_properties = runpy.run_path(unknown_proplist_path)[
        "COUNTED_UNKNOWN_PROPERTIES"
    ]
    for prop in unknown_properties:
        enum_name = f"eUseCounter_unknown_property_{to_camel_case(prop)}"
        prop_name = to_snake_case(prop)
        css_page.append(
            (
                enum_name,
                prop_name,
                f"Whether a page used the (unknown, counted) CSS property {prop}.",
            )
        )
        css_doc.append(
            (
                enum_name,
                prop_name,
                f"Whether a document used the (unknown, counted) CSS property {prop}.",
            )
        )

    return (page, doc, dedicated, shared, service, ops_page, ops_doc, css_page, css_doc)


def to_snake_case(kebab_or_pascal):
    """
    Takes `kebab_or_pascal` which is in PascalCase or kebab-case
    and conjugates it to "snake_case" (all lowercase, "_"-delimited).
    """
    return (
        re.sub("([A-Z]+)", r"_\1", kebab_or_pascal).replace("-", "_").lower().strip("_")
    )


def metric_map(f, *inputs):
    """
    Parses all use counters and outputs UseCounter.cpp which contains implementations
    for two functions defined in UseCounter.h:
      * const char* IncrementUseCounter(UseCounter aUseCounter, bool aIsPage)
      * const char* IncrementWorkerUseCounter(UseCounterWorker aUseCounter, dom::WorkerKind aKind)

    (Basically big switch statements mapping from enums to glean metrics, calling Add())
    """

    (
        page,
        doc,
        dedicated,
        shared,
        service,
        ops_page,
        ops_doc,
        css_page,
        css_doc,
    ) = parse_use_counters()

    f.write(
        """\
/* AUTOGENERATED by usecounters.py. DO NOT EDIT */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "mozilla/dom/UseCounterMetrics.h"

#include "mozilla/dom/WorkerPrivate.h"
#include "mozilla/glean/GleanMetrics.h"

namespace mozilla::dom {

const char* IncrementUseCounter(UseCounter aUseCounter, bool aIsPage) {
  static constexpr struct {
    const char* name;
    glean::impl::CounterMetric doc_metric;
    glean::impl::CounterMetric page_metric;
  } kEntries[] = {
"""
    )

    # This order must match the order UseCounter is defined,
    # (and we guarantee it via the MOZ_ASSERT below at runtime).
    assert len(page) == len(doc)
    assert len(ops_page) == len(ops_doc)
    assert len(css_page) == len(css_doc)

    index = 0
    static_asserts = []
    for pc, dc in zip(page, doc):
        assert pc[0] == dc[0]
        assert pc[1] == dc[1]
        static_asserts.append(f"static_assert({index} == size_t(UseCounter::{pc[0]}));")
        f.write(
            f"""\
        {{
          "{pc[1]}",
          glean::use_counter_doc::{pc[1]},
          glean::use_counter_page::{pc[1]},
        }},
"""
        )
        index += 1

    for pc, dc in zip(ops_page, ops_doc):
        assert pc[0] == dc[0]
        assert pc[1] == dc[1]
        static_asserts.append(f"static_assert({index} == size_t(UseCounter::{pc[0]}));")
        f.write(
            f"""\
        {{
          "deprecated_ops.{pc[1]}",
          glean::use_counter_deprecated_ops_doc::{pc[1]},
          glean::use_counter_deprecated_ops_page::{pc[1]},
        }},
"""
        )
        index += 1

    for pc, dc in zip(css_page, css_doc):
        assert pc[0] == dc[0]
        assert pc[1] == dc[1]
        static_asserts.append(f"static_assert({index} == size_t(UseCounter::{pc[0]}));")
        f.write(
            f"""\
        {{
          "css.{pc[1]}",
          glean::use_counter_css_doc::{pc[1]},
          glean::use_counter_css_page::{pc[1]},
        }},
"""
        )
        index += 1

    f.write("};\n")
    f.write("\n".join(static_asserts))
    f.write(
        """\
  MOZ_ASSERT(size_t(aUseCounter) < std::size(kEntries));
  const auto& entry = kEntries[size_t(aUseCounter)];
  (aIsPage ? entry.page_metric : entry.doc_metric).Add();
  return entry.name;
}

const char* IncrementWorkerUseCounter(UseCounterWorker aUseCounter, WorkerKind aKind) {
  static constexpr struct {
    const char* name;
    glean::impl::CounterMetric dedicated_metric;
    glean::impl::CounterMetric shared_metric;
    glean::impl::CounterMetric service_metric;
  } kEntries[] = {
"""
    )
    assert len(dedicated) == len(shared)
    assert len(dedicated) == len(service)
    index = 0
    static_asserts = []
    for dc, sc, servicec in zip(dedicated, shared, service):
        assert dc[0] == sc[0]
        assert dc[1] == sc[1]
        assert dc[0] == servicec[0]
        assert dc[1] == servicec[1]
        static_asserts.append(
            f"static_assert({index} == size_t(UseCounterWorker::{dc[0]}));"
        )
        f.write(
            f"""\
        {{
          "{dc[1]}",
          glean::use_counter_worker_dedicated::{dc[1]},
          glean::use_counter_worker_shared::{dc[1]},
          glean::use_counter_worker_service::{dc[1]},
        }},
"""
        )
        index += 1
    f.write("};\n")
    f.write("\n".join(static_asserts))
    f.write(
        """\
  MOZ_ASSERT(size_t(aUseCounter) < std::size(kEntries));
  const auto& entry = kEntries[size_t(aUseCounter)];
  switch (aKind) {
    case WorkerKind::WorkerKindDedicated:
      entry.dedicated_metric.Add();
      break;
    case WorkerKind::WorkerKindShared:
      entry.shared_metric.Add();
      break;
    case WorkerKind::WorkerKindService:
      entry.service_metric.Add();
      break;
  }
  return entry.name;
}

}  // namespace mozilla
"""
    )