Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2025-02-25 20:01:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Backed out changeset 1bfb1774ed03 (bug 1889404) for causing bc failures @ browser_contentrelevancy_telemetry CLOSED TREE

Browse Source
This commit is contained in:
Sandor Molnar 2024-04-26 06:00:43 +03:00
parent fbd32a8f3e
commit c7f67706fc
10 changed files with 4 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
toolkit/components/contentrelevancy/ContentRelevancyManager.sys.mjs
View File

@ -212,8 +212,6 @@ class RelevancyManager {
// exit points & success.
this.#isInProgress = true;
let timerId;
try {
lazy.log.info("Fetching input data for interest classification");
@ -228,38 +226,17 @@ class RelevancyManager {
const urls = await lazy.getFrecentRecentCombinedUrls(maxUrls);
if (urls.length < minUrls) {
lazy.log.info("Aborting interest classification: insufficient input");
Glean.relevancyClassify.fail.record({ reason: "insufficient-input" });
return;
}
lazy.log.info("Starting interest classification");
timerId = Glean.relevancyClassify.duration.start();
await this.#doClassificationHelper(urls);
Glean.relevancyClassify.duration.stopAndAccumulate(timerId);
Glean.relevancyClassify.succeed.record({
input_size: urls.length,
// TODO(nanj): Fill out the actual counters once the classification is enabled.
input_classified_size: 0,
input_inconclusive_size: 0,
output_interest_size: 0,
interest_top_1_hits: 0,
interest_top_2_hits: 0,
interest_top_3_hits: 0,
});
} catch (error) {
let reason;
if (error instanceof StoreNotAvailableError) {
lazy.log.error("#store became null, aborting interest classification");
reason = "store-not-ready";
} else {
lazy.log.error("Classification error: " + (error.reason ?? error));
reason = "component-errors";
}
Glean.relevancyClassify.fail.record({ reason });
Glean.relevancyClassify.duration.cancel(timerId); // No error is recorded if `start` was not called.
} finally {
this.#isInProgress = false;
}
@ -267,13 +244,6 @@ class RelevancyManager {
lazy.log.info("Finished interest classification");
}
/**
* Exposed for testing.
*/
async _test_doClassification() {
await this.#doClassification();
}
/**
* Classification helper. Use the getter `this.#store` rather than `#_store`
* to access the store so that when it becomes null, a `StoreNotAvailableError`
@ -302,7 +272,7 @@ class RelevancyManager {
/**
* Exposed for testing.
*/
async _test_doClassificationHelper(urls) {
async _test_doClassification(urls) {
await this.#doClassificationHelper(urls);
}

8
toolkit/components/contentrelevancy/docs/index.md
View File

@ -1,11 +1,3 @@
# Content Relevancy
This is the home for the project: Interest-based Content Relevance Ranking & Personalization for Firefox, a client-based privacy preserving approach to enhancing content experience of Firefox.
```{toctree}
:titlesonly:
:maxdepth: 1
:glob:
telemetry.md
```

62
toolkit/components/contentrelevancy/docs/telemetry.md
View File

@ -1,62 +0,0 @@
# Telemetry
This document serves as a complementary doc for all the telemetry we collect for `contentrelevancy`.
Note that we use FoG ([Firefox on Glean][FoG]) to record telemetry,
all the metrics described below follow the standard Glean metric types.
You can reference `metrics.yaml` or [Glean Dictionary][glean-dictionary] for more details for each metric.
## Classification Metrics - `relevancyClassify`
When the `contentrelevancy` feature is enabled, it will periodically perform classifications in the background.
The following metrics will be recorded for each classification.
### Events
#### `succeed`
This is recorded when a classification is successfully performed.
Various counters are recorded in the `extra_keys` to measure the classification results.
Example:
```
"extra_keys": {
"input_size": 100, // The input size was 100 items
"input_classified_size": 50, // 50 items (out of 100) were classified with at least one conclusive interest
"input_inconclusive_size": 10, // 10 items were classified with the inconclusive interest
"output_interest_size": 4 // The total number of unique interests classified
"interest_top_1_hits": 20, // 20 items were classified with the interest with the most hits
"interest_top_2_hits": 15, // 15 items were classified with the interest with the 2nd most hits
"interest_top_3_hits": 10, // 10 items were classified with the interest with the 3rd most hits
}
```
#### `fail`
This is recorded when a classification is failed or aborted.
The `reason` of the failure is recorded in the `extra_keys`.
```
"extra_keys": {
"reason": "insufficient-input" // The classification was aborted due to insufficient input.
// `store-not-ready` indicates the store is null.
// `component-errors` indicates an error in the Rust component.
}
```
### Timing Distribution
#### `duration`
This records the time duration (in milliseconds) of a successful classification.
The durations of unsuccessful classifications are not measured.
## Changelog
### 2024-04
* [Bug 1889404][bug-1889404]: Added basic metrics for relevancy manager
[FoG]: https://firefox-source-docs.mozilla.org/toolkit/components/glean/index.html
[glean-dictionary]: https://dictionary.telemetry.mozilla.org/
[bug-1889404]: https://bugzilla.mozilla.org/show_bug.cgi?id=1889404

103
toolkit/components/contentrelevancy/metrics.yaml
View File

@ -1,103 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Adding a new metric? We have docs for that!
# https://firefox-source-docs.mozilla.org/toolkit/components/glean/user/new_definitions_file.html
---
$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0
$tags:
- "Application Services :: Relevancy"
relevancy.classify:
succeed:
type: event
description: >
Record an event of a successful user interest classification.
bugs:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1889404
data_reviews:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1889404
data_sensitivity:
- interaction
notification_emails:
- disco-team@mozilla.com
- najiang@mozilla.com
expires: never
extra_keys:
input_size:
description: >
The total number of input elements used for classification.
type: quantity
input_classified_size:
description: >
The total number of input elements classified with at least one conclusive interest.
type: quantity
input_inconclusive_size:
description: >
The total number of input elements classified with the inconclusive interest.
type: quantity
output_interest_size:
description: >
The total number of output interests for classification.
type: quantity
interest_top_1_hits:
description: >
The total number of input elements for the interest with the most hits.
Defaults to 0. This, along with `input_size` and `input_classified_size`,
can be used to calculate the top-N classification percentages and top-N
hit percentages.
type: quantity
interest_top_2_hits:
description: >
The total number of input elements for the interest with the 2nd most hits.
Defaults to 0. This, along with `input_size` and `input_classified_size`,
can be used to calculate the top-N classification percentages and top-N
hit percentages.
type: quantity
interest_top_3_hits:
description: >
The total number of input elements for the interest with the 3rd most hits.
Defaults to 0. This, along with `input_size` and `input_classified_size`,
can be used to calculate the top-N classification percentages and top-N
hit percentages.
type: quantity
fail:
type: event
description: >
Record an event of a failed / aborted user interest classification.
bugs:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1889404
data_reviews:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1889404
data_sensitivity:
- interaction
notification_emails:
- disco-team@mozilla.com
- najiang@mozilla.com
expires: never
extra_keys:
reason:
description: >
The reason of the failed / aborted classification.
One of
* `insufficient-input`
* `component-errors`
* `store-not-ready`
type: string
duration:
type: timing_distribution
time_unit: millisecond
description: >
Record the duration (in ms) of a successful classification.
bugs:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1889404
data_reviews:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1889404
data_sensitivity:
- interaction
notification_emails:
- disco-team@mozilla.com
- najiang@mozilla.com
expires: never

4
toolkit/components/contentrelevancy/tests/browser/browser.toml
View File

@ -4,7 +4,3 @@ prefs = [
]
["browser_contentrelevancy_nimbus.js"]
lineno = "6"
["browser_contentrelevancy_telemetry.js"]
lineno = "9"

162
toolkit/components/contentrelevancy/tests/browser/browser_contentrelevancy_telemetry.js
View File

@ -1,162 +0,0 @@
/* Any copyright is dedicated to the Public Domain.
http://creativecommons.org/publicdomain/zero/1.0/ */
"use strict";
const { ExperimentAPI } = ChromeUtils.importESModule(
"resource://nimbus/ExperimentAPI.sys.mjs"
);
const { ExperimentFakes } = ChromeUtils.importESModule(
"resource://testing-common/NimbusTestUtils.sys.mjs"
);
const { ContentRelevancyManager } = ChromeUtils.importESModule(
"resource://gre/modules/ContentRelevancyManager.sys.mjs"
);
/**
* Test classification metrics - succeed.
*/
add_task(async function test_classify_succeed() {
Services.fog.testResetFOG();
await ExperimentAPI.ready();
const doExperimentCleanup = await ExperimentFakes.enrollWithFeatureConfig({
featureId: "contentRelevancy",
value: {
enabled: true,
minInputUrls: 0,
maxInputUrls: 3,
timerInterval: 3600,
},
});
await TestUtils.waitForCondition(
() => ContentRelevancyManager.shouldEnable,
"Should enable it via Nimbus"
);
Assert.equal(undefined, Glean.relevancyClassify.succeed.testGetValue());
Assert.equal(undefined, Glean.relevancyClassify.duration.testGetValue());
await TestUtils.waitForCondition(
() => !ContentRelevancyManager.isInProgress,
"Ensure no classification is in-progress"
);
await ContentRelevancyManager._test_doClassification();
Assert.deepEqual(
{
input_size: 0,
input_classified_size: 0,
input_inconclusive_size: 0,
output_interest_size: 0,
interest_top_1_hits: 0,
interest_top_2_hits: 0,
interest_top_3_hits: 0,
},
Glean.relevancyClassify.succeed.testGetValue()[0].extra,
"Should record the succeed event"
);
Assert.ok(
Glean.relevancyClassify.duration.testGetValue().sum > 0,
"Should record the duration"
);
await doExperimentCleanup();
});
/**
* Test classification metrics - fail - insufficient-input.
*/
add_task(async function test_classify_fail_case1() {
Services.fog.testResetFOG();
await ExperimentAPI.ready();
const doExperimentCleanup = await ExperimentFakes.enrollWithFeatureConfig({
featureId: "contentRelevancy",
value: {
enabled: true,
// Require at least 1 input to trigger the failure.
minInputUrls: 1,
maxInputUrls: 3,
timerInterval: 3600,
},
});
await TestUtils.waitForCondition(
() => ContentRelevancyManager.shouldEnable,
"Should enable it via Nimbus"
);
Assert.equal(undefined, Glean.relevancyClassify.fail.testGetValue());
Assert.equal(undefined, Glean.relevancyClassify.duration.testGetValue());
await TestUtils.waitForCondition(
() => !ContentRelevancyManager.isInProgress,
"Ensure no classification is in-progress"
);
await ContentRelevancyManager._test_doClassification();
Assert.deepEqual(
{
reason: "insufficient-input",
},
Glean.relevancyClassify.fail.testGetValue()[0].extra,
"Should record the fail event"
);
Assert.equal(
undefined,
Glean.relevancyClassify.duration.testGetValue(),
"Should not record the duration"
);
await doExperimentCleanup();
});
/**
* Test classification metrics - fail - store-not-ready.
*/
add_task(async function test_classify_fail_case2() {
Services.fog.testResetFOG();
await ExperimentAPI.ready();
const doExperimentCleanup = await ExperimentFakes.enrollWithFeatureConfig({
featureId: "contentRelevancy",
value: {
// Disable the feature so the `store` won't be ready.
enabled: false,
minInputUrls: 0,
maxInputUrls: 3,
timerInterval: 3600,
},
});
await TestUtils.waitForCondition(
() => !ContentRelevancyManager.shouldEnable,
"Should be disabled it via Nimbus"
);
Assert.equal(undefined, Glean.relevancyClassify.fail.testGetValue());
Assert.equal(undefined, Glean.relevancyClassify.duration.testGetValue());
await TestUtils.waitForCondition(
() => !ContentRelevancyManager.isInProgress,
"Ensure no classification is in-progress"
);
await ContentRelevancyManager._test_doClassification();
Assert.deepEqual(
{
reason: "store-not-ready",
},
Glean.relevancyClassify.fail.testGetValue()[0].extra,
"Should record the fail event"
);
Assert.equal(
undefined,
Glean.relevancyClassify.duration.testGetValue(),
"Should not record the duration"
);
await doExperimentCleanup();
});

6
toolkit/components/contentrelevancy/tests/xpcshell/test_ContentRelevancyManager.js
View File

@ -100,17 +100,17 @@ add_task(async function test_call_disable_twice() {
Services.prefs.clearUserPref(PREF_CONTENT_RELEVANCY_ENABLED);
});
add_task(async function test_doClassificationHelper() {
add_task(async function test_doClassification() {
Services.prefs.setBoolPref(PREF_CONTENT_RELEVANCY_ENABLED, true);
await TestUtils.waitForCondition(() => ContentRelevancyManager._isStoreReady);
await ContentRelevancyManager._test_doClassificationHelper([]);
await ContentRelevancyManager._test_doClassification([]);
// Disable it to reset the store.
Services.prefs.setBoolPref(PREF_CONTENT_RELEVANCY_ENABLED, false);
await TestUtils.waitForTick();
await Assert.rejects(
ContentRelevancyManager._test_doClassificationHelper([]),
ContentRelevancyManager._test_doClassification([]),
/Store is not available/,
"Should throw with an unset store"
);

2
toolkit/components/contentrelevancy/tests/xpcshell/xpcshell.toml
View File

@ -4,8 +4,6 @@ firefox-appdir = "browser"
["test_ContentRelevancyManager.js"]
skip-if = ["os == 'android'"] # bug 1886601
lineno = "5"
["test_InputUtils.js"]
skip-if = ["os == 'android'"] # bug 1886601
lineno = "9"

1
toolkit/components/glean/metrics_index.py
View File

@ -67,7 +67,6 @@ firefox_desktop_metrics = [
"browser/components/urlbar/metrics.yaml",
"browser/modules/metrics.yaml",
"dom/media/platforms/wmf/metrics.yaml",
"toolkit/components/contentrelevancy/metrics.yaml",
"toolkit/components/crashes/metrics.yaml",
"toolkit/components/nimbus/metrics.yaml",
"toolkit/components/search/metrics.yaml",

2
toolkit/components/glean/tags.yaml
View File

@ -9,8 +9,6 @@
---
$schema: moz://mozilla.org/schemas/glean/tags/1-0-0
'Application Services :: Relevancy':
description: The Bugzilla component which applies to this object.
'Cloud Services :: Firefox: Common':
description: The Bugzilla component which applies to this object.
'Conduit :: mots':