Bug 1886221 - Translations Telemetry r=android-reviewers,Roger

This bug adds additional telemetry including:
* When a translation is requested, including to/from languages
* When a translation is successful
* When a translation fails
* Whether or not the translations engine is supported

Differential Revision: https://phabricator.services.mozilla.com/D206048
This commit is contained in:
ohall-m 2024-04-01 18:05:56 +00:00
parent 34629a24b2
commit 319baa7d14
3 changed files with 209 additions and 1 deletions

View File

@ -904,7 +904,89 @@ translations:
notification_emails:
- android-probes@mozilla.com
expires: never
translate_requested:
type: event
description: |
The user requested a translation of the page.
extra_keys:
to_language:
description: |
A string containing the BCP 47 that the user is translating the page to. Predefined based
on published translation models. For example, possible values are:
"en", "es", "fr", and all other translations engine "to" language codes.
type: string
from_language:
description: |
A string containing the BCP 47 that the user is translating the page from. Predefined based
on published translation models. For example, possible values are:
"en", "es", "fr", and all other translations engine "from" language codes.
type: string
bugs:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1886221
data_reviews:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1886221#c3
data_sensitivity:
- interaction
notification_emails:
- android-probes@mozilla.com
expires: never
translate_success:
type: event
description: |
The user requested a translation of the page and it completed successfully.
bugs:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1886221
data_reviews:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1886221#c3
data_sensitivity:
- interaction
notification_emails:
- android-probes@mozilla.com
expires: never
translate_failed:
type: event
description: |
The user requested a translation of the page and it failed.
extra_keys:
error:
description: |
A string containing the name of the error that occurred. Possible values are predefined
in TranslationError and include:
unknown, unexpected-null, missing-session-coordinator, engine-not-supported, unknown-engine-support
could-not-translate, could-not-restore, could-not-determine-translation-download-size, could-not-load-languages,
could-not-load-settings, could-not-load-language-settings, could-not-load-never-translate-sites,
language-not-supported, model-could-not-retrieve, model-could-not-delete, model-could-not-download,
model-language-required, model-download-required
type: string
bugs:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1886221
data_reviews:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1886221#c3
data_sensitivity:
- interaction
notification_emails:
- android-probes@mozilla.com
expires: never
engine_supported:
type: event
description: |
The translations engine for translating pages is only supported on certain devices.
This probe indicates if the device supports the translations engine or not.
extra_keys:
support:
description: |
A string describing the support of the engine, either:
supported, unsupported,
type: string
bugs:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1886221
data_reviews:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1886221#c3
data_sensitivity:
- technical
notification_emails:
- android-probes@mozilla.com
expires: never
splash_screen:
first_launch_extended:
type: event

View File

@ -20,6 +20,7 @@ import mozilla.components.browser.state.selector.privateTabs
import mozilla.components.browser.state.state.BrowserState
import mozilla.components.browser.state.state.SessionState
import mozilla.components.concept.base.crash.CrashReporting
import mozilla.components.concept.engine.translate.TranslationOperation
import mozilla.components.lib.state.Middleware
import mozilla.components.lib.state.MiddlewareContext
import mozilla.components.support.base.log.logger.Logger
@ -161,6 +162,31 @@ class TelemetryMiddleware(
is TranslationsAction.TranslateExpectedAction -> {
Translations.offerEvent.record(Translations.OfferEventExtra("expected"))
}
is TranslationsAction.TranslateAction -> {
Translations.translateRequested.record(
Translations.TranslateRequestedExtra(
fromLanguage = action.fromLanguage,
toLanguage = action.toLanguage,
),
)
}
is TranslationsAction.TranslateSuccessAction -> {
if (action.operation == TranslationOperation.TRANSLATE) {
Translations.translateSuccess.record()
}
}
is TranslationsAction.TranslateExceptionAction -> {
if (action.operation == TranslationOperation.TRANSLATE) {
Translations.translateFailed.record(
Translations.TranslateFailedExtra(action.translationError.errorName),
)
}
}
is TranslationsAction.SetEngineSupportedAction -> {
Translations.engineSupported.record(
Translations.EngineSupportedExtra(if (action.isEngineSupported) "supported" else "unsupported"),
)
}
else -> {
// no-op
}

View File

@ -22,6 +22,8 @@ import mozilla.components.browser.state.state.recover.RecoverableTab
import mozilla.components.browser.state.state.recover.TabState
import mozilla.components.browser.state.store.BrowserStore
import mozilla.components.concept.engine.Engine
import mozilla.components.concept.engine.translate.TranslationError
import mozilla.components.concept.engine.translate.TranslationOperation
import mozilla.components.service.glean.testing.GleanTestRule
import mozilla.components.support.base.android.Clock
import mozilla.components.support.test.ext.joinBlocking
@ -471,6 +473,104 @@ class TelemetryMiddlewareTest {
val telemetry = Translations.offerEvent.testGetValue()?.firstOrNull()
assertEquals("expected", telemetry?.extra?.get("item"))
}
@Test
fun `WHEN TranslateAction is dispatched THEN update telemetry`() {
assertNull(Translations.translateRequested.testGetValue())
store.dispatch(
TranslationsAction.TranslateAction(
tabId = "1",
fromLanguage = "en",
toLanguage = "es",
options = null,
),
).joinBlocking()
val telemetry = Translations.translateRequested.testGetValue()?.firstOrNull()
assertEquals("es", telemetry?.extra?.get("to_language"))
assertEquals("en", telemetry?.extra?.get("from_language"))
}
@Test
fun `WHEN TranslateSuccessAction is dispatched THEN update telemetry`() {
assertNull(Translations.translateSuccess.testGetValue())
// Shouldn't record other operations
store.dispatch(
TranslationsAction.TranslateSuccessAction(
tabId = "1",
operation = TranslationOperation.FETCH_SUPPORTED_LANGUAGES,
),
).joinBlocking()
assertNull(Translations.translateSuccess.testGetValue())
// Should record translate operations
store.dispatch(
TranslationsAction.TranslateSuccessAction(
tabId = "1",
operation = TranslationOperation.TRANSLATE,
),
).joinBlocking()
val telemetry = Translations.translateSuccess.testGetValue()?.firstOrNull()
assertNotNull(telemetry)
}
@Test
fun `WHEN TranslateExceptionAction for Translate operation is dispatched THEN update telemetry`() {
assertNull(Translations.translateFailed.testGetValue())
// Shouldn't record other operations
store.dispatch(
TranslationsAction.TranslateExceptionAction(
tabId = "1",
operation = TranslationOperation.FETCH_SUPPORTED_LANGUAGES,
translationError = TranslationError.UnknownError(IllegalStateException()),
),
).joinBlocking()
assertNull(Translations.translateFailed.testGetValue())
// Should record translate operations
store.dispatch(
TranslationsAction.TranslateExceptionAction(
tabId = "1",
operation = TranslationOperation.TRANSLATE,
translationError = TranslationError.CouldNotTranslateError(null),
),
).joinBlocking()
val telemetry = Translations.translateFailed.testGetValue()?.firstOrNull()
assertEquals(TranslationError.CouldNotTranslateError(cause = null).errorName, telemetry?.extra?.get("error"))
}
@Test
fun `WHEN SetEngineSupportedAction is dispatched AND supported THEN update telemetry`() {
assertNull(Translations.engineSupported.testGetValue())
store.dispatch(
TranslationsAction.SetEngineSupportedAction(
isEngineSupported = true,
),
).joinBlocking()
val telemetry = Translations.engineSupported.testGetValue()?.firstOrNull()
assertEquals("supported", telemetry?.extra?.get("support"))
}
@Test
fun `WHEN SetEngineSupportedAction is dispatched AND unsupported THEN update telemetry`() {
assertNull(Translations.engineSupported.testGetValue())
store.dispatch(
TranslationsAction.SetEngineSupportedAction(
isEngineSupported = false,
),
).joinBlocking()
val telemetry = Translations.engineSupported.testGetValue()?.firstOrNull()
assertEquals("unsupported", telemetry?.extra?.get("support"))
}
}
internal class FakeClock : Clock.Delegate {