Bug 1848419 - Make custom-car android available in CI selection. r=perftest-reviewers,taskgraph-reviewers,AlexandruIonescu,bhearsum

Chromium-as-Release (CaR) is a custom upstream chromium build (configured as "release")
On the desktop platform we have these builds running already.
This patch adds support for custom-car Android builds to be useable in performance testing.
Since android tests are run on a linux host, we make use of the existing chromium linux chromedriver fetch.
At the moment we only want the performance tests to run on speedometer.

Differential Revision: https://phabricator.services.mozilla.com/D191212

taskcluster/ci/config.yml

@@ -44,6 +44,7 @@ treeherder:
         'Btime-refbrow': 'Browsertime performance tests on the reference browser'
         'Btime-Saf': 'Browsertime performance tests on Safari'
         'Btime-CaR': 'Browsertime performance tests on Chromium-as-Release'
+        'Btime-nofis-CaR': 'Browsertime performance tests on Chromium-as-Release without fission enabled'
         'Fetch': 'Fetch and store content'
         'Fxfn': 'Firefox functional tests'
         'Fxfn-nofis': 'Firefox functional tests without fission enabled'

taskcluster/ci/test/browsertime-mobile.yml

@@ -57,7 +57,19 @@ job-defaults:
         by-app:
             chrome-m:
                 - .*shippable-qr.*
+            cstm-car-m:
+                - .*shippable-qr.*
             default: []
+    fetches:
+        toolchain:
+            by-app:
+                cstm-car-m:
+                    - android-custom-car
+                    - linux64-minidump-stackwalk
+                    - linux64-fix-stacks
+                default:
+                    - linux64-minidump-stackwalk
+                    - linux64-fix-stacks
 
 browsertime-tp6m:
     <<: &tp6m-defaults
@@ -296,7 +308,7 @@ browsertime-benchmark-speedometer-mobile:
     description: Browsertime Speedometer on Android
     run-without-variant: false
     raptor:
-        apps: [geckoview, fenix, chrome-m]
+        apps: [geckoview, fenix, chrome-m, cstm-car-m]
     tier:
         by-app:
             fenix: 1
@@ -318,7 +330,7 @@ browsertime-benchmark-speedometer3-mobile:
     description: Browsertime Speedometer 3 on Android
     run-without-variant: false
     raptor:
-        apps: [geckoview, fenix, chrome-m]
+        apps: [geckoview, fenix, chrome-m, cstm-car-m]
     tier:
         by-app:
             fenix: 2

taskcluster/gecko_taskgraph/transforms/test/other.py

@@ -369,6 +369,7 @@ def setup_browsertime(config, tasks):
             "windows.*aarch64.*": ["win32-chromium"],
             "windows.*-32.*": ["win32-chromium"],
             "windows.*-64.*": ["win64-chromium"],
+            "android.*": ["linux64-chromium"],
         }
 
         cd_extracted_name = {
@@ -381,7 +382,11 @@ def setup_browsertime(config, tasks):
             # Only add the chromedriver fetches when chrome is running
             for platform in cd_fetches:
                 fs["by-test-platform"][platform].extend(cd_fetches[platform])
-        if "--app=chromium" in extra_options or "--app=custom-car" in extra_options:
+        if (
+            "--app=chromium" in extra_options
+            or "--app=custom-car" in extra_options
+            or "--app=cstm-car-m" in extra_options
+        ):
             for platform in chromium_fetches:
                 fs["by-test-platform"][platform].extend(chromium_fetches[platform])
 
@@ -395,7 +400,7 @@ def setup_browsertime(config, tasks):
             }
 
         # Disable the Raptor install step
-        if "--app=chrome-m" in extra_options:
+        if "--app=chrome-m" in extra_options or "--app=cstm-car-m" in extra_options:
             extra_options.append("--noinstall")
 
         task.setdefault("fetches", {}).setdefault("fetch", []).extend(
@@ -455,7 +460,7 @@ def get_mobile_project(task):
     if not task["build-platform"].startswith("android"):
         return
 
-    mobile_projects = ("fenix", "geckoview", "refbrow", "chrome-m")
+    mobile_projects = ("fenix", "geckoview", "refbrow", "chrome-m", "cstm-car-m")
 
     for name in mobile_projects:
         if name in task["test-name"]:

taskcluster/gecko_taskgraph/transforms/test/raptor.py

@@ -80,6 +80,7 @@ def split_apps(config, tests):
         "refbrow": "refbrow",
         "safari": "Saf",
         "custom-car": "CaR",
+        "cstm-car-m": "CaR",
     }
 
     for test in tests:
@@ -90,9 +91,13 @@ def split_apps(config, tests):
 
         for app in apps:
             # Ignore variants for non-Firefox or non-mobile applications.
-            if app not in ["firefox", "geckoview", "fenix", "chrome-m"] and test[
-                "attributes"
-            ].get("unittest_variant"):
+            if app not in [
+                "firefox",
+                "geckoview",
+                "fenix",
+                "chrome-m",
+                "cstm-car-m",
+            ] and test["attributes"].get("unittest_variant"):
                 continue
 
             atest = copy_task(test)

testing/mozharness/external_tools/performance-artifact-schema.json

@@ -14,7 +14,8 @@
             "refbrow",
             "fenix",
             "safari",
-            "custom-car"
+            "custom-car",
+            "cstm-car-m"
           ],
           "maxLength": 10,
           "type": "string"

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -196,6 +196,7 @@ class Raptor(
                         "fenix",
                         "safari",
                         "custom-car",
+                        "cstm-car-m",
                     ],
                     "dest": "app",
                     "help": "Name of the application we are testing (default: firefox).",
@@ -718,7 +719,10 @@ class Raptor(
         self.debug_mode = self.config.get("debug_mode", False)
         self.chromium_dist_path = None
         self.firefox_android_browsers = ["fennec", "geckoview", "refbrow", "fenix"]
-        self.android_browsers = self.firefox_android_browsers + ["chrome-m"]
+        self.android_browsers = self.firefox_android_browsers + [
+            "chrome-m",
+            "cstm-car-m",
+        ]
         self.browsertime_visualmetrics = self.config.get("browsertime_visualmetrics")
         self.browsertime_node = self.config.get("browsertime_node")
         self.browsertime_user_args = self.config.get("browsertime_user_args")
@@ -778,7 +782,7 @@ class Raptor(
 
     def install_chrome_android(self):
         """Install Google Chrome for Android in production from tooltool"""
-        if self.app != "chrome-m":
+        if self.app not in ("chrome-m", "cstm-car-m"):
             self.info("Google Chrome for Android not required")
             return
         if self.config.get("run_local"):
@@ -822,9 +826,19 @@ class Raptor(
     def install_chromium_distribution(self):
         """Install Google Chromium distribution in production"""
         linux, mac, win = "linux", "mac", "win"
-        chrome, chromium, chromium_release = "chrome", "chromium", "custom-car"
+        chrome, chromium, chromium_release, chromium_release_android = (
+            "chrome",
+            "chromium",
+            "custom-car",
+            "cstm-car-m",
+        )
 
-        available_chromium_dists = [chrome, chromium, chromium_release]
+        available_chromium_dists = [
+            chrome,
+            chromium,
+            chromium_release,
+            chromium_release_android,
+        ]
         binary_location = {
             chromium: {
                 linux: ["chrome-linux", "chrome"],
@@ -836,6 +850,9 @@ class Raptor(
                 win: ["chromium", "Default", "chrome.exe"],
                 mac: ["chromium", "Chromium.app", "Contents", "MacOS", "chromium"],
             },
+            chromium_release_android: {
+                linux: ["chromium", "apks", "ChromePublic.apk"],
+            },
         }
 
         if self.app not in available_chromium_dists:

testing/perfdocs/generated/raptor.rst

@@ -1649,6 +1649,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
         - ❌
         - ❌
         - ❌
+      * - **browsertime-benchmark-speedometer-mobile-cstm-car-m-nofis**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
       * - **browsertime-benchmark-speedometer-mobile-fenix-nofis**
         - ❌
         - ❌
@@ -1937,7 +1942,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
    **Owner**: SpiderMonkey Team
 
    * **alert threshold**: 2.0
-   * **apps**: fenix, geckoview, refbrow, chrome-m
+   * **apps**: fenix, geckoview, refbrow, chrome-m, cstm-car-m
    * **expected**: pass
    * **gecko profile entries**: 14000000
    * **gecko profile interval**: 1
@@ -1986,6 +1991,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
         - ❌
         - ❌
         - ❌
+      * - **browsertime-benchmark-speedometer-mobile-cstm-car-m-nofis**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
       * - **browsertime-benchmark-speedometer-mobile-fenix-nofis**
         - ❌
         - ❌
@@ -2337,6 +2347,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
         - ❌
         - ❌
         - ❌
+      * - **browsertime-benchmark-speedometer3-mobile-cstm-car-m-nofis**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
       * - **browsertime-benchmark-speedometer3-mobile-fenix-nofis**
         - ❌
         - ❌
@@ -2615,7 +2630,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
    **Owner**: Performance Team
 
    * **alert threshold**: 2.0
-   * **apps**: fenix, geckoview, refbrow, chrome-m
+   * **apps**: fenix, geckoview, refbrow, chrome-m, cstm-car-m
    * **browsertime args**: --browsertime.speedometer_iterations=5
    * **custom data**: true
    * **expected**: pass
@@ -2673,6 +2688,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
         - ❌
         - ❌
         - ❌
+      * - **browsertime-benchmark-speedometer3-mobile-cstm-car-m-nofis**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
       * - **browsertime-benchmark-speedometer3-mobile-fenix-nofis**
         - ❌
         - ❌

testing/raptor/raptor/tests/benchmarks/speedometer-mobile.ini

@@ -6,7 +6,7 @@
 
 [DEFAULT]
 alert_threshold = 2.0
-apps = fenix, geckoview, refbrow, chrome-m
+apps = fenix, geckoview, refbrow, chrome-m, cstm-car-m
 gecko_profile_entries = 14000000
 gecko_profile_interval = 1
 lower_is_better = false