Bug 1525851: Add semi-automated test. r=jdescottes

Differential Revision: https://phabricator.services.mozilla.com/D20117

--HG--
extra : moz-landing-system : lando

devtools/client/aboutdebugging-new/documentation/TESTS_REAL_DEVICES.md

@@ -0,0 +1,67 @@
+# Running Tests with real USB connected devices for the new about:debugging
+
+This document explains how to test with real USB connected devices.
+
+## Tests overview
+
+The tests that use a real device are located in `devtools/client/aboutdebugging-new/test/browser/`and the name of tests starts with `browser_aboutdebugging_real`. These are normal mochitest, but we need to setup the environment before starting tests.
+
+## Setup environment
+### Real device side
+1. Enable USB debugging on your device
+2. Launch Firefox
+3. Enable USB debugging on your Firefox
+4. Connect to your PC via USB
+
+You can refer to https://developer.mozilla.org/docs/Tools/Remote_Debugging/Debugging_Firefox_for_Android_with_WebIDE#Setting_up_the_Android_device.
+
+### PC side
+Setup the real device information to evaluate the validity in tests.
+
+1. Copy a sample file which is located at `devtools/client/aboutdebugging-new/test/browser/real/usb-runtimes-sample.json` and rename it for example to `devtools/client/aboutdebugging-new/test/browser/real/local-usb-runtimes.json`.
+2. Edit the file.
+
+   This is a JSON file like below, write your real device information in here. This example indicates that there should be one USB device and should be displayed `Pixel 2` as device name and `Firefox Nightly` as short name on the sidebar of about:debugging. Regarding the other information, please see `Detail of config file` section of this document.
+
+```
+[
+  {
+    "sidebarInfo": {
+      "deviceName": "Pixel 2",
+      "shortName": "Firefox Nightly"
+    },
+    ...
+  },
+  ...
+]
+```
+
+## Test
+Normally, although we do test `./mach mochitest <path>`, to load the real device information created, specify the path as `USB_RUNTIME` environment variable, then do test.
+For example, if the name of the saved file is `devtools/client/aboutdebugging-new/test/browser/real/local-usb-runtimes.json`, run all real device test with a command like the one below:
+
+```
+USB_RUNTIMES=devtools/client/aboutdebugging-new/test/browser/real/local-usb-runtimes.json ./mach mochitest devtools/client/aboutdebugging-new/test/browser/browser_aboutdebugging_real
+```
+
+If there is no `USB_RUNTIMES` environment variable, the tests will not run. This is also to avoid to run on try-server and so on.
+
+## Detail of config file
+
+```
+[
+  {
+    "sidebarInfo": {
+      "deviceName": "Pixel 2", // This should display as device name on the sidebar.
+      "shortName": "Firefox Nightly" // This should display as short name on the sidebar.
+    },
+    "runtimeDetails": {
+      "info": {
+        "name": "Mozilla Nightly", // This should display on the runtime info of runtime page.
+        "version": "64.0a1" // This should display on the runtime info of runtime page.
+      }
+    }
+  }
+  // Of course, you can append additional USB devices. Some tests can do with multiple devices.
+]
+```

devtools/client/aboutdebugging-new/test/browser/browser.ini

@@ -11,6 +11,7 @@ support-files =
   helper-addons.js
   helper-collapsibilities.js
   helper-mocks.js
+  helper-real-usb.js
   helper-serviceworker.js
   helper-telemetry.js
   mocks/*
@@ -62,6 +63,8 @@ skip-if = (os == "win" && ccov) # Bug 1521349
 [browser_aboutdebugging_navigate.js]
 [browser_aboutdebugging_persist_connection.js]
 [browser_aboutdebugging_profiler_dialog.js]
+[browser_aboutdebugging_real_usb_runtime_page_runtime_info.js]
+[browser_aboutdebugging_real_usb_sidebar.js]
 [browser_aboutdebugging_routes.js]
 [browser_aboutdebugging_runtime_compatibility_warning.js]
 [browser_aboutdebugging_runtime_remote_runtime_buttons.js]

devtools/client/aboutdebugging-new/test/browser/browser_aboutdebugging_real_usb_runtime_page_runtime_info.js

@@ -0,0 +1,47 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/* import-globals-from helper-real-usb.js */
+Services.scriptloader.loadSubScript(CHROME_URL_ROOT + "helper-real-usb.js", this);
+
+// Test that runtime info of USB runtime appears on main pane.
+// Documentation for real usb tests in /documentation/TESTS_REAL_DEVICES.md
+add_task(async function() {
+  if (!isAvailable()) {
+    ok(true, "Real usb runtime test is not available");
+    return;
+  }
+
+  const { document, tab } = await openAboutDebuggingWithADB();
+
+  const expectedRuntime = await getExpectedRuntime();
+  const { runtimeDetails, sidebarInfo } = expectedRuntime;
+
+  info("Connect a USB runtime");
+  await Promise.race([
+    connectToRuntime(sidebarInfo.deviceName, document),
+    /* eslint-disable mozilla/no-arbitrary-setTimeout */
+    new Promise(resolve => setTimeout(() => {
+      ok(false,
+         "Failed to connect, did you disable the connection prompt for this runtime?");
+      resolve();
+    }, 5000)),
+    /* eslint-enable mozilla/no-arbitrary-setTimeout */
+  ]);
+
+  info("Select a USB runtime");
+  await selectRuntime(sidebarInfo.deviceName, runtimeDetails.info.name, document);
+
+  info("Check that runtime info is properly displayed");
+  const runtimeInfo = document.querySelector(".js-runtime-info");
+  ok(runtimeInfo, "Runtime info is displayed");
+  const runtimeInfoText = runtimeInfo.textContent;
+  ok(runtimeInfoText.includes(runtimeDetails.info.name),
+     "Runtime info shows the correct runtime name: " + runtimeInfoText);
+  ok(runtimeInfoText.includes(runtimeDetails.info.version),
+     "Runtime info shows the correct version number: " + runtimeInfoText);
+
+  await removeTab(tab);
+});

devtools/client/aboutdebugging-new/test/browser/browser_aboutdebugging_real_usb_sidebar.js

@@ -0,0 +1,29 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/* import-globals-from helper-real-usb.js */
+Services.scriptloader.loadSubScript(CHROME_URL_ROOT + "helper-real-usb.js", this);
+
+// Test that USB runtimes appear from the sidebar.
+// Documentation for real usb tests in /documentation/TESTS_REAL_DEVICES.md
+add_task(async function() {
+  if (!isAvailable()) {
+    ok(true, "Real usb runtime test is not available");
+    return;
+  }
+
+  const { document, tab } = await openAboutDebuggingWithADB();
+
+  for (const { sidebarInfo } of await getExpectedRuntimeAll()) {
+    const { deviceName, shortName } = sidebarInfo;
+    await waitUntil(() => findSidebarItemByText(deviceName, document));
+    const usbRuntimeSidebarItem = findSidebarItemByText(deviceName, document);
+    ok(usbRuntimeSidebarItem.textContent.includes(shortName),
+       "The device name and short name of the usb runtime are visible in sidebar item " +
+       `[${usbRuntimeSidebarItem.textContent}]`);
+  }
+
+  await removeTab(tab);
+});

devtools/client/aboutdebugging-new/test/browser/helper-real-usb.js

@@ -0,0 +1,51 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/* import-globals-from helper-adb.js */
+Services.scriptloader.loadSubScript(CHROME_URL_ROOT + "helper-adb.js", this);
+
+async function getExpectedRuntime() {
+  const runtimes = await getExpectedRuntimeAll();
+  return runtimes[0];
+}
+/* exported getExpectedRuntime */
+
+async function getExpectedRuntimeAll() {
+  const runtimesPath = _getExpectedRuntimesPath();
+  const currentPath = env.get("PWD");
+  const path = `${currentPath}/${runtimesPath}`;
+  info(`Load ${ path }`);
+  const buffer = await OS.File.read(path);
+  const data = new TextDecoder().decode(buffer);
+  return JSON.parse(data);
+}
+/* exported getExpectedRuntimeAll */
+
+function isAvailable() {
+  return !!_getExpectedRuntimesPath();
+}
+/* exported isAvailable */
+
+async function openAboutDebuggingWithADB() {
+  const { document, tab, window } = await openAboutDebugging();
+
+  await pushPref("devtools.remote.adb.extensionURL",
+                 CHROME_URL_ROOT + "resources/test-adb-extension/adb-extension-#OS#.xpi");
+  await checkAdbNotRunning();
+
+  const { adbAddon } = require("devtools/shared/adb/adb-addon");
+  adbAddon.install("internal");
+  const usbStatusElement = document.querySelector(".js-sidebar-usb-status");
+  await waitUntil(() => usbStatusElement.textContent.includes("USB devices enabled"));
+  await waitForAdbStart();
+
+  return { document, tab, window };
+}
+/* exported openAboutDebuggingWithADB */
+
+function _getExpectedRuntimesPath() {
+  const env = Cc["@mozilla.org/process/environment;1"].getService(Ci.nsIEnvironment);
+  return env.get("USB_RUNTIMES");
+}

devtools/client/aboutdebugging-new/test/browser/resources/real/usb-runtimes-sample.json

@@ -0,0 +1,14 @@
+[
+  {
+    "sidebarInfo": {
+      "deviceName": "Pixel 2",
+      "shortName": "Firefox Nightly"
+    },
+    "runtimeDetails": {
+      "info": {
+        "name": "Mozilla Nightly",
+        "version": "64.0a1"
+      }
+    }
+  }
+]