From f9b94c2b0cd65a19bdf73874b94746bdd98b34c0 Mon Sep 17 00:00:00 2001 From: Kershaw Chang Date: Fri, 23 Aug 2024 08:08:59 +0000 Subject: [PATCH] Bug 1895359 - Setup servers for running upload test locally, r=perftest-reviewers,sparky Differential Revision: https://phabricator.services.mozilla.com/D218585 --- .../gecko_taskgraph/transforms/test/worker.py | 2 +- .../kinds/test/browsertime-desktop.yml | 14 + taskcluster/kinds/test/test-sets.yml | 1 + testing/perfdocs/generated/raptor.rst | 188 ++++++++++++- .../support-scripts/network_bench.py | 260 ++++++++++++++++++ testing/raptor/browsertime/upload.js | 6 +- .../raptor/browsertime/utils/http2-cert.key | 28 ++ .../raptor/browsertime/utils/http2-cert.pem | 19 ++ .../browsertime/utils/upload_backend.js | 109 ++++++++ testing/raptor/raptor/base_python_support.py | 4 + testing/raptor/raptor/browsertime/base.py | 4 + testing/raptor/raptor/perfdocs/config.yml | 5 +- testing/raptor/raptor/raptor.toml | 3 + .../custom/browsertime-network-bench.toml | 26 ++ .../tests/custom/browsertime-upload.toml | 4 +- 15 files changed, 663 insertions(+), 10 deletions(-) create mode 100644 testing/raptor/browsertime/support-scripts/network_bench.py create mode 100644 testing/raptor/browsertime/utils/http2-cert.key create mode 100644 testing/raptor/browsertime/utils/http2-cert.pem create mode 100644 testing/raptor/browsertime/utils/upload_backend.js create mode 100644 testing/raptor/raptor/tests/custom/browsertime-network-bench.toml diff --git a/taskcluster/gecko_taskgraph/transforms/test/worker.py b/taskcluster/gecko_taskgraph/transforms/test/worker.py index b344a0b20cd9..b87225391f2a 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/worker.py +++ b/taskcluster/gecko_taskgraph/transforms/test/worker.py @@ -204,7 +204,7 @@ def set_worker_type(config, tasks): elif task.get("suite", "") in ["talos", "raptor"] and not task[ "build-platform" ].startswith("linux64-ccov"): - if "browsertime-upload" in task.get("test-name"): + if "browsertime-network-bench" in task.get("test-name"): task["worker-type"] = "t-linux-netperf-1804" else: task["worker-type"] = "t-linux-talos-1804" diff --git a/taskcluster/kinds/test/browsertime-desktop.yml b/taskcluster/kinds/test/browsertime-desktop.yml index 7e6fd5b0bb71..12b05c16e440 100644 --- a/taskcluster/kinds/test/browsertime-desktop.yml +++ b/taskcluster/kinds/test/browsertime-desktop.yml @@ -876,6 +876,20 @@ browsertime-upload: fetch: - upload-speed-test-file +browsertime-network-bench: + raptor: + apps: [firefox] + subtests: + - h3-upload + description: Raptor (browsertime) networking download/upload performance test + max-run-time: 2700 + run-on-projects: [] + tier: 3 + treeherder-symbol: Btime(network-bench) + fetches: + fetch: + - upload-speed-test-file + browsertime-pageload-benchmark: raptor: apps: [firefox] diff --git a/taskcluster/kinds/test/test-sets.yml b/taskcluster/kinds/test/test-sets.yml index a75ed1542d54..3a82715ab4be 100644 --- a/taskcluster/kinds/test/test-sets.yml +++ b/taskcluster/kinds/test/test-sets.yml @@ -113,6 +113,7 @@ browsertime: - browsertime-pageload-benchmark - browsertime-indexeddb - browsertime-video-playback-latency + - browsertime-network-bench browsertime-clang: - browsertime-tp6 diff --git a/testing/perfdocs/generated/raptor.rst b/testing/perfdocs/generated/raptor.rst index 5b6b1e709437..bd326ead9976 100644 --- a/testing/perfdocs/generated/raptor.rst +++ b/testing/perfdocs/generated/raptor.rst @@ -13696,6 +13696,186 @@ Browsertime tests that use a custom pageload test script. These use the pageload +.. dropdown:: h3-upload + :class-container: anchor-id-h3-upload-c + + * Command to Run Locally + + .. code-block:: + + ./mach raptor -t h3-upload + + **Owner**: Network Team + + **Description**: Measures HTTP/3 file upload throughput with a local server + + * **alert threshold**: 2.0 + * **apps**: firefox + * **browser cycles**: 1 + * **browsertime args**: --browsertime.upload_iterations=10 --firefox.preference=network.http.http3.enable:true + * **custom data**: true + * **expected**: pass + * **gecko profile interval**: 1 + * **link searchfox**: ``__ + * **lower is better**: false + * **output timeout**: 2000000 + * **page cycles**: 1 + * **page timeout**: 1800000 + * **subtest lower is better**: false + * **subtest name filters**: tests/,iterations/ + * **subtest unit**: mbps + * **support class**: network_bench.py + * **test script**: upload.js + * **test url**: ``__ + * **type**: pageload + * **unit**: mbps + * **Test Task**: + + .. list-table:: **test-linux1804-64-nightlyasrelease-qr/opt** + :widths: 30 15 15 15 15 + :header-rows: 1 + + * - **Test Name** + - mozilla-central + - autoland + - mozilla-release + - mozilla-beta + * - **browsertime-network-bench-firefox-h3-upload** + - ❌ + - ❌ + - ❌ + - ❌ + + + .. list-table:: **test-linux1804-64-qr/opt** + :widths: 30 15 15 15 15 + :header-rows: 1 + + * - **Test Name** + - mozilla-central + - autoland + - mozilla-release + - mozilla-beta + * - **browsertime-network-bench-firefox-h3-upload** + - ❌ + - ❌ + - ❌ + - ❌ + + + .. list-table:: **test-linux1804-64-shippable-qr/opt** + :widths: 30 15 15 15 15 + :header-rows: 1 + + * - **Test Name** + - mozilla-central + - autoland + - mozilla-release + - mozilla-beta + * - **browsertime-network-bench-firefox-h3-upload** + - ❌ + - ❌ + - ❌ + - ❌ + + + .. list-table:: **test-macosx1015-64-nightlyasrelease-qr/opt** + :widths: 30 15 15 15 15 + :header-rows: 1 + + * - **Test Name** + - mozilla-central + - autoland + - mozilla-release + - mozilla-beta + * - **browsertime-network-bench-firefox-h3-upload** + - ❌ + - ❌ + - ❌ + - ❌ + + + .. list-table:: **test-macosx1015-64-qr/opt** + :widths: 30 15 15 15 15 + :header-rows: 1 + + * - **Test Name** + - mozilla-central + - autoland + - mozilla-release + - mozilla-beta + * - **browsertime-network-bench-firefox-h3-upload** + - ❌ + - ❌ + - ❌ + - ❌ + + + .. list-table:: **test-macosx1015-64-shippable-qr/opt** + :widths: 30 15 15 15 15 + :header-rows: 1 + + * - **Test Name** + - mozilla-central + - autoland + - mozilla-release + - mozilla-beta + * - **browsertime-network-bench-firefox-h3-upload** + - ❌ + - ❌ + - ❌ + - ❌ + + + .. list-table:: **test-windows11-64-nightlyasrelease-qr/opt** + :widths: 30 15 15 15 15 + :header-rows: 1 + + * - **Test Name** + - mozilla-central + - autoland + - mozilla-release + - mozilla-beta + * - **browsertime-network-bench-firefox-h3-upload** + - ❌ + - ❌ + - ❌ + - ❌ + + + .. list-table:: **test-windows11-64-qr/opt** + :widths: 30 15 15 15 15 + :header-rows: 1 + + * - **Test Name** + - mozilla-central + - autoland + - mozilla-release + - mozilla-beta + * - **browsertime-network-bench-firefox-h3-upload** + - ❌ + - ❌ + - ❌ + - ❌ + + + .. list-table:: **test-windows11-64-shippable-qr/opt** + :widths: 30 15 15 15 15 + :header-rows: 1 + + * - **Test Name** + - mozilla-central + - autoland + - mozilla-release + - mozilla-beta + * - **browsertime-network-bench-firefox-h3-upload** + - ❌ + - ❌ + - ❌ + - ❌ + + + .. dropdown:: process-switch :class-container: anchor-id-process-switch-c @@ -14285,12 +14465,12 @@ Browsertime tests that use a custom pageload test script. These use the pageload **Owner**: Network Team - **Description**: Measures http/2 file upload throughput + **Description**: Measures HTTP/2 file upload throughput with a remote server * **alert threshold**: 2.0 * **apps**: firefox, chrome, safari * **browser cycles**: 1 - * **browsertime args**: --browsertime.upload_iterations=10 --firefox.preference=network.http.http3.enable:false --chrome.args disable-quic + * **browsertime args**: --browsertime.upload_iterations=10 --browsertime.upload_url=https://uploadtest-381620.uc.r.appspot.com --firefox.preference=network.http.http3.enable:false --chrome.args disable-quic * **custom data**: true * **expected**: pass * **gecko profile interval**: 1 @@ -14485,12 +14665,12 @@ Browsertime tests that use a custom pageload test script. These use the pageload **Owner**: Network Team - **Description**: Measures http/3 file upload throughput + **Description**: Measures HTTP/3 file upload throughput with a remote server * **alert threshold**: 2.0 * **apps**: firefox, chrome, safari * **browser cycles**: 1 - * **browsertime args**: --browsertime.upload_iterations=10 --firefox.preference=network.http.http3.enable:true + * **browsertime args**: --browsertime.upload_iterations=10 --browsertime.upload_url=https://uploadtest-381620.uc.r.appspot.com --firefox.preference=network.http.http3.enable:true * **custom data**: true * **expected**: pass * **gecko profile interval**: 1 diff --git a/testing/raptor/browsertime/support-scripts/network_bench.py b/testing/raptor/browsertime/support-scripts/network_bench.py new file mode 100644 index 000000000000..fe816752724d --- /dev/null +++ b/testing/raptor/browsertime/support-scripts/network_bench.py @@ -0,0 +1,260 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import re +import signal +import socket +import subprocess +import tempfile +import threading +from pathlib import Path +from subprocess import PIPE, Popen + +from base_python_support import BasePythonSupport +from logger.logger import RaptorLogger + +LOG = RaptorLogger(component="raptor-browsertime") + + +class NetworkBench(BasePythonSupport): + def __init__(self, **kwargs): + self._is_chrome = False + self.browsertime_node = None + self.backend_server = None + self.backend_port = None + self.caddy_port = None + self.caddy_server = None + self.caddy_stdout = None + self.caddy_stderr = None + self.http_version = "h1" + + def setup_test(self, test, args): + from cmdline import CHROME_ANDROID_APPS, CHROMIUM_DISTROS + + LOG.info("setup_test: '%s'" % test) + + self._is_chrome = ( + args.app in CHROMIUM_DISTROS or args.app in CHROME_ANDROID_APPS + ) + + prefix = test.get("name").split("-")[0] + if prefix == "h3": + self.http_version = "h3" + elif prefix == "h2": + self.http_version = "h2" + LOG.info("http_version: '%s'" % self.http_version) + + def check_caddy_installed(self): + try: + result = subprocess.run( + ["caddy", "version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode == 0: + LOG.info("Caddy is installed. Version: %s" % result.stdout.strip()) + return True + else: + LOG.error("Caddy is not installed.") + except FileNotFoundError: + LOG.error("Caddy is not installed.") + return False + + def start_backend_server(self, path): + if self.browsertime_node is None or not self.browsertime_node.exists(): + return None + + LOG.info("node bin: %s" % self.browsertime_node) + + server_path = ( + Path(__file__).parent / ".." / ".." / "browsertime" / "utils" / path + ) + LOG.info("server_path: %s" % server_path) + + if not server_path.exists(): + return None + + process = Popen( + [self.browsertime_node, server_path], + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True, + start_new_session=True, + ) + msg = process.stdout.readline() + LOG.info("server msg: %s" % msg) + match = re.search(r"Server is running on http://[^:]+:(\d+)", msg) + if match: + self.backend_port = match.group(1) + LOG.info("backend port: %s" % self.backend_port) + return process + return None + + def find_free_port(self, socket_type): + with socket.socket(socket.AF_INET, socket_type) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("localhost", 0)) + return s.getsockname()[1] + + def start_caddy(self): + if not self.check_caddy_installed(): + return None + if self.caddy_port is None or not (1 <= self.caddy_port <= 65535): + return None + + cert_path = Path(__file__).parent / ".." / ".." / "browsertime" / "utils" + + if not cert_path.exists(): + return None + + key_path = cert_path / "http2-cert.key" + LOG.info("key_path: %s" % key_path) + if not key_path.exists(): + return None + + pem_path = cert_path / "http2-cert.pem" + LOG.info("pem_path: %s" % pem_path) + if not pem_path.exists(): + return None + + upstream = f"localhost:{self.backend_port}" + port_str = f":{self.caddy_port}" + caddyfile_content = { + "admin": {"disabled": True}, + "apps": { + "http": { + "servers": { + "server1": { + "listen": [port_str], + "protocols": ["h1", "h2", "h3"], + "routes": [ + { + "handle": [ + { + "handler": "reverse_proxy", + "upstreams": [{"dial": upstream}], + } + ] + } + ], + "tls_connection_policies": [ + {"certificate_selection": {"any_tag": ["cert1"]}} + ], + "automatic_https": {"disable": True}, + } + }, + }, + "tls": { + "certificates": { + "load_files": [ + { + "certificate": str(pem_path), + "key": str(key_path), + "tags": ["cert1"], + } + ] + } + }, + }, + } + + LOG.info("caddyfile_content: %s" % caddyfile_content) + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".json" + ) as temp_json_file: + json.dump(caddyfile_content, temp_json_file, indent=2) + temp_json_file_path = temp_json_file.name + + LOG.info("temp_json_file_path: %s" % temp_json_file_path) + command = ["caddy", "run", "--config", temp_json_file_path] + + def read_output(pipe, log_func): + for line in iter(pipe.readline, ""): + log_func(line) + + process = Popen( + command, + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True, + start_new_session=True, + ) + self.caddy_stdout = threading.Thread( + target=read_output, args=(process.stdout, LOG.info) + ) + self.caddy_stderr = threading.Thread( + target=read_output, args=(process.stderr, LOG.info) + ) + self.caddy_stdout.start() + self.caddy_stderr.start() + return process + + def modify_command(self, cmd, test): + if not self._is_chrome: + cmd += [ + "--firefox.acceptInsecureCerts", + "true", + ] + if self.http_version == "h3": + self.caddy_port = self.find_free_port(socket.SOCK_DGRAM) + cmd += [ + "--firefox.preference", + f"network.http.http3.alt-svc-mapping-for-testing:localhost;h3=:{self.caddy_port}", + ] + else: + self.caddy_port = self.find_free_port(socket.SOCK_STREAM) + + cmd += [ + "--browsertime.upload_url", + f"https://localhost:{self.caddy_port}", + ] + + LOG.info("modify_command: %s" % cmd) + + # We know that cmd[0] is the path to nodejs. + self.browsertime_node = Path(cmd[0]) + self.backend_server = self.start_backend_server("upload_backend.js") + if self.backend_server: + self.caddy_server = self.start_caddy() + if self.caddy_server is None: + raise Exception("Failed to start test servers") + + def handle_result(self, bt_result, raw_result, last_result=False, **kwargs): + # TODO + LOG.info("handle_result: %s" % raw_result) + + def _build_subtest(self, measurement_name, replicates, test): + # TODO + LOG.info("_build_subtest") + + def summarize_test(self, test, suite, **kwargs): + # TODO + LOG.info("summarize_test") + + def shutdown_server(self, name, proc): + LOG.info("%s server shutting down ..." % name) + if proc.poll() is not None: + LOG.info("server already dead %s" % proc.poll()) + else: + LOG.info("server pid is %s" % str(proc.pid)) + try: + os.killpg(proc.pid, signal.SIGTERM) + except Exception as e: + LOG.error("Failed during kill: " + str(e)) + + def clean_up(self): + if self.caddy_server: + self.shutdown_server("Caddy", self.caddy_server) + if self.backend_server: + self.shutdown_server("Backend", self.backend_server) + if self.caddy_stdout: + self.caddy_stdout.join() + if self.caddy_stderr: + self.caddy_stderr.join() diff --git a/testing/raptor/browsertime/upload.js b/testing/raptor/browsertime/upload.js index a07ad7e79217..88b1afa38a23 100644 --- a/testing/raptor/browsertime/upload.js +++ b/testing/raptor/browsertime/upload.js @@ -24,6 +24,10 @@ async function waitForUpload(timeout, commands, context) { `return document.getElementById('upload_status').innerHTML;` ); + if (status.startsWith("success")) { + status = "success"; + } + context.log.info("context.log test: " + status); console.log("test: " + status); } @@ -38,7 +42,7 @@ async function waitForUpload(timeout, commands, context) { } module.exports = logTest("upload test", async function (context, commands) { - let uploadSiteUrl = "https://uploadtest-381620.uc.r.appspot.com"; + let uploadSiteUrl = `${context.options.browsertime.upload_url}`; let iterations = `${context.options.browsertime.upload_iterations}`; await commands.measure.start(uploadSiteUrl); diff --git a/testing/raptor/browsertime/utils/http2-cert.key b/testing/raptor/browsertime/utils/http2-cert.key new file mode 100644 index 000000000000..09e044f5e087 --- /dev/null +++ b/testing/raptor/browsertime/utils/http2-cert.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6iFGoRI4W1kH9 +braIBjYQPTwT2erkNUq07PVoV2wke8HHJajg2B+9sZwGm24ahvJr4q9adWtqZHEI +eqVap0WH9xzVJJwCfs1D/B5p0DggKZOrIMNJ5Nu5TMJrbA7tFYIP8X6taRqx0wI6 +iypB7qdw4A8Njf1mCyuwJJKkfbmIYXmQsVeQPdI7xeC4SB+oN9OIQ+8nFthVt2Za +qn4CkC86exCABiTMHGyXrZZhW7filhLAdTGjDJHdtMr3/K0dJdMJ77kXDqdo4bN7 +LyJvaeO0ipVhHe4m1iWdq5EITjbLHCQELL8Wiy/l8Y+ZFzG4s/5JI/pyUcQx1QOs +2hgKNe2NAgMBAAECggEBAJ7LzjhhpFTsseD+j4XdQ8kvWCXOLpl4hNDhqUnaosWs +VZskBFDlrJ/gw+McDu+mUlpl8MIhlABO4atGPd6e6CKHzJPnRqkZKcXmrD2IdT9s +JbpZeec+XY+yOREaPNq4pLDN9fnKsF8SM6ODNcZLVWBSXn47kq18dQTPHcfLAFeI +r8vh6Pld90AqFRUw1YCDRoZOs3CqeZVqWHhiy1M3kTB/cNkcltItABppAJuSPGgz +iMnzbLm16+ZDAgQceNkIIGuHAJy4yrrK09vbJ5L7kRss9NtmA1hb6a4Mo7jmQXqg +SwbkcOoaO1gcoDpngckxW2KzDmAR8iRyWUbuxXxtlEECgYEA3W4dT//r9o2InE0R +TNqqnKpjpZN0KGyKXCmnF7umA3VkTVyqZ0xLi8cyY1hkYiDkVQ12CKwn1Vttt0+N +gSfvj6CQmLaRR94GVXNEfhg9Iv59iFrOtRPZWB3V4HwakPXOCHneExNx7O/JznLp +xD3BJ9I4GQ3oEXc8pdGTAfSMdCsCgYEA16dz2evDgKdn0v7Ak0rU6LVmckB3Gs3r +ta15b0eP7E1FmF77yVMpaCicjYkQL63yHzTi3UlA66jAnW0fFtzClyl3TEMnXpJR +3b5JCeH9O/Hkvt9Go5uLODMo70rjuVuS8gcK8myefFybWH/t3gXo59hspXiG+xZY +EKd7mEW8MScCgYEAlkcrQaYQwK3hryJmwWAONnE1W6QtS1oOtOnX6zWBQAul3RMs +2xpekyjHu8C7sBVeoZKXLt+X0SdR2Pz2rlcqMLHqMJqHEt1OMyQdse5FX8CT9byb +WS11bmYhR08ywHryL7J100B5KzK6JZC7smGu+5WiWO6lN2VTFb6cJNGRmS0CgYAo +tFCnp1qFZBOyvab3pj49lk+57PUOOCPvbMjo+ibuQT+LnRIFVA8Su+egx2got7pl +rYPMpND+KiIBFOGzXQPVqFv+Jwa9UPzmz83VcbRspiG47UfWBbvnZbCqSgZlrCU2 +TaIBVAMuEgS4VZ0+NPtbF3yaVv+TUQpaSmKHwVHeLQKBgCgGe5NVgB0u9S36ltit +tYlnPPjuipxv9yruq+nva+WKT0q/BfeIlH3IUf2qNFQhR6caJGv7BU7naqNGq80m +ks/J5ExR5vBpxzXgc7oBn2pyFJYckbJoccrqv48GRBigJpDjmo1f8wZ7fNt/ULH1 +NBinA5ZsT8d0v3QCr2xDJH9D +-----END PRIVATE KEY----- diff --git a/testing/raptor/browsertime/utils/http2-cert.pem b/testing/raptor/browsertime/utils/http2-cert.pem new file mode 100644 index 000000000000..1f89de1a4551 --- /dev/null +++ b/testing/raptor/browsertime/utils/http2-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDEzCCAfugAwIBAgIUCTTdK3eSofAM6mNwAi4Z4YUn8WEwDQYJKoZIhvcNAQEL +BQAwGTEXMBUGA1UEAwwOIEhUVFAyIFRlc3QgQ0EwIhgPMjAxNzAxMDEwMDAwMDBa +GA8yMDI3MDEwMTAwMDAwMFowGzEZMBcGA1UEAwwQIEhUVFAyIFRlc3QgQ2VydDCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALqIUahEjhbWQf1utogGNhA9 +PBPZ6uQ1SrTs9WhXbCR7wcclqODYH72xnAabbhqG8mvir1p1a2pkcQh6pVqnRYf3 +HNUknAJ+zUP8HmnQOCApk6sgw0nk27lMwmtsDu0Vgg/xfq1pGrHTAjqLKkHup3Dg +Dw2N/WYLK7AkkqR9uYhheZCxV5A90jvF4LhIH6g304hD7ycW2FW3ZlqqfgKQLzp7 +EIAGJMwcbJetlmFbt+KWEsB1MaMMkd20yvf8rR0l0wnvuRcOp2jhs3svIm9p47SK +lWEd7ibWJZ2rkQhONsscJAQsvxaLL+Xxj5kXMbiz/kkj+nJRxDHVA6zaGAo17Y0C +AwEAAaNNMEswSQYDVR0RBEIwQIIJbG9jYWxob3N0gg9mb28uZXhhbXBsZS5jb22C +EGFsdDEuZXhhbXBsZS5jb22CEGFsdDIuZXhhbXBsZS5jb20wDQYJKoZIhvcNAQEL +BQADggEBAE5aEiXOkvEYeWpMhkGheeeaKwgr44qiWJKC5N/8t+NprB3vNCbTMzE9 +09iWQh9EXbwMjMQ8H0uZwedek2sryxsTzxsdTC5qmEtxs/kbf0rTNUwQDjGHvzMk +gO+ULESdLTcIFJ57olHaZaXtPGm2ELJAOiEpsYFTafmCEPXZ/b+UkGsSkuVLSOIA +ClaIJgjff/ucvCvRwl79GzGDCoh3qpqhvxQpC/Fcdz1iQDYEVAmjgUrYJe1lTfj8 +ZozM1WIq8fQ3SCXTJK82CnX818tJio2PWq3uzb9vhpuxJJif7WoMP88Jpdh8zcEb +YL15XPzhQMyor2p6XfwNI3J6347fd7U= +-----END CERTIFICATE----- diff --git a/testing/raptor/browsertime/utils/upload_backend.js b/testing/raptor/browsertime/utils/upload_backend.js new file mode 100644 index 000000000000..1578c5ea5b98 --- /dev/null +++ b/testing/raptor/browsertime/utils/upload_backend.js @@ -0,0 +1,109 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +/* globals require, process */ + +const http = require("http"); + +const html = ` + + + + Upload test + + + +
+ Upload test +
+ +

+ + + + + +`; + +const server = http.createServer((req, res) => { + if (req.url === "/saveFile" && req.method.toLowerCase() === "post") { + let totalSize = 0; + req.on("data", chunk => { + totalSize += chunk.length; + }); + + req.on("end", () => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ status: "success", size: totalSize })); + }); + } else if (req.url === "/" && req.method.toLowerCase() === "get") { + res.writeHead(200, { "Content-Type": "text/html" }); + res.end(html); + } else { + res.writeHead(404, { "Content-Type": "text/plain" }); + res.end("Not Found"); + } +}); + +server.listen(() => { + console.log(`Server is running on http://localhost:${server.address().port}`); +}); diff --git a/testing/raptor/raptor/base_python_support.py b/testing/raptor/raptor/base_python_support.py index 45e4f9433b7e..5a9860aacfd9 100644 --- a/testing/raptor/raptor/base_python_support.py +++ b/testing/raptor/raptor/base_python_support.py @@ -180,3 +180,7 @@ class BasePythonSupport: test is successfull, True should be returned (which is the default). """ return True + + def clean_up(self): + """Perform cleanup operations to release resources.""" + pass diff --git a/testing/raptor/raptor/browsertime/base.py b/testing/raptor/raptor/browsertime/base.py index e2aeabba92d7..8242ab580471 100644 --- a/testing/raptor/raptor/browsertime/base.py +++ b/testing/raptor/raptor/browsertime/base.py @@ -256,6 +256,10 @@ class Browsertime(Perftest): if self.benchmark: self.benchmark.stop_http_server() + if test.get("support_class", None): + LOG.info("Test support class is cleaning up...") + test.get("support_class").clean_up() + def check_for_crashes(self): super(Browsertime, self).check_for_crashes() diff --git a/testing/raptor/raptor/perfdocs/config.yml b/testing/raptor/raptor/perfdocs/config.yml index 670a6f49cd4a..50157138eaa0 100644 --- a/testing/raptor/raptor/perfdocs/config.yml +++ b/testing/raptor/raptor/perfdocs/config.yml @@ -235,8 +235,9 @@ suites: addkBLN: "Use add API to send a couple of large Blobs to IndexedDB independently" constant-regression: "Generates a constant value that can be changed to induce a regression." getkeyrng: "Measures the speed of key-range based reads of IndexedDB" - upload: "Measures http/2 file upload throughput" - upload-h3: "Measures http/3 file upload throughput" + upload: "Measures HTTP/2 file upload throughput with a remote server" + upload-h3: "Measures HTTP/3 file upload throughput with a remote server" + h3-upload: "Measures HTTP/3 file upload throughput with a local server" throttled: "Pageload test using a throttled network environment" sample-python-support: "A sample test that uses a python support file to modify the test command." connect: "Verifies that connection time on a pageload is zero when a speculative connection has first been made" diff --git a/testing/raptor/raptor/raptor.toml b/testing/raptor/raptor/raptor.toml index 003a5ad5698f..d9e4c1d2b135 100644 --- a/testing/raptor/raptor/raptor.toml +++ b/testing/raptor/raptor/raptor.toml @@ -79,6 +79,9 @@ ["include:tests/custom/browsertime-upload.toml"] # Custom upload speed test +["include:tests/custom/browsertime-network-bench.toml"] +# Custom download/upload performance test + ["include:tests/custom/browsertime-video-playback-latency.toml"] # Video playback latency tests diff --git a/testing/raptor/raptor/tests/custom/browsertime-network-bench.toml b/testing/raptor/raptor/tests/custom/browsertime-network-bench.toml new file mode 100644 index 000000000000..14a487d4e7ba --- /dev/null +++ b/testing/raptor/raptor/tests/custom/browsertime-network-bench.toml @@ -0,0 +1,26 @@ +[DEFAULT] +apps = "firefox" +alert_threshold = 2.0 +browser_cycles = 1 +gecko_profile_interval = 1 +subtest_lower_is_better = false +owner = "Network Team" +page_cycles = 1 +page_timeout = 1800000 +output_timeout = 2000000 +subtest_unit = "mbps" +type = "pageload" +unit = "mbps" +custom_data = true +lower_is_better = false +subtest_name_filters = "tests/,iterations/" +test_url = "None" +support_class = "network_bench.py" + +# upload/download performance custom tests + +# http/3 + +["h3-upload"] +browsertime_args = "--browsertime.upload_iterations=10 --firefox.preference=network.http.http3.enable:true" +test_script = "upload.js" diff --git a/testing/raptor/raptor/tests/custom/browsertime-upload.toml b/testing/raptor/raptor/tests/custom/browsertime-upload.toml index 2dcd1d20bbc1..93c551f6d121 100644 --- a/testing/raptor/raptor/tests/custom/browsertime-upload.toml +++ b/testing/raptor/raptor/tests/custom/browsertime-upload.toml @@ -18,7 +18,7 @@ use_live_sites = true # http/2 ["upload"] -browsertime_args = "--browsertime.upload_iterations=10 --firefox.preference=network.http.http3.enable:false --chrome.args disable-quic" +browsertime_args = "--browsertime.upload_iterations=10 --browsertime.upload_url=https://uploadtest-381620.uc.r.appspot.com --firefox.preference=network.http.http3.enable:false --chrome.args disable-quic" custom_data = true unit = "mbps" lower_is_better = false @@ -29,7 +29,7 @@ test_url = "None" # http/3 ["upload-h3"] -browsertime_args = "--browsertime.upload_iterations=10 --firefox.preference=network.http.http3.enable:true" +browsertime_args = "--browsertime.upload_iterations=10 --browsertime.upload_url=https://uploadtest-381620.uc.r.appspot.com --firefox.preference=network.http.http3.enable:true" custom_data = true unit = "mbps" lower_is_better = false