From 950f616c3e10e915707831935c16779ee3b83383 Mon Sep 17 00:00:00 2001 From: andrej Date: Wed, 9 Mar 2022 20:53:29 +0000 Subject: [PATCH] Bug 1741971 - Add MozPerfTest Testing Section to WPT Layer r=perftest-reviewers,sparky Depends on D133785 Differential Revision: https://phabricator.services.mozilla.com/D137459 --- .../mozperftest/mozperftest/tests/support.py | 1 + .../mozperftest/tests/test_webpagetest.py | 211 ++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 python/mozperftest/mozperftest/tests/test_webpagetest.py diff --git a/python/mozperftest/mozperftest/tests/support.py b/python/mozperftest/mozperftest/tests/support.py index b9b988a84352..e9af4fa52c50 100644 --- a/python/mozperftest/mozperftest/tests/support.py +++ b/python/mozperftest/mozperftest/tests/support.py @@ -20,6 +20,7 @@ EXAMPLE_TESTS_DIR = os.path.join(HERE, "data", "samples") EXAMPLE_TEST = os.path.join(EXAMPLE_TESTS_DIR, "perftest_example.js") EXAMPLE_XPCSHELL_TEST = Path(EXAMPLE_TESTS_DIR, "test_xpcshell.js") EXAMPLE_XPCSHELL_TEST2 = Path(EXAMPLE_TESTS_DIR, "test_xpcshell_flavor2.js") +EXAMPLE_WPT_TEST = Path(EXAMPLE_TESTS_DIR, "WPT_init_file.js") BT_DATA = Path(HERE, "data", "browsertime-results", "browsertime.json") BT_DATA_VIDEO = Path(HERE, "data", "browsertime-results-video", "browsertime.json") DMG = Path(HERE, "data", "firefox.dmg") diff --git a/python/mozperftest/mozperftest/tests/test_webpagetest.py b/python/mozperftest/mozperftest/tests/test_webpagetest.py new file mode 100644 index 000000000000..b8103bcb1073 --- /dev/null +++ b/python/mozperftest/mozperftest/tests/test_webpagetest.py @@ -0,0 +1,211 @@ +import json +from mozperftest.tests.support import ( + get_running_env, + EXAMPLE_WPT_TEST, +) +import requests +from unittest import mock +import random +import mozperftest.test.webpagetest as webpagetest +import pytest +from mozperftest.test.webpagetest import ( + WPTTimeOutError, + WPTBrowserSelectionError, + WPTInvalidURLError, + WPTLocationSelectionError, + WPT_METRICS, + WPTInvalidConnectionSelection, + ACCEPTED_STATISTICS, + WPTInvalidStatisticsError, + WPTErrorWithWebsite, + WPTDataProcessingError, +) + + +class WPTTests: + def __init__(self, log): + self.log = log + + def runTests(self, args): + return True + + +def running_env(**kw): + return get_running_env(flavor="webpagetest", **kw) + + +def init_placeholder_wpt_data(fvonly=False, invalid_results=False): + views = {"firstView": {}} + if not fvonly: + views["repeatView"] = {} + placeholder_data = { + "data": { + "summary": "websitelink.com", + "location": "ec2-us-west-1:Firefox", + "testRuns": 3, + "successfulFVRuns": 3, + "successfulRVRuns": 3, + "fvonly": fvonly, + "average": views, + "standardDeviation": views, + "median": views, + } + } + exclude_metrics = 0 if not invalid_results else 2 + for metric in WPT_METRICS[exclude_metrics:]: + for view in views: + for stat in ACCEPTED_STATISTICS: + placeholder_data["data"][stat][view][metric] = random.randint(0, 10000) + placeholder_data["data"][stat][view][metric] = random.randint(0, 10000) + placeholder_data["data"][stat][view][metric] = random.randint(0, 10000) + return placeholder_data + + +def init_mocked_request(status_code, **kwargs): + mock_data = { + "data": { + "ec2-us-west-1": {"PendingTests": {"Queued": 3}, "Label": "California"}, + "jsonUrl": "mock_test.com", + "summary": "Just a pageload test", + }, + "statusCode": status_code, + } + for key, value in kwargs.items(): + mock_data["data"][key] = value + mock_request = requests.Response() + mock_request._content = json.dumps(mock_data).encode("utf-8") + return mock_request + + +@mock.patch( + "mozperftest.test.webpagetest.WebPageTest.get_WPT_results", + return_value=init_placeholder_wpt_data(), +) +@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"}) +def test_webpagetest_no_issues_mocked_results(*mocked): + mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) + test = webpagetest.WebPageTest(env, mach_cmd) + metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1 + metadata.script["options"]["test_parameters"]["first_view_only"] = 0 + metadata.script["options"]["test_parameters"]["test_list"] = ["google.ca"] + test.run(metadata) + + +@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"}) +def test_webpagetest_test_invalid_browser(*mocked): + mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) + metadata.script["options"]["test_parameters"]["browser"] = "Invalid Browser" + test = webpagetest.WebPageTest(env, mach_cmd) + with pytest.raises(WPTBrowserSelectionError): + test.run(metadata) + + +@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"}) +def test_webpagetest_test_invalid_connection(*mocked): + mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) + test = webpagetest.WebPageTest(env, mach_cmd) + metadata.script["options"]["test_parameters"]["connection"] = "Invalid Connection" + with pytest.raises(WPTInvalidConnectionSelection): + test.run(metadata) + + +@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"}) +def test_webpagetest_test_invalid_url(*mocked): + mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) + test = webpagetest.WebPageTest(env, mach_cmd) + metadata.script["options"]["test_list"] = ["InvalidUrl"] + with pytest.raises(WPTInvalidURLError): + test.run(metadata) + + +@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"}) +def test_webpagetest_test_invalid_statistic(*mocked): + mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) + test = webpagetest.WebPageTest(env, mach_cmd) + metadata.script["options"]["test_parameters"]["statistics"] = ["Invalid Statistic"] + with pytest.raises(WPTInvalidStatisticsError): + test.run(metadata) + assert True + + +@mock.patch("requests.get", return_value=init_mocked_request(200)) +@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"}) +def test_webpagetest_test_invalid_location(*mocked): + mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) + test = webpagetest.WebPageTest(env, mach_cmd) + metadata.script["options"]["test_parameters"]["location"] = "Invalid Location" + with pytest.raises(WPTLocationSelectionError): + test.run(metadata) + + +@mock.patch("requests.get", return_value=init_mocked_request(400)) +@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"}) +def test_webpagetest_test_timeout(*mocked): + mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) + test = webpagetest.WebPageTest(env, mach_cmd) + metadata.script["options"]["test_parameters"]["timeout_limit"] = 2 + metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1 + with pytest.raises(WPTTimeOutError): + test.run(metadata) + assert True + + +@mock.patch( + "requests.get", + return_value=init_mocked_request(200, testRuns=3, successfulFVRuns=2, fvonly=True), +) +@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"}) +def test_webpagetest_test_fv_count_discrepancy(*mocked): + mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) + metadata.script["options"]["test_list"] = ["google.ca"] + metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1 + test = webpagetest.WebPageTest(env, mach_cmd) + with pytest.raises(WPTErrorWithWebsite): + test.run(metadata) + assert True + + +@mock.patch( + "requests.get", + return_value=init_mocked_request( + 200, testRuns=3, successfulFVRuns=3, successfulRVRuns=2, fvonly=False + ), +) +@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"}) +def test_webpagetest_test_rv_count_discrepancy(*mocked): + mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) + metadata.script["options"]["test_list"] = ["google.ca"] + metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1 + test = webpagetest.WebPageTest(env, mach_cmd) + with pytest.raises(WPTErrorWithWebsite): + test.run(metadata) + + +@mock.patch( + "requests.get", + return_value=init_mocked_request( + 200, testRuns=3, successfulFVRuns=3, fvonly=True, location="BadLocation" + ), +) +@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"}) +def test_webpagetest_test_wrong_browserlocation(*mocked): + mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) + metadata.script["options"]["test_list"] = ["google.ca"] + metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1 + test = webpagetest.WebPageTest(env, mach_cmd) + with pytest.raises(WPTBrowserSelectionError): + test.run(metadata) + + +@mock.patch( + "mozperftest.test.webpagetest.WebPageTest.get_WPT_results", + return_value=init_placeholder_wpt_data(invalid_results=True), +) +@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"}) +def test_webpagetest_test_metric_not_found(*mocked): + mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) + metadata.script["options"]["test_list"] = ["google.ca"] + metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1 + test = webpagetest.WebPageTest(env, mach_cmd) + with pytest.raises(WPTDataProcessingError): + test.run(metadata)