mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-24 21:31:04 +00:00
Bug 1741971 - Add MozPerfTest Testing Section to WPT Layer r=perftest-reviewers,sparky
Depends on D133785 Differential Revision: https://phabricator.services.mozilla.com/D137459
This commit is contained in:
andrej
parent
c601667cc8
commit
950f616c3e
@ -20,6 +20,7 @@ EXAMPLE_TESTS_DIR = os.path.join(HERE, "data", "samples")
|
||||
EXAMPLE_TEST = os.path.join(EXAMPLE_TESTS_DIR, "perftest_example.js")
|
||||
EXAMPLE_XPCSHELL_TEST = Path(EXAMPLE_TESTS_DIR, "test_xpcshell.js")
|
||||
EXAMPLE_XPCSHELL_TEST2 = Path(EXAMPLE_TESTS_DIR, "test_xpcshell_flavor2.js")
|
||||
EXAMPLE_WPT_TEST = Path(EXAMPLE_TESTS_DIR, "WPT_init_file.js")
|
||||
BT_DATA = Path(HERE, "data", "browsertime-results", "browsertime.json")
|
||||
BT_DATA_VIDEO = Path(HERE, "data", "browsertime-results-video", "browsertime.json")
|
||||
DMG = Path(HERE, "data", "firefox.dmg")
|
||||
|
||||
211
python/mozperftest/mozperftest/tests/test_webpagetest.py
Normal file
211
python/mozperftest/mozperftest/tests/test_webpagetest.py
Normal file
@ -0,0 +1,211 @@
|
||||
import json
|
||||
from mozperftest.tests.support import (
|
||||
get_running_env,
|
||||
EXAMPLE_WPT_TEST,
|
||||
)
|
||||
import requests
|
||||
from unittest import mock
|
||||
import random
|
||||
import mozperftest.test.webpagetest as webpagetest
|
||||
import pytest
|
||||
from mozperftest.test.webpagetest import (
|
||||
WPTTimeOutError,
|
||||
WPTBrowserSelectionError,
|
||||
WPTInvalidURLError,
|
||||
WPTLocationSelectionError,
|
||||
WPT_METRICS,
|
||||
WPTInvalidConnectionSelection,
|
||||
ACCEPTED_STATISTICS,
|
||||
WPTInvalidStatisticsError,
|
||||
WPTErrorWithWebsite,
|
||||
WPTDataProcessingError,
|
||||
)
|
||||
|
||||
|
||||
class WPTTests:
|
||||
def __init__(self, log):
|
||||
self.log = log
|
||||
|
||||
def runTests(self, args):
|
||||
return True
|
||||
|
||||
|
||||
def running_env(**kw):
|
||||
return get_running_env(flavor="webpagetest", **kw)
|
||||
|
||||
|
||||
def init_placeholder_wpt_data(fvonly=False, invalid_results=False):
|
||||
views = {"firstView": {}}
|
||||
if not fvonly:
|
||||
views["repeatView"] = {}
|
||||
placeholder_data = {
|
||||
"data": {
|
||||
"summary": "websitelink.com",
|
||||
"location": "ec2-us-west-1:Firefox",
|
||||
"testRuns": 3,
|
||||
"successfulFVRuns": 3,
|
||||
"successfulRVRuns": 3,
|
||||
"fvonly": fvonly,
|
||||
"average": views,
|
||||
"standardDeviation": views,
|
||||
"median": views,
|
||||
}
|
||||
}
|
||||
exclude_metrics = 0 if not invalid_results else 2
|
||||
for metric in WPT_METRICS[exclude_metrics:]:
|
||||
for view in views:
|
||||
for stat in ACCEPTED_STATISTICS:
|
||||
placeholder_data["data"][stat][view][metric] = random.randint(0, 10000)
|
||||
placeholder_data["data"][stat][view][metric] = random.randint(0, 10000)
|
||||
placeholder_data["data"][stat][view][metric] = random.randint(0, 10000)
|
||||
return placeholder_data
|
||||
|
||||
|
||||
def init_mocked_request(status_code, **kwargs):
|
||||
mock_data = {
|
||||
"data": {
|
||||
"ec2-us-west-1": {"PendingTests": {"Queued": 3}, "Label": "California"},
|
||||
"jsonUrl": "mock_test.com",
|
||||
"summary": "Just a pageload test",
|
||||
},
|
||||
"statusCode": status_code,
|
||||
}
|
||||
for key, value in kwargs.items():
|
||||
mock_data["data"][key] = value
|
||||
mock_request = requests.Response()
|
||||
mock_request._content = json.dumps(mock_data).encode("utf-8")
|
||||
return mock_request
|
||||
|
||||
|
||||
@mock.patch(
|
||||
"mozperftest.test.webpagetest.WebPageTest.get_WPT_results",
|
||||
return_value=init_placeholder_wpt_data(),
|
||||
)
|
||||
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
||||
def test_webpagetest_no_issues_mocked_results(*mocked):
|
||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||
metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
|
||||
metadata.script["options"]["test_parameters"]["first_view_only"] = 0
|
||||
metadata.script["options"]["test_parameters"]["test_list"] = ["google.ca"]
|
||||
test.run(metadata)
|
||||
|
||||
|
||||
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
||||
def test_webpagetest_test_invalid_browser(*mocked):
|
||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||
metadata.script["options"]["test_parameters"]["browser"] = "Invalid Browser"
|
||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||
with pytest.raises(WPTBrowserSelectionError):
|
||||
test.run(metadata)
|
||||
|
||||
|
||||
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
||||
def test_webpagetest_test_invalid_connection(*mocked):
|
||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||
metadata.script["options"]["test_parameters"]["connection"] = "Invalid Connection"
|
||||
with pytest.raises(WPTInvalidConnectionSelection):
|
||||
test.run(metadata)
|
||||
|
||||
|
||||
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
||||
def test_webpagetest_test_invalid_url(*mocked):
|
||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||
metadata.script["options"]["test_list"] = ["InvalidUrl"]
|
||||
with pytest.raises(WPTInvalidURLError):
|
||||
test.run(metadata)
|
||||
|
||||
|
||||
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
||||
def test_webpagetest_test_invalid_statistic(*mocked):
|
||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||
metadata.script["options"]["test_parameters"]["statistics"] = ["Invalid Statistic"]
|
||||
with pytest.raises(WPTInvalidStatisticsError):
|
||||
test.run(metadata)
|
||||
assert True
|
||||
|
||||
|
||||
@mock.patch("requests.get", return_value=init_mocked_request(200))
|
||||
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
||||
def test_webpagetest_test_invalid_location(*mocked):
|
||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||
metadata.script["options"]["test_parameters"]["location"] = "Invalid Location"
|
||||
with pytest.raises(WPTLocationSelectionError):
|
||||
test.run(metadata)
|
||||
|
||||
|
||||
@mock.patch("requests.get", return_value=init_mocked_request(400))
|
||||
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
||||
def test_webpagetest_test_timeout(*mocked):
|
||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||
metadata.script["options"]["test_parameters"]["timeout_limit"] = 2
|
||||
metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
|
||||
with pytest.raises(WPTTimeOutError):
|
||||
test.run(metadata)
|
||||
assert True
|
||||
|
||||
|
||||
@mock.patch(
|
||||
"requests.get",
|
||||
return_value=init_mocked_request(200, testRuns=3, successfulFVRuns=2, fvonly=True),
|
||||
)
|
||||
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
||||
def test_webpagetest_test_fv_count_discrepancy(*mocked):
|
||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||
metadata.script["options"]["test_list"] = ["google.ca"]
|
||||
metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
|
||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||
with pytest.raises(WPTErrorWithWebsite):
|
||||
test.run(metadata)
|
||||
assert True
|
||||
|
||||
|
||||
@mock.patch(
|
||||
"requests.get",
|
||||
return_value=init_mocked_request(
|
||||
200, testRuns=3, successfulFVRuns=3, successfulRVRuns=2, fvonly=False
|
||||
),
|
||||
)
|
||||
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
||||
def test_webpagetest_test_rv_count_discrepancy(*mocked):
|
||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||
metadata.script["options"]["test_list"] = ["google.ca"]
|
||||
metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
|
||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||
with pytest.raises(WPTErrorWithWebsite):
|
||||
test.run(metadata)
|
||||
|
||||
|
||||
@mock.patch(
|
||||
"requests.get",
|
||||
return_value=init_mocked_request(
|
||||
200, testRuns=3, successfulFVRuns=3, fvonly=True, location="BadLocation"
|
||||
),
|
||||
)
|
||||
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
||||
def test_webpagetest_test_wrong_browserlocation(*mocked):
|
||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||
metadata.script["options"]["test_list"] = ["google.ca"]
|
||||
metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
|
||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||
with pytest.raises(WPTBrowserSelectionError):
|
||||
test.run(metadata)
|
||||
|
||||
|
||||
@mock.patch(
|
||||
"mozperftest.test.webpagetest.WebPageTest.get_WPT_results",
|
||||
return_value=init_placeholder_wpt_data(invalid_results=True),
|
||||
)
|
||||
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
||||
def test_webpagetest_test_metric_not_found(*mocked):
|
||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||
metadata.script["options"]["test_list"] = ["google.ca"]
|
||||
metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
|
||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||
with pytest.raises(WPTDataProcessingError):
|
||||
test.run(metadata)
|
||||
Loading…
Reference in New Issue
Block a user