mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-24 21:31:04 +00:00
Bug 1741975 - Have WPT tests available in taskcluster r=perftest-reviewers,sparky
Differential Revision: https://phabricator.services.mozilla.com/D140843
This commit is contained in:
andrej
parent
bff6a5c9a7
commit
6cbd9c7da7
@ -152,6 +152,7 @@ class WebPageTestData:
|
||||
{"file": "webpagetest", "value": value, "xaxis": xaxis}
|
||||
for xaxis, value in enumerate(data["values"])
|
||||
],
|
||||
"shouldAlert": True,
|
||||
}
|
||||
|
||||
def transform(self, data):
|
||||
@ -322,8 +323,8 @@ class WebPageTest(Layer):
|
||||
2) If repeat view is enabled and if testRuns requested does not equal successfulFVRuns
|
||||
and successfulRVRuns
|
||||
"""
|
||||
# TODO: establish a threshold for failures, maybe fail after data processing
|
||||
raise WPTErrorWithWebsite(
|
||||
# TODO: establish a threshold for failures, and consider failing see bug 1762470
|
||||
self.warning(
|
||||
f"Something went wrong with firstview/repeat view runs for: {url}"
|
||||
)
|
||||
self.confirm_correct_browser_and_location(
|
||||
@ -341,12 +342,14 @@ class WebPageTest(Layer):
|
||||
metadata.add_result(
|
||||
{
|
||||
"name": ("WebPageTest:" + re.match(r"(^.\w+)", website)[0]),
|
||||
"framework": {"name": "mozperftest:"},
|
||||
"framework": {"name": "mozperftest"},
|
||||
"transformer": "mozperftest.test.webpagetest:WebPageTestData",
|
||||
"shouldAlert": True,
|
||||
"results": [
|
||||
{
|
||||
"values": [int(metric_value)],
|
||||
"name": metric_name,
|
||||
"shouldAlert": True,
|
||||
}
|
||||
for metric_name, metric_value in requested_values.items()
|
||||
],
|
||||
|
||||
@ -1,51 +0,0 @@
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
/* eslint-env node */
|
||||
"use strict";
|
||||
|
||||
module.exports = {
|
||||
setUp,
|
||||
tearDown,
|
||||
test,
|
||||
owner: "Performance Testing Team",
|
||||
name: "WebpageTest Parameter list",
|
||||
description: "These are the arguments used when running the webpagetest Layer",
|
||||
options: {
|
||||
test_parameters : {
|
||||
location: "ec2-us-west-1",
|
||||
browser: "Firefox",
|
||||
connection: "Cable",
|
||||
timeout_limit: 21600,
|
||||
wait_between_requests: 5,
|
||||
statistics: ["average", "median", "standardDeviation"],
|
||||
label: "",
|
||||
runs: 3,
|
||||
fvonly: 0,
|
||||
private: 1,
|
||||
web10: 0,
|
||||
script: "",
|
||||
block: "",
|
||||
video: 1,
|
||||
tcpdump: 0,
|
||||
noimages: 0,
|
||||
keepua: 1,
|
||||
uastring: "",
|
||||
htmlbody: 0,
|
||||
custom: "",
|
||||
ignoreSSL: 0,
|
||||
appendua: "",
|
||||
injectScript: "",
|
||||
disableAVIF: 0,
|
||||
disableWEBP: 0,
|
||||
disableJXL: 0,
|
||||
},
|
||||
test_list: ["Google.com", "Youtube.com", "Facebook.com", "Qq.com",
|
||||
"Baidu.com", "Sohu.com", "360.cn", "Jd.com", "Amazon.com", "Yahoo.com", "Zoom.us",
|
||||
"Weibo.com", "Sina.com.cn", "Live.com", "Reddit.com", "Netflix.com",
|
||||
"Microsoft.com", "Instagram.com", "Panda.tv", "Google.com.hk", "Csdn.net",
|
||||
"Bing.com", "Vk.com", "Yahoo.co.jp", "Twitter.com", "Naver.com", "Canva.com", "Ebay.com",
|
||||
"Force.com", "Amazon.in", "Adobe.com", "Aliexpress.com", "Linkedin.com", "Tianya.cn",
|
||||
"Yy.com", "Huanqiu.com", "Amazon.co.jp", "Okezone.com"]
|
||||
},
|
||||
};
|
||||
@ -20,7 +20,7 @@ EXAMPLE_TESTS_DIR = os.path.join(HERE, "data", "samples")
|
||||
EXAMPLE_TEST = os.path.join(EXAMPLE_TESTS_DIR, "perftest_example.js")
|
||||
EXAMPLE_XPCSHELL_TEST = Path(EXAMPLE_TESTS_DIR, "test_xpcshell.js")
|
||||
EXAMPLE_XPCSHELL_TEST2 = Path(EXAMPLE_TESTS_DIR, "test_xpcshell_flavor2.js")
|
||||
EXAMPLE_WPT_TEST = Path(EXAMPLE_TESTS_DIR, "WPT_init_file.js")
|
||||
EXAMPLE_WPT_TEST = Path(ROOT, "testing", "performance", "perftest_WPT_init_file.js")
|
||||
BT_DATA = Path(HERE, "data", "browsertime-results", "browsertime.json")
|
||||
BT_DATA_VIDEO = Path(HERE, "data", "browsertime-results-video", "browsertime.json")
|
||||
DMG = Path(HERE, "data", "firefox.dmg")
|
||||
|
||||
@ -17,7 +17,6 @@ from mozperftest.test.webpagetest import (
|
||||
WPTInvalidConnectionSelection,
|
||||
ACCEPTED_STATISTICS,
|
||||
WPTInvalidStatisticsError,
|
||||
WPTErrorWithWebsite,
|
||||
WPTDataProcessingError,
|
||||
)
|
||||
|
||||
@ -41,7 +40,7 @@ def init_placeholder_wpt_data(fvonly=False, invalid_results=False):
|
||||
placeholder_data = {
|
||||
"data": {
|
||||
"summary": "websitelink.com",
|
||||
"location": "ec2-us-west-1:Firefox",
|
||||
"location": "ec2-us-east-1:Firefox",
|
||||
"testRuns": 3,
|
||||
"successfulFVRuns": 3,
|
||||
"successfulRVRuns": 3,
|
||||
@ -64,7 +63,7 @@ def init_placeholder_wpt_data(fvonly=False, invalid_results=False):
|
||||
def init_mocked_request(status_code, **kwargs):
|
||||
mock_data = {
|
||||
"data": {
|
||||
"ec2-us-west-1": {"PendingTests": {"Queued": 3}, "Label": "California"},
|
||||
"ec2-us-east-1": {"PendingTests": {"Queued": 3}, "Label": "California"},
|
||||
"jsonUrl": "mock_test.com",
|
||||
"summary": "Just a pageload test",
|
||||
},
|
||||
@ -150,37 +149,6 @@ def test_webpagetest_test_timeout(*mocked):
|
||||
assert True
|
||||
|
||||
|
||||
@mock.patch(
|
||||
"requests.get",
|
||||
return_value=init_mocked_request(200, testRuns=3, successfulFVRuns=2, fvonly=True),
|
||||
)
|
||||
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
||||
def test_webpagetest_test_fv_count_discrepancy(*mocked):
|
||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||
metadata.script["options"]["test_list"] = ["google.ca"]
|
||||
metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
|
||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||
with pytest.raises(WPTErrorWithWebsite):
|
||||
test.run(metadata)
|
||||
assert True
|
||||
|
||||
|
||||
@mock.patch(
|
||||
"requests.get",
|
||||
return_value=init_mocked_request(
|
||||
200, testRuns=3, successfulFVRuns=3, successfulRVRuns=2, fvonly=False
|
||||
),
|
||||
)
|
||||
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
||||
def test_webpagetest_test_rv_count_discrepancy(*mocked):
|
||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||
metadata.script["options"]["test_list"] = ["google.ca"]
|
||||
metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
|
||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||
with pytest.raises(WPTErrorWithWebsite):
|
||||
test.run(metadata)
|
||||
|
||||
|
||||
@mock.patch(
|
||||
"requests.get",
|
||||
return_value=init_mocked_request(
|
||||
|
||||
@ -34,6 +34,7 @@ suites:
|
||||
pageload: ""
|
||||
JSConf (warm): ""
|
||||
perfstats: ""
|
||||
webpagetest: ""
|
||||
browser/base/content/test:
|
||||
description: "Performance tests from the 'browser/base/content/test' folder."
|
||||
tests:
|
||||
|
||||
@ -37,6 +37,26 @@ try-xpcshell:
|
||||
--xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node
|
||||
--xpcshell-xre-path ${MOZ_FETCHES_DIR}/firefox
|
||||
|
||||
webpagetest:
|
||||
description: Run webpagetest performance pageload test against Alexa top 50 websites
|
||||
treeherder:
|
||||
symbol: perftest(linux-webpagetest)
|
||||
attributes:
|
||||
cron: true
|
||||
scopes:
|
||||
- secrets:get:project/perftest/gecko/level-{level}/perftest-login
|
||||
run:
|
||||
command: >-
|
||||
mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
|
||||
cd $MOZ_FETCHES_DIR &&
|
||||
python3.8 python/mozperftest/mozperftest/runner.py
|
||||
testing/performance/perftest_WPT_init_file.js
|
||||
--flavor=webpagetest
|
||||
--webpagetest
|
||||
--perfherder
|
||||
--perfherder-metrics median
|
||||
--output $MOZ_FETCHES_DIR/../artifacts
|
||||
|
||||
try-browsertime:
|
||||
description: Run ./mach perftest on Linux
|
||||
treeherder:
|
||||
|
||||
@ -211,5 +211,15 @@ perftest_perfstats.js
|
||||
|
||||
This test launches browsertime with the perfStats option (will collect low-overhead timings, see Bug 1553254). The test currently runs a short user journey. A selection of popular sites are visited, first as cold pageloads, and then as warm.
|
||||
|
||||
perftest_WPT_init_file.js
|
||||
=========================
|
||||
|
||||
:owner: Performance Testing Team
|
||||
:name: webpagetest
|
||||
|
||||
**Run webpagetest performance pageload test against Alexa top 50 websites**
|
||||
|
||||
This mozperftest gets webpagetest to run pageload tests against the 50 most popular websites and provide data. The full list of data returned from webpagetest: firstContentfulPaint, timeToContentfulPaint, visualComplete90, firstPaint, visualComplete99, visualComplete, SpeedIndex, bytesIn,bytesOut,TTFB,fullyLoadedCPUms, fullyLoadedCPUpct, domElements, domContentLoadedEventStart, domContentLoadedEventEnd, loadEventStart, loadEventEnd
|
||||
|
||||
|
||||
If you have any questions, please see this `wiki page <https://wiki.mozilla.org/TestEngineering/Performance#Where_to_find_us>`_.
|
||||
|
||||
@ -8,3 +8,4 @@
|
||||
[perftest_perfstats.js]
|
||||
[perftest_politico_link.js]
|
||||
[perftest_youtube_link.js]
|
||||
[perftest_WPT_init_file.js]
|
||||
|
||||
94
testing/performance/perftest_WPT_init_file.js
Normal file
94
testing/performance/perftest_WPT_init_file.js
Normal file
@ -0,0 +1,94 @@
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
/* eslint-env node */
|
||||
"use strict";
|
||||
function setUp() {}
|
||||
|
||||
function tearDown() {}
|
||||
|
||||
function test() {}
|
||||
|
||||
module.exports = {
|
||||
setUp,
|
||||
tearDown,
|
||||
test,
|
||||
owner: "Performance Testing Team",
|
||||
name: "webpagetest",
|
||||
description:
|
||||
"Run webpagetest performance pageload test against Alexa top 50 websites",
|
||||
longDescription: `This mozperftest gets webpagetest to run pageload tests against the 50 most popular websites and provide data.
|
||||
The full list of data returned from webpagetest: firstContentfulPaint, timeToContentfulPaint, visualComplete90,
|
||||
firstPaint, visualComplete99, visualComplete, SpeedIndex, bytesIn,bytesOut,TTFB,fullyLoadedCPUms, fullyLoadedCPUpct,
|
||||
domElements, domContentLoadedEventStart, domContentLoadedEventEnd, loadEventStart, loadEventEnd`,
|
||||
options: {
|
||||
test_parameters: {
|
||||
location: "ec2-us-east-1",
|
||||
browser: "Firefox",
|
||||
connection: "Cable",
|
||||
timeout_limit: 21600,
|
||||
wait_between_requests: 5,
|
||||
statistics: ["average", "median", "standardDeviation"],
|
||||
label: "",
|
||||
runs: 3,
|
||||
fvonly: 0,
|
||||
private: 1,
|
||||
web10: 0,
|
||||
script: "",
|
||||
block: "",
|
||||
video: 1,
|
||||
tcpdump: 0,
|
||||
noimages: 0,
|
||||
keepua: 1,
|
||||
uastring: "",
|
||||
htmlbody: 0,
|
||||
custom: "",
|
||||
ignoreSSL: 0,
|
||||
appendua: "",
|
||||
injectScript: "",
|
||||
disableAVIF: 0,
|
||||
disableWEBP: 0,
|
||||
disableJXL: 0,
|
||||
},
|
||||
test_list: [
|
||||
"google.com",
|
||||
"youtube.com",
|
||||
"facebook.com",
|
||||
"qq.com",
|
||||
"baidu.com",
|
||||
"sohu.com",
|
||||
"360.cn",
|
||||
"jd.com",
|
||||
"amazon.com",
|
||||
"yahoo.com",
|
||||
"zoom.us",
|
||||
"weibo.com",
|
||||
"sina.com.cn",
|
||||
"live.com",
|
||||
"reddit.com",
|
||||
"netflix.com",
|
||||
"microsoft.com",
|
||||
"instagram.com",
|
||||
"panda.tv",
|
||||
"google.com.hk",
|
||||
"csdn.net",
|
||||
"bing.com",
|
||||
"vk.com",
|
||||
"yahoo.co.jp",
|
||||
"twitter.com",
|
||||
"naver.com",
|
||||
"canva.com",
|
||||
"ebay.com",
|
||||
"force.com",
|
||||
"amazon.in",
|
||||
"adobe.com",
|
||||
"aliexpress.com",
|
||||
"linkedin.com",
|
||||
"tianya.cn",
|
||||
"yy.com",
|
||||
"huanqiu.com",
|
||||
"amazon.co.jp",
|
||||
"okezone.com",
|
||||
],
|
||||
},
|
||||
};
|
||||
Loading…
Reference in New Issue
Block a user