diff --git a/python/mozperftest/mozperftest/test/webpagetest.py b/python/mozperftest/mozperftest/test/webpagetest.py index 41f782cc4582..eb1e49395005 100644 --- a/python/mozperftest/mozperftest/test/webpagetest.py +++ b/python/mozperftest/mozperftest/test/webpagetest.py @@ -152,6 +152,7 @@ class WebPageTestData: {"file": "webpagetest", "value": value, "xaxis": xaxis} for xaxis, value in enumerate(data["values"]) ], + "shouldAlert": True, } def transform(self, data): @@ -322,8 +323,8 @@ class WebPageTest(Layer): 2) If repeat view is enabled and if testRuns requested does not equal successfulFVRuns and successfulRVRuns """ - # TODO: establish a threshold for failures, maybe fail after data processing - raise WPTErrorWithWebsite( + # TODO: establish a threshold for failures, and consider failing see bug 1762470 + self.warning( f"Something went wrong with firstview/repeat view runs for: {url}" ) self.confirm_correct_browser_and_location( @@ -341,12 +342,14 @@ class WebPageTest(Layer): metadata.add_result( { "name": ("WebPageTest:" + re.match(r"(^.\w+)", website)[0]), - "framework": {"name": "mozperftest:"}, + "framework": {"name": "mozperftest"}, "transformer": "mozperftest.test.webpagetest:WebPageTestData", + "shouldAlert": True, "results": [ { "values": [int(metric_value)], "name": metric_name, + "shouldAlert": True, } for metric_name, metric_value in requested_values.items() ], diff --git a/python/mozperftest/mozperftest/tests/data/samples/WPT_init_file.js b/python/mozperftest/mozperftest/tests/data/samples/WPT_init_file.js deleted file mode 100644 index 7e854172e4bf..000000000000 --- a/python/mozperftest/mozperftest/tests/data/samples/WPT_init_file.js +++ /dev/null @@ -1,51 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. -/* eslint-env node */ -"use strict"; - -module.exports = { - setUp, - tearDown, - test, - owner: "Performance Testing Team", - name: "WebpageTest Parameter list", - description: "These are the arguments used when running the webpagetest Layer", - options: { - test_parameters : { - location: "ec2-us-west-1", - browser: "Firefox", - connection: "Cable", - timeout_limit: 21600, - wait_between_requests: 5, - statistics: ["average", "median", "standardDeviation"], - label: "", - runs: 3, - fvonly: 0, - private: 1, - web10: 0, - script: "", - block: "", - video: 1, - tcpdump: 0, - noimages: 0, - keepua: 1, - uastring: "", - htmlbody: 0, - custom: "", - ignoreSSL: 0, - appendua: "", - injectScript: "", - disableAVIF: 0, - disableWEBP: 0, - disableJXL: 0, - }, - test_list: ["Google.com", "Youtube.com", "Facebook.com", "Qq.com", - "Baidu.com", "Sohu.com", "360.cn", "Jd.com", "Amazon.com", "Yahoo.com", "Zoom.us", - "Weibo.com", "Sina.com.cn", "Live.com", "Reddit.com", "Netflix.com", - "Microsoft.com", "Instagram.com", "Panda.tv", "Google.com.hk", "Csdn.net", - "Bing.com", "Vk.com", "Yahoo.co.jp", "Twitter.com", "Naver.com", "Canva.com", "Ebay.com", - "Force.com", "Amazon.in", "Adobe.com", "Aliexpress.com", "Linkedin.com", "Tianya.cn", - "Yy.com", "Huanqiu.com", "Amazon.co.jp", "Okezone.com"] - }, -}; diff --git a/python/mozperftest/mozperftest/tests/support.py b/python/mozperftest/mozperftest/tests/support.py index e9af4fa52c50..c113050014e4 100644 --- a/python/mozperftest/mozperftest/tests/support.py +++ b/python/mozperftest/mozperftest/tests/support.py @@ -20,7 +20,7 @@ EXAMPLE_TESTS_DIR = os.path.join(HERE, "data", "samples") EXAMPLE_TEST = os.path.join(EXAMPLE_TESTS_DIR, "perftest_example.js") EXAMPLE_XPCSHELL_TEST = Path(EXAMPLE_TESTS_DIR, "test_xpcshell.js") EXAMPLE_XPCSHELL_TEST2 = Path(EXAMPLE_TESTS_DIR, "test_xpcshell_flavor2.js") -EXAMPLE_WPT_TEST = Path(EXAMPLE_TESTS_DIR, "WPT_init_file.js") +EXAMPLE_WPT_TEST = Path(ROOT, "testing", "performance", "perftest_WPT_init_file.js") BT_DATA = Path(HERE, "data", "browsertime-results", "browsertime.json") BT_DATA_VIDEO = Path(HERE, "data", "browsertime-results-video", "browsertime.json") DMG = Path(HERE, "data", "firefox.dmg") diff --git a/python/mozperftest/mozperftest/tests/test_webpagetest.py b/python/mozperftest/mozperftest/tests/test_webpagetest.py index b8103bcb1073..1d7910423b4a 100644 --- a/python/mozperftest/mozperftest/tests/test_webpagetest.py +++ b/python/mozperftest/mozperftest/tests/test_webpagetest.py @@ -17,7 +17,6 @@ from mozperftest.test.webpagetest import ( WPTInvalidConnectionSelection, ACCEPTED_STATISTICS, WPTInvalidStatisticsError, - WPTErrorWithWebsite, WPTDataProcessingError, ) @@ -41,7 +40,7 @@ def init_placeholder_wpt_data(fvonly=False, invalid_results=False): placeholder_data = { "data": { "summary": "websitelink.com", - "location": "ec2-us-west-1:Firefox", + "location": "ec2-us-east-1:Firefox", "testRuns": 3, "successfulFVRuns": 3, "successfulRVRuns": 3, @@ -64,7 +63,7 @@ def init_placeholder_wpt_data(fvonly=False, invalid_results=False): def init_mocked_request(status_code, **kwargs): mock_data = { "data": { - "ec2-us-west-1": {"PendingTests": {"Queued": 3}, "Label": "California"}, + "ec2-us-east-1": {"PendingTests": {"Queued": 3}, "Label": "California"}, "jsonUrl": "mock_test.com", "summary": "Just a pageload test", }, @@ -150,37 +149,6 @@ def test_webpagetest_test_timeout(*mocked): assert True -@mock.patch( - "requests.get", - return_value=init_mocked_request(200, testRuns=3, successfulFVRuns=2, fvonly=True), -) -@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"}) -def test_webpagetest_test_fv_count_discrepancy(*mocked): - mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) - metadata.script["options"]["test_list"] = ["google.ca"] - metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1 - test = webpagetest.WebPageTest(env, mach_cmd) - with pytest.raises(WPTErrorWithWebsite): - test.run(metadata) - assert True - - -@mock.patch( - "requests.get", - return_value=init_mocked_request( - 200, testRuns=3, successfulFVRuns=3, successfulRVRuns=2, fvonly=False - ), -) -@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"}) -def test_webpagetest_test_rv_count_discrepancy(*mocked): - mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) - metadata.script["options"]["test_list"] = ["google.ca"] - metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1 - test = webpagetest.WebPageTest(env, mach_cmd) - with pytest.raises(WPTErrorWithWebsite): - test.run(metadata) - - @mock.patch( "requests.get", return_value=init_mocked_request( diff --git a/python/mozperftest/perfdocs/config.yml b/python/mozperftest/perfdocs/config.yml index 7b806547856b..7422344be8eb 100644 --- a/python/mozperftest/perfdocs/config.yml +++ b/python/mozperftest/perfdocs/config.yml @@ -34,6 +34,7 @@ suites: pageload: "" JSConf (warm): "" perfstats: "" + webpagetest: "" browser/base/content/test: description: "Performance tests from the 'browser/base/content/test' folder." tests: diff --git a/taskcluster/ci/perftest/linux.yml b/taskcluster/ci/perftest/linux.yml index caac30c66bad..38eb2dbe0984 100644 --- a/taskcluster/ci/perftest/linux.yml +++ b/taskcluster/ci/perftest/linux.yml @@ -37,6 +37,26 @@ try-xpcshell: --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node --xpcshell-xre-path ${MOZ_FETCHES_DIR}/firefox +webpagetest: + description: Run webpagetest performance pageload test against Alexa top 50 websites + treeherder: + symbol: perftest(linux-webpagetest) + attributes: + cron: true + scopes: + - secrets:get:project/perftest/gecko/level-{level}/perftest-login + run: + command: >- + mkdir -p $MOZ_FETCHES_DIR/../artifacts && + cd $MOZ_FETCHES_DIR && + python3.8 python/mozperftest/mozperftest/runner.py + testing/performance/perftest_WPT_init_file.js + --flavor=webpagetest + --webpagetest + --perfherder + --perfherder-metrics median + --output $MOZ_FETCHES_DIR/../artifacts + try-browsertime: description: Run ./mach perftest on Linux treeherder: diff --git a/testing/perfdocs/generated/mozperftest.rst b/testing/perfdocs/generated/mozperftest.rst index 4d553ef5fb0d..1438b8e85967 100644 --- a/testing/perfdocs/generated/mozperftest.rst +++ b/testing/perfdocs/generated/mozperftest.rst @@ -211,5 +211,15 @@ perftest_perfstats.js This test launches browsertime with the perfStats option (will collect low-overhead timings, see Bug 1553254). The test currently runs a short user journey. A selection of popular sites are visited, first as cold pageloads, and then as warm. +perftest_WPT_init_file.js +========================= + +:owner: Performance Testing Team +:name: webpagetest + +**Run webpagetest performance pageload test against Alexa top 50 websites** + +This mozperftest gets webpagetest to run pageload tests against the 50 most popular websites and provide data. The full list of data returned from webpagetest: firstContentfulPaint, timeToContentfulPaint, visualComplete90, firstPaint, visualComplete99, visualComplete, SpeedIndex, bytesIn,bytesOut,TTFB,fullyLoadedCPUms, fullyLoadedCPUpct, domElements, domContentLoadedEventStart, domContentLoadedEventEnd, loadEventStart, loadEventEnd + If you have any questions, please see this `wiki page `_. diff --git a/testing/performance/perftest.ini b/testing/performance/perftest.ini index b7cbf2bc8f95..62b28f335721 100644 --- a/testing/performance/perftest.ini +++ b/testing/performance/perftest.ini @@ -8,3 +8,4 @@ [perftest_perfstats.js] [perftest_politico_link.js] [perftest_youtube_link.js] +[perftest_WPT_init_file.js] diff --git a/testing/performance/perftest_WPT_init_file.js b/testing/performance/perftest_WPT_init_file.js new file mode 100644 index 000000000000..7c1ec72e8b1d --- /dev/null +++ b/testing/performance/perftest_WPT_init_file.js @@ -0,0 +1,94 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +/* eslint-env node */ +"use strict"; +function setUp() {} + +function tearDown() {} + +function test() {} + +module.exports = { + setUp, + tearDown, + test, + owner: "Performance Testing Team", + name: "webpagetest", + description: + "Run webpagetest performance pageload test against Alexa top 50 websites", + longDescription: `This mozperftest gets webpagetest to run pageload tests against the 50 most popular websites and provide data. + The full list of data returned from webpagetest: firstContentfulPaint, timeToContentfulPaint, visualComplete90, + firstPaint, visualComplete99, visualComplete, SpeedIndex, bytesIn,bytesOut,TTFB,fullyLoadedCPUms, fullyLoadedCPUpct, + domElements, domContentLoadedEventStart, domContentLoadedEventEnd, loadEventStart, loadEventEnd`, + options: { + test_parameters: { + location: "ec2-us-east-1", + browser: "Firefox", + connection: "Cable", + timeout_limit: 21600, + wait_between_requests: 5, + statistics: ["average", "median", "standardDeviation"], + label: "", + runs: 3, + fvonly: 0, + private: 1, + web10: 0, + script: "", + block: "", + video: 1, + tcpdump: 0, + noimages: 0, + keepua: 1, + uastring: "", + htmlbody: 0, + custom: "", + ignoreSSL: 0, + appendua: "", + injectScript: "", + disableAVIF: 0, + disableWEBP: 0, + disableJXL: 0, + }, + test_list: [ + "google.com", + "youtube.com", + "facebook.com", + "qq.com", + "baidu.com", + "sohu.com", + "360.cn", + "jd.com", + "amazon.com", + "yahoo.com", + "zoom.us", + "weibo.com", + "sina.com.cn", + "live.com", + "reddit.com", + "netflix.com", + "microsoft.com", + "instagram.com", + "panda.tv", + "google.com.hk", + "csdn.net", + "bing.com", + "vk.com", + "yahoo.co.jp", + "twitter.com", + "naver.com", + "canva.com", + "ebay.com", + "force.com", + "amazon.in", + "adobe.com", + "aliexpress.com", + "linkedin.com", + "tianya.cn", + "yy.com", + "huanqiu.com", + "amazon.co.jp", + "okezone.com", + ], + }, +};