Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-23 12:51:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1855674 - Modify pdfpaint test to run more PDFs. r=aglavic,perftest-reviewers

Browse Source 
This patch modifies the pdfpaint test to run more pdfs that are found in the Mozilla pdf.js repository. The pdfpaint test is also moved to it's own suite due to the number of PDFs now being tested. These PDFs are pulled in locally from a toolchain task called talos-pdfs. The *ignore files are modified since the pdfpaint folder now contains a symbolic link to the local PDFs that should not be commit in-tree.

To handle running the large number of PDFs, chunking is added to the test with the chunk size being 100 PDFs. Each chunk runs each of the 100 PDFs 5 times. A CLI option is also added for local runs so that users can select a specific pdfpaint PDF to test. An additional issue with the subtest/pdf file name parsing is also fixed for this to work.

Differential Revision: https://phabricator.services.mozilla.com/D205824
This commit is contained in:
Greg Mierzwinski 2024-04-08 11:47:05 +00:00
parent d24c07b079
commit 7948524fe0
10 changed files with 148 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -262,6 +262,7 @@ testing/talos/talos/fis/tp5n
testing/talos/talos/tests/tp5n.zip
testing/talos/talos/tests/tp5n.tar.gz
testing/talos/talos/tests/tp5n
testing/talos/talos/tests/pdfpaint/pdfs
testing/talos/talos/tests/devtools/damp.manifest.develop
testing/talos/talos/startup_test/startup_about_home_paint/startup_about_home_paint.manifest.develop
testing/talos/talos/webextensions/

1
.hgignore
View File

@ -258,6 +258,7 @@ _OPT\.OBJ/
^testing/talos/talos/tests/tp5n\.zip
^testing/talos/talos/tests/tp5n\.tar\.gz
^testing/talos/talos/tests/tp5n
^testing/talos/talos/tests/pdfpaint/pdfs
^testing/talos/talos/tests/devtools/damp\.manifest\.develop
^testing/talos/talos/startup_test/startup_about_home_paint/startup_about_home_paint\.manifest\.develop
^testing/talos/talos/webextensions/

17
testing/mozharness/mozharness/mozilla/testing/talos.py
View File

@ -280,6 +280,18 @@ class Talos(
"help": "Take a screenshot when the test fails.",
},
],
[
["--pdfPaintChunk"],
{
"type": "int",
"dest": "pdfpaint_chunk",
"default": None,
"help": (
"Chunk of the pdfpaint test to run (each chunk runs at most 100 pdfs). "
"Defaults to None to run all the pdfs at the same time."
),
},
],
]
+ testing_config_options
+ copy.deepcopy(code_coverage_config_options)
@ -558,6 +570,8 @@ class Talos(
kw_options["symbolsPath"] = self.symbols_path
if self.config.get("project", None):
kw_options["project"] = self.config["project"]
if self.config.get("pdfpaint_chunk", None):
kw_options["pdfPaintChunk"] = str(self.config["pdfpaint_chunk"])
kw_options.update(kw)
# talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg'
@ -850,6 +864,9 @@ class Talos(
env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"]
if not self.run_local:
env["MINIDUMP_STACKWALK"] = self.query_minidump_stackwalk()
env["MOZ_FETCHES_DIR"] = os.environ.get("MOZ_FETCHES_DIR")
else:
env["MOZBUILD_PATH"] = self.config.get("mozbuild_path")
env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"]
env["RUST_BACKTRACE"] = "full"
if not os.path.isdir(env["MOZ_UPLOAD_DIR"]):

182
testing/perfdocs/generated/talos.rst
View File

@ -7992,11 +7992,18 @@ For the sample commands found below, note that the capitalization used is import
* source:
* type: `Page load`_
* reporting: time from *performance.timing.navigationStart* to *pagerendered* event in ms (lower is better)
* data: load a PDF 20 times
* data: loads a PDF 5 times
* description:
Runs through a set of chunks. Each chunk runs 100 PDFs with 5 iterations each.
If --pdfPaintChunk is not used when running the test locally, all PDFs will be tested
by default with only 1 cycle each. The PDFs that are run are found in the Mozilla pdf.js
repository, and this test pulls those in for testing locally through a toolchain artifact
called talos-pdfs.
* pdfpaint: True
* timeout: 600
* timeout: 1800
* tpmanifest: ${talos}/tests/pdfpaint/pdfpaint.manifest
* tppagecycles: 20
* tppagecycles: 1
* tptimeout: 60000
* unit: ms
* Command
@ -8004,175 +8011,6 @@ For the sample commands found below, note that the capitalization used is import
./mach talos-test -a pdfpaint
* **Test Task**:
.. list-table:: **test-linux1804-64-qr/opt**
:widths: 30 15 15 15 15
:header-rows: 1
* - **Test Name**
- mozilla-central
- autoland
- mozilla-release
- mozilla-beta
* - **talos-other**
- ❌
- ❌
- ❌
- ❌
* - **talos-other-swr**
- ❌
- ❌
- ❌
- ❌
.. list-table:: **test-linux1804-64-shippable-qr/opt**
:widths: 30 15 15 15 15
:header-rows: 1
* - **Test Name**
- mozilla-central
- autoland
- mozilla-release
- mozilla-beta
* - **talos-other**
- ✅
- ✅
- ❌
- ❌
* - **talos-other-profiling**
- ❌
- ❌
- ❌
- ❌
* - **talos-other-swr**
- ✅
- ✅
- ❌
- ❌
.. list-table:: **test-macosx1015-64-shippable-qr/opt**
:widths: 30 15 15 15 15
:header-rows: 1
* - **Test Name**
- mozilla-central
- autoland
- mozilla-release
- mozilla-beta
* - **talos-other**
- ✅
- ✅
- ❌
- ❌
* - **talos-other-profiling**
- ❌
- ❌
- ❌
- ❌
* - **talos-other-swr**
- ✅
- ✅
- ❌
- ❌
.. list-table:: **test-windows10-32-qr/opt**
:widths: 30 15 15 15 15
:header-rows: 1
* - **Test Name**
- mozilla-central
- autoland
- mozilla-release
- mozilla-beta
* - **talos-other**
- ❌
- ❌
- ❌
- ❌
* - **talos-other-swr**
- ❌
- ❌
- ❌
- ❌
.. list-table:: **test-windows10-32-shippable-qr/opt**
:widths: 30 15 15 15 15
:header-rows: 1
* - **Test Name**
- mozilla-central
- autoland
- mozilla-release
- mozilla-beta
* - **talos-other**
- ❌
- ❌
- ❌
- ❌
* - **talos-other-profiling**
- ❌
- ❌
- ❌
- ❌
* - **talos-other-swr**
- ❌
- ❌
- ❌
- ❌
.. list-table:: **test-windows10-64-qr/opt**
:widths: 30 15 15 15 15
:header-rows: 1
* - **Test Name**
- mozilla-central
- autoland
- mozilla-release
- mozilla-beta
* - **talos-other**
- ❌
- ❌
- ❌
- ❌
* - **talos-other-swr**
- ❌
- ❌
- ❌
- ❌
.. list-table:: **test-windows10-64-shippable-qr/opt**
:widths: 30 15 15 15 15
:header-rows: 1
* - **Test Name**
- mozilla-central
- autoland
- mozilla-release
- mozilla-beta
* - **talos-other**
- ✅
- ✅
- ❌
- ❌
* - **talos-other-profiling**
- ❌
- ❌
- ❌
- ❌
* - **talos-other-swr**
- ✅
- ✅
- ❌
- ❌
.. dropdown:: perf_reftest
:class-container: anchor-id-perf_reftest

8
testing/talos/perfdocs/config.yml
View File

@ -607,7 +607,13 @@ suites:
- source:
- type: `Page load`_
- reporting: time from *performance.timing.navigationStart* to *pagerendered* event in ms (lower is better)
- data: load a PDF 20 times
- data: loads a PDF 5 times
- description:
Runs through a set of chunks. Each chunk runs 100 PDFs with 5 iterations each.
If --pdfPaintChunk is not used when running the test locally, all PDFs will be tested
by default with only 1 cycle each. The PDFs that are run are found in the Mozilla pdf.js
repository, and this test pulls those in for testing locally through a toolchain artifact
called talos-pdfs.
perf_reftest: >
- contact: :emilio and css/layout team
- source: `perf-reftest <https://dxr.mozilla.org/mozilla-central/source/testing/talos/talos/tests/perf-reftest>`__

4
testing/talos/talos.json
View File

@ -19,7 +19,6 @@
"tabpaint",
"cpstartup",
"startup_about_home_paint",
"pdfpaint",
"cross_origin_pageload",
"startup_about_home_paint_cached"
]
@ -121,6 +120,9 @@
"realworld-webextensions": {
"tests": ["startup_about_home_paint_realworld_webextensions"],
"webextensions_zip": "webextensions.zip"
},
"pdfpaint": {
"tests": ["pdfpaint"]
}
}
}

20
testing/talos/talos/cmdline.py
View File

@ -223,6 +223,26 @@ def create_parser(mach_interface=False):
dest="pdfpaint",
help="Wait for the first page of a PDF to be rendered",
)
add_arg(
"--pdfPaintChunk",
type=int,
default=None,
dest="pdfpaint_chunk",
help=(
"Chunk of the pdfpaint test to run (each chunk runs at most 100 pdfs). "
"Defaults to None to run all the pdfs at the same time."
),
)
add_arg(
"--pdfPaintName",
type=str,
default=None,
dest="pdfpaint_name",
help=(
"Name of a pdfpaint test to run (e.g. xfa_imm5257e.pdf). Chunking will be "
"ignored/disabled if this option is used."
),
)
add_arg("--webServer", dest="webserver", help="DEPRECATED")
if not mach_interface:
add_arg(

81
testing/talos/talos/config.py
View File

@ -2,7 +2,9 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import copy
import json
import os
import pathlib
import re
import sys
@ -198,6 +200,77 @@ def get_global_overrides(config):
return global_overrides
def setup_pdfpaint_test(config, test_instance):
# Get the root of the location of the PDFs artifact
if os.environ.get("MOZ_FETCHES_DIR", None):
pdfs_root = pathlib.Path(os.environ.get("MOZ_FETCHES_DIR"), "talos-pdfs")
else:
pdfs_root = pathlib.Path(os.environ.get("MOZBUILD_PATH"), "talos-pdfs")
if not pdfs_root.exists():
raise Exception(f"Cannot find webserver root: {pdfs_root}")
pdfpaint_manifest_path = pathlib.Path(pdfs_root, "pdfpaint.manifest")
test_manifest_path = pathlib.Path(pdfs_root, "test_manifest.json")
test_manifest = json.loads(test_manifest_path.read_text(encoding="utf8"))
# If a pdfpaint test was specified, prevent any chunking
chunk_number = config.get("pdfpaint_chunk", None)
pdfpaint_test = None
if config.get("pdfpaint_name") is not None:
chunk_number = None
pdfpaint_test = config["pdfpaint_name"]
# Gather all the pdf files that can be used in the test, and write
# all the pdfs to be tested to the manifest file
start_ind = 0
end_ind = None
pdfs_per_chunk = 100
if chunk_number is not None:
start_ind = pdfs_per_chunk * (chunk_number - 1)
end_ind = pdfs_per_chunk * chunk_number
pdf_files = set()
for pdf_info in test_manifest:
if pdf_info.get("password", None) is not None:
# PDFs that require passwords cause timeouts
continue
pdf_name = pathlib.Path(pdf_info["file"]).name
if (
config.get("pdfpaint_name", None) is not None
and config["pdfpaint_name"] != pdf_name
):
# If a user passed a name of a pdf, skip all the others
continue
pdf_files.add(pdf_name)
if start_ind > len(pdf_files):
raise ConfigurationError(
f"Chunk {chunk_number} contains no PDFs to test. "
f"For {len(pdf_files)} PDFs, the max chunk is "
f"{int((len(pdf_files)-1)/pdfs_per_chunk)+1}."
)
with pdfpaint_manifest_path.open("w") as f:
for pdf_file in sorted(list(pdf_files))[start_ind:end_ind]:
print(f"http://localhost/tests/pdfpaint/pdfs/{pdf_file}{os.linesep}")
f.write(f"http://localhost/tests/pdfpaint/pdfs/{pdf_file}{os.linesep}")
# Make a symbolic link to the mozbuild pdf folder since the talos
# webserver depends on having the doc root as the talos folder for getInfo.html
symlink_dest = pathlib.Path(__file__).parent / "tests" / "pdfpaint" / "pdfs"
symlink_dest.unlink(missing_ok=True)
symlink_dest.symlink_to(pdfs_root, target_is_directory=True)
test_instance.tpmanifest = str(pdfpaint_manifest_path)
# Increase the pagecycles for each pdf to 5 if we're running chunks, otherwise
# it can take a very long time to complete testing of all pdfs
if chunk_number is not None or pdfpaint_test is not None:
print("Setting pdfpaint tppagecycles to 5")
test_instance.tppagecycles = 5
def get_test_host(manifest_line):
match = re.match(r"^http://localhost/page_load_test/tp5n/([^/]+)/", manifest_line)
host = match.group(1)
@ -248,6 +321,9 @@ def get_test(config, global_overrides, counters, test_instance):
if pdfPaint is not None:
test_instance.pdfpaint = pdfPaint
if test_instance.pdfpaint:
setup_pdfpaint_test(config, test_instance)
# fix up url
url = getattr(test_instance, "url", None)
if url:
@ -359,6 +435,11 @@ def get_config(argv=None):
elif not cli_opts.activeTests:
raise ConfigurationError("--activeTests or --suite required!")
if cli_opts.pdfpaint_chunk is not None and cli_opts.pdfpaint_chunk < 1:
raise ConfigurationError(
"pdfpaint chunk must be a positive integer greater than or equal to 1"
)
cli_opts = parse_args(argv=argv)
setup_logging("talos", cli_opts, {"tbpl": sys.stdout})
config = copy.deepcopy(DEFAULTS)

5
testing/talos/talos/pageloader/chrome/report.js
View File

@ -4,6 +4,11 @@
// given an array of strings, finds the longest common prefix
function findCommonPrefixLength(strs) {
if (strs.every(str => str.includes("/pdfs/"))) {
// In all cases for pdfpaint PDFs, return the full file name
return strs[0].lastIndexOf("/") + 1;
}
if (strs.length < 2) {
// only one page in the manifest
// i.e. http://localhost/tests/perf-reftest/bloom-basic.html

5
testing/talos/talos/test.py
View File

@ -419,8 +419,9 @@ class pdfpaint(PageloaderTest):
"""
tpmanifest = "${talos}/tests/pdfpaint/pdfpaint.manifest"
tppagecycles = 20
timeout = 600
tppagecycles = 1
timeout = 1800
tptimeout = 60000
pdfpaint = True
unit = "ms"