mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-23 12:51:06 +00:00
Bug 1920813 - Add performance test for WebCodecs video encoding r=perftest-reviewers,media-playback-reviewers,aosmond,padenot,sparky
This patch introduces performance tests to evaluate the video encoding performance in two usage scenarios: *Realtime* and *Record*. Each scenario assesses the encoding using two key metrics: 1. Frame Encoding Latency: The time from encoding a single frame to receiving its encoded result 2. Total Encoding Time: The duration from the first encoding request to the reception of the last encoded result In the *Realtime* scenario, minimizing the latency for each frame is crucial. This metric is essential for applications like live streaming and video conferencing, where timely delivery of each frame impacts user experience. In contrast, the *Record* scenarios prioritizes the total encoding time. Here, the focus is on processing the entire sequence efficiently to produce the final output. By adding these tests, we aim to thoroughly assess and optimize the encoder's performance under both scenarios, ensuring Gecko provides the experience users expect. Differential Revision: https://phabricator.services.mozilla.com/D224999
This commit is contained in:
Chun-Min Chang
parent
cba99bcabc
commit
f64dbf04f0
@ -7,7 +7,10 @@
|
||||
with Files("*"):
|
||||
BUG_COMPONENT = ("Core", "Audio/Video: Web Codecs")
|
||||
|
||||
MOCHITEST_MANIFESTS += ["test/mochitest.toml"]
|
||||
MOCHITEST_MANIFESTS += [
|
||||
"test/mochitest.toml",
|
||||
"test/performance/perftest.toml",
|
||||
]
|
||||
CRASHTEST_MANIFESTS += ["crashtests/crashtests.list"]
|
||||
|
||||
# For mozilla/layers/ImageBridgeChild.h
|
||||
|
||||
57
dom/media/webcodecs/test/performance/encode_from_canvas.js
Normal file
57
dom/media/webcodecs/test/performance/encode_from_canvas.js
Normal file
@ -0,0 +1,57 @@
|
||||
let encoder;
|
||||
let encodeTimes = [];
|
||||
let outputTimes = [];
|
||||
|
||||
function reset() {
|
||||
encoder = null;
|
||||
encodeTimes.length = 0;
|
||||
outputTimes.length = 0;
|
||||
}
|
||||
|
||||
self.onmessage = async event => {
|
||||
if (event.data.command === "configure") {
|
||||
const { codec, width, height, latencyMode, avcFormat } = event.data;
|
||||
|
||||
encoder = new VideoEncoder({
|
||||
output: (chunk, _) => {
|
||||
const endTime = performance.now();
|
||||
outputTimes.push({
|
||||
timestamp: chunk.timestamp,
|
||||
time: endTime,
|
||||
type: chunk.type,
|
||||
});
|
||||
},
|
||||
error: _ => {},
|
||||
});
|
||||
|
||||
const config = {
|
||||
codec,
|
||||
width,
|
||||
height,
|
||||
bitrate: 1000000, // 1 Mbps
|
||||
framerate: 30,
|
||||
latencyMode,
|
||||
};
|
||||
|
||||
// Add H264 specific configuration
|
||||
if (codec.startsWith("avc1")) {
|
||||
config.avc = { format: avcFormat };
|
||||
}
|
||||
|
||||
encoder.configure(config);
|
||||
} else if (event.data.command === "encode") {
|
||||
const { frame, isKey } = event.data;
|
||||
const encodeTime = performance.now();
|
||||
encoder.encode(frame, { keyFrame: isKey });
|
||||
encodeTimes.push({
|
||||
timestamp: frame.timestamp,
|
||||
time: encodeTime,
|
||||
type: isKey ? "key" : "delta",
|
||||
});
|
||||
frame.close();
|
||||
} else if (event.data.command === "flush") {
|
||||
await encoder.flush();
|
||||
self.postMessage({ command: "result", encodeTimes, outputTimes });
|
||||
reset();
|
||||
}
|
||||
};
|
||||
9
dom/media/webcodecs/test/performance/perftest.toml
Normal file
9
dom/media/webcodecs/test/performance/perftest.toml
Normal file
@ -0,0 +1,9 @@
|
||||
[DEFAULT]
|
||||
prefs = ["dom.media.webcodecs.enabled=true"]
|
||||
scheme = "https"
|
||||
support-files = [
|
||||
"encode_from_canvas.js",
|
||||
]
|
||||
|
||||
["test_encode_from_canvas.html"]
|
||||
skip-if = ["os != 'mac'"]
|
||||
@ -0,0 +1,410 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>WebCodecs performance test: Video Encoding</title>
|
||||
</head>
|
||||
<script src="/tests/SimpleTest/SimpleTest.js"></script>
|
||||
<script>
|
||||
const REALTIME = "realtime";
|
||||
const QUALITY = "quality";
|
||||
|
||||
const RESULTS = {};
|
||||
RESULTS[REALTIME] = [
|
||||
{ name: "frame-to-frame mean (key)", value: Infinity },
|
||||
{ name: "frame-to-frame stddev (key)", value: Infinity },
|
||||
{ name: "frame-dropping rate (key)", value: Infinity },
|
||||
{ name: "frame-to-frame mean (non key)", value: Infinity },
|
||||
{ name: "frame-to-frame stddev (non key)", value: Infinity },
|
||||
{ name: "frame-dropping rate (non key)", value: Infinity },
|
||||
];
|
||||
RESULTS[QUALITY] = [
|
||||
{ name: "first encode to last output", value: Infinity },
|
||||
];
|
||||
|
||||
var perfMetadata = {
|
||||
owner: "Media Team",
|
||||
name: "WebCodecs Video Encoding",
|
||||
description: "Test webcodecs video encoding performance",
|
||||
options: {
|
||||
default: {
|
||||
perfherder: true,
|
||||
perfherder_metrics: [
|
||||
{
|
||||
name: "realtime - frame-to-frame mean (key)",
|
||||
unit: "ms",
|
||||
shouldAlert: true,
|
||||
},
|
||||
{
|
||||
name: "realtime - frame-to-frame stddev (key)",
|
||||
unit: "ms",
|
||||
shouldAlert: true,
|
||||
},
|
||||
{
|
||||
name: "realtime - frame-dropping rate (key)",
|
||||
unit: "ratio",
|
||||
shouldAlert: true,
|
||||
},
|
||||
{
|
||||
name: "realtime - frame-to-frame mean (non key)",
|
||||
unit: "ms",
|
||||
shouldAlert: true,
|
||||
},
|
||||
{
|
||||
name: "realtime - frame-to-frame stddev (non key)",
|
||||
unit: "ms",
|
||||
shouldAlert: true,
|
||||
},
|
||||
{
|
||||
name: "realtime - frame-dropping rate (non key)",
|
||||
unit: "ratio",
|
||||
shouldAlert: true,
|
||||
},
|
||||
{
|
||||
name: "quality - first encode to last output",
|
||||
unit: "ms",
|
||||
shouldAlert: true,
|
||||
},
|
||||
],
|
||||
verbose: true,
|
||||
manifest: "perftest.toml",
|
||||
manifest_flavor: "plain",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
function createCanvas(width, height) {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
document.body.appendChild(canvas);
|
||||
return canvas;
|
||||
}
|
||||
|
||||
function removeCanvas(canvas) {
|
||||
const ctx = canvas.getContext("2d");
|
||||
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
||||
document.body.removeChild(canvas);
|
||||
}
|
||||
|
||||
function drawClock(canvas) {
|
||||
const ctx = canvas.getContext("2d");
|
||||
ctx.save();
|
||||
|
||||
ctx.fillStyle = "#dfdacd";
|
||||
ctx.fillRect(0, 0, canvas.width, canvas.height);
|
||||
|
||||
let radius = canvas.height / 2;
|
||||
ctx.translate(radius, radius);
|
||||
radius = radius * 0.7;
|
||||
|
||||
drawFace(ctx, radius);
|
||||
markHours(ctx, radius);
|
||||
markMinutes(ctx, radius);
|
||||
drawTime(ctx, radius);
|
||||
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
function drawFace(ctx, radius) {
|
||||
ctx.save();
|
||||
ctx.beginPath();
|
||||
ctx.arc(0, 0, radius, 0, 2 * Math.PI);
|
||||
ctx.fillStyle = "#feefde";
|
||||
ctx.fill();
|
||||
ctx.strokeStyle = "#6e6d6e";
|
||||
ctx.lineWidth = radius * 0.1;
|
||||
ctx.stroke();
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
function markHours(ctx, radius) {
|
||||
ctx.save();
|
||||
ctx.strokeStyle = "#947360";
|
||||
ctx.lineWidth = radius * 0.05;
|
||||
for (let i = 0; i < 12; i++) {
|
||||
ctx.beginPath();
|
||||
ctx.rotate(Math.PI / 6);
|
||||
ctx.moveTo(radius * 0.7, 0);
|
||||
ctx.lineTo(radius * 0.9, 0);
|
||||
ctx.stroke();
|
||||
}
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
function markMinutes(ctx, radius) {
|
||||
ctx.save();
|
||||
ctx.strokeStyle = "#947360";
|
||||
ctx.lineWidth = radius * 0.01;
|
||||
for (let i = 0; i < 60; i++) {
|
||||
if (i % 5 !== 0) {
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(radius * 0.8, 0);
|
||||
ctx.lineTo(radius * 0.85, 0);
|
||||
ctx.stroke();
|
||||
}
|
||||
ctx.rotate(Math.PI / 30);
|
||||
}
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
function drawTime(ctx, radius) {
|
||||
ctx.save();
|
||||
const now = new Date();
|
||||
let hour = now.getHours();
|
||||
let minute = now.getMinutes();
|
||||
let second = now.getSeconds() + now.getMilliseconds() / 1000;
|
||||
|
||||
hour = hour % 12;
|
||||
hour =
|
||||
(hour * Math.PI) / 6 +
|
||||
(minute * Math.PI) / (6 * 60) +
|
||||
(second * Math.PI) / (360 * 60);
|
||||
drawHand(ctx, hour, radius * 0.5, radius * 0.07, "#a1afa0");
|
||||
|
||||
minute = (minute * Math.PI) / 30 + (second * Math.PI) / (30 * 60);
|
||||
drawHand(ctx, minute, radius * 0.8, radius * 0.07, "#a1afa0");
|
||||
|
||||
second = (second * Math.PI) / 30;
|
||||
drawHand(ctx, second, radius * 0.9, radius * 0.02, "#970c10");
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
function drawHand(ctx, pos, length, width, color = "black") {
|
||||
ctx.save();
|
||||
ctx.strokeStyle = color;
|
||||
ctx.beginPath();
|
||||
ctx.lineWidth = width;
|
||||
ctx.lineCap = "round";
|
||||
ctx.moveTo(0, 0);
|
||||
ctx.rotate(pos);
|
||||
ctx.lineTo(0, -length);
|
||||
ctx.stroke();
|
||||
ctx.rotate(-pos);
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
function configureEncoder(
|
||||
worker,
|
||||
width,
|
||||
height,
|
||||
codec,
|
||||
latencyMode,
|
||||
avcFormat
|
||||
) {
|
||||
worker.postMessage({
|
||||
command: "configure",
|
||||
codec,
|
||||
width,
|
||||
height,
|
||||
latencyMode,
|
||||
avcFormat,
|
||||
});
|
||||
}
|
||||
|
||||
async function encodeCanvas(
|
||||
worker,
|
||||
canvas,
|
||||
fps,
|
||||
totalDuration,
|
||||
keyFrameIntervalInFrames
|
||||
) {
|
||||
const frameDuration = Math.round(1000 / fps); // ms
|
||||
let encodeDuration = 0;
|
||||
let frameCount = 0;
|
||||
let intervalId;
|
||||
|
||||
return new Promise((resolve, _) => {
|
||||
// first callback happens after frameDuration.
|
||||
intervalId = setInterval(() => {
|
||||
if (encodeDuration > totalDuration) {
|
||||
clearInterval(intervalId);
|
||||
resolve(encodeDuration);
|
||||
return;
|
||||
}
|
||||
drawClock(canvas);
|
||||
const frame = new VideoFrame(canvas, { timestamp: encodeDuration });
|
||||
worker.postMessage({
|
||||
command: "encode",
|
||||
frame,
|
||||
isKey: frameCount % keyFrameIntervalInFrames == 0,
|
||||
});
|
||||
frameCount += 1;
|
||||
encodeDuration += frameDuration;
|
||||
}, frameDuration);
|
||||
});
|
||||
}
|
||||
|
||||
async function getEncoderResults(worker) {
|
||||
worker.postMessage({ command: "flush" });
|
||||
return new Promise((resolve, _) => {
|
||||
worker.onmessage = event => {
|
||||
if (event.data.command === "result") {
|
||||
const { encodeTimes, outputTimes } = event.data;
|
||||
resolve({ encodeTimes, outputTimes });
|
||||
}
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function getTotalDuration(encodeTimes, outputTimes) {
|
||||
if (!outputTimes.length || encodeTimes.length < outputTimes.length) {
|
||||
return Infinity;
|
||||
}
|
||||
return outputTimes[outputTimes.length - 1].time - encodeTimes[0].time;
|
||||
}
|
||||
|
||||
function calculateRoundTripTimes(encodeTimes, outputTimes) {
|
||||
let roundTripTimes = [];
|
||||
let encodeIndex = 0;
|
||||
let outputIndex = 0;
|
||||
while (
|
||||
encodeIndex < encodeTimes.length &&
|
||||
outputIndex < outputTimes.length
|
||||
) {
|
||||
const encodeEntry = encodeTimes[encodeIndex];
|
||||
const outputEntry = outputTimes[outputIndex];
|
||||
|
||||
if (encodeEntry.timestamp === outputEntry.timestamp) {
|
||||
const roundTripTime = outputEntry.time - encodeEntry.time;
|
||||
roundTripTimes.push({
|
||||
timestamp: outputEntry.timestamp,
|
||||
time: roundTripTime,
|
||||
});
|
||||
encodeIndex++;
|
||||
outputIndex++;
|
||||
} else if (encodeEntry.timestamp < outputEntry.timestamp) {
|
||||
encodeIndex++;
|
||||
} else {
|
||||
outputIndex++;
|
||||
}
|
||||
}
|
||||
return roundTripTimes;
|
||||
}
|
||||
|
||||
function getMeanAndStandardDeviation(values) {
|
||||
if (!values.length) {
|
||||
return { mean: 0, stddev: 0 };
|
||||
}
|
||||
const mean = values.reduce((a, b) => a + b, 0) / values.length;
|
||||
const stddev = Math.sqrt(
|
||||
values.map(x => Math.pow(x - mean, 2)).reduce((a, b) => a + b) /
|
||||
values.length
|
||||
);
|
||||
return { mean, stddev };
|
||||
}
|
||||
|
||||
function reportMetrics(results) {
|
||||
const metrics = {};
|
||||
let text = "\nResults (ms)\n";
|
||||
for (const mode in results) {
|
||||
for (const r of results[mode]) {
|
||||
const name = mode + " - " + r.name;
|
||||
metrics[name] = r.value;
|
||||
text += " " + mode + " " + r.name + " : " + r.value + "\n";
|
||||
}
|
||||
}
|
||||
dump(text);
|
||||
info("perfMetrics", JSON.stringify(metrics));
|
||||
}
|
||||
|
||||
add_task(async () => {
|
||||
const width = 640;
|
||||
const height = 480;
|
||||
const fps = 30;
|
||||
const totalDuration = 5000; // ms
|
||||
const keyFrameInterval = 15; // 1 key every 15 frames
|
||||
|
||||
const worker = new Worker("encode_from_canvas.js");
|
||||
const h264main = "avc1.4D001E";
|
||||
configureEncoder(worker, width, height, h264main, REALTIME, "annexb");
|
||||
|
||||
const canvas = createCanvas(width, height);
|
||||
await encodeCanvas(worker, canvas, fps, totalDuration, keyFrameInterval);
|
||||
let { encodeTimes, outputTimes } = await getEncoderResults(worker);
|
||||
|
||||
ok(
|
||||
encodeTimes.length >= outputTimes.length,
|
||||
"Should have more encoded samples than outputs"
|
||||
);
|
||||
|
||||
let results = { key: {}, delta: {} };
|
||||
results.key.encodeTimes = encodeTimes.filter(x => x.type == "key");
|
||||
results.delta.encodeTimes = encodeTimes.filter(x => x.type != "key");
|
||||
results.key.outputTimes = outputTimes.filter(x => x.type == "key");
|
||||
results.delta.outputTimes = outputTimes.filter(x => x.type != "key");
|
||||
ok(
|
||||
results.key.encodeTimes.length >= results.key.outputTimes.length,
|
||||
"Should have more encoded samples than outputs (key)"
|
||||
);
|
||||
ok(
|
||||
results.delta.encodeTimes.length >= results.delta.outputTimes.length,
|
||||
"Should have more encoded samples than outputs (delta)"
|
||||
);
|
||||
|
||||
results.key.frameDroppingRate =
|
||||
(results.key.encodeTimes.length - results.key.outputTimes.length) /
|
||||
results.key.encodeTimes.length;
|
||||
results.delta.frameDroppingRate =
|
||||
(results.delta.encodeTimes.length - results.delta.outputTimes.length) /
|
||||
results.delta.encodeTimes.length;
|
||||
|
||||
results.key.roundTripTimes = calculateRoundTripTimes(
|
||||
results.key.encodeTimes,
|
||||
results.key.outputTimes
|
||||
);
|
||||
results.key.roundTripResult = getMeanAndStandardDeviation(
|
||||
results.key.roundTripTimes.map(x => x.time)
|
||||
);
|
||||
|
||||
results.delta.roundTripTimes = calculateRoundTripTimes(
|
||||
results.delta.encodeTimes,
|
||||
results.delta.outputTimes
|
||||
);
|
||||
results.delta.roundTripResult = getMeanAndStandardDeviation(
|
||||
results.delta.roundTripTimes.map(x => x.time)
|
||||
);
|
||||
|
||||
RESULTS[REALTIME][0].value = results.key.roundTripResult.mean;
|
||||
RESULTS[REALTIME][1].value = results.key.roundTripResult.stddev;
|
||||
RESULTS[REALTIME][2].value = results.key.frameDroppingRate;
|
||||
|
||||
RESULTS[REALTIME][3].value = results.delta.roundTripResult.mean;
|
||||
RESULTS[REALTIME][4].value = results.delta.roundTripResult.stddev;
|
||||
RESULTS[REALTIME][5].value = results.delta.frameDroppingRate;
|
||||
|
||||
removeCanvas(canvas);
|
||||
worker.terminate();
|
||||
});
|
||||
|
||||
add_task(async () => {
|
||||
const width = 640;
|
||||
const height = 480;
|
||||
const fps = 30;
|
||||
const totalDuration = 5000; // ms
|
||||
const keyFrameInterval = 15; // 1 key every 15 frames
|
||||
|
||||
const worker = new Worker("encode_from_canvas.js");
|
||||
const h264main = "avc1.4D001E";
|
||||
configureEncoder(worker, width, height, h264main, QUALITY, "annexb");
|
||||
|
||||
const canvas = createCanvas(width, height);
|
||||
await encodeCanvas(worker, canvas, fps, totalDuration, keyFrameInterval);
|
||||
let { encodeTimes, outputTimes } = await getEncoderResults(worker);
|
||||
|
||||
is(
|
||||
encodeTimes.length,
|
||||
outputTimes.length,
|
||||
`frame cannot be dropped in ${QUALITY} mode`
|
||||
);
|
||||
RESULTS[QUALITY][0].value = getTotalDuration(encodeTimes, outputTimes);
|
||||
|
||||
removeCanvas(canvas);
|
||||
worker.terminate();
|
||||
});
|
||||
|
||||
add_task(async () => {
|
||||
reportMetrics(RESULTS);
|
||||
});
|
||||
</script>
|
||||
<body></body>
|
||||
</html>
|
||||
@ -48,3 +48,8 @@ suites:
|
||||
"Service Worker Fetch": ""
|
||||
"Service Worker Registration": ""
|
||||
"Service Worker Update": ""
|
||||
|
||||
dom/media/webcodecs/test/performance:
|
||||
description: "Performance tests running through Mochitest for WebCodecs"
|
||||
tests:
|
||||
"WebCodecs Video Encoding": ""
|
||||
|
||||
@ -95,6 +95,26 @@ service-worker:
|
||||
--flavor mochitest
|
||||
--output $MOZ_FETCHES_DIR/../artifacts
|
||||
|
||||
webcodecs:
|
||||
description: Run webcodecs tests
|
||||
treeherder:
|
||||
symbol: perftest(macosx-webcodecs)
|
||||
tier: 2
|
||||
attributes:
|
||||
batch: false
|
||||
cron: false
|
||||
run-on-projects: [autoland, mozilla-central]
|
||||
run:
|
||||
command: >-
|
||||
mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
|
||||
cd $MOZ_FETCHES_DIR &&
|
||||
python3 -m venv . &&
|
||||
bin/python3 python/mozperftest/mozperftest/runner.py
|
||||
dom/media/webcodecs/test/performance/test_encode_from_canvas.html
|
||||
--mochitest-binary ${MOZ_FETCHES_DIR}/target.dmg
|
||||
--flavor mochitest
|
||||
--output $MOZ_FETCHES_DIR/../artifacts
|
||||
|
||||
domcount:
|
||||
description: Run DOM test on macOS
|
||||
treeherder:
|
||||
|
||||
@ -29,6 +29,28 @@ perftest_browser_xhtml_dom.js
|
||||
**Measures the size of the DOM**
|
||||
|
||||
|
||||
dom/media/webcodecs/test/performance
|
||||
------------------------------------
|
||||
Performance tests running through Mochitest for WebCodecs
|
||||
|
||||
test_encode_from_canvas.html
|
||||
============================
|
||||
|
||||
:owner: Media Team
|
||||
:name: WebCodecs Video Encoding
|
||||
:Default options:
|
||||
|
||||
::
|
||||
|
||||
--perfherder
|
||||
--perfherder-metrics name:realtime - frame-to-frame mean (key),unit:ms,shouldAlert:True, name:realtime - frame-to-frame stddev (key),unit:ms,shouldAlert:True, name:realtime - frame-dropping rate (key),unit:ratio,shouldAlert:True, name:realtime - frame-to-frame mean (non key),unit:ms,shouldAlert:True, name:realtime - frame-to-frame stddev (non key),unit:ms,shouldAlert:True, name:realtime - frame-dropping rate (non key),unit:ratio,shouldAlert:True, name:quality - first encode to last output,unit:ms,shouldAlert:True
|
||||
--verbose
|
||||
--manifest perftest.toml
|
||||
--manifest-flavor plain
|
||||
|
||||
**Test webcodecs video encoding performance**
|
||||
|
||||
|
||||
dom/serviceworkers/test/performance
|
||||
-----------------------------------
|
||||
Performance tests running through Mochitest for Service Workers
|
||||
|
||||
Loading…
Reference in New Issue
Block a user