mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-24 05:11:16 +00:00
277 lines
9.9 KiB
HTML
277 lines
9.9 KiB
HTML
<!DOCTYPE HTML>
|
|
<html>
|
|
<head>
|
|
<title>A/V sync test for stream capturing</title>
|
|
<script src="/tests/SimpleTest/SimpleTest.js"></script>
|
|
<link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
|
|
<p>Following canvas will capture and show the video frame when the video becomes audible</p><br>
|
|
<canvas id="canvas" width="640" height="480"></canvas>
|
|
<script type="application/javascript">
|
|
|
|
/**
|
|
* This test will capture stream before the video starts playing, and check if
|
|
* A/V sync will keep sync during playing.
|
|
*/
|
|
add_task(async function testAVSyncForStreamCapturing() {
|
|
createVideo();
|
|
captureStreamFromVideo();
|
|
await playMedia();
|
|
await testAVSync();
|
|
destroyVideo();
|
|
});
|
|
|
|
/**
|
|
* This test will check if A/V is still on sync after we switch the media sink
|
|
* from playback-based sink to mediatrack-based sink.
|
|
*/
|
|
add_task(async function testAVSyncWhenSwitchingMediaSink() {
|
|
createVideo();
|
|
await playMedia({resolveAfterReceivingTimeupdate : 5});
|
|
captureStreamFromVideo();
|
|
await testAVSync();
|
|
destroyVideo();
|
|
});
|
|
|
|
/**
|
|
* This test will check if A/V is still on sync after we change the playback
|
|
* rate on the captured stream.
|
|
*/
|
|
add_task(async function testAVSyncWhenChangingPlaybackRate() {
|
|
createVideo();
|
|
captureStreamFromVideo();
|
|
await playMedia();
|
|
const playbackRates = [0.25, 0.5, 1.0, 1.5, 2.0];
|
|
for (let rate of playbackRates) {
|
|
setPlaybackRate(rate);
|
|
// TODO : when playback rate set to 1.5+x, the A/V will become less stable
|
|
// in testing so we raise the fuzzy frames for that, but also increase the
|
|
// test times. As at that speed, precise A/V becomes trivial because we
|
|
// can't really tell the difference. But it would be good for us to
|
|
// investigate if we could make A/V sync work better at that extreme high
|
|
// rate.
|
|
if (rate >= 1.5) {
|
|
await testAVSync({ expectedAVSyncTestTimes : 4, fuzzyFrames : 10});
|
|
} else {
|
|
await testAVSync({ expectedAVSyncTestTimes : 2 });
|
|
}
|
|
}
|
|
destroyVideo();
|
|
});
|
|
|
|
/**
|
|
* Following are helper functions
|
|
*/
|
|
const DEBUG = false;
|
|
function info_debug(msg) {
|
|
if (DEBUG) {
|
|
info(msg);
|
|
}
|
|
}
|
|
|
|
function createVideo() {
|
|
const video = document.createElement("video");
|
|
// This video is special for testing A/V sync, it only produce audible sound
|
|
// once per second, and when the sound comes out, you can check the position
|
|
// of the square to know if the A/V keeps sync.
|
|
video.src = "sync.webm";
|
|
video.loop = true;
|
|
video.controls = true;
|
|
video.width = 640;
|
|
video.height = 480;
|
|
video.id = "video";
|
|
document.body.appendChild(video);
|
|
}
|
|
|
|
function destroyVideo() {
|
|
const video = document.getElementById("video");
|
|
video.src = null;
|
|
video.remove();
|
|
}
|
|
|
|
async function playMedia({ resolveAfterReceivingTimeupdate } = {}) {
|
|
const video = document.getElementById("video");
|
|
ok(await video.play().then(_=>true,_=>false), "video started playing");
|
|
if (resolveAfterReceivingTimeupdate > 0) {
|
|
// Play it for a while to ensure the clock growing on the normal audio sink.
|
|
for (let idx = 0; idx < resolveAfterReceivingTimeupdate; idx++) {
|
|
await new Promise(r => video.ontimeupdate = r);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function captureStreamFromVideo() {
|
|
const video = document.getElementById("video");
|
|
let ac = new AudioContext();
|
|
let analyser = ac.createAnalyser();
|
|
analyser.frequencyBuf = new Float32Array(analyser.frequencyBinCount);
|
|
analyser.smoothingTimeConstant = 0;
|
|
analyser.fftSize = 2048; // 1024 bins
|
|
let sourceNode = ac.createMediaElementSource(video);
|
|
sourceNode.connect(analyser);
|
|
analyser.connect(ac.destination);
|
|
video.analyser = analyser;
|
|
}
|
|
|
|
// This method will capture the stream from the video element, and check if A/V
|
|
// keeps sync during capturing. `callback` parameter will be executed after
|
|
// finishing capturing.
|
|
// @param [optional] expectedAVSyncTestTimes
|
|
// The amount of times that A/V sync test performs.
|
|
// @param [optional] fuzzyFrames
|
|
// This will fuzz the result from +0 (perfect sync) to -X to +X frames.
|
|
async function testAVSync({ expectedAVSyncTestTimes = 5, fuzzyFrames = 5} = {}) {
|
|
return new Promise(r => {
|
|
const analyser = document.getElementById("video").analyser;
|
|
let testIdx = 0;
|
|
let hasDetectedAudibleFrame = false;
|
|
// As we only want to detect the audible frame at the first moment when
|
|
// sound becomes audible, so we always skip the first audible frame because
|
|
// it might not be the start, but the tail part (where audio is being
|
|
// decaying to silence) when we start detecting.
|
|
let hasSkippedFirstFrame = false;
|
|
analyser.notifyAnalysis = () => {
|
|
let {frequencyBuf} = analyser;
|
|
analyser.getFloatFrequencyData(frequencyBuf);
|
|
if (checkIfBufferIsSilent(frequencyBuf)) {
|
|
info_debug("no need to paint the silent frame");
|
|
hasDetectedAudibleFrame = false;
|
|
requestAnimationFrame(analyser.notifyAnalysis);
|
|
return;
|
|
}
|
|
if (hasDetectedAudibleFrame) {
|
|
info_debug("detected audible frame already");
|
|
requestAnimationFrame(analyser.notifyAnalysis);
|
|
return;
|
|
}
|
|
hasDetectedAudibleFrame = true;
|
|
if (!hasSkippedFirstFrame) {
|
|
info("skip the first audible frame");
|
|
hasSkippedFirstFrame = true;
|
|
requestAnimationFrame(analyser.notifyAnalysis);
|
|
return;
|
|
}
|
|
const video = document.getElementById("video");
|
|
info(`paint audible frame`);
|
|
const cvs = document.getElementById("canvas");
|
|
let context = cvs.getContext('2d');
|
|
context.drawImage(video, 0, 0, 640, 480);
|
|
if (checkIfAVIsOnSyncFuzzy(context, fuzzyFrames)) {
|
|
ok(true, `test ${testIdx++} times, a/v is in sync!`);
|
|
} else {
|
|
ok(false, `test ${testIdx++} times, a/v is out of sync!`);
|
|
}
|
|
if (testIdx == expectedAVSyncTestTimes) {
|
|
r();
|
|
return;
|
|
}
|
|
requestAnimationFrame(analyser.notifyAnalysis);
|
|
}
|
|
analyser.notifyAnalysis();
|
|
});
|
|
}
|
|
|
|
function checkIfBufferIsSilent(buffer) {
|
|
for (let data of buffer) {
|
|
// when sound is audible, its values are around -200 and the silence values
|
|
// are around -800.
|
|
if (data > -200) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// This function will check the pixel data from the `context` to see if the
|
|
// square appears in the right place. As we can't control the exact timing
|
|
// of rendering video frames in the compositor, so the result would be fuzzy.
|
|
function checkIfAVIsOnSyncFuzzy(context, fuzzyFrames) {
|
|
const squareLength = 48;
|
|
// Canvas is 640*480, so perfect sync is the left-top corner when the square
|
|
// shows up in the middle.
|
|
const perfectSync =
|
|
{ x: 320 - squareLength/2.0 ,
|
|
y: 240 - squareLength/2.0 };
|
|
let isAVSyncFuzzy = false;
|
|
// Get the whole partial section of image and detect where the square is.
|
|
let imageData = context.getImageData(0, perfectSync.y, 640, squareLength);
|
|
for (let i = 0; i < imageData.data.length; i += 4) {
|
|
// If the pixel's color is red, then this position will be the left-top
|
|
// corner of the square.
|
|
if (isPixelColorRed(imageData.data[i], imageData.data[i+1],
|
|
imageData.data[i+2])) {
|
|
const pos = ImageIdxToRelativeCoordinate(imageData, i);
|
|
let diff = calculateFrameDiffenceInXAxis(pos.x, perfectSync.x);
|
|
info(`find the square in diff=${diff}`);
|
|
// Maybe we check A/V sync too early or too late, try to adjust the diff
|
|
// to guess the previous correct position where the square should be.
|
|
if (diff > fuzzyFrames) {
|
|
diff = adjustFrameDiffBasedOnMediaTime(diff);
|
|
const video = document.getElementById("video");
|
|
info(`adjusted diff to ${diff} (time=${video.currentTime})`);
|
|
}
|
|
if (diff <= fuzzyFrames) {
|
|
isAVSyncFuzzy = true;
|
|
}
|
|
context.putImageData(imageData, 0, 0);
|
|
break;
|
|
}
|
|
}
|
|
if (!isAVSyncFuzzy) {
|
|
const ctx = document.getElementById('canvas');
|
|
info(ctx.toDataURL());
|
|
}
|
|
return isAVSyncFuzzy;
|
|
}
|
|
|
|
// Input an imageData and its idx, then return a relative coordinate on the
|
|
// range of given imageData.
|
|
function ImageIdxToRelativeCoordinate(imageData, idx) {
|
|
const offset = idx / 4; // RGBA
|
|
return { x: offset % imageData.width, y: offset / imageData.width };
|
|
}
|
|
|
|
function calculateFrameDiffenceInXAxis(squareX, targetX) {
|
|
const offsetX = Math.abs(targetX - squareX);
|
|
const xSpeedPerFrame = 640 / 60; // video is 60fps
|
|
return offsetX / xSpeedPerFrame;
|
|
}
|
|
|
|
function isPixelColorRed(r, g, b) {
|
|
// As the rendering color would vary in the screen on different platforms, so
|
|
// we won't strict the R should be 255, just check if it's larger than a
|
|
// certain threshold.
|
|
return r > 200 && g < 10 && b < 10;
|
|
}
|
|
|
|
function setPlaybackRate(rate) {
|
|
const video = document.getElementById("video");
|
|
info(`change playback rate from ${video.playbackRate} to ${rate}`);
|
|
document.getElementById("video").playbackRate = rate;
|
|
}
|
|
|
|
function adjustFrameDiffBasedOnMediaTime(currentDiff) {
|
|
// The audio wave can be simply regarded as being composed by "start", "peak"
|
|
// and "tail". The "start" part is the sound gradually becoming louder and the
|
|
// "tail" is gradually becoming silent. We want to check the "peak" part which
|
|
// should happn on evert second regularly (1s, 2s, 3s ...) However, this check
|
|
// is triggered by `requestAnimationFrame()` and we can't guarantee that
|
|
// we're checking the peak part while the function is being called. Therefore,
|
|
// we have to do an adjustment by the video time, to know if we're checking
|
|
// the audio wave too early or too late in order to get a consistent result.
|
|
const video = document.getElementById("video");
|
|
const videoCurrentTimeFloatPortion = video.currentTime % 1;
|
|
const timeOffset =
|
|
videoCurrentTimeFloatPortion > 0.5 ?
|
|
1 - videoCurrentTimeFloatPortion : // too early
|
|
videoCurrentTimeFloatPortion; // too late
|
|
const frameOffset = timeOffset / 0.016; // 60fps, 1 frame=0.016s
|
|
info(`timeOffset=${timeOffset}, frameOffset=${frameOffset}`);
|
|
return Math.abs(currentDiff - frameOffset);
|
|
}
|
|
|
|
</script>
|
|
</head>
|
|
<body>
|
|
</body>
|
|
</html>
|