Bug 1580095 - Add audio scrubber to PiP. r=pip-reviewers,desktop-theme-reviewers,kpatenio,dao

Differential Revision: https://phabricator.services.mozilla.com/D179556
This commit is contained in:
Niklas Baumgardner 2023-06-17 17:01:58 +00:00
parent 78718dabb9
commit f7058e9858
8 changed files with 457 additions and 38 deletions

View File

@ -122,7 +122,7 @@ add_task(async function test_volume_change_with_keyboard() {
// Decrease volume with arrow down
EventUtils.synthesizeKey("KEY_ArrowDown", {}, pipWin);
ok(!(await isVideoMuted(browser, videoID)), "The audio is playing.");
ok(await isVideoMuted(browser, videoID), "The audio is not playing.");
// Increase volume with arrow up
EventUtils.synthesizeKey("KEY_ArrowUp", {}, pipWin);

View File

@ -224,6 +224,7 @@ export class PictureInPictureLauncherChild extends JSWindowActorChild {
webVTTSubtitles: !!video.textTracks?.length,
scrubberPosition,
timestamp,
volume: PictureInPictureChild.videoWrapper.getVolume(video),
});
let args = {
@ -1939,6 +1940,9 @@ export class PictureInPictureChild extends JSWindowActorChild {
} else {
this.sendAsyncMessage("PictureInPicture:Unmuting");
}
this.sendAsyncMessage("PictureInPicture:VolumeChange", {
volume: this.videoWrapper.getVolume(video),
});
break;
}
case "resize": {
@ -2143,6 +2147,12 @@ export class PictureInPictureChild extends JSWindowActorChild {
this.setVideoTime(scrubberPosition, wasPlaying);
break;
}
case "PictureInPicture:SetVolume": {
const { volume } = message.data;
let video = this.getWeakVideo();
this.videoWrapper.setVolume(video, volume);
break;
}
}
}
@ -2565,12 +2575,16 @@ export class PictureInPictureChild extends JSWindowActorChild {
this.closePictureInPicture({ reason: "closePlayerShortcut" });
break;
case "downArrow" /* Volume decrease */:
if (this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.VOLUME)) {
if (
this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.VOLUME) ||
this.videoWrapper.isMuted(video)
) {
return;
}
oldval = this.videoWrapper.getVolume(video);
this.videoWrapper.setVolume(video, oldval < 0.1 ? 0 : oldval - 0.1);
this.videoWrapper.setMuted(video, false);
newval = oldval < 0.1 ? 0 : oldval - 0.1;
this.videoWrapper.setVolume(video, newval);
this.videoWrapper.setMuted(video, newval === 0);
break;
case "upArrow" /* Volume increase */:
if (this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.VOLUME)) {

View File

@ -186,6 +186,12 @@ export class PictureInPictureParent extends JSWindowActorParent {
player.setScrubberPosition(scrubberPosition);
break;
}
case "PictureInPicture:VolumeChange": {
let { volume } = aMessage.data;
let player = PictureInPicture.getWeakPipPlayer(this);
player.setVolume(volume);
break;
}
}
}
}
@ -815,6 +821,7 @@ export var PictureInPicture = {
win.setScrubberPosition(videoData.scrubberPosition);
win.setTimestamp(videoData.timestamp);
win.setVolume(videoData.volume);
Services.prefs.setBoolPref(TOGGLE_HAS_USED_PREF, true);

View File

@ -115,6 +115,10 @@ function setTimestamp(timeString) {
Player.setTimestamp(timeString);
}
function setVolume(volume) {
Player.setVolume(volume);
}
/**
* The Player object handles initializing the player, holds state, and handles
* events for updating state.
@ -236,6 +240,19 @@ let Player = {
this.handleScrubbingDone(event);
});
this.audioScrubber.addEventListener("input", event => {
this.audioScrubbing = true;
this.handleAudioScrubbing(event.target.value);
});
this.audioScrubber.addEventListener("change", event => {
this.audioScrubbing = false;
});
this.audioScrubber.addEventListener("pointerdown", event => {
if (this.isMuted) {
this.audioScrubber.max = 1;
}
});
for (let radio of document.querySelectorAll(
'input[type=radio][name="cc-size"]'
)) {
@ -259,6 +276,9 @@ let Player = {
if (Services.prefs.getBoolPref(AUDIO_TOGGLE_ENABLED_PREF, false)) {
const audioButton = document.getElementById("audio");
audioButton.hidden = false;
const audioScrubber = document.getElementById("audio-scrubber");
audioScrubber.hidden = false;
}
if (Services.prefs.getBoolPref(CAPTIONS_ENABLED_PREF, false)) {
@ -489,7 +509,8 @@ let Player = {
handleScrubbing(event) {
// When using the keyboard to scrub, we get both a keydown and an input
// event. The input event is fired after the keydown and we have already
// handle the keydown event in onKeyDown and we don't want to handle it twice
// handled the keydown event in onKeyDown so we set preventNextInputEvent
// to true in onKeyDown as to not set the current time twice.
if (this.preventNextInputEvent) {
this.preventNextInputEvent = false;
return;
@ -522,6 +543,36 @@ let Player = {
this.scrubbing = false;
},
/**
* Set the volume on the video and unmute if the video was muted.
* If the volume is changed via the keyboard, onKeyDown will set
* this.preventNextInputEvent to true.
* @param {Number} volume A number between 0 and 1 that represents the volume
*/
handleAudioScrubbing(volume) {
// When using the keyboard to adjust the volume, we get both a keydown and
// an input event. The input event is fired after the keydown event and we
// have already handled the keydown event in onKeyDown so we set
// preventNextInputEvent to true in onKeyDown as to not set the volume twice.
if (this.preventNextInputEvent) {
this.preventNextInputEvent = false;
return;
}
if (this.isMuted) {
this.isMuted = false;
this.actor.sendAsyncMessage("PictureInPicture:Unmute");
}
if (volume == 0) {
this.actor.sendAsyncMessage("PictureInPicture:Mute");
}
this.actor.sendAsyncMessage("PictureInPicture:SetVolume", {
volume,
});
},
getScrubberPositionFromEvent(event) {
return event.target.value;
},
@ -549,6 +600,14 @@ let Player = {
this.timestamp.hidden = timestamp === undefined;
},
setVolume(volume) {
if (volume < Number.EPSILON) {
this.actor.sendAsyncMessage("PictureInPicture:Mute");
}
this.audioScrubber.value = volume;
},
closePipWindow(closeData) {
// Set the subtitles font size prefs
Services.prefs.setBoolPref(
@ -577,11 +636,7 @@ let Player = {
onClick(event) {
switch (event.target.id) {
case "audio": {
if (this.isMuted) {
this.actor.sendAsyncMessage("PictureInPicture:Unmute");
} else {
this.actor.sendAsyncMessage("PictureInPicture:Mute");
}
this.toggleMute();
break;
}
@ -717,6 +772,20 @@ let Player = {
}
},
/**
* Toggle the mute state of the video
*/
toggleMute() {
if (this.isMuted) {
// We unmute in handleAudioScrubbing so no need to also do it here
this.audioScrubber.max = 1;
this.handleAudioScrubbing(this.lastVolume ?? 1);
} else {
this.lastVolume = this.audioScrubber.value;
this.actor.sendAsyncMessage("PictureInPicture:Mute");
}
},
resizeToVideo(rect) {
if (this.isFullscreen) {
// We store the size and position because resizing the PiP window
@ -749,7 +818,7 @@ let Player = {
};
// If the up or down arrow is pressed while the scrubber is focused then we
// want to hijack these keydown events to act as left or right arrow
// want to hijack these keydown events to act as left or right arrows
// respectively to correctly seek the video.
if (
event.target.id === "scrubber" &&
@ -763,17 +832,33 @@ let Player = {
eventKeys.keyCode = window.KeyEvent.DOM_VK_LEFT;
}
// If the keydown event was one of the arrow keys and the scrubber was
// focused then we will also get an input event that will overwrite the
// keydown event if we dont' prevent the input event.
// If the left or right arrow is pressed while the audio scrubber is focused
// then we want to hijack these keydown events to act as up or down arrows
// respectively to correctly change the volume.
if (
event.target.id === "scrubber" &&
[
window.KeyEvent.DOM_VK_LEFT,
window.KeyEvent.DOM_VK_RIGHT,
window.KeyEvent.DOM_VK_UP,
window.KeyEvent.DOM_VK_DOWN,
].includes(event.keyCode)
event.target.id === "audio-scrubber" &&
event.keyCode === window.KeyEvent.DOM_VK_RIGHT
) {
eventKeys.keyCode = window.KeyEvent.DOM_VK_UP;
} else if (
event.target.id === "audio-scrubber" &&
event.keyCode === window.KeyEvent.DOM_VK_LEFT
) {
eventKeys.keyCode = window.KeyEvent.DOM_VK_DOWN;
}
// If the keydown event was one of the arrow keys and the scrubber or the
// audio scrubber was focused then we want to prevent the subsequent input
// event from overwriting the keydown event.
if (
event.target.id === "audio-scrubber" ||
(event.target.id === "scrubber" &&
[
window.KeyEvent.DOM_VK_LEFT,
window.KeyEvent.DOM_VK_RIGHT,
window.KeyEvent.DOM_VK_UP,
window.KeyEvent.DOM_VK_DOWN,
].includes(event.keyCode))
) {
this.preventNextInputEvent = true;
}
@ -1075,6 +1160,11 @@ let Player = {
return (this.scrubber = document.getElementById("scrubber"));
},
get audioScrubber() {
delete this.audioScrubber;
return (this.audioScrubber = document.getElementById("audio-scrubber"));
},
get timestamp() {
delete this.timestamp;
return (this.timestamp = document.getElementById("timestamp"));
@ -1143,6 +1233,11 @@ let Player = {
set isMuted(isMuted) {
this._isMuted = isMuted;
if (!isMuted) {
this.audioScrubber.max = 1;
} else if (!this.audioScrubbing) {
this.audioScrubber.max = 0;
}
this.controls.classList.toggle("muted", isMuted);
let strId = isMuted
? `pictureinpicture-unmute-btn`

View File

@ -45,25 +45,25 @@
class="control-item control-button tooltip-under-controls" data-l10n-attrs="tooltip"
#ifdef XP_MACOSX
mac="true"
tabindex="8"
#else
tabindex="9"
#else
tabindex="10"
#endif
/>
<button id="unpip"
class="control-item control-button tooltip-under-controls" data-l10n-id="pictureinpicture-unpip-btn" data-l10n-attrs="tooltip"
#ifdef XP_MACOSX
mac="true"
tabindex="9"
tabindex="10"
#else
tabindex="8"
tabindex="9"
#endif
/>
<div id="controls-bottom-gradient" class="control-item"></div>
<div id="controls-bottom">
<div class="controls-bottom-upper">
<div class="scrubber-no-drag">
<input id="scrubber" class="control-item" min="0" max="1" step=".001" type="range" tabindex="10" hidden="true"/>
<input id="scrubber" class="control-item" min="0" max="1" step=".001" type="range" tabindex="11" hidden="true"/>
</div>
</div>
<div class="controls-bottom-lower">
@ -71,21 +71,22 @@
<div id="timestamp" class="control-item" hidden="true"></div>
</div>
<div class="center-controls">
<button id="seekBackward" class="control-item control-button tooltip-over-controls center-tooltip" hidden="true" data-l10n-id="pictureinpicture-seekbackward-btn" data-l10n-attrs="tooltip" tabindex="11"></button>
<button id="seekBackward" class="control-item control-button tooltip-over-controls center-tooltip" hidden="true" data-l10n-id="pictureinpicture-seekbackward-btn" data-l10n-attrs="tooltip" tabindex="12"></button>
<button id="playpause" class="control-item control-button tooltip-over-controls center-tooltip" tabindex="1"
data-l10n-id="pictureinpicture-pause-btn" data-l10n-attrs="tooltip"/>
<button id="seekForward" class="control-item control-button tooltip-over-controls center-tooltip" hidden="true" data-l10n-id="pictureinpicture-seekforward-btn" data-l10n-attrs="tooltip" tabindex="2"></button>
</div>
<div class="end-controls">
<button id="audio" class="control-item control-button tooltip-over-controls center-tooltip" hidden="true" data-l10n-attrs="tooltip" tabindex="3"/>
<button id="closed-caption" class="control-item control-button tooltip-over-controls center-tooltip closed-caption" hidden="true" disabled="true" data-l10n-id="pictureinpicture-subtitles-btn" data-l10n-attrs="tooltip" tabindex="4"></button>
<input id="audio-scrubber" class="control-item" min="0" max="1" step=".001" type="range" tabindex="4" hidden="true"/>
<button id="closed-caption" class="control-item control-button tooltip-over-controls center-tooltip closed-caption" hidden="true" disabled="true" data-l10n-id="pictureinpicture-subtitles-btn" data-l10n-attrs="tooltip" tabindex="5"></button>
<div id="settings" class="hide panel">
<fieldset class="box panel-fieldset">
<legend class="a11y-only panel-legend" data-l10n-id="pictureinpicture-subtitles-panel-accessible"></legend>
<div class="subtitle-grid">
<label id="subtitles-toggle-label" data-l10n-id="pictureinpicture-subtitles-label" class="bold" for="subtitles-toggle"></label>
<label class="switch">
<input id="subtitles-toggle" type="checkbox" tabindex="5" checked=""/>
<input id="subtitles-toggle" type="checkbox" tabindex="6" checked=""/>
<span class="slider" role="presentation"></span>
</label>
</div>
@ -93,22 +94,22 @@
<fieldset class="font-size-selection panel-fieldset">
<legend data-l10n-id="pictureinpicture-font-size-label" class="bold panel-legend"></legend>
<div id="font-size-selection-radio-small" class="font-size-selection-radio">
<input id="small" type="radio" name="cc-size" tabindex="6"/>
<input id="small" type="radio" name="cc-size" tabindex="7"/>
<label data-l10n-id="pictureinpicture-font-size-small" for="small"></label>
</div>
<div id="font-size-selection-radio-medium" class="font-size-selection-radio">
<input id="medium" type="radio" name="cc-size" tabindex="6"/>
<input id="medium" type="radio" name="cc-size" tabindex="7"/>
<label data-l10n-id="pictureinpicture-font-size-medium" for="medium"></label>
</div>
<div id="font-size-selection-radio-large" class="font-size-selection-radio">
<input id="large" type="radio" name="cc-size" tabindex="6"/>
<input id="large" type="radio" name="cc-size" tabindex="7"/>
<label data-l10n-id="pictureinpicture-font-size-large" for="large"></label>
</div>
</fieldset>
</fieldset>
<div class="arrow"></div>
</div>
<button id="fullscreen" class="control-item control-button tooltip-over-controls inline-end-tooltip" hidden="true" data-l10n-attrs="tooltip" tabindex="7"></button>
<button id="fullscreen" class="control-item control-button tooltip-over-controls inline-end-tooltip" hidden="true" data-l10n-attrs="tooltip" tabindex="8"></button>
</div>
</div>
</div>

View File

@ -44,6 +44,7 @@ prefs =
[browser_aaa_run_first_firstTimePiPToggleEvents.js]
[browser_aaa_telemetry_togglePiP.js]
[browser_audioScrubber.js]
[browser_backgroundTab.js]
[browser_cannotTriggerFromContent.js]
[browser_changePiPSrcInFullscreen.js]

View File

@ -0,0 +1,258 @@
/* Any copyright is dedicated to the Public Domain.
http://creativecommons.org/publicdomain/zero/1.0/ */
"use strict";
const IMPROVED_CONTROLS_ENABLED_PREF =
"media.videocontrols.picture-in-picture.improved-video-controls.enabled";
const videoID = "with-controls";
async function getVideoVolume(browser, videoID) {
return SpecialPowers.spawn(browser, [videoID], async videoID => {
return content.document.getElementById(videoID).volume;
});
}
async function getVideoMuted(browser, videoID) {
return SpecialPowers.spawn(browser, [videoID], async videoID => {
return content.document.getElementById(videoID).muted;
});
}
add_task(async function test_audioScrubber() {
await SpecialPowers.pushPrefEnv({
set: [[IMPROVED_CONTROLS_ENABLED_PREF, true]],
});
await BrowserTestUtils.withNewTab(
{
url: TEST_PAGE,
gBrowser,
},
async browser => {
await ensureVideosReady(browser);
let pipWin = await triggerPictureInPicture(browser, videoID);
ok(pipWin, "Got Picture-in-Picture window.");
// Resize the PiP window so we know the audio scrubber is visible
let resizePromise = BrowserTestUtils.waitForEvent(pipWin, "resize");
let ratio = pipWin.innerWidth * pipWin.innerHeight;
pipWin.resizeTo(750 * ratio, 750);
await resizePromise;
let audioScrubber = pipWin.document.getElementById("audio-scrubber");
audioScrubber.focus();
// Volume should be 1 and video should not be muted when opening this video
let actualVolume = await getVideoVolume(browser, videoID);
let expectedVolume = 1;
let actualMuted = await getVideoMuted(browser, videoID);
isfuzzy(
actualVolume,
expectedVolume,
Number.EPSILON,
`The actual volume ${actualVolume}. The expected volume ${expectedVolume}`
);
ok(!actualMuted, "Video is not muted");
let volumeChangePromise = BrowserTestUtils.waitForContentEvent(
browser,
"volumechange",
true
);
// Test that left arrow key changes volume by -0.1
EventUtils.synthesizeKey("KEY_ArrowLeft", {}, pipWin);
await volumeChangePromise;
actualVolume = await getVideoVolume(browser, videoID);
expectedVolume = 0.9;
isfuzzy(
actualVolume,
expectedVolume,
Number.EPSILON,
`The actual volume ${actualVolume}. The expected volume ${expectedVolume}`
);
// Test that right arrow key changes volume by +0.1 and does not exceed 1
EventUtils.synthesizeKey("KEY_ArrowRight", {}, pipWin);
EventUtils.synthesizeKey("KEY_ArrowRight", {}, pipWin);
EventUtils.synthesizeKey("KEY_ArrowRight", {}, pipWin);
actualVolume = await getVideoVolume(browser, videoID);
expectedVolume = 1;
isfuzzy(
actualVolume,
expectedVolume,
Number.EPSILON,
`The actual volume ${actualVolume}. The expected volume ${expectedVolume}`
);
// Test that the mouse can move the audio scrubber to 0.5
let rect = audioScrubber.getBoundingClientRect();
volumeChangePromise = BrowserTestUtils.waitForContentEvent(
browser,
"volumechange",
true
);
EventUtils.synthesizeMouse(
audioScrubber,
rect.width / 2,
rect.height / 2,
{},
pipWin
);
await volumeChangePromise;
actualVolume = await getVideoVolume(browser, videoID);
expectedVolume = 0.5;
isfuzzy(
actualVolume,
expectedVolume,
0.01,
`The actual volume ${actualVolume}. The expected volume ${expectedVolume}`
);
// Test muting and unmuting the video
let muteButton = pipWin.document.getElementById("audio");
volumeChangePromise = BrowserTestUtils.waitForContentEvent(
browser,
"volumechange",
true
);
muteButton.click();
await volumeChangePromise;
ok(
await getVideoMuted(browser, videoID),
"The video is muted aftering clicking the mute button"
);
is(
audioScrubber.max,
"0",
"The max of the audio scrubber is 0, so the volume slider appears that the volume is 0"
);
volumeChangePromise = BrowserTestUtils.waitForContentEvent(
browser,
"volumechange",
true
);
muteButton.click();
await volumeChangePromise;
ok(
!(await getVideoMuted(browser, videoID)),
"The video is muted aftering clicking the mute button"
);
isfuzzy(
actualVolume,
expectedVolume,
0.01,
`The volume is still ${actualVolume}.`
);
volumeChangePromise = BrowserTestUtils.waitForContentEvent(
browser,
"volumechange",
true
);
// Test that the audio scrubber can be dragged from 0.5 to 0 and the video gets muted
EventUtils.synthesizeMouse(
audioScrubber,
rect.width / 2,
rect.height / 2,
{ type: "mousedown" },
pipWin
);
EventUtils.synthesizeMouse(
audioScrubber,
0,
rect.height / 2,
{ type: "mousemove" },
pipWin
);
EventUtils.synthesizeMouse(
audioScrubber,
0,
rect.height / 2,
{ type: "mouseup" },
pipWin
);
await volumeChangePromise;
actualVolume = await getVideoVolume(browser, videoID);
expectedVolume = 0;
isfuzzy(
actualVolume,
expectedVolume,
Number.EPSILON,
`The actual volume ${actualVolume}. The expected volume ${expectedVolume}`
);
ok(
await getVideoMuted(browser, videoID),
"Video is now muted because slider was moved to 0"
);
// Test that the left arrow key does not unmute the video and the volume remains at 0
EventUtils.synthesizeKey("KEY_ArrowLeft", {}, pipWin);
EventUtils.synthesizeKey("KEY_ArrowLeft", {}, pipWin);
actualVolume = await getVideoVolume(browser, videoID);
isfuzzy(
actualVolume,
expectedVolume,
Number.EPSILON,
`The actual volume ${actualVolume}. The expected volume ${expectedVolume}`
);
ok(
await getVideoMuted(browser, videoID),
"Video is now muted because slider is still at 0"
);
volumeChangePromise = BrowserTestUtils.waitForContentEvent(
browser,
"volumechange",
true
);
// Test that the right arrow key will increase the volume by +0.1 and will unmute the video
EventUtils.synthesizeKey("KEY_ArrowRight", {}, pipWin);
await volumeChangePromise;
actualVolume = await getVideoVolume(browser, videoID);
expectedVolume = 0.1;
isfuzzy(
actualVolume,
expectedVolume,
Number.EPSILON,
`The actual volume ${actualVolume}. The expected volume ${expectedVolume}`
);
ok(
!(await getVideoMuted(browser, videoID)),
"Video is no longer muted because we moved the slider"
);
}
);
});

View File

@ -104,8 +104,8 @@ browser {
.end-controls {
display: grid;
grid-template-columns: repeat(3, 1fr);
grid-template-areas: "audio closedcaption fullscreen";
grid-template-columns: 1fr 2fr 1fr 1fr;
grid-template-areas: "audio audio-scrubber closedcaption fullscreen";
justify-self: end;
gap: 8px;
}
@ -253,8 +253,7 @@ body:fullscreen #controls[showing]:hover .control-item:not(:hover),
/* Background gradient is the only control item with a fixed opacity value. */
#controls[keying] .control-item#controls-bottom-gradient,
#controls[showing] .control-item#controls-bottom-gradient,
#controls:hover .control-item#controls-bottom-gradient:hover
#controls[donthide] .control-item#controls-bottom-gradient {
#controls:hover .control-item#controls-bottom-gradient:hover #controls[donthide] .control-item#controls-bottom-gradient {
opacity: 0.8;
}
@ -306,6 +305,39 @@ body:fullscreen #controls[showing]:hover {
grid-area: audio;
}
#audio-scrubber {
grid-area: audio-scrubber;
width: 64px;
background-color: transparent;
padding: 15px 0;
margin: 0;
}
#audio-scrubber::-moz-range-thumb {
border-radius: 8px;
background-color: #FFFFFF;
position: relative;
width: 8px;
height: 8px;
bottom: 24px;
padding: 0;
}
#audio-scrubber::-moz-range-track {
background-color: #969696;
}
#audio-scrubber::-moz-range-progress {
background-color: #FFFFFF;
}
#audio-scrubber,
#audio-scrubber::-moz-range-track,
#audio-scrubber::-moz-range-progress {
height: 2px;
border-radius: 4px;
}
#fullscreen {
grid-area: fullscreen;
}
@ -386,7 +418,7 @@ body:not(:fullscreen) #fullscreen {
font-weight: 590;
}
.box > input[type="radio"]{
.box > input[type="radio"] {
background-color: red;
fill: currentColor;
}
@ -591,6 +623,17 @@ input:checked + .slider::before {
content: unset;
}
@media (width <= 630px) {
#audio-scrubber {
display: none;
}
.end-controls {
grid-template-columns: repeat(3, 1fr);
grid-template-areas: "audio closedcaption fullscreen";
}
}
@media (height <= 325px), (width <= 475px) {
#closed-caption {
display: none;