Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-07 18:04:46 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1749795 - create documentation for adding and testing site-specific wrappers. r=mhowell

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D142011
This commit is contained in:
Katherine Patenio 2022-04-22 20:46:11 +00:00
parent 6b6840fbbf
commit c4f6478c5d
6 changed files with 306 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/conf.py
View File

@ -56,6 +56,7 @@ js_source_path = [
"../browser/components/uitour",
"../browser/components/urlbar",
"../remote/marionette",
"../toolkit/actors",
"../toolkit/components/extensions",
"../toolkit/components/extensions/parent",
"../toolkit/components/featuregates",

114
toolkit/actors/PictureInPictureChild.jsm
View File

@ -2235,8 +2235,7 @@ class PictureInPictureChild extends JSWindowActorChild {
*
* - The "site wrapper" script must export a class called "PictureInPictureVideoWrapper"
* - Method names on a site wrapper class should match its caller's name
* (i.e: PictureInPictureChildVideoWrapper.play will only call `play` on a site-wrapper,
* if available)
* (i.e: PictureInPictureChildVideoWrapper.play will only call `play` on a site-wrapper, if available)
*/
class PictureInPictureChildVideoWrapper {
#sandbox;
@ -2252,12 +2251,14 @@ class PictureInPictureChildVideoWrapper {
* commanding the original <video>.
* @param {HTMLVideoElement} video
* The original <video> we want to create a wrapper class for.
* @param {Object} pipChild
* Reference to PictureInPictureChild class calling this function.
*/
constructor(videoWrapperScriptPath, video, piPChild) {
constructor(videoWrapperScriptPath, video, pipChild) {
this.#sandbox = videoWrapperScriptPath
? this.#createSandbox(videoWrapperScriptPath, video)
: null;
this.#PictureInPictureChild = piPChild;
this.#PictureInPictureChild = pipChild;
}
/**
@ -2278,7 +2279,6 @@ class PictureInPictureChildVideoWrapper {
* return null.
*
* @returns The expected output of the wrapper function.
*
*/
#callWrapperMethod({ name, args = [], fallback = () => {}, validateRetVal }) {
try {
@ -2362,6 +2362,9 @@ class PictureInPictureChildVideoWrapper {
return typeof val === "number";
}
/**
* Destroys the sandbox for the site wrapper class
*/
destroy() {
if (this.#sandbox) {
Cu.nukeSandbox(this.#sandbox);
@ -2381,6 +2384,13 @@ class PictureInPictureChildVideoWrapper {
/* Video methods to be used for video controls from the PiP window. */
/**
* OVERRIDABLE - calls the play() method defined in the site wrapper script. Runs a fallback implementation
* if the method does not exist or if an error is thrown while calling it. This method is meant to handle video
* behaviour when a video is played.
* @param {HTMLVideoElement} video
* The originating video source element
*/
play(video) {
return this.#callWrapperMethod({
name: "play",
@ -2390,6 +2400,13 @@ class PictureInPictureChildVideoWrapper {
});
}
/**
* OVERRIDABLE - calls the pause() method defined in the site wrapper script. Runs a fallback implementation
* if the method does not exist or if an error is thrown while calling it. This method is meant to handle video
* behaviour when a video is paused.
* @param {HTMLVideoElement} video
* The originating video source element
*/
pause(video) {
return this.#callWrapperMethod({
name: "pause",
@ -2399,6 +2416,14 @@ class PictureInPictureChildVideoWrapper {
});
}
/**
* OVERRIDABLE - calls the getPaused() method defined in the site wrapper script. Runs a fallback implementation
* if the method does not exist or if an error is thrown while calling it. This method is meant to determine if
* a video is paused or not.
* @param {HTMLVideoElement} video
* The originating video source element
* @returns {Boolean} Boolean value true if paused, or false if video is still playing
*/
getPaused(video) {
return this.#callWrapperMethod({
name: "getPaused",
@ -2408,6 +2433,14 @@ class PictureInPictureChildVideoWrapper {
});
}
/**
* OVERRIDABLE - calls the getEnded() method defined in the site wrapper script. Runs a fallback implementation
* if the method does not exist or if an error is thrown while calling it. This method is meant to determine if
* video playback or streaming has stopped.
* @param {HTMLVideoElement} video
* The originating video source element
* @returns {Boolean} Boolean value true if the video has ended, or false if still playing
*/
getEnded(video) {
return this.#callWrapperMethod({
name: "getEnded",
@ -2417,6 +2450,14 @@ class PictureInPictureChildVideoWrapper {
});
}
/**
* OVERRIDABLE - calls the getDuration() method defined in the site wrapper script. Runs a fallback implementation
* if the method does not exist or if an error is thrown while calling it. This method is meant to get the current
* duration of a video in seconds.
* @param {HTMLVideoElement} video
* The originating video source element
* @returns {Number} Duration of the video in seconds
*/
getDuration(video) {
return this.#callWrapperMethod({
name: "getDuration",
@ -2426,6 +2467,14 @@ class PictureInPictureChildVideoWrapper {
});
}
/**
* OVERRIDABLE - calls the getCurrentTime() method defined in the site wrapper script. Runs a fallback implementation
* if the method does not exist or if an error is thrown while calling it. This method is meant to get the current
* time of a video in seconds.
* @param {HTMLVideoElement} video
* The originating video source element
* @returns {Number} Current time of the video in seconds
*/
getCurrentTime(video) {
return this.#callWrapperMethod({
name: "getCurrentTime",
@ -2435,6 +2484,15 @@ class PictureInPictureChildVideoWrapper {
});
}
/**
* OVERRIDABLE - calls the setCurrentTime() method defined in the site wrapper script. Runs a fallback implementation
* if the method does not exist or if an error is thrown while calling it. This method is meant to set the current
* time of a video.
* @param {HTMLVideoElement} video
* The originating video source element
* @param {Number} position
* The current playback time of the video
*/
setCurrentTime(video, position) {
return this.#callWrapperMethod({
name: "setCurrentTime",
@ -2446,6 +2504,14 @@ class PictureInPictureChildVideoWrapper {
});
}
/**
* OVERRIDABLE - calls the getVolume() method defined in the site wrapper script. Runs a fallback implementation
* if the method does not exist or if an error is thrown while calling it. This method is meant to get the volume
* value of a video.
* @param {HTMLVideoElement} video
* The originating video source element
* @returns {Number} Volume of the video between 0 (muted) and 1 (loudest)
*/
getVolume(video) {
return this.#callWrapperMethod({
name: "getVolume",
@ -2455,6 +2521,15 @@ class PictureInPictureChildVideoWrapper {
});
}
/**
* OVERRIDABLE - calls the setVolume() method defined in the site wrapper script. Runs a fallback implementation
* if the method does not exist or if an error is thrown while calling it. This method is meant to set the volume
* value of a video.
* @param {HTMLVideoElement} video
* The originating video source element
* @param {Number} volume
* Value between 0 (muted) and 1 (loudest)
*/
setVolume(video, volume) {
return this.#callWrapperMethod({
name: "setVolume",
@ -2466,6 +2541,15 @@ class PictureInPictureChildVideoWrapper {
});
}
/**
* OVERRIDABLE - calls the setMuted() method defined in the site wrapper script. Runs a fallback implementation
* if the method does not exist or if an error is thrown while calling it. This method is meant to mute or unmute
* a video.
* @param {HTMLVideoElement} video
* The originating video source element
* @param {Boolean} shouldMute
* Boolean value true to mute the video, or false to unmute the video
*/
setMuted(video, shouldMute) {
return this.#callWrapperMethod({
name: "setMuted",
@ -2477,7 +2561,17 @@ class PictureInPictureChildVideoWrapper {
});
}
setCaptionContainerObserver(video) {
/**
* OVERRIDABLE - calls the setCaptionContainerObserver() method defined in the site wrapper script. Runs a fallback implementation
* if the method does not exist or if an error is thrown while calling it. This method is meant to listen for any cue changes in a
* video's caption container and execute a callback function responsible for updating the pip window's text tracks container whenever
* a cue change is triggered {@see updatePiPTextTracks()}.
* @param {HTMLVideoElement} video
* The originating video source element
* @param {Function} callback
* The callback function to be executed when cue changes are detected
*/
setCaptionContainerObserver(video, callback) {
return this.#callWrapperMethod({
name: "setCaptionContainerObserver",
args: [
@ -2491,6 +2585,14 @@ class PictureInPictureChildVideoWrapper {
});
}
/**
* OVERRIDABLE - calls the shouldHideToggle() method defined in the site wrapper script. Runs a fallback implementation
* if the method does not exist or if an error is thrown while calling it. This method is meant to determine if the pip toggle
* for a video should be hidden by the site wrapper.
* @param {HTMLVideoElement} video
* The originating video source element
* @returns {Boolean} Boolean value true if the pip toggle should be hidden by the site wrapper, or false if it should not
*/
shouldHideToggle(video) {
return this.#callWrapperMethod({
name: "shouldHideToggle",

6
toolkit/actors/docs/picture-in-picture-child-video-wrapper-api.rst Normal file
View File

@ -0,0 +1,6 @@
.. _picture_in_picture_child_video_wrapper_api:
PictureInPictureChildVideoWrapper Reference
===========================================
.. js:autoclass:: PictureInPictureChildVideoWrapper
:members:

2
toolkit/actors/moz.build
View File

@ -19,6 +19,8 @@ with Files("KeyPressEventModelCheckerChild.jsm"):
with Files("PictureInPictureChild.jsm"):
BUG_COMPONENT = ("Toolkit", "Picture-in-Picture")
SPHINX_TREES["actors"] = "docs"
TESTING_JS_MODULES += [
"TestProcessActorChild.jsm",
"TestProcessActorParent.jsm",

2
toolkit/components/pictureinpicture/PictureInPicture.jsm
View File

@ -153,7 +153,7 @@ var PictureInPicture = {
* @param {PictureInPictureParent} pipActorRef
* Reference to the calling PictureInPictureParent actor
*
* @return {DOM Window} the player window if it exists and is not in the
* @returns {Window} the player window if it exists and is not in the
* process of being closed. Returns null otherwise.
*/
getWeakPipPlayer(pipActorRef) {

189
toolkit/components/pictureinpicture/docs/index.rst
View File

@ -189,10 +189,197 @@ If ``video`` is being cloned visually to another element, calling this method wi
A read-only value that returns ``true`` if ``video`` is being cloned visually.
Site-specific video wrappers
============================
A site-specific video wrapper allows for the creation of custom scripts that the Picture-in-Picture component can utilize when videos are loaded in specific domains. Currently, some uses of video wrappers include:
* Integration of captions and subtitles support on certain video streaming sites
* Fixing inconsistent video behaviour when using Picture-in-Picture controls
* Hiding the Picture-in-Picture toggle for videos on particular areas of a page, given a URL (rather than hiding the toggle for all videos on a page)
``PictureInPictureChildVideoWrapper`` and ``videoWrapperScriptPath``
--------------------------------------------------------------------
``PictureInPictureChildVideoWrapper`` is a special class that represents a video wrapper. It is defined in ``PictureInPictureChild.jsm`` and maps to a ``videoWrapperScriptPath``, which is the path of the custom wrapper script to use.
``videoWrapperScriptPath`` is defined in `browser/extensions/pictureinpicture/data/picture_in_picture_overrides.js <https://searchfox.org/mozilla-central/source/browser/extensions/pictureinpicture/data/picture_in_picture_overrides.js>`_ for a domain,
and custom wrapper scripts are defined in `browser/extensions/pictureinpicture/video-wrappers <https://searchfox.org/mozilla-central/source/browser/extensions/pictureinpicture/video-wrappers>`_.
If a ``videoWrapperScriptPath`` is detected while initializing the Picture-in-Picture toggle or window, we immediately create a new instance of ``PictureInPictureChildVideoWrapper`` based on the given path, allowing us to run our custom scripts.
API
^^^
See the full list of methods at `API References <#toolkit-actors-pictureinpicturechild-jsm>`_.
Sandbox
^^^^^^^
Performing video control operations on the originating video requires executing code in the browser content. For security reasons, we utilize a *sandbox* to isolate these operations and prevent direct access to ``PictureInPictureChild``. In other words, we run content code within the sandbox itself.
However, it is necessary to waive :ref:`xray vision <Waiving_Xray_vision>` so that we can execute the video control operations. This is done by reading the wrapper’s ``.wrappedJSObject`` property.
Adding a new site-specific video wrapper
----------------------------------------
Creating a new wrapper script file
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Add a new JS file for the new video wrapper in `browser/extensions/pictureinpicture/video-wrappers <https://searchfox.org/mozilla-central/source/browser/extensions/pictureinpicture/video-wrappers>`_.
The file must meet several requirements to get the wrapper working.
**Script file requirements**:
* Defined class ``PictureInPictureVideoWrapper``
* Assigned ``this.PictureInPictureVideoWrapper = PictureInPictureVideoWrapper``
**PictureInPictureVideoWrapper class requirements**:
* Implementation of at least one overridable method (see :ref:`picture_in_picture_child_video_wrapper_api`)
**Overriden method requirements**:
* Return value with a type that corresponds to ``validateRetVal`` in ``PictureInPictureChildVideoWrapper.#callWrapperMethod()``
Below is an example of a script file ``mock-wrapper.js`` that overrides an existing method ``setMuted()`` in ``PictureInPictureChildVideoWrapper``:
.. code-block:: js
// sample file `mock-wrapper.js`
class PictureInPictureVideoWrapper {
setMuted(video, shouldMute) {
if (video.muted !== shouldMute) {
let muteButton = document.querySelector("#player .mute-button");
if (muteButton) {
muteButton.click();
} else {
video.muted = shouldMute;
}
}
}
}
this.PictureInPictureVideoWrapper = PictureInPictureVideoWrapper
.. note::
If a new ``PictureInPictureChildVideoWrapper`` video control method is needed, see `Adding a new video control method`_.
Declaring ``videoWrapperScriptPath``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Declare a property ``videoWrapperScriptPath`` for the site at `browser/extensions/pictureinpicture/data/picture_in_picture_overrides.js <https://searchfox.org/mozilla-central/source/browser/extensions/pictureinpicture/data/picture_in_picture_overrides.js>`_:
.. code-block:: js
someWebsite: {
"https://*.somewebsite.com/*": {
videoWrapperScriptPath: "video-wrappers/mock-wrapper.js",
},
}
In this example, the URL pattern ``https://*.somewebsite.com/*`` is provided for a site named ``someWebsite``.
Picture-in-Picture checks for any overrides upon initialization, and it will load scripts specified by ``videoWrapperScriptPath``.
The scripts located at ``video-wrappers/mock-wrapper.js`` will therefore run whenever we view a video from a URL matching ``somewebsite.com``.
Registering the new wrapper in ``moz.build``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
We should update `browser/extensions/pictureinpicture/moz.build <https://searchfox.org/mozilla-central/source/browser/extensions/pictureinpicture/moz.build>`_ by adding the path of the newly created wrapper:
.. code-block:: js
FINAL_TARGET_FILES.features["pictureinpicture@mozilla.org"]["video-wrappers"] += [
"video-wrappers/mock-wrapper.js",
"video-wrappers/netflix.js",
"video-wrappers/youtube.js",
]
As expected for any ``moz.build`` file, order matters. Registered paths should be listed in alphabetical order. Otherwise, the build will fail.
Adding a new video control method
---------------------------------
If none of the existing overridable methods in ``PictureInPictureChildVideoWrapper`` are applicable for a bug fix or feature enhancement,
we can create a new one by calling ``#callWrapperMethod()``. Below is an example of how we would define a new overridable method ``setMuted()``:
.. code-block:: js
// class PictureInPictureChildVideoWrapper in PictureInPictureChild.jsm
setMuted(video, shouldMute) {
return this.#callWrapperMethod({
name: "setMuted",
args: [video, shouldMute],
fallback: () => {
video.muted = shouldMute;
},
validateRetVal: retVal => retVal == null,
});
}
The new method passes to ``#callWrapperMethod()``:
#. The method name
#. The expected arguments that a wrapper script may use
#. A fallback function
#. A conditional expression that validates the return value
The fallback function only executes if a wrapper script fails or if the method is not overriden.
``validateRetVal`` checks the type of the return value and ensures it matches the expected type. If there is no return value, simply validate if type is ``null``.
.. note::
Generic method names are preferred so that they can be used for any video wrapper.
For example: instead of naming a method ``updateCaptionsContainerForSiteA()``, use ``updateCaptionsContainer()``.
Using the new video control method
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Once the new method is defined, it can be used throughout ``PictureInPictureChild.jsm``. In the current example, we call
``PictureInPictureChildVideoWrapper.setMuted()`` to mute or unmute a video. ``this.videoWrapper`` is an instance of
``PictureInPictureChildVideoWrapper``:
.. code-block:: js
// class PictureInPictureChild in PictureInPictureChild.jsm
mute() {
let video = this.getWeakVideo();
if (video && this.videoWrapper) {
this.videoWrapper.setMuted(video, true);
}
}
unmute() {
let video = this.getWeakVideo();
if (video && this.videoWrapper) {
this.videoWrapper.setMuted(video, false);
}
}
Testing site-specific video wrappers
------------------------------------
Automated Tests
^^^^^^^^^^^^^^^
Automated tests for site specific wrappers are currently limited. New tests can be made in `browser/extensions/pictureinpicture/tests/browser <https://searchfox.org/mozilla-central/source/browser/extensions/pictureinpicture/tests/browser>`_ to ensure
general functionality, but these are restricted to Firefox Nightly and do not test functionality on specific sites.
Some challenges with writing tests include:
* Accessing DRM content
* Log-in credentials if a site requires a user account
* Detecting modifications to a web page or video player that render a wrapper script obsolete
Manual Tests
^^^^^^^^^^^^
The go-to approach right now is to test video wrappers manually, in tandem with reviews provided by the phabricator group `#pip-reviewers <https://phabricator.services.mozilla.com/project/profile/163/>`_. Below are some questions that reviewers will consider:
* Does Picture-in-Picture crash or freeze?
* Does the wrapper work on Windows, MacOS, and Linux?
* Do Picture-in-Picture features work as expected? (Picture-in-Picture toggle, text tracks, video controls, etc.)
* Do existing automated tests work as they should?
.. warning::
DRM content may not load for all local Firefox builds. One possible solution is to test the video wrapper in a try build (ex. Linux).
Depending on the changes made, we may also require the script to run under a temporary pref such as ``media.videocontrols.picture-in-picture.WIP.someWebsiteWrapper`` for the purpose of testing changes in Firefox Nightly.
API References
====================================
==============
``toolkit/components/pictureinpicture``
---------------------------------------
.. toctree::
:maxdepth: 1
picture-in-picture-api
player-api
``toolkit/actors/PictureInPictureChild.jsm``
--------------------------------------------
* :ref:`picture_in_picture_child_video_wrapper_api`