Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-08 02:14:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1781526 - Implement means to use VideoFrame with WebGL. r=webidl,smaug,lsalzman

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D181118
This commit is contained in:
Andrew Osmond 2023-09-20 22:40:24 +00:00
parent 664deabf64
commit 34f04947f7
8 changed files with 119 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
dom/canvas/ClientWebGLContext.cpp
View File

@ -36,6 +36,7 @@
#include "TexUnpackBlob.h"
#include "WebGLMethodDispatcher.h"
#include "WebGLChild.h"
#include "WebGLTextureUpload.h"
#include "WebGLValidateStrings.h"
namespace mozilla {
@ -4186,22 +4187,6 @@ void ClientWebGLContext::TexStorage(uint8_t funcDims, GLenum texTarget,
internalFormat, CastUvec3(size));
}
namespace webgl {
// TODO: Move these definitions into statics here.
Maybe<webgl::TexUnpackBlobDesc> FromImageBitmap(
GLenum target, Maybe<uvec3> size, const dom::ImageBitmap& imageBitmap,
ErrorResult* const out_rv);
Maybe<webgl::TexUnpackBlobDesc> FromOffscreenCanvas(
const ClientWebGLContext&, GLenum target, Maybe<uvec3> size,
const dom::OffscreenCanvas& src, ErrorResult* const out_error);
Maybe<webgl::TexUnpackBlobDesc> FromDomElem(const ClientWebGLContext&,
GLenum target, Maybe<uvec3> size,
const dom::Element& src,
ErrorResult* const out_error);
} // namespace webgl
// -
void webgl::TexUnpackBlobDesc::Shrink(const webgl::PackingInfo& pi) {
@ -4366,6 +4351,11 @@ void ClientWebGLContext::TexImage(uint8_t funcDims, GLenum imageTarget,
*this, imageTarget, size, *(src.mOffscreenCanvas), src.mOut_error);
}
if (src.mVideoFrame) {
return webgl::FromVideoFrame(*this, imageTarget, size, *(src.mVideoFrame),
src.mOut_error);
}
if (src.mDomElem) {
return webgl::FromDomElem(*this, imageTarget, size, *(src.mDomElem),
src.mOut_error);

6
dom/canvas/ClientWebGLContext.h
View File

@ -731,6 +731,12 @@ struct TexImageSourceAdapter final : public TexImageSource {
mOut_error = out_error;
}
TexImageSourceAdapter(const dom::VideoFrame* videoFrame,
ErrorResult* const out_error) {
mVideoFrame = videoFrame;
mOut_error = out_error;
}
TexImageSourceAdapter(const dom::Element* domElem,
ErrorResult* const out_error) {
mDomElem = domElem;

72
dom/canvas/WebGLTextureUpload.cpp
View File

@ -3,6 +3,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "WebGLTextureUpload.h"
#include "WebGLTexture.h"
#include <algorithm>
@ -37,6 +38,15 @@
namespace mozilla {
namespace webgl {
// The canvas spec says that drawImage should draw the first frame of
// animated images. The webgl spec doesn't mention the issue, so we do the
// same as drawImage.
static constexpr uint32_t kDefaultSurfaceFromElementFlags =
nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE |
nsLayoutUtils::SFE_USE_ELEMENT_SIZE_IF_VECTOR |
nsLayoutUtils::SFE_EXACT_SIZE_SURFACE |
nsLayoutUtils::SFE_ALLOW_NON_PREMULT;
Maybe<TexUnpackBlobDesc> FromImageBitmap(const GLenum target, Maybe<uvec3> size,
const dom::ImageBitmap& imageBitmap,
ErrorResult* const out_rv) {
@ -119,44 +129,19 @@ Maybe<webgl::TexUnpackBlobDesc> FromOffscreenCanvas(
return {};
}
// The canvas spec says that drawImage should draw the first frame of
// animated images. The webgl spec doesn't mention the issue, so we do the
// same as drawImage.
uint32_t flags = nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE;
auto sfer = nsLayoutUtils::SurfaceFromOffscreenCanvas(
const_cast<dom::OffscreenCanvas*>(&canvas), flags);
const_cast<dom::OffscreenCanvas*>(&canvas),
kDefaultSurfaceFromElementFlags);
return FromSurfaceFromElementResult(webgl, target, size, sfer, out_error);
}
RefPtr<gfx::DataSourceSurface> dataSurf;
if (sfer.GetSourceSurface()) {
dataSurf = sfer.GetSourceSurface()->GetDataSurface();
}
if (!dataSurf) {
webgl.EnqueueWarning("Resource has no data (yet?). Uploading zeros.");
if (!size) {
size.emplace(0, 0, 1);
}
return Some(
TexUnpackBlobDesc{target, size.value(), gfxAlphaType::NonPremult});
}
// We checked this above before we requested the surface.
MOZ_RELEASE_ASSERT(!sfer.mIsWriteOnly);
uvec2 canvasSize = *uvec2::FromSize(dataSurf->GetSize());
if (!size) {
size.emplace(canvasSize.x, canvasSize.y, 1);
}
return Some(TexUnpackBlobDesc{target,
size.value(),
sfer.mAlphaType,
{},
{},
Some(canvasSize),
{},
{},
dataSurf});
Maybe<webgl::TexUnpackBlobDesc> FromVideoFrame(
const ClientWebGLContext& webgl, const GLenum target, Maybe<uvec3> size,
const dom::VideoFrame& videoFrame, ErrorResult* const out_error) {
auto sfer = nsLayoutUtils::SurfaceFromVideoFrame(
const_cast<dom::VideoFrame*>(&videoFrame),
kDefaultSurfaceFromElementFlags);
return FromSurfaceFromElementResult(webgl, target, size, sfer, out_error);
}
Maybe<webgl::TexUnpackBlobDesc> FromDomElem(const ClientWebGLContext& webgl,
@ -173,13 +158,7 @@ Maybe<webgl::TexUnpackBlobDesc> FromDomElem(const ClientWebGLContext& webgl,
}
}
// The canvas spec says that drawImage should draw the first frame of
// animated images. The webgl spec doesn't mention the issue, so we do the
// same as drawImage.
uint32_t flags = nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE |
nsLayoutUtils::SFE_USE_ELEMENT_SIZE_IF_VECTOR |
nsLayoutUtils::SFE_EXACT_SIZE_SURFACE |
nsLayoutUtils::SFE_ALLOW_NON_PREMULT;
uint32_t flags = kDefaultSurfaceFromElementFlags;
const auto& unpacking = webgl.State().mPixelUnpackState;
if (unpacking.colorspaceConversion == LOCAL_GL_NONE) {
flags |= nsLayoutUtils::SFE_NO_COLORSPACE_CONVERSION;
@ -188,9 +167,12 @@ Maybe<webgl::TexUnpackBlobDesc> FromDomElem(const ClientWebGLContext& webgl,
RefPtr<gfx::DrawTarget> idealDrawTarget = nullptr; // Don't care for now.
auto sfer = nsLayoutUtils::SurfaceFromElement(
const_cast<dom::Element*>(&elem), flags, idealDrawTarget);
return FromSurfaceFromElementResult(webgl, target, size, sfer, out_error);
}
//////
Maybe<webgl::TexUnpackBlobDesc> FromSurfaceFromElementResult(
const ClientWebGLContext& webgl, const GLenum target, Maybe<uvec3> size,
SurfaceFromElementResult& sfer, ErrorResult* const out_error) {
uvec2 elemSize;
const auto& layersImage = sfer.mLayersImage;

47
dom/canvas/WebGLTextureUpload.h Normal file
View File

@ -0,0 +1,47 @@
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef WEBGLTEXTUREUPLOAD_H_
#define WEBGLTEXTUREUPLOAD_H_
#include "WebGLTypes.h"
namespace mozilla {
struct SurfaceFromElementResult;
namespace dom {
class Element;
class OffscreenCanvas;
} // namespace dom
namespace webgl {
Maybe<TexUnpackBlobDesc> FromImageBitmap(GLenum target, Maybe<uvec3> size,
const dom::ImageBitmap& imageBitmap,
ErrorResult* const out_rv);
Maybe<TexUnpackBlobDesc> FromOffscreenCanvas(const ClientWebGLContext& webgl,
GLenum target, Maybe<uvec3> size,
const dom::OffscreenCanvas& canvas,
ErrorResult* const out_error);
Maybe<TexUnpackBlobDesc> FromVideoFrame(const ClientWebGLContext& webgl,
GLenum target, Maybe<uvec3> size,
const dom::VideoFrame& videoFrame,
ErrorResult* const out_error);
Maybe<TexUnpackBlobDesc> FromDomElem(const ClientWebGLContext& webgl,
GLenum target, Maybe<uvec3> size,
const dom::Element& elem,
ErrorResult* const out_error);
Maybe<TexUnpackBlobDesc> FromSurfaceFromElementResult(
const ClientWebGLContext& webgl, GLenum target, Maybe<uvec3> size,
SurfaceFromElementResult& sfer, ErrorResult* const out_error);
} // namespace webgl
} // namespace mozilla
#endif

3
dom/canvas/WebGLTypes.h
View File

@ -951,6 +951,7 @@ class Element;
class ImageBitmap;
class ImageData;
class OffscreenCanvas;
class VideoFrame;
} // namespace dom
struct TexImageSource {
@ -965,6 +966,8 @@ struct TexImageSource {
const dom::OffscreenCanvas* mOffscreenCanvas = nullptr;
const dom::VideoFrame* mVideoFrame = nullptr;
const dom::Element* mDomElem = nullptr;
ErrorResult* mOut_error = nullptr;
};

22
dom/webidl/WebGL2RenderingContext.webidl
View File

@ -379,6 +379,9 @@ interface mixin WebGL2RenderingContextBase
[Throws] // Another overhead throws.
undefined texImage2D(GLenum target, GLint level, GLint internalformat,
GLenum format, GLenum type, OffscreenCanvas source);
[Throws] // Another overhead throws.
undefined texImage2D(GLenum target, GLint level, GLint internalformat,
GLenum format, GLenum type, VideoFrame source);
[Throws] // Another overhead throws.
undefined texSubImage2D(GLenum target, GLint level, GLint xoffset, GLint yoffset,
@ -402,6 +405,9 @@ interface mixin WebGL2RenderingContextBase
[Throws] // Another overhead throws.
undefined texSubImage2D(GLenum target, GLint level, GLint xoffset, GLint yoffset,
GLenum format, GLenum type, OffscreenCanvas source);
[Throws] // Another overhead throws.
undefined texSubImage2D(GLenum target, GLint level, GLint xoffset, GLint yoffset,
GLenum format, GLenum type, VideoFrame source);
// WebGL2 entrypoints:
[Throws] // Another overhead throws.
@ -432,6 +438,10 @@ interface mixin WebGL2RenderingContextBase
GLint border, GLenum format, GLenum type,
OffscreenCanvas source);
[Throws] // Another overhead throws.
undefined texImage2D(GLenum target, GLint level, GLint internalformat, GLsizei width, GLsizei height,
GLint border, GLenum format, GLenum type,
VideoFrame source);
[Throws] // Another overhead throws.
undefined texImage2D(GLenum target, GLint level, GLint internalformat, GLsizei width, GLsizei height,
GLint border, GLenum format, GLenum type, [AllowShared] ArrayBufferView srcData,
GLuint srcOffset);
@ -464,6 +474,10 @@ interface mixin WebGL2RenderingContextBase
GLsizei depth, GLint border, GLenum format, GLenum type,
OffscreenCanvas source);
[Throws] // Another overhead throws.
undefined texImage3D(GLenum target, GLint level, GLint internalformat, GLsizei width, GLsizei height,
GLsizei depth, GLint border, GLenum format, GLenum type,
VideoFrame source);
[Throws] // Another overhead throws.
undefined texImage3D(GLenum target, GLint level, GLint internalformat, GLsizei width, GLsizei height,
GLsizei depth, GLint border, GLenum format, GLenum type, [AllowShared] ArrayBufferView? srcData);
[Throws] // Another overhead throws.
@ -499,6 +513,10 @@ interface mixin WebGL2RenderingContextBase
GLsizei height, GLenum format, GLenum type,
OffscreenCanvas source);
[Throws] // Another overhead throws.
undefined texSubImage2D(GLenum target, GLint level, GLint xoffset, GLint yoffset, GLsizei width,
GLsizei height, GLenum format, GLenum type,
VideoFrame source);
[Throws] // Another overhead throws.
undefined texSubImage2D(GLenum target, GLint level, GLint xoffset, GLint yoffset, GLsizei width,
GLsizei height, GLenum format, GLenum type, [AllowShared] ArrayBufferView srcData,
GLuint srcOffset);
@ -532,6 +550,10 @@ interface mixin WebGL2RenderingContextBase
GLsizei width, GLsizei height, GLsizei depth, GLenum format, GLenum type,
OffscreenCanvas source);
[Throws] // Another overhead throws.
undefined texSubImage3D(GLenum target, GLint level, GLint xoffset, GLint yoffset, GLint zoffset,
GLsizei width, GLsizei height, GLsizei depth, GLenum format, GLenum type,
VideoFrame source);
[Throws] // Another overhead throws.
undefined texSubImage3D(GLenum target, GLint level, GLint xoffset, GLint yoffset, GLint zoffset,
GLsizei width, GLsizei height, GLsizei depth, GLenum format, GLenum type,
[AllowShared] ArrayBufferView? srcData, optional GLuint srcOffset = 0);

6
dom/webidl/WebGLRenderingContext.webidl
View File

@ -770,6 +770,9 @@ interface WebGLRenderingContext {
[Throws]
undefined texImage2D(GLenum target, GLint level, GLint internalformat,
GLenum format, GLenum type, OffscreenCanvas canvas); // May throw DOMException
[Throws]
undefined texImage2D(GLenum target, GLint level, GLint internalformat,
GLenum format, GLenum type, VideoFrame videoFrame); // May throw DOMException
// texSubImage2D has WebGL2 overloads.
[Throws] // Can't actually throw.
@ -794,6 +797,9 @@ interface WebGLRenderingContext {
[Throws]
undefined texSubImage2D(GLenum target, GLint level, GLint xoffset, GLint yoffset,
GLenum format, GLenum type, OffscreenCanvas canvas); // May throw DOMException
[Throws]
undefined texSubImage2D(GLenum target, GLint level, GLint xoffset, GLint yoffset,
GLenum format, GLenum type, VideoFrame videoFrame); // May throw DOMException
// uniform*fv have WebGL2 overloads, or rather extensions, that are not
// distinguishable from the WebGL1 versions when called with two arguments.

41
testing/web-platform/meta/webcodecs/videoFrame-texImage.any.js.ini
View File

@ -1,42 +1,5 @@
[videoFrame-texImage.any.worker.html]
expected:
if (os == "android") and fission: [OK, TIMEOUT]
[texImage2D with 48x36 srgb VideoFrame.]
expected: FAIL
[texSubImage2D with 48x36 srgb VideoFrame.]
expected: FAIL
[texImage2D with 480x360 srgb VideoFrame.]
expected: FAIL
[texSubImage2D with 480x360 srgb VideoFrame.]
expected: FAIL
[texImage2D with a closed VideoFrame.]
expected: FAIL
[texSubImage2D with a closed VideoFrame.]
expected: FAIL
prefs: [dom.media.webcodecs.enabled:true]
[videoFrame-texImage.any.html]
expected:
if (os == "android") and fission: [OK, TIMEOUT]
[texImage2D with 48x36 srgb VideoFrame.]
expected: FAIL
[texSubImage2D with 48x36 srgb VideoFrame.]
expected: FAIL
[texImage2D with 480x360 srgb VideoFrame.]
expected: FAIL
[texSubImage2D with 480x360 srgb VideoFrame.]
expected: FAIL
[texImage2D with a closed VideoFrame.]
expected: FAIL
[texSubImage2D with a closed VideoFrame.]
expected: FAIL
prefs: [dom.media.webcodecs.enabled:true]