Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-27 14:52:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1784109 - Override SurfaceTexture transform for videos on Mediatek 6735. r=gfx-reviewers,media-playback-reviewers,lsalzman,alwu

Browse Source 
On Android, SurfaceTextures provide a transform matrix that should be
applied to texture coordinates when sampling from the texture. Prior
to bug 1731980 we ignored this value, and simply y-flipped the video
instead. On most devices the transform is just a y-flip, so this
produced the correct results. However, on some devices the transform
included a scale as well as the y-flip, meaning that we rendered
videos at an incorrect size.

The fix for bug 1731980 was to correctly apply the transformation.
However, it now appears that on Mediatek 6735 devices the transform
provided by the system is incorrect. On these devices, videos were
rendered correctly when we ignored the transform and just did a
y-flip, and now that we apply the transform videos are rendered at the
wrong size.

This patch makes it so that we override the system-provided transform
on these devices with a simple y-flip. The existing mIgnoreTransform
flag has been changed to an optional "transform override" value to
achieve this. We ensure that we only override the transform for
SurfaceTextures that are output from a MediaCodec, to ensure that we
don't accidentally apply the wrong transform to SurfaceTextures
attached to other sources.

Differential Revision: https://phabricator.services.mozilla.com/D155706
This commit is contained in:
Jamie Nicol 2022-09-09 14:43:21 +00:00
parent e97396d469
commit 50310d33ee
13 changed files with 89 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
dom/media/platforms/android/RemoteDataDecoder.cpp
View File

@ -7,6 +7,7 @@
#include <jni.h>
#include "AndroidBridge.h"
#include "AndroidBuild.h"
#include "AndroidDecoderModule.h"
#include "EMEDecoderModule.h"
#include "GLImages.h"
@ -17,6 +18,7 @@
#include "SimpleMap.h"
#include "VPXDecoder.h"
#include "VideoUtils.h"
#include "mozilla/gfx/Matrix.h"
#include "mozilla/java/CodecProxyWrappers.h"
#include "mozilla/java/GeckoSurfaceWrappers.h"
#include "mozilla/java/SampleBufferWrappers.h"
@ -170,6 +172,16 @@ class RemoteVideoDecoder : public RemoteDataDecoder {
mIsCodecSupportAdaptivePlayback =
mJavaDecoder->IsAdaptivePlaybackSupported();
mIsHardwareAccelerated = mJavaDecoder->IsHardwareAccelerated();
// On Mediatek 6735 devices we have observed that the transform obtained
// from SurfaceTexture.getTransformMatrix() is incorrect for surfaces
// produced by a MediaCodec. We therefore override the transform to be a
// simple y-flip to ensure it is rendered correctly.
if (java::sdk::Build::HARDWARE()->ToString().EqualsASCII("mt6735")) {
mTransformOverride = Some(
gfx::Matrix4x4::Scaling(1.0, -1.0, 1.0).PostTranslate(0.0, 1.0, 0.0));
}
return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__);
}
@ -315,7 +327,7 @@ class RemoteVideoDecoder : public RemoteDataDecoder {
if (ok && (size > 0 || presentationTimeUs >= 0)) {
RefPtr<layers::Image> img = new layers::SurfaceTextureImage(
mSurfaceHandle, inputInfo.mImageSize, false /* NOT continuous */,
gl::OriginPos::BottomLeft, mConfig.HasAlpha());
gl::OriginPos::BottomLeft, mConfig.HasAlpha(), mTransformOverride);
img->AsSurfaceTextureImage()->RegisterSetCurrentCallback(
std::move(releaseSample));
@ -341,6 +353,9 @@ class RemoteVideoDecoder : public RemoteDataDecoder {
const VideoInfo mConfig;
java::GeckoSurface::GlobalRef mSurface;
AndroidSurfaceTextureHandle mSurfaceHandle;
// Used to override the SurfaceTexture transform on some devices where the
// decoder provides a buggy value.
Maybe<gfx::Matrix4x4> mTransformOverride;
// Only accessed on reader's task queue.
bool mIsCodecSupportAdaptivePlayback = false;
// Can be accessed on any thread, but only written on during init.

2
gfx/gl/SharedSurfaceEGL.cpp
View File

@ -254,7 +254,7 @@ Maybe<layers::SurfaceDescriptor>
SharedSurface_SurfaceTexture::ToSurfaceDescriptor() {
return Some(layers::SurfaceTextureDescriptor(
mSurface->GetHandle(), mDesc.size, gfx::SurfaceFormat::R8G8B8A8,
false /* NOT continuous */, false /* Do not ignore transform */));
false /* NOT continuous */, Nothing() /* Do not override transform */));
}
SurfaceFactory_SurfaceTexture::SurfaceFactory_SurfaceTexture(GLContext& gl)

14
gfx/layers/GLImages.cpp
View File

@ -72,17 +72,17 @@ already_AddRefed<gfx::SourceSurface> GLImage::GetAsSourceSurface() {
}
#ifdef MOZ_WIDGET_ANDROID
SurfaceTextureImage::SurfaceTextureImage(AndroidSurfaceTextureHandle aHandle,
const gfx::IntSize& aSize,
bool aContinuous,
gl::OriginPos aOriginPos,
bool aHasAlpha /* = true */)
SurfaceTextureImage::SurfaceTextureImage(
AndroidSurfaceTextureHandle aHandle, const gfx::IntSize& aSize,
bool aContinuous, gl::OriginPos aOriginPos, bool aHasAlpha,
Maybe<gfx::Matrix4x4> aTransformOverride)
: GLImage(ImageFormat::SURFACE_TEXTURE),
mHandle(aHandle),
mSize(aSize),
mContinuous(aContinuous),
mOriginPos(aOriginPos),
mHasAlpha(aHasAlpha) {
mHasAlpha(aHasAlpha),
mTransformOverride(aTransformOverride) {
MOZ_ASSERT(mHandle);
}
@ -90,7 +90,7 @@ Maybe<SurfaceDescriptor> SurfaceTextureImage::GetDesc() {
SurfaceDescriptor sd = SurfaceTextureDescriptor(
mHandle, mSize,
mHasAlpha ? gfx::SurfaceFormat::R8G8B8A8 : gfx::SurfaceFormat::R8G8B8X8,
false /* NOT continuous */, false /* do not ignore transform */);
false /* NOT continuous */, mTransformOverride);
return Some(sd);
}
#endif

17
gfx/layers/GLImages.h
View File

@ -9,10 +9,12 @@
#include "GLContextTypes.h"
#include "GLTypes.h"
#include "ImageContainer.h" // for Image
#include "ImageTypes.h" // for ImageFormat::SHARED_GLTEXTURE
#include "nsCOMPtr.h" // for already_AddRefed
#include "mozilla/gfx/Point.h" // for IntSize
#include "ImageContainer.h" // for Image
#include "ImageTypes.h" // for ImageFormat::SHARED_GLTEXTURE
#include "nsCOMPtr.h" // for already_AddRefed
#include "mozilla/Maybe.h" // for Maybe
#include "mozilla/gfx/Matrix.h" // for Matrix4x4
#include "mozilla/gfx/Point.h" // for IntSize
#ifdef MOZ_WIDGET_ANDROID
# include "AndroidSurfaceTexture.h"
@ -42,13 +44,17 @@ class SurfaceTextureImage : public GLImage {
SurfaceTextureImage(AndroidSurfaceTextureHandle aHandle,
const gfx::IntSize& aSize, bool aContinuous,
gl::OriginPos aOriginPos, bool aHasAlpha = true);
gl::OriginPos aOriginPos, bool aHasAlpha,
Maybe<gfx::Matrix4x4> aTransformOverride);
gfx::IntSize GetSize() const override { return mSize; }
AndroidSurfaceTextureHandle GetHandle() const { return mHandle; }
bool GetContinuous() const { return mContinuous; }
gl::OriginPos GetOriginPos() const { return mOriginPos; }
bool GetHasAlpha() const { return mHasAlpha; }
const Maybe<gfx::Matrix4x4>& GetTransformOverride() const {
return mTransformOverride;
}
already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() override {
// We can implement this, but currently don't want to because it will cause
@ -78,6 +84,7 @@ class SurfaceTextureImage : public GLImage {
bool mContinuous;
gl::OriginPos mOriginPos;
const bool mHasAlpha;
const Maybe<gfx::Matrix4x4> mTransformOverride;
UniquePtr<SetCurrentCallback> mSetCurrentCallback;
};

1
gfx/layers/client/ImageClient.cpp
View File

@ -120,6 +120,7 @@ already_AddRefed<TextureClient> ImageClient::CreateTextureClientForImage(
texture = AndroidSurfaceTextureData::CreateTextureClient(
typedImage->GetHandle(), size, typedImage->GetContinuous(),
typedImage->GetOriginPos(), typedImage->GetHasAlpha(),
typedImage->GetTransformOverride(),
aKnowsCompositor->GetTextureForwarder(), TextureFlags::DEFAULT);
#endif
} else {

3
gfx/layers/ipc/LayersSurfaces.ipdlh
View File

@ -19,6 +19,7 @@ using mozilla::gfx::ColorSpace2 from "mozilla/gfx/Types.h";
using mozilla::gfx::SurfaceFormat from "mozilla/gfx/Types.h";
using mozilla::gfx::IntRect from "mozilla/gfx/Rect.h";
using mozilla::gfx::IntSize from "mozilla/gfx/Point.h";
using mozilla::gfx::Matrix4x4 from "mozilla/gfx/Matrix.h";
[MoveOnly] using mozilla::ipc::SharedMemoryBasic::Handle from "mozilla/ipc/SharedMemoryBasic.h";
using gfxImageFormat from "gfxTypes.h";
using mozilla::layers::MaybeVideoBridgeSource from "mozilla/layers/VideoBridgeUtils.h";
@ -80,7 +81,7 @@ namespace layers {
IntSize size;
SurfaceFormat format;
bool continuous;
bool ignoreTransform;
Matrix4x4? transformOverride;
};
[Comparable] struct SurfaceDescriptorAndroidHardwareBuffer {

21
gfx/layers/opengl/TextureClientOGL.cpp
View File

@ -38,24 +38,27 @@ class CompositableForwarder;
already_AddRefed<TextureClient> AndroidSurfaceTextureData::CreateTextureClient(
AndroidSurfaceTextureHandle aHandle, gfx::IntSize aSize, bool aContinuous,
gl::OriginPos aOriginPos, bool aHasAlpha, LayersIPCChannel* aAllocator,
gl::OriginPos aOriginPos, bool aHasAlpha,
Maybe<gfx::Matrix4x4> aTransformOverride, LayersIPCChannel* aAllocator,
TextureFlags aFlags) {
if (aOriginPos == gl::OriginPos::BottomLeft) {
aFlags |= TextureFlags::ORIGIN_BOTTOM_LEFT;
}
return TextureClient::CreateWithData(
new AndroidSurfaceTextureData(aHandle, aSize, aContinuous, aHasAlpha),
new AndroidSurfaceTextureData(aHandle, aSize, aContinuous, aHasAlpha,
aTransformOverride),
aFlags, aAllocator);
}
AndroidSurfaceTextureData::AndroidSurfaceTextureData(
AndroidSurfaceTextureHandle aHandle, gfx::IntSize aSize, bool aContinuous,
bool aHasAlpha)
bool aHasAlpha, Maybe<gfx::Matrix4x4> aTransformOverride)
: mHandle(aHandle),
mSize(aSize),
mContinuous(aContinuous),
mHasAlpha(aHasAlpha) {
mHasAlpha(aHasAlpha),
mTransformOverride(aTransformOverride) {
MOZ_ASSERT(mHandle);
}
@ -73,7 +76,7 @@ bool AndroidSurfaceTextureData::Serialize(SurfaceDescriptor& aOutDescriptor) {
aOutDescriptor = SurfaceTextureDescriptor(
mHandle, mSize,
mHasAlpha ? gfx::SurfaceFormat::R8G8B8A8 : gfx::SurfaceFormat::R8G8B8X8,
mContinuous, false /* do not ignore transform */);
mContinuous, mTransformOverride);
return true;
}
@ -140,7 +143,7 @@ AndroidNativeWindowTextureData::AndroidNativeWindowTextureData(
// Ideally here we'd call ANativeWindow_setBuffersTransform() with the
// identity transform, but that is only available on api level >= 26.
// Instead use the SurfaceDescriptor's ignoreTransform flag when serializing.
// Instead use SurfaceDescriptor's transformOverride flag when serializing.
}
void AndroidNativeWindowTextureData::FillInfo(TextureData::Info& aInfo) const {
@ -154,9 +157,9 @@ void AndroidNativeWindowTextureData::FillInfo(TextureData::Info& aInfo) const {
bool AndroidNativeWindowTextureData::Serialize(
SurfaceDescriptor& aOutDescriptor) {
aOutDescriptor = SurfaceTextureDescriptor(mSurface->GetHandle(), mSize,
mFormat, false /* not continuous */,
true /* ignore transform */);
aOutDescriptor = SurfaceTextureDescriptor(
mSurface->GetHandle(), mSize, mFormat, false /* not continuous */,
Some(gfx::Matrix4x4()) /* always use identity transform */);
return true;
}

7
gfx/layers/opengl/TextureClientOGL.h
View File

@ -41,7 +41,8 @@ class AndroidSurfaceTextureData : public TextureData {
public:
static already_AddRefed<TextureClient> CreateTextureClient(
AndroidSurfaceTextureHandle aHandle, gfx::IntSize aSize, bool aContinuous,
gl::OriginPos aOriginPos, bool aHasAlpha, LayersIPCChannel* aAllocator,
gl::OriginPos aOriginPos, bool aHasAlpha,
Maybe<gfx::Matrix4x4> aTransformOverride, LayersIPCChannel* aAllocator,
TextureFlags aFlags);
virtual ~AndroidSurfaceTextureData();
@ -61,12 +62,14 @@ class AndroidSurfaceTextureData : public TextureData {
protected:
AndroidSurfaceTextureData(AndroidSurfaceTextureHandle aHandle,
gfx::IntSize aSize, bool aContinuous,
bool aHasAlpha);
bool aHasAlpha,
Maybe<gfx::Matrix4x4> aTransformOverride);
const AndroidSurfaceTextureHandle mHandle;
const gfx::IntSize mSize;
const bool mContinuous;
const bool mHasAlpha;
const Maybe<gfx::Matrix4x4> mTransformOverride;
};
class AndroidNativeWindowTextureData : public TextureData {

25
gfx/layers/opengl/TextureHostOGL.cpp
View File

@ -69,7 +69,7 @@ already_AddRefed<TextureHost> CreateTextureHostOGL(
result = new SurfaceTextureHost(aFlags, surfaceTexture, desc.size(),
desc.format(), desc.continuous(),
desc.ignoreTransform());
desc.transformOverride());
break;
}
case SurfaceDescriptor::TSurfaceDescriptorAndroidHardwareBuffer: {
@ -437,14 +437,14 @@ SurfaceTextureSource::SurfaceTextureSource(
TextureSourceProvider* aProvider,
mozilla::java::GeckoSurfaceTexture::Ref& aSurfTex,
gfx::SurfaceFormat aFormat, GLenum aTarget, GLenum aWrapMode,
gfx::IntSize aSize, bool aIgnoreTransform)
gfx::IntSize aSize, Maybe<gfx::Matrix4x4> aTransformOverride)
: mGL(aProvider->GetGLContext()),
mSurfTex(aSurfTex),
mFormat(aFormat),
mTextureTarget(aTarget),
mWrapMode(aWrapMode),
mSize(aSize),
mIgnoreTransform(aIgnoreTransform) {}
mTransformOverride(aTransformOverride) {}
void SurfaceTextureSource::BindTexture(GLenum aTextureUnit,
gfx::SamplingFilter aSamplingFilter) {
@ -468,11 +468,14 @@ gfx::Matrix4x4 SurfaceTextureSource::GetTextureTransform() {
gfx::Matrix4x4 ret;
// GetTransformMatrix() returns the transform set by the producer side of
// the SurfaceTexture. We should ignore this if we know the transform should
// be identity but the producer couldn't set it correctly, like is the
// case for AndroidNativeWindowTextureData.
if (!mIgnoreTransform) {
// GetTransformMatrix() returns the transform set by the producer side of the
// SurfaceTexture that must be applied to texture coordinates when
// sampling. In some cases we may have set an override value, such as in
// AndroidNativeWindowTextureData where we own the producer side, or for
// MediaCodec output on devices where where we know the value is incorrect.
if (mTransformOverride) {
ret = *mTransformOverride;
} else {
const auto& surf = java::sdk::SurfaceTexture::LocalRef(
java::sdk::SurfaceTexture::Ref::From(mSurfTex));
AndroidSurfaceTexture::GetTransformMatrix(surf, &ret);
@ -488,13 +491,13 @@ void SurfaceTextureSource::DeallocateDeviceData() { mSurfTex = nullptr; }
SurfaceTextureHost::SurfaceTextureHost(
TextureFlags aFlags, mozilla::java::GeckoSurfaceTexture::Ref& aSurfTex,
gfx::IntSize aSize, gfx::SurfaceFormat aFormat, bool aContinuousUpdate,
bool aIgnoreTransform)
Maybe<Matrix4x4> aTransformOverride)
: TextureHost(TextureHostType::AndroidSurfaceTexture, aFlags),
mSurfTex(aSurfTex),
mSize(aSize),
mFormat(aFormat),
mContinuousUpdate(aContinuousUpdate),
mIgnoreTransform(aIgnoreTransform) {
mTransformOverride(aTransformOverride) {
if (!mSurfTex) {
return;
}
@ -563,7 +566,7 @@ void SurfaceTextureHost::CreateRenderTexture(
const wr::ExternalImageId& aExternalImageId) {
RefPtr<wr::RenderTextureHost> texture =
new wr::RenderAndroidSurfaceTextureHost(
mSurfTex, mSize, mFormat, mContinuousUpdate, mIgnoreTransform);
mSurfTex, mSize, mFormat, mContinuousUpdate, mTransformOverride);
wr::RenderThread::Get()->RegisterExternalImage(aExternalImageId,
texture.forget());
}

9
gfx/layers/opengl/TextureHostOGL.h
View File

@ -347,7 +347,7 @@ class SurfaceTextureSource : public TextureSource, public TextureSourceOGL {
java::GeckoSurfaceTexture::Ref& aSurfTex,
gfx::SurfaceFormat aFormat, GLenum aTarget,
GLenum aWrapMode, gfx::IntSize aSize,
bool aIgnoreTransform);
Maybe<gfx::Matrix4x4> aTransformOverride);
const char* Name() const override { return "SurfaceTextureSource"; }
@ -379,7 +379,7 @@ class SurfaceTextureSource : public TextureSource, public TextureSourceOGL {
const GLenum mTextureTarget;
const GLenum mWrapMode;
const gfx::IntSize mSize;
const bool mIgnoreTransform;
const Maybe<gfx::Matrix4x4> mTransformOverride;
};
class SurfaceTextureHost : public TextureHost {
@ -387,7 +387,8 @@ class SurfaceTextureHost : public TextureHost {
SurfaceTextureHost(TextureFlags aFlags,
mozilla::java::GeckoSurfaceTexture::Ref& aSurfTex,
gfx::IntSize aSize, gfx::SurfaceFormat aFormat,
bool aContinuousUpdate, bool aIgnoreTransform);
bool aContinuousUpdate,
Maybe<gfx::Matrix4x4> aTransformOverride);
virtual ~SurfaceTextureHost();
@ -439,7 +440,7 @@ class SurfaceTextureHost : public TextureHost {
const gfx::IntSize mSize;
const gfx::SurfaceFormat mFormat;
bool mContinuousUpdate;
const bool mIgnoreTransform;
const Maybe<gfx::Matrix4x4> mTransformOverride;
RefPtr<CompositorOGL> mCompositor;
RefPtr<SurfaceTextureSource> mTextureSource;
};

18
gfx/webrender_bindings/RenderAndroidSurfaceTextureHost.cpp
View File

@ -17,12 +17,13 @@ namespace wr {
RenderAndroidSurfaceTextureHost::RenderAndroidSurfaceTextureHost(
const java::GeckoSurfaceTexture::GlobalRef& aSurfTex, gfx::IntSize aSize,
gfx::SurfaceFormat aFormat, bool aContinuousUpdate, bool aIgnoreTransform)
gfx::SurfaceFormat aFormat, bool aContinuousUpdate,
Maybe<gfx::Matrix4x4> aTransformOverride)
: mSurfTex(aSurfTex),
mSize(aSize),
mFormat(aFormat),
mContinuousUpdate(aContinuousUpdate),
mIgnoreTransform(aIgnoreTransform),
mTransformOverride(aTransformOverride),
mPrepareStatus(STATUS_NONE),
mAttachedToGLContext(false) {
MOZ_COUNT_CTOR_INHERITED(RenderAndroidSurfaceTextureHost, RenderTextureHost);
@ -273,11 +274,14 @@ std::pair<gfx::Point, gfx::Point> RenderAndroidSurfaceTextureHost::GetUvCoords(
gfx::IntSize aTextureSize) const {
gfx::Matrix4x4 transform;
// GetTransformMatrix() returns the transform set by the producer side of
// the SurfaceTexture. We should ignore this if we know the transform should
// be identity but the producer couldn't set it correctly, like is the
// case for AndroidNativeWindowTextureData.
if (mSurfTex && !mIgnoreTransform) {
// GetTransformMatrix() returns the transform set by the producer side of the
// SurfaceTexture that must be applied to texture coordinates when
// sampling. In some cases we may have set an override value, such as in
// AndroidNativeWindowTextureData where we own the producer side, or for
// MediaCodec output on devices where where we know the value is incorrect.
if (mTransformOverride) {
transform = *mTransformOverride;
} else if (mSurfTex) {
const auto& surf = java::sdk::SurfaceTexture::LocalRef(
java::sdk::SurfaceTexture::Ref::From(mSurfTex));
gl::AndroidSurfaceTexture::GetTransformMatrix(surf, &transform);

4
gfx/webrender_bindings/RenderAndroidSurfaceTextureHost.h
View File

@ -24,7 +24,7 @@ class RenderAndroidSurfaceTextureHost final : public RenderTextureHostSWGL {
explicit RenderAndroidSurfaceTextureHost(
const java::GeckoSurfaceTexture::GlobalRef& aSurfTex, gfx::IntSize aSize,
gfx::SurfaceFormat aFormat, bool aContinuousUpdate,
bool aIgnoreTransform);
Maybe<gfx::Matrix4x4> aTransformOverride);
wr::WrExternalImage Lock(uint8_t aChannelIndex, gl::GLContext* aGL,
wr::ImageRendering aRendering) override;
@ -59,7 +59,7 @@ class RenderAndroidSurfaceTextureHost final : public RenderTextureHostSWGL {
// mContinuousUpdate was used for rendering video in the past.
// It is not used on current gecko.
const bool mContinuousUpdate;
const bool mIgnoreTransform;
const Maybe<gfx::Matrix4x4> mTransformOverride;
private:
virtual ~RenderAndroidSurfaceTextureHost();

2
gfx/webrender_bindings/RenderCompositorOGLSWGL.cpp
View File

@ -200,7 +200,7 @@ void RenderCompositorOGLSWGL::HandleExternalImage(
// since the effect doesn't hold a strong reference.
RefPtr<SurfaceTextureSource> layer = new SurfaceTextureSource(
(TextureSourceProvider*)mCompositor, host->mSurfTex, host->mFormat,
target, wrapMode, host->mSize, host->mIgnoreTransform);
target, wrapMode, host->mSize, host->mTransformOverride);
RefPtr<TexturedEffect> texturedEffect =
CreateTexturedEffect(host->mFormat, layer, aFrameSurface.mFilter,
/* isAlphaPremultiplied */ true);