Bug 1625363 - AVIF (AV1 Image File Format): experimental support. r=aosmond,necko-reviewers,valentin

There are many limitations currently, but this prototype should successfully
render most basic AVIF images. Known limitations:

- No support for any derived image items (crop, rotate, etc.)
- No support for alpha planes
- No support for ICC profiles (bug 1634741)
- The primary image item must be an av01 (no grid support)
- HDR images aren't tone-mapped

Differential Revision: https://phabricator.services.mozilla.com/D68498
This commit is contained in:
Jon Bauman 2020-05-01 22:56:04 +00:00
parent 1a902cc7ab
commit 2db43533f3
16 changed files with 455 additions and 1 deletions

View File

@ -20,6 +20,7 @@
#include "nsICODecoder.h"
#include "nsIconDecoder.h"
#include "nsWebPDecoder.h"
#include "nsAVIFDecoder.h"
namespace mozilla {
@ -76,6 +77,11 @@ DecoderType DecoderFactory::GetDecoderType(const char* aMimeType) {
} else if (!strcmp(aMimeType, IMAGE_WEBP) &&
StaticPrefs::image_webp_enabled()) {
type = DecoderType::WEBP;
// AVIF
} else if (!strcmp(aMimeType, IMAGE_AVIF) &&
StaticPrefs::image_avif_enabled()) {
type = DecoderType::AVIF;
}
return type;
@ -115,6 +121,9 @@ already_AddRefed<Decoder> DecoderFactory::GetDecoder(DecoderType aType,
case DecoderType::WEBP:
decoder = new nsWebPDecoder(aImage);
break;
case DecoderType::AVIF:
decoder = new nsAVIFDecoder(aImage);
break;
default:
MOZ_ASSERT_UNREACHABLE("Unknown decoder type");
}

View File

@ -38,6 +38,7 @@ enum class DecoderType {
ICO,
ICON,
WEBP,
AVIF,
UNKNOWN
};

View File

@ -22,6 +22,7 @@ elif toolkit == 'android':
UNIFIED_SOURCES += [
'EXIF.cpp',
'iccjpeg.c',
'nsAVIFDecoder.cpp',
'nsBMPDecoder.cpp',
'nsGIFDecoder2.cpp',
'nsICODecoder.cpp',

View File

@ -0,0 +1,339 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "ImageLogging.h" // Must appear first
#include "nsAVIFDecoder.h"
#include "aom/aomdx.h"
#include "mozilla/gfx/Types.h"
#include "YCbCrUtils.h"
#include "SurfacePipeFactory.h"
using namespace mozilla::gfx;
namespace mozilla {
namespace image {
static LazyLogModule sAVIFLog("AVIFDecoder");
// Wrapper to allow rust to call our read adaptor.
intptr_t nsAVIFDecoder::read_source(uint8_t* aDestBuf, uintptr_t aDestBufSize,
void* aUserData) {
MOZ_ASSERT(aDestBuf);
MOZ_ASSERT(aUserData);
MOZ_LOG(sAVIFLog, LogLevel::Verbose,
("AVIF read_source, aDestBufSize: %zu", aDestBufSize));
auto* decoder = reinterpret_cast<nsAVIFDecoder*>(aUserData);
MOZ_ASSERT(decoder->mReadCursor);
size_t bufferLength = decoder->mBufferedData.end() - decoder->mReadCursor;
size_t n_bytes = std::min(aDestBufSize, bufferLength);
MOZ_LOG(sAVIFLog, LogLevel::Verbose,
("AVIF read_source, %zu bytes ready, copying %zu", bufferLength,
n_bytes));
memcpy(aDestBuf, decoder->mReadCursor, n_bytes);
decoder->mReadCursor += n_bytes;
return n_bytes;
}
nsAVIFDecoder::nsAVIFDecoder(RasterImage* aImage)
: Decoder(aImage), mParser(nullptr) {
MOZ_LOG(sAVIFLog, LogLevel::Debug,
("[this=%p] nsAVIFDecoder::nsAVIFDecoder", this));
}
nsAVIFDecoder::~nsAVIFDecoder() {
MOZ_LOG(sAVIFLog, LogLevel::Debug,
("[this=%p] nsAVIFDecoder::~nsAVIFDecoder", this));
if (mParser) {
mp4parse_avif_free(mParser);
}
if (mCodecContext) {
aom_codec_err_t res = aom_codec_destroy(mCodecContext.ptr());
MOZ_LOG(sAVIFLog, LogLevel::Debug,
("[this=%p] aom_codec_destroy -> %d", this, res));
}
}
LexerResult nsAVIFDecoder::DoDecode(SourceBufferIterator& aIterator,
IResumable* aOnResume) {
MOZ_LOG(sAVIFLog, LogLevel::Debug,
("[this=%p] nsAVIFDecoder::DoDecode", this));
// Since the SourceBufferIterator doesn't guarantee a contiguous buffer,
// but the current mp4parse-rust implementation requires it, always buffer
// locally. This keeps the code simpler at the cost of some performance, but
// this implementation is only experimental, so we don't want to spend time
// optimizing it prematurely.
while (!mReadCursor) {
SourceBufferIterator::State state =
aIterator.AdvanceOrScheduleResume(SIZE_MAX, aOnResume);
MOZ_LOG(sAVIFLog, LogLevel::Debug,
("[this=%p] After advance, iterator state is %d", this, state));
switch (state) {
case SourceBufferIterator::WAITING:
return LexerResult(Yield::NEED_MORE_DATA);
case SourceBufferIterator::COMPLETE:
mReadCursor = mBufferedData.begin();
break;
case SourceBufferIterator::READY: { // copy new data to buffer
MOZ_LOG(sAVIFLog, LogLevel::Debug,
("[this=%p] SourceBufferIterator ready, %zu bytes available",
this, aIterator.Length()));
bool appendSuccess =
mBufferedData.append(aIterator.Data(), aIterator.Length());
if (!appendSuccess) {
MOZ_LOG(sAVIFLog, LogLevel::Error,
("[this=%p] Failed to append %zu bytes to buffer", this,
aIterator.Length()));
}
break;
}
default:
MOZ_ASSERT_UNREACHABLE("unexpected SourceBufferIterator state");
}
}
Mp4parseIo io = {nsAVIFDecoder::read_source, this};
if (!mParser) {
Mp4parseStatus status = mp4parse_avif_new(&io, &mParser);
MOZ_LOG(sAVIFLog, LogLevel::Debug,
("[this=%p] mp4parse_avif_new status: %d", this, status));
}
if (!mParser) {
return LexerResult(TerminalState::FAILURE);
}
Mp4parseByteData mdat = {}; // change the name to 'primary_item' or something
Mp4parseStatus status = mp4parse_avif_get_primary_item(mParser, &mdat);
MOZ_LOG(sAVIFLog, LogLevel::Debug,
("[this=%p] mp4parse_avif_get_primary_item -> %d", this, status));
if (status != MP4PARSE_STATUS_OK) {
return LexerResult(TerminalState::FAILURE);
}
aom_codec_iface_t* iface = aom_codec_av1_dx();
aom_codec_ctx_t ctx;
aom_codec_err_t res =
aom_codec_dec_init(&ctx, iface, /* cfg = */ nullptr, /* flags = */ 0);
MOZ_LOG(
sAVIFLog, LogLevel::Error,
("[this=%p] aom_codec_dec_init -> %d, name = %s", this, res, ctx.name));
if (res == AOM_CODEC_OK) {
mCodecContext = Some(ctx);
} else {
return LexerResult(TerminalState::FAILURE);
}
res = aom_codec_decode(mCodecContext.ptr(), mdat.data, mdat.length, nullptr);
if (res != AOM_CODEC_OK) {
MOZ_LOG(sAVIFLog, LogLevel::Error,
("[this=%p] aom_codec_decode -> %d", this, res));
return LexerResult(TerminalState::FAILURE);
}
MOZ_LOG(sAVIFLog, LogLevel::Debug,
("[this=%p] aom_codec_decode -> %d", this, res));
aom_codec_iter_t iter = nullptr;
const aom_image_t* img = aom_codec_get_frame(mCodecContext.ptr(), &iter);
if (img == nullptr) {
MOZ_LOG(sAVIFLog, LogLevel::Error,
("[this=%p] aom_codec_get_frame -> %p", this, img));
return LexerResult(TerminalState::FAILURE);
}
const CheckedInt<int> decoded_width = img->d_w;
const CheckedInt<int> decoded_height = img->d_h;
if (!decoded_height.isValid() || !decoded_width.isValid()) {
MOZ_LOG(
sAVIFLog, LogLevel::Debug,
("[this=%p] image dimensions can't be stored in int: d_w: %u, d_h: %u",
this, img->d_w, img->d_h));
return LexerResult(TerminalState::FAILURE);
}
PostSize(decoded_width.value(), decoded_height.value());
// TODO: This doesn't account for the alpha plane in a separate frame
const bool hasAlpha = false;
if (hasAlpha) {
PostHasTransparency();
}
if (IsMetadataDecode()) {
return LexerResult(TerminalState::SUCCESS);
}
MOZ_ASSERT(img->stride[AOM_PLANE_Y] == img->stride[AOM_PLANE_ALPHA]);
MOZ_ASSERT(img->stride[AOM_PLANE_Y] >= aom_img_plane_width(img, AOM_PLANE_Y));
MOZ_ASSERT(img->stride[AOM_PLANE_U] == img->stride[AOM_PLANE_V]);
MOZ_ASSERT(img->stride[AOM_PLANE_U] >= aom_img_plane_width(img, AOM_PLANE_U));
MOZ_ASSERT(img->stride[AOM_PLANE_V] >= aom_img_plane_width(img, AOM_PLANE_V));
MOZ_ASSERT(aom_img_plane_width(img, AOM_PLANE_U) ==
aom_img_plane_width(img, AOM_PLANE_V));
MOZ_ASSERT(aom_img_plane_height(img, AOM_PLANE_U) ==
aom_img_plane_height(img, AOM_PLANE_V));
layers::PlanarYCbCrData data;
data.mYChannel = img->planes[AOM_PLANE_Y];
data.mYStride = img->stride[AOM_PLANE_Y];
data.mYSize = gfx::IntSize(aom_img_plane_width(img, AOM_PLANE_Y),
aom_img_plane_height(img, AOM_PLANE_Y));
data.mYSkip =
img->stride[AOM_PLANE_Y] - aom_img_plane_width(img, AOM_PLANE_Y);
data.mCbChannel = img->planes[AOM_PLANE_U];
data.mCrChannel = img->planes[AOM_PLANE_V];
data.mCbCrStride = img->stride[AOM_PLANE_U];
data.mCbCrSize = gfx::IntSize(aom_img_plane_width(img, AOM_PLANE_U),
aom_img_plane_height(img, AOM_PLANE_U));
data.mCbSkip =
img->stride[AOM_PLANE_U] - aom_img_plane_width(img, AOM_PLANE_U);
data.mCrSkip =
img->stride[AOM_PLANE_V] - aom_img_plane_width(img, AOM_PLANE_V);
data.mPicX = 0;
data.mPicY = 0;
data.mPicSize = gfx::IntSize(decoded_width.value(), decoded_height.value());
data.mStereoMode = StereoMode::MONO;
data.mColorDepth = ColorDepthForBitDepth(img->bit_depth);
switch (img->cp) {
case AOM_CICP_CP_BT_601:
data.mYUVColorSpace = gfx::YUVColorSpace::BT601;
break;
case AOM_CICP_CP_BT_709:
data.mYUVColorSpace = gfx::YUVColorSpace::BT709;
break;
case AOM_CICP_CP_BT_2020:
data.mYUVColorSpace = gfx::YUVColorSpace::BT2020;
break;
default:
MOZ_LOG(sAVIFLog, LogLevel::Debug,
("[this=%p] unsupported aom_color_primaries value: %u", this,
img->cp));
data.mYUVColorSpace = gfx::YUVColorSpace::UNKNOWN;
}
switch (img->range) {
case AOM_CR_STUDIO_RANGE:
data.mColorRange = gfx::ColorRange::LIMITED;
break;
case AOM_CR_FULL_RANGE:
data.mColorRange = gfx::ColorRange::FULL;
break;
default:
MOZ_ASSERT_UNREACHABLE("unknown color range");
}
gfx::SurfaceFormat format =
hasAlpha ? SurfaceFormat::OS_RGBA : SurfaceFormat::OS_RGBX;
const IntSize intrinsicSize = Size();
IntSize rgbSize = intrinsicSize;
gfx::GetYCbCrToRGBDestFormatAndSize(data, format, rgbSize);
const int bytesPerPixel = BytesPerPixel(format);
const CheckedInt rgbStride = CheckedInt<int>(rgbSize.width) * bytesPerPixel;
const CheckedInt rgbBufLength = rgbStride * rgbSize.height;
if (!rgbStride.isValid() || !rgbBufLength.isValid()) {
MOZ_LOG(sAVIFLog, LogLevel::Debug,
("[this=%p] overflow calculating rgbBufLength: rbgSize.width: %d, "
"rgbSize.height: %d, "
"bytesPerPixel: %u",
this, rgbSize.width, rgbSize.height, bytesPerPixel));
return LexerResult(TerminalState::FAILURE);
}
UniquePtr<uint8_t[]> rgbBuf = MakeUnique<uint8_t[]>(rgbBufLength.value());
const uint8_t* endOfRgbBuf = {rgbBuf.get() + rgbBufLength.value()};
if (!rgbBuf) {
MOZ_LOG(sAVIFLog, LogLevel::Debug,
("[this=%p] allocation of %u-byte rgbBuf failed", this,
rgbBufLength.value()));
return LexerResult(TerminalState::FAILURE);
}
gfx::ConvertYCbCrToRGB(data, format, rgbSize, rgbBuf.get(),
rgbStride.value());
Maybe<SurfacePipe> pipe = SurfacePipeFactory::CreateSurfacePipe(
this, rgbSize, OutputSize(), FullFrame(), format, format, Nothing(),
nullptr, SurfacePipeFlags());
if (!pipe) {
MOZ_LOG(sAVIFLog, LogLevel::Debug,
("[this=%p] could not initialize surface pipe", this));
return LexerResult(TerminalState::FAILURE);
}
WriteState writeBufferResult = WriteState::NEED_MORE_DATA;
for (uint8_t* rowPtr = rgbBuf.get(); rowPtr < endOfRgbBuf;
rowPtr += rgbStride.value()) {
writeBufferResult = pipe->WriteBuffer(reinterpret_cast<uint32_t*>(rowPtr));
Maybe<SurfaceInvalidRect> invalidRect = pipe->TakeInvalidRect();
if (invalidRect) {
PostInvalidation(invalidRect->mInputSpaceRect,
Some(invalidRect->mOutputSpaceRect));
}
if (writeBufferResult == WriteState::FAILURE) {
MOZ_LOG(sAVIFLog, LogLevel::Debug,
("[this=%p] error writing rowPtr to surface pipe", this));
} else if (writeBufferResult == WriteState::FINISHED) {
MOZ_ASSERT(rowPtr + rgbStride.value() == endOfRgbBuf);
}
}
// We don't support image sequences yet
DebugOnly<aom_image_t*> next_img =
aom_codec_get_frame(mCodecContext.ptr(), &iter);
MOZ_ASSERT(next_img == nullptr);
if (writeBufferResult == WriteState::FINISHED) {
PostFrameStop(hasAlpha ? Opacity::SOME_TRANSPARENCY
: Opacity::FULLY_OPAQUE);
PostDecodeDone();
return LexerResult(TerminalState::SUCCESS);
}
return LexerResult(TerminalState::FAILURE);
}
} // namespace image
} // namespace mozilla

View File

@ -0,0 +1,51 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef mozilla_image_decoders_nsAVIFDecoder_h
#define mozilla_image_decoders_nsAVIFDecoder_h
#include "Decoder.h"
#include "mp4parse.h"
#include "SurfacePipe.h"
#include "aom/aom_decoder.h"
namespace mozilla {
namespace image {
class RasterImage;
class nsAVIFDecoder final : public Decoder {
public:
virtual ~nsAVIFDecoder();
DecoderType GetType() const override { return DecoderType::AVIF; }
protected:
LexerResult DoDecode(SourceBufferIterator& aIterator,
IResumable* aOnResume) override;
private:
friend class DecoderFactory;
// Decoders should only be instantiated via DecoderFactory.
explicit nsAVIFDecoder(RasterImage* aImage);
static intptr_t read_source(uint8_t* aDestBuf, uintptr_t aDestBufSize,
void* aUserData);
Mp4parseAvifParser* mParser;
Maybe<aom_codec_ctx_t> mCodecContext;
Vector<uint8_t> mBufferedData;
/// Pointer to the next place to read from mBufferedData
const uint8_t* mReadCursor = nullptr;
};
} // namespace image
} // namespace mozilla
#endif // mozilla_image_decoders_nsAVIFDecoder_h

View File

@ -40,6 +40,10 @@ AutoInitializeImageLib::AutoInitializeImageLib() {
nsresult rv = Preferences::SetBool("image.webp.enabled", true);
EXPECT_TRUE(rv == NS_OK);
// Ensure AVIF is enabled to run decoder tests.
rv = Preferences::SetBool("image.avif.enabled", true);
EXPECT_TRUE(rv == NS_OK);
// Ensure that ImageLib services are initialized.
nsCOMPtr<imgITools> imgTools =
do_CreateInstance("@mozilla.org/image/tools;1");
@ -133,7 +137,6 @@ already_AddRefed<nsIInputStream> LoadFile(const char* aRelativePath) {
rv = dirService->Get(NS_OS_CURRENT_WORKING_DIR, NS_GET_IID(nsIFile),
getter_AddRefs(file));
ASSERT_TRUE_OR_RETURN(NS_SUCCEEDED(rv), nullptr);
// Construct the final path by appending the working path to the current
// working directory.
file->AppendNative(nsDependentCString(aRelativePath));
@ -422,11 +425,23 @@ ImageTestCase GreenWebPTestCase() {
return ImageTestCase("green.webp", "image/webp", IntSize(100, 100));
}
// Forcing sRGB is required until nsAVIFDecoder supports ICC profiles
// See bug 1634741
ImageTestCase GreenAVIFTestCase() {
return ImageTestCase("green.avif", "image/avif", IntSize(100, 100))
.WithSurfaceFlags(SurfaceFlags::TO_SRGB_COLORSPACE);
}
ImageTestCase LargeWebPTestCase() {
return ImageTestCase("large.webp", "image/webp", IntSize(1200, 660),
TEST_CASE_IGNORE_OUTPUT);
}
ImageTestCase LargeAVIFTestCase() {
return ImageTestCase("large.avif", "image/avif", IntSize(1200, 660),
TEST_CASE_IGNORE_OUTPUT);
}
ImageTestCase GreenWebPIccSrgbTestCase() {
return ImageTestCase("green.icc_srgb.webp", "image/webp", IntSize(100, 100));
}
@ -597,6 +612,11 @@ ImageTestCase DownscaledWebPTestCase() {
IntSize(20, 20));
}
ImageTestCase DownscaledAVIFTestCase() {
return ImageTestCase("downscaled.avif", "image/avif", IntSize(100, 100),
IntSize(20, 20));
}
ImageTestCase DownscaledTransparentICOWithANDMaskTestCase() {
// This test case is an ICO with AND mask transparency. We want to ensure that
// we can downscale it without crashing or triggering ASAN failures, but its

View File

@ -463,6 +463,7 @@ ImageTestCase GreenBMPTestCase();
ImageTestCase GreenICOTestCase();
ImageTestCase GreenIconTestCase();
ImageTestCase GreenWebPTestCase();
ImageTestCase GreenAVIFTestCase();
ImageTestCase LargeWebPTestCase();
ImageTestCase GreenWebPIccSrgbTestCase();

View File

@ -653,6 +653,26 @@ TEST_F(ImageDecoders, WebPTransparentNoAlphaHeaderSingleChunk) {
CheckDecoderSingleChunk(TransparentNoAlphaHeaderWebPTestCase());
}
TEST_F(ImageDecoders, AVIFSingleChunk) {
CheckDecoderSingleChunk(GreenAVIFTestCase());
}
TEST_F(ImageDecoders, AVIFDelayedChunk) {
CheckDecoderDelayedChunk(GreenAVIFTestCase());
}
TEST_F(ImageDecoders, AVIFMultiChunk) {
CheckDecoderMultiChunk(GreenAVIFTestCase());
}
TEST_F(ImageDecoders, AVIFLargeMultiChunk) {
CheckDecoderMultiChunk(LargeAVIFTestCase(), /* aChunkSize */ 64);
}
TEST_F(ImageDecoders, AVIFDownscaleDuringDecode) {
CheckDownscaleDuringDecode(DownscaledAVIFTestCase());
}
TEST_F(ImageDecoders, AnimatedGIFSingleChunk) {
CheckDecoderSingleChunk(GreenFirstFrameAnimatedGIFTestCase());
}

Binary file not shown.

BIN
image/test/gtest/green.avif Normal file

Binary file not shown.

BIN
image/test/gtest/large.avif Normal file

Binary file not shown.

View File

@ -53,6 +53,7 @@ TEST_HARNESS_FILES.gtest += [
'corrupt-with-bad-bmp-width.ico',
'corrupt-with-bad-ico-bpp.ico',
'corrupt.jpg',
'downscaled.avif',
'downscaled.bmp',
'downscaled.gif',
'downscaled.ico',
@ -68,6 +69,7 @@ TEST_HARNESS_FILES.gtest += [
'green-large-bmp.ico',
'green-large-png.ico',
'green-multiple-sizes.ico',
'green.avif',
'green.bmp',
'green.gif',
'green.icc_srgb.webp',
@ -77,6 +79,7 @@ TEST_HARNESS_FILES.gtest += [
'green.png',
'green.webp',
'invalid-truncated-metadata.bmp',
'large.avif',
'large.webp',
'no-frame-delay.gif',
'perf_cmyk.jpg',

View File

@ -38,6 +38,7 @@ content_types = [
'text/xml',
'image/apng',
'image/avif',
'image/bmp',
'image/gif',
'image/icon',

View File

@ -4595,6 +4595,12 @@
value: true
mirror: always
# Whether we attempt to decode AVIF images or not.
- name: image.avif.enabled
type: RelaxedAtomicBool
value: false
mirror: always
#---------------------------------------------------------------------------
# Prefs starting with "intl."
#---------------------------------------------------------------------------

View File

@ -119,6 +119,7 @@
#define IMAGE_JNG "image/x-jng"
#define IMAGE_SVG_XML "image/svg+xml"
#define IMAGE_WEBP "image/webp"
#define IMAGE_AVIF "image/avif"
#define MESSAGE_EXTERNAL_BODY "message/external-body"
#define MESSAGE_NEWS "message/news"

View File

@ -495,6 +495,7 @@ static const nsExtraMimeTypeEntry extraMimeEntries[] = {
{IMAGE_XBM, "xbm", "XBM Image"},
{IMAGE_SVG_XML, "svg", "Scalable Vector Graphics"},
{IMAGE_WEBP, "webp", "WebP Image"},
{IMAGE_AVIF, "avif", "AV1 Image File"},
{MESSAGE_RFC822, "eml", "RFC-822 data"},
{TEXT_PLAIN, "txt,text", "Text File"},
{APPLICATION_JSON, "json", "JavaScript Object Notation"},