Bug 1788119 - Part 3 - Add initial support for animated AVIF sequences. r=tnikkel

Depends on D156651

Differential Revision: https://phabricator.services.mozilla.com/D156652

dom/media/gtest/mp4_demuxer/TestMP4.cpp

@@ -92,12 +92,12 @@ TEST(rust, MP4MetadataEmpty)
   io = {vector_reader, &buf};
   rv = mp4parse_new(&io, &parser);
   ASSERT_EQ(parser, nullptr);
-  EXPECT_EQ(rv, MP4PARSE_STATUS_NO_MOOV);
+  EXPECT_EQ(rv, MP4PARSE_STATUS_MOOV_MISSING);
 
   buf.buffer.reserve(4097);
   rv = mp4parse_new(&io, &parser);
   ASSERT_EQ(parser, nullptr);
-  EXPECT_EQ(rv, MP4PARSE_STATUS_NO_MOOV);
+  EXPECT_EQ(rv, MP4PARSE_STATUS_MOOV_MISSING);
 
   // Empty buffers should fail.
   buf.buffer.resize(4097, 0);

dom/media/mp4/SampleIterator.cpp

@@ -78,6 +78,8 @@ SampleIterator::SampleIterator(MP4SampleIndex* aIndex)
 
 SampleIterator::~SampleIterator() { mIndex->UnregisterIterator(this); }
 
+bool SampleIterator::HasNext() { return !!Get(); }
+
 already_AddRefed<MediaRawData> SampleIterator::GetNext() {
   Sample* s(Get());
   if (!s) {

dom/media/mp4/SampleIterator.h

@@ -27,6 +27,7 @@ class SampleIterator {
  public:
   explicit SampleIterator(MP4SampleIndex* aIndex);
   ~SampleIterator();
+  bool HasNext();
   already_AddRefed<mozilla::MediaRawData> GetNext();
   void Seek(Microseconds aTime);
   Microseconds GetNextKeyframeTime();

image/DecoderFactory.cpp

@@ -217,7 +217,7 @@ nsresult DecoderFactory::CreateAnimationDecoder(
   }
 
   MOZ_ASSERT(aType == DecoderType::GIF || aType == DecoderType::PNG ||
-                 aType == DecoderType::WEBP,
+                 aType == DecoderType::WEBP || aType == DecoderType::AVIF,
              "Calling CreateAnimationDecoder for non-animating DecoderType");
 
   // Create an anonymous decoder. Interaction with the SurfaceCache and the
@@ -272,7 +272,7 @@ already_AddRefed<Decoder> DecoderFactory::CloneAnimationDecoder(
   // rediscover it is animated).
   DecoderType type = aDecoder->GetType();
   MOZ_ASSERT(type == DecoderType::GIF || type == DecoderType::PNG ||
-                 type == DecoderType::WEBP,
+                 type == DecoderType::WEBP || type == DecoderType::AVIF,
              "Calling CloneAnimationDecoder for non-animating DecoderType");
 
   RefPtr<Decoder> decoder = GetDecoder(type, nullptr, /* aIsRedecode = */ true);

image/decoders/nsAVIFDecoder.h

@@ -8,8 +8,10 @@
 #define mozilla_image_decoders_nsAVIFDecoder_h
 
 #include "Decoder.h"
-#include "mp4parse.h"
 #include "mozilla/gfx/Types.h"
+#include "MP4Metadata.h"
+#include "mp4parse.h"
+#include "SampleIterator.h"
 #include "SurfacePipe.h"
 
 #include "aom/aom_decoder.h"
@@ -20,6 +22,7 @@
 namespace mozilla {
 namespace image {
 class RasterImage;
+class AVIFDecoderStream;
 class AVIFParser;
 class AVIFDecoderInterface;
 
@@ -37,6 +40,7 @@ class nsAVIFDecoder final : public Decoder {
  private:
   friend class DecoderFactory;
   friend class AVIFDecoderInterface;
+  friend class AVIFParser;
 
   // Decoders should only be instantiated via DecoderFactory.
   explicit nsAVIFDecoder(RasterImage* aImage);
@@ -49,8 +53,8 @@ class nsAVIFDecoder final : public Decoder {
   typedef Variant<aom_codec_err_t, NonAOMCodecError> AOMResult;
   enum class NonDecoderResult {
     NeedMoreData,
-    MetadataOk,
-    NoPrimaryItem,
+    OutputAvailable,
+    Complete,
     SizeOverflow,
     OutOfMemory,
     PipeInitError,
@@ -59,10 +63,12 @@ class nsAVIFDecoder final : public Decoder {
     AlphaYColorDepthMismatch,
     MetadataImageSizeMismatch,
     InvalidCICP,
+    NoSamples,
   };
   using DecodeResult =
       Variant<Mp4parseStatus, NonDecoderResult, Dav1dResult, AOMResult>;
   Mp4parseStatus CreateParser();
+  DecodeResult CreateDecoder();
   DecodeResult Decode(SourceBufferIterator& aIterator, IResumable* aOnResume);
 
   static bool IsDecodeSuccess(const DecodeResult& aResult);
@@ -70,96 +76,169 @@ class nsAVIFDecoder final : public Decoder {
   void RecordDecodeResultTelemetry(const DecodeResult& aResult);
 
   Vector<uint8_t> mBufferedData;
+  UniquePtr<AVIFDecoderStream> mBufferStream;
 
   /// Pointer to the next place to read from mBufferedData
   const uint8_t* mReadCursor = nullptr;
 
   UniquePtr<AVIFParser> mParser = nullptr;
   UniquePtr<AVIFDecoderInterface> mDecoder = nullptr;
+
+  bool mIsAnimated = false;
+  bool mHasAlpha = false;
+};
+
+class AVIFDecoderStream : public ByteStream {
+ public:
+  explicit AVIFDecoderStream(Vector<uint8_t>* aBuffer) { mBuffer = aBuffer; }
+
+  virtual bool ReadAt(int64_t offset, void* data, size_t size,
+                      size_t* bytes_read) override;
+  virtual bool CachedReadAt(int64_t offset, void* data, size_t size,
+                            size_t* bytes_read) override {
+    return ReadAt(offset, data, size, bytes_read);
+  };
+  virtual bool Length(int64_t* size) override;
+  virtual const uint8_t* GetContiguousAccess(int64_t aOffset,
+                                             size_t aSize) override;
+
+ private:
+  Vector<uint8_t>* mBuffer;
+};
+
+struct AVIFImage {
+  uint32_t mFrameNum = 0;
+  FrameTimeout mDuration = FrameTimeout::Zero();
+  RefPtr<MediaRawData> mColorImage = nullptr;
+  RefPtr<MediaRawData> mAlphaImage = nullptr;
 };
 
 class AVIFParser {
  public:
-  static Mp4parseStatus Create(const Mp4parseIo* aIo,
+  static Mp4parseStatus Create(const Mp4parseIo* aIo, ByteStream* aBuffer,
                                UniquePtr<AVIFParser>& aParserOut);
 
   ~AVIFParser();
 
-  Mp4parseAvifImage* GetImage();
+  const Mp4parseAvifInfo& GetInfo() const { return mInfo; }
+
+  nsAVIFDecoder::DecodeResult GetImage(AVIFImage& aImage);
+
+  bool IsAnimated() const;
 
  private:
   explicit AVIFParser(const Mp4parseIo* aIo);
 
-  Mp4parseStatus Init();
+  Mp4parseStatus Init(ByteStream* aBuffer);
 
   struct FreeAvifParser {
     void operator()(Mp4parseAvifParser* aPtr) { mp4parse_avif_free(aPtr); }
   };
 
   const Mp4parseIo* mIo;
-  UniquePtr<Mp4parseAvifParser, FreeAvifParser> mParser;
-  Maybe<Mp4parseAvifImage> mAvifImage;
+  UniquePtr<Mp4parseAvifParser, FreeAvifParser> mParser = nullptr;
+  Mp4parseAvifInfo mInfo = {};
+
+  UniquePtr<SampleIterator> mColorSampleIter = nullptr;
+  UniquePtr<SampleIterator> mAlphaSampleIter = nullptr;
+  uint32_t mFrameNum = 0;
+};
+
+struct Dav1dPictureUnref {
+  void operator()(Dav1dPicture* aPtr) {
+    dav1d_picture_unref(aPtr);
+    delete aPtr;
+  }
+};
+
+using OwnedDav1dPicture = UniquePtr<Dav1dPicture, Dav1dPictureUnref>;
+
+class OwnedAOMImage {
+ public:
+  ~OwnedAOMImage();
+
+  static OwnedAOMImage* CopyFrom(aom_image_t* aImage, bool aIsAlpha);
+
+  aom_image_t* GetImage() { return mImage.isSome() ? mImage.ptr() : nullptr; }
+
+ private:
+  OwnedAOMImage();
+
+  bool CloneFrom(aom_image_t* aImage, bool aIsAlpha);
+
+  // The mImage's planes are referenced to mBuffer
+  Maybe<aom_image_t> mImage;
+  UniquePtr<uint8_t[]> mBuffer;
 };
 
-// CICP values (either from the BMFF container or the AV1 sequence header) are
-// used to create the colorspace transform. CICP::MatrixCoefficients is only
-// stored for the sake of telemetry, since the relevant information for YUV ->
-// RGB conversion is stored in mYUVColorSpace.
-//
-// There are three potential sources of color information for an AVIF:
-// 1. ICC profile via a ColourInformationBox (colr) defined in [ISOBMFF]
-//    § 12.1.5 "Colour information" and [MIAF] § 7.3.6.4 "Colour information
-//    property"
-// 2. NCLX (AKA CICP see [ITU-T H.273]) values in the same ColourInformationBox
-//    which can have an ICC profile or NCLX values, not both).
-// 3. NCLX values in the AV1 bitstream
-//
-// The 'colr' box is optional, but there are always CICP values in the AV1
-// bitstream, so it is possible to have both. Per ISOBMFF § 12.1.5.1
-// > If colour information is supplied in both this box, and also in the
-// > video bitstream, this box takes precedence, and over-rides the
-// > information in the bitstream.
-//
-// If present, the ICC profile takes precedence over CICP values, but only
-// specifies the color space, not the matrix coefficients necessary to convert
-// YCbCr data (as most AVIF are encoded) to RGB. The matrix coefficients are
-// always derived from the CICP values for matrix_coefficients (and potentially
-// colour_primaries, but in that case only the CICP values for colour_primaries
-// will be used, not anything harvested from the ICC profile).
-//
-// If there is no ICC profile, the color space transform will be based on the
-// CICP values either from the 'colr' box, or if absent/unspecified, the
-// decoded AV1 sequence header.
-//
-// For values that are 2 (meaning unspecified) after trying both, the
-// fallback values are:
-// - CP:  1 (BT.709/sRGB)
-// - TC: 13 (sRGB)
-// - MC:  6 (BT.601)
-// - Range: Full
-//
-// Additional details here:
-// <https://github.com/AOMediaCodec/libavif/wiki/CICP#unspecified>. Note
-// that this contradicts the current version of [MIAF] § 7.3.6.4 which
-// specifies MC=1 (BT.709). This is revised in [MIAF DAMD2] and confirmed by
-// <https://github.com/AOMediaCodec/av1-avif/issues/77#issuecomment-676526097>
-//
-// The precedence for applying the various values and defaults in the event
-// no valid values are found are managed by the following functions.
-//
-// References:
-// [ISOBMFF]: ISO/IEC 14496-12:2020 <https://www.iso.org/standard/74428.html>
-// [MIAF]: ISO/IEC 23000-22:2019 <https://www.iso.org/standard/74417.html>
-// [MIAF DAMD2]: ISO/IEC 23000-22:2019/FDAmd 2
-// <https://www.iso.org/standard/81634.html>
-// [ITU-T H.273]: Rec. ITU-T H.273 (12/2016)
-//     <https://www.itu.int/rec/T-REC-H.273-201612-I/en>
 struct AVIFDecodedData : layers::PlanarYCbCrData {
+ public:
+  OrientedIntRect mRenderRect = {};
   gfx::CICP::ColourPrimaries mColourPrimaries = gfx::CICP::CP_UNSPECIFIED;
   gfx::CICP::TransferCharacteristics mTransferCharacteristics =
       gfx::CICP::TC_UNSPECIFIED;
   gfx::CICP::MatrixCoefficients mMatrixCoefficients = gfx::CICP::MC_UNSPECIFIED;
 
+  OwnedDav1dPicture mColorDav1d;
+  OwnedDav1dPicture mAlphaDav1d;
+  UniquePtr<OwnedAOMImage> mColorAOM;
+  UniquePtr<OwnedAOMImage> mAlphaAOM;
+
+  // CICP values (either from the BMFF container or the AV1 sequence header) are
+  // used to create the colorspace transform. CICP::MatrixCoefficients is only
+  // stored for the sake of telemetry, since the relevant information for YUV ->
+  // RGB conversion is stored in mYUVColorSpace.
+  //
+  // There are three potential sources of color information for an AVIF:
+  // 1. ICC profile via a ColourInformationBox (colr) defined in [ISOBMFF]
+  //    § 12.1.5 "Colour information" and [MIAF] § 7.3.6.4 "Colour information
+  //    property"
+  // 2. NCLX (AKA CICP see [ITU-T H.273]) values in the same
+  // ColourInformationBox
+  //    which can have an ICC profile or NCLX values, not both).
+  // 3. NCLX values in the AV1 bitstream
+  //
+  // The 'colr' box is optional, but there are always CICP values in the AV1
+  // bitstream, so it is possible to have both. Per ISOBMFF § 12.1.5.1
+  // > If colour information is supplied in both this box, and also in the
+  // > video bitstream, this box takes precedence, and over-rides the
+  // > information in the bitstream.
+  //
+  // If present, the ICC profile takes precedence over CICP values, but only
+  // specifies the color space, not the matrix coefficients necessary to convert
+  // YCbCr data (as most AVIF are encoded) to RGB. The matrix coefficients are
+  // always derived from the CICP values for matrix_coefficients (and
+  // potentially colour_primaries, but in that case only the CICP values for
+  // colour_primaries will be used, not anything harvested from the ICC
+  // profile).
+  //
+  // If there is no ICC profile, the color space transform will be based on the
+  // CICP values either from the 'colr' box, or if absent/unspecified, the
+  // decoded AV1 sequence header.
+  //
+  // For values that are 2 (meaning unspecified) after trying both, the
+  // fallback values are:
+  // - CP:  1 (BT.709/sRGB)
+  // - TC: 13 (sRGB)
+  // - MC:  6 (BT.601)
+  // - Range: Full
+  //
+  // Additional details here:
+  // <https://github.com/AOMediaCodec/libavif/wiki/CICP#unspecified>. Note
+  // that this contradicts the current version of [MIAF] § 7.3.6.4 which
+  // specifies MC=1 (BT.709). This is revised in [MIAF DAMD2] and confirmed by
+  // <https://github.com/AOMediaCodec/av1-avif/issues/77#issuecomment-676526097>
+  //
+  // The precedence for applying the various values and defaults in the event
+  // no valid values are found are managed by the following functions.
+  //
+  // References:
+  // [ISOBMFF]: ISO/IEC 14496-12:2020 <https://www.iso.org/standard/74428.html>
+  // [MIAF]: ISO/IEC 23000-22:2019 <https://www.iso.org/standard/74417.html>
+  // [MIAF DAMD2]: ISO/IEC 23000-22:2019/FDAmd 2
+  // <https://www.iso.org/standard/81634.html>
+  // [ITU-T H.273]: Rec. ITU-T H.273 (12/2016)
+  //     <https://www.itu.int/rec/T-REC-H.273-201612-I/en>
   void SetCicpValues(
       const Mp4parseNclxColourInformation* aNclx,
       const gfx::CICP::ColourPrimaries aAv1ColourPrimaries,
@@ -180,22 +259,23 @@ class AVIFDecoderInterface {
 
   // Set the mDecodedData if Decode() succeeds
   virtual DecodeResult Decode(bool aIsMetadataDecode,
-                              const Mp4parseAvifImage& parsedImg) = 0;
-  // Must be called after Decode() succeeds
-  AVIFDecodedData& GetDecodedData() {
-    MOZ_ASSERT(mDecodedData.isSome());
-    return mDecodedData.ref();
+                              const Mp4parseAvifInfo& aAVIFInfo,
+                              const AVIFImage& aSamples) = 0;
+  // Must be called only once after Decode() succeeds
+  UniquePtr<AVIFDecodedData> GetDecodedData() {
+    MOZ_ASSERT(mDecodedData);
+    return std::move(mDecodedData);
   }
 
  protected:
-  explicit AVIFDecoderInterface() {}
+  explicit AVIFDecoderInterface() = default;
 
   inline static bool IsDecodeSuccess(const DecodeResult& aResult) {
     return nsAVIFDecoder::IsDecodeSuccess(aResult);
   }
 
   // The mDecodedData is valid after Decode() succeeds
-  Maybe<AVIFDecodedData> mDecodedData;
+  UniquePtr<AVIFDecodedData> mDecodedData;
 };
 
 }  // namespace image

modules/libpref/init/StaticPrefList.yaml

@@ -7039,6 +7039,19 @@
   value: true
   mirror: always
 
+# Whether to allow decoding of animated AVIF sequences.
+- name: image.avif.sequence.enabled
+  type: RelaxedAtomicBool
+  value: false
+  mirror: always
+
+# Whether AVIF files containing sequences should be animated even when the
+# major brand is set to 'avif'.
+- name: image.avif.sequence.animate_avif_major_branded_images
+  type: RelaxedAtomicBool
+  value: false
+  mirror: always
+
 # Whether we attempt to decode JXL images or not.
 - name: image.jxl.enabled
   type: RelaxedAtomicBool

toolkit/components/telemetry/Histograms.json

@@ -1791,6 +1791,7 @@
       "alpha_y_bpc_mismatch",
       "ispe_mismatch",
       "invalid_cicp",
+      "no_samples",
       "invalid_parse_status",
       "missing_brand",
       "ftyp_not_first",
@@ -1805,7 +1806,8 @@
       "item_type_missing",
       "construction_method",
       "item_loc_not_found",
-      "no_item_data_box"
+      "no_item_data_box",
+      "uncategorized"
     ],
     "description": "Decode result of AVIF image",
     "bug_numbers": [1670827]