Bug 1235605 - Use CheckedInt in Deinterlacer and make its buffer allocation fallible. r=tn

image/Deinterlacer.cpp

@@ -6,16 +6,24 @@
 
 
 #include "Downscaler.h"
+#include "mozilla/UniquePtrExtensions.h"
 
 namespace mozilla {
 namespace image {
 
 Deinterlacer::Deinterlacer(const nsIntSize& aImageSize)
   : mImageSize(aImageSize)
-  , mBuffer(MakeUnique<uint8_t[]>(mImageSize.width *
-                                  mImageSize.height *
-                                  sizeof(uint32_t)))
-{ }
+{
+  CheckedInt<size_t> bufferSize = mImageSize.width;
+  bufferSize *= mImageSize.height;
+  bufferSize *= sizeof(uint32_t);
+
+  if (!bufferSize.isValid()) {
+    return;
+  }
+
+  mBuffer = MakeUniqueFallible<uint8_t[]>(bufferSize.value());
+}
 
 uint32_t
 Deinterlacer::RowSize() const
@@ -27,6 +35,7 @@ uint8_t*
 Deinterlacer::RowBuffer(uint32_t aRow)
 {
   uint32_t offset = aRow * RowSize();
+  MOZ_ASSERT(IsValid(), "Deinterlacer in invalid state");
   MOZ_ASSERT(offset < mImageSize.width * mImageSize.height * sizeof(uint32_t),
              "Row is outside of image");
   return mBuffer.get() + offset;
@@ -35,6 +44,7 @@ Deinterlacer::RowBuffer(uint32_t aRow)
 void
 Deinterlacer::PropagatePassToDownscaler(Downscaler& aDownscaler)
 {
+  MOZ_ASSERT(IsValid(), "Deinterlacer in invalid state");
   for (int32_t row = 0 ; row < mImageSize.height ; ++row) {
     memcpy(aDownscaler.RowBuffer(), RowBuffer(row), RowSize());
     aDownscaler.CommitRow();

image/Deinterlacer.h

@@ -32,6 +32,7 @@ class Deinterlacer
 {
 public:
   explicit Deinterlacer(const nsIntSize& aImageSize);
+  bool IsValid() { return !!mBuffer; }
 
   uint8_t* RowBuffer(uint32_t aRow);
   void PropagatePassToDownscaler(Downscaler& aDownscaler);

image/decoders/nsGIFDecoder2.cpp

@@ -1178,6 +1178,12 @@ nsGIFDecoder2::WriteInternal(const char* aBuffer, uint32_t aCount)
         mGIFStruct.ipass = 1;
         if (mDownscaler) {
           mDeinterlacer.emplace(mDownscaler->FrameSize());
+
+          if (!mDeinterlacer->IsValid()) {
+            mDeinterlacer.reset();
+            mGIFStruct.state = gif_error;
+            break;
+          }
         }
       } else {
         mGIFStruct.interlaced = false;

image/test/crashtests/crashtests.list

@@ -14,6 +14,7 @@ load 856616.gif
 skip-if(B2G) load 944353.jpg
 load 1205923-1.html
 load 1212954-1.svg
+load 1235605.gif
 load colormap-range.gif
 HTTP load delayedframe.sjs # A 3-frame animated GIF with an inordinate delay between the second and third frame