Bug 1348941. r=n.nethercote

image/encoders/bmp/nsBMPEncoder.cpp

@@ -10,6 +10,7 @@
 #include "nsString.h"
 #include "nsStreamUtils.h"
 #include "nsTArray.h"
+#include "mozilla/CheckedInt.h"
 
 using namespace mozilla;
 using namespace mozilla::image;
@@ -58,6 +59,11 @@ nsBMPEncoder::InitFromData(const uint8_t* aData,
     return NS_ERROR_INVALID_ARG;
   }
 
+  CheckedInt32 check = CheckedInt32(aWidth) * 4;
+  if (MOZ_UNLIKELY(!check.isValid())) {
+    return NS_ERROR_INVALID_ARG;
+  }
+
   // Stride is the padded width of each row, so it better be longer
   if ((aInputFormat == INPUT_FORMAT_RGB &&
        aStride < aWidth * 3) ||
@@ -86,19 +92,19 @@ nsBMPEncoder::InitFromData(const uint8_t* aData,
 
 // Just a helper method to make it explicit in calculations that we are dealing
 // with bytes and not bits
-static inline uint32_t
-BytesPerPixel(uint32_t aBPP)
+static inline uint16_t
+BytesPerPixel(uint16_t aBPP)
 {
   return aBPP / 8;
 }
 
 // Calculates the number of padding bytes that are needed per row of image data
 static inline uint32_t
-PaddingBytes(uint32_t aBPP, uint32_t aWidth)
+PaddingBytes(uint16_t aBPP, uint32_t aWidth)
 {
   uint32_t rowSize = aWidth * BytesPerPixel(aBPP);
   uint8_t paddingSize = 0;
-  if(rowSize % 4) {
+  if (rowSize % 4) {
     paddingSize = (4 - (rowSize % 4));
   }
   return paddingSize;
@@ -125,14 +131,21 @@ nsBMPEncoder::StartImageEncode(uint32_t aWidth,
 
   // parse and check any provided output options
   Version version;
-  uint32_t bpp;
+  uint16_t bpp;
   nsresult rv = ParseOptions(aOutputOptions, version, bpp);
   if (NS_FAILED(rv)) {
     return rv;
   }
+  MOZ_ASSERT(bpp <= 32);
 
-  InitFileHeader(version, bpp, aWidth, aHeight);
-  InitInfoHeader(version, bpp, aWidth, aHeight);
+  rv = InitFileHeader(version, bpp, aWidth, aHeight);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
+  rv = InitInfoHeader(version, bpp, aWidth, aHeight);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
 
   mImageBufferSize = mBMPFileHeader.filesize;
   mImageBufferStart = static_cast<uint8_t*>(malloc(mImageBufferSize));
@@ -187,12 +200,26 @@ nsBMPEncoder::AddImageFrame(const uint8_t* aData,
     return NS_ERROR_INVALID_ARG;
   }
 
-  auto row = MakeUniqueFallible<uint8_t[]>(mBMPInfoHeader.width *
-                                           BytesPerPixel(mBMPInfoHeader.bpp));
+  if (mBMPInfoHeader.width < 0) {
+    return NS_ERROR_ILLEGAL_VALUE;
+  }
+
+  CheckedUint32 size =
+    CheckedUint32(mBMPInfoHeader.width) * CheckedUint32(BytesPerPixel(mBMPInfoHeader.bpp));
+  if (MOZ_UNLIKELY(!size.isValid())) {
+    return NS_ERROR_FAILURE;
+  }
+
+  auto row = MakeUniqueFallible<uint8_t[]>(size.value());
   if (!row) {
     return NS_ERROR_OUT_OF_MEMORY;
   }
 
+  CheckedUint32 check = CheckedUint32(mBMPInfoHeader.height) * aStride;
+  if (MOZ_UNLIKELY(!check.isValid())) {
+    return NS_ERROR_FAILURE;
+  }
+
   // write each row: if we add more input formats, we may want to
   // generalize the conversions
   if (aInputFormat == INPUT_FORMAT_HOSTARGB) {
@@ -256,7 +283,7 @@ nsBMPEncoder::EndImageEncode()
 // See InitFromData for a description of the parse options
 nsresult
 nsBMPEncoder::ParseOptions(const nsAString& aOptions, Version& aVersionOut,
-                           uint32_t& aBppOut)
+                           uint16_t& aBppOut)
 {
   aVersionOut = VERSION_3;
   aBppOut = 24;
@@ -424,7 +451,7 @@ nsBMPEncoder::ConvertHostARGBRow(const uint8_t* aSrc,
                                  const UniquePtr<uint8_t[]>& aDest,
                                  uint32_t aPixelWidth)
 {
-  int bytes = BytesPerPixel(mBMPInfoHeader.bpp);
+  uint16_t bytes = BytesPerPixel(mBMPInfoHeader.bpp);
 
   if (mBMPInfoHeader.bpp == 32) {
     for (uint32_t x = 0; x < aPixelWidth; x++) {
@@ -473,8 +500,8 @@ nsBMPEncoder::NotifyListener()
 }
 
 // Initializes the BMP file header mBMPFileHeader to the passed in values
-void
-nsBMPEncoder::InitFileHeader(Version aVersion, uint32_t aBPP, uint32_t aWidth,
+nsresult
+nsBMPEncoder::InitFileHeader(Version aVersion, uint16_t aBPP, uint32_t aWidth,
                              uint32_t aHeight)
 {
   memset(&mBMPFileHeader, 0, sizeof(mBMPFileHeader));
@@ -491,13 +518,25 @@ nsBMPEncoder::InitFileHeader(Version aVersion, uint32_t aBPP, uint32_t aWidth,
   if (aBPP <= 8) {
     uint32_t numColors = 1 << aBPP;
     mBMPFileHeader.dataoffset += 4 * numColors;
-    mBMPFileHeader.filesize = mBMPFileHeader.dataoffset + aWidth * aHeight;
+    CheckedUint32 filesize =
+      CheckedUint32(mBMPFileHeader.dataoffset) + CheckedUint32(aWidth) * aHeight;
+    if (MOZ_UNLIKELY(!filesize.isValid())) {
+      return NS_ERROR_INVALID_ARG;
+    }
+    mBMPFileHeader.filesize = filesize.value();
   } else {
-    mBMPFileHeader.filesize = mBMPFileHeader.dataoffset +
-      (aWidth * BytesPerPixel(aBPP) + PaddingBytes(aBPP, aWidth)) * aHeight;
+    CheckedUint32 filesize =
+      CheckedUint32(mBMPFileHeader.dataoffset) +
+        (CheckedUint32(aWidth) * BytesPerPixel(aBPP) + PaddingBytes(aBPP, aWidth)) * aHeight;
+    if (MOZ_UNLIKELY(!filesize.isValid())) {
+      return NS_ERROR_INVALID_ARG;
+    }
+    mBMPFileHeader.filesize = filesize.value();
   }
 
   mBMPFileHeader.reserved = 0;
+
+  return NS_OK;
 }
 
 #define ENCODE(pImageBufferCurr, value) \
@@ -505,8 +544,8 @@ nsBMPEncoder::InitFileHeader(Version aVersion, uint32_t aBPP, uint32_t aWidth,
     *pImageBufferCurr += sizeof value;
 
 // Initializes the bitmap info header mBMPInfoHeader to the passed in values
-void
-nsBMPEncoder::InitInfoHeader(Version aVersion, uint32_t aBPP, uint32_t aWidth,
+nsresult
+nsBMPEncoder::InitInfoHeader(Version aVersion, uint16_t aBPP, uint32_t aWidth,
                              uint32_t aHeight)
 {
   memset(&mBMPInfoHeader, 0, sizeof(mBMPInfoHeader));
@@ -516,18 +555,39 @@ nsBMPEncoder::InitInfoHeader(Version aVersion, uint32_t aBPP, uint32_t aWidth,
     MOZ_ASSERT(aVersion == VERSION_5);
     mBMPInfoHeader.bihsize = InfoHeaderLength::WIN_V5;
   }
-  mBMPInfoHeader.width = aWidth;
-  mBMPInfoHeader.height = aHeight;
+
+  CheckedInt32 width(aWidth);
+  CheckedInt32 height(aHeight);
+  if (MOZ_UNLIKELY(!width.isValid() || !height.isValid())) {
+    return NS_ERROR_INVALID_ARG;
+  }
+  mBMPInfoHeader.width = width.value();
+  mBMPInfoHeader.height = height.value();
+
   mBMPInfoHeader.planes = 1;
   mBMPInfoHeader.bpp = aBPP;
   mBMPInfoHeader.compression = 0;
   mBMPInfoHeader.colors = 0;
   mBMPInfoHeader.important_colors = 0;
+
+  CheckedUint32 check = CheckedUint32(aWidth) * BytesPerPixel(aBPP);
+  if (MOZ_UNLIKELY(check.isValid())) {
+    return NS_ERROR_INVALID_ARG;
+  }
+
   if (aBPP <= 8) {
-    mBMPInfoHeader.image_size = aWidth * aHeight;
+    CheckedUint32 imagesize = CheckedUint32(aWidth) * aHeight;
+    if (MOZ_UNLIKELY(imagesize.isValid())) {
+      return NS_ERROR_INVALID_ARG;
+    }
+    mBMPInfoHeader.image_size = imagesize.value();
   } else {
-    mBMPInfoHeader.image_size =
-      (aWidth * BytesPerPixel(aBPP) + PaddingBytes(aBPP, aWidth)) * aHeight;
+    CheckedUint32 imagesize =
+      CheckedUint32(aWidth) * BytesPerPixel(aBPP) + PaddingBytes(aBPP, aWidth) * CheckedUint32(aHeight);
+    if (MOZ_UNLIKELY(imagesize.isValid())) {
+      return NS_ERROR_INVALID_ARG;
+    }
+    mBMPInfoHeader.image_size = imagesize.value();
   }
   mBMPInfoHeader.xppm = 0;
   mBMPInfoHeader.yppm = 0;
@@ -554,6 +614,8 @@ nsBMPEncoder::InitInfoHeader(Version aVersion, uint32_t aBPP, uint32_t aWidth,
       mBMPInfoHeader.profile_size = 0;
       mBMPInfoHeader.reserved = 0;
   }
+
+  return NS_OK;
 }
 
 // Encodes the BMP file header mBMPFileHeader

image/encoders/bmp/nsBMPEncoder.h

@@ -104,7 +104,7 @@ protected:
 
   // See InitData in the cpp for valid parse options
   nsresult ParseOptions(const nsAString& aOptions, Version& aVersionOut,
-                        uint32_t& aBppOut);
+                        uint16_t& aBppOut);
   // Obtains data with no alpha in machine-independent byte order
   void ConvertHostARGBRow(const uint8_t* aSrc,
                           const mozilla::UniquePtr<uint8_t[]>& aDest,
@@ -113,10 +113,10 @@ protected:
   void NotifyListener();
 
   // Initializes the bitmap file header member mBMPFileHeader
-  void InitFileHeader(Version aVersion, uint32_t aBPP, uint32_t aWidth,
+  nsresult InitFileHeader(Version aVersion, uint16_t aBPP, uint32_t aWidth,
                           uint32_t aHeight);
   // Initializes the bitmap info header member mBMPInfoHeader
-  void InitInfoHeader(Version aVersion, uint32_t aBPP, uint32_t aWidth,
+  nsresult InitInfoHeader(Version aVersion, uint16_t aBPP, uint32_t aWidth,
                           uint32_t aHeight);
 
   // Encodes the bitmap file header member mBMPFileHeader

image/encoders/ico/nsICOEncoder.cpp

@@ -228,10 +228,11 @@ nsICOEncoder::StartImageEncode(uint32_t aWidth,
   }
 
   // parse and check any provided output options
-  uint32_t bpp = 24;
+  uint16_t bpp = 24;
   bool usePNG = true;
   nsresult rv = ParseOptions(aOutputOptions, bpp, usePNG);
   NS_ENSURE_SUCCESS(rv, rv);
+  MOZ_ASSERT(bpp <= 32);
 
   mUsePNG = usePNG;
 
@@ -265,7 +266,7 @@ nsICOEncoder::EndImageEncode()
 // Parses the encoder options and sets the bits per pixel to use and PNG or BMP
 // See InitFromData for a description of the parse options
 nsresult
-nsICOEncoder::ParseOptions(const nsAString& aOptions, uint32_t& aBppOut,
+nsICOEncoder::ParseOptions(const nsAString& aOptions, uint16_t& aBppOut,
                            bool& aUsePNGOut)
 {
   // If no parsing options just use the default of 24BPP and PNG yes
@@ -469,7 +470,7 @@ nsICOEncoder::InitFileHeader()
 
 // Initializes the icon directory info header mICODirEntry
 void
-nsICOEncoder::InitInfoHeader(uint32_t aBPP, uint8_t aWidth, uint8_t aHeight)
+nsICOEncoder::InitInfoHeader(uint16_t aBPP, uint8_t aWidth, uint8_t aHeight)
 {
   memset(&mICODirEntry, 0, sizeof(mICODirEntry));
   mICODirEntry.mBitCount = aBPP;

image/encoders/ico/nsICOEncoder.h

@@ -50,14 +50,14 @@ public:
 protected:
   ~nsICOEncoder();
 
-  nsresult ParseOptions(const nsAString& aOptions, uint32_t& aBppOut,
+  nsresult ParseOptions(const nsAString& aOptions, uint16_t& aBppOut,
                         bool& aUsePNGOut);
   void NotifyListener();
 
   // Initializes the icon file header mICOFileHeader
   void InitFileHeader();
   // Initializes the icon directory info header mICODirEntry
-  void InitInfoHeader(uint32_t aBPP, uint8_t aWidth, uint8_t aHeight);
+  void InitInfoHeader(uint16_t aBPP, uint8_t aWidth, uint8_t aHeight);
   // Encodes the icon file header mICOFileHeader
   void EncodeFileHeader();
   // Encodes the icon directory info header mICODirEntry