Bug 1333858 - Intersect should be fallible on overflow. r=daoshengmu

MozReview-Commit-ID: 6lmIKKyXXah

dom/canvas/WebGLContext.cpp

@@ -2225,25 +2225,47 @@ ScopedLazyBind::UnwrapImpl()
 
 ////////////////////////////////////////
 
-void
+bool
-Intersect(uint32_t srcSize, int32_t dstStartInSrc, uint32_t dstSize,
+Intersect(const int32_t srcSize, const int32_t read0, const int32_t readSize,
-          uint32_t* const out_intStartInSrc, uint32_t* const out_intStartInDst,
+          int32_t* const out_intRead0, int32_t* const out_intWrite0,
-          uint32_t* const out_intSize)
+          int32_t* const out_intSize)
 {
-    // Only >0 if dstStartInSrc is >0:
+    MOZ_ASSERT(srcSize >= 0);
-    // 0  3          // src coords
+    MOZ_ASSERT(readSize >= 0);
-    // |  [========] // dst box
+    const auto read1 = int64_t(read0) + readSize;
-    // ^--^
-    *out_intStartInSrc = std::max<int32_t>(0, dstStartInSrc);
 
-    // Only >0 if dstStartInSrc is <0:
+    int32_t intRead0 = read0; // Clearly doesn't need validation.
-    //-6     0       // src coords
+    int64_t intWrite0 = 0;
-    // [=====|==]    // dst box
+    int64_t intSize = readSize;
-    // ^-----^
-    *out_intStartInDst = std::max<int32_t>(0, 0 - dstStartInSrc);
 
-    int32_t intEndInSrc = std::min<int32_t>(srcSize, dstStartInSrc + dstSize);
+    if (read1 <= 0 || read0 >= srcSize) {
-    *out_intSize = std::max<int32_t>(0, intEndInSrc - *out_intStartInSrc);
+        // Disjoint ranges.
+        intSize = 0;
+    } else {
+        if (read0 < 0) {
+            const auto diff = int64_t(0) - read0;
+            MOZ_ASSERT(diff >= 0);
+            intRead0 = 0;
+            intWrite0 = diff;
+            intSize -= diff;
+        }
+        if (read1 > srcSize) {
+            const auto diff = int64_t(read1) - srcSize;
+            MOZ_ASSERT(diff >= 0);
+            intSize -= diff;
+        }
+
+        if (!CheckedInt<int32_t>(intWrite0).isValid() ||
+            !CheckedInt<int32_t>(intSize).isValid())
+        {
+            return false;
+        }
+    }
+
+    *out_intRead0 = intRead0;
+    *out_intWrite0 = intWrite0;
+    *out_intSize = intSize;
+    return true;
 }
 
 ////////////////////////////////////////////////////////////////////////////////

dom/canvas/WebGLContext.h

@@ -2164,10 +2164,9 @@ private:
 
 ////
 
-void
+bool
-Intersect(uint32_t srcSize, int32_t dstStartInSrc, uint32_t dstSize,
+Intersect(int32_t srcSize, int32_t read0, int32_t readSize, int32_t* out_intRead0,
-          uint32_t* const out_intStartInSrc, uint32_t* const out_intStartInDst,
+          int32_t* out_intWrite0, int32_t* out_intSize);
-          uint32_t* const out_intSize);
 
 ////
 

dom/canvas/WebGLContextGL.cpp

@@ -1556,18 +1556,32 @@ WebGLContext::ReadPixelsImpl(GLint x, GLint y, GLsizei rawWidth, GLsizei rawHeig
         return;
     }
 
+    ////
+
+    int32_t readX, readY;
+    int32_t writeX, writeY;
+    int32_t rwWidth, rwHeight;
+    if (!Intersect(srcWidth, x, width, &readX, &writeX, &rwWidth) ||
+        !Intersect(srcHeight, y, height, &readY, &writeY, &rwHeight))
+    {
+        ErrorOutOfMemory("readPixels: Bad subrect selection.");
+        return;
+    }
+
     ////////////////
     // Now that the errors are out of the way, on to actually reading!
 
     OnBeforeReadCall();
 
-    uint32_t readX, readY;
+    if (!rwWidth || !rwHeight) {
-    uint32_t writeX, writeY;
+        // Disjoint rects, so we're done already.
-    uint32_t rwWidth, rwHeight;
+        DummyReadFramebufferOperation("readPixels");
-    Intersect(srcWidth, x, width, &readX, &writeX, &rwWidth);
+        return;
-    Intersect(srcHeight, y, height, &readY, &writeY, &rwHeight);
+    }
 
-    if (rwWidth == uint32_t(width) && rwHeight == uint32_t(height)) {
+    if (uint32_t(rwWidth) == width &&
+        uint32_t(rwHeight) == height)
+    {
         DoReadPixelsAndConvert(srcFormat->format, x, y, width, height, packFormat,
                                packType, dest, dataLen, rowStride);
         return;
@@ -1586,12 +1600,6 @@ WebGLContext::ReadPixelsImpl(GLint x, GLint y, GLsizei rawWidth, GLsizei rawHeig
     ////////////////////////////////////
     // Read only the in-bounds pixels.
 
-    if (!rwWidth || !rwHeight) {
-        // There aren't any, so we're 'done'.
-        DummyReadFramebufferOperation("readPixels");
-        return;
-    }
-
     if (IsWebGL2()) {
         if (!mPixelStore_PackRowLength) {
             gl->fPixelStorei(LOCAL_GL_PACK_ROW_LENGTH,
@@ -1611,7 +1619,7 @@ WebGLContext::ReadPixelsImpl(GLint x, GLint y, GLsizei rawWidth, GLsizei rawHeig
 
         uint8_t* row = (uint8_t*)dest + writeX * bytesPerPixel;
         row += writeY * rowStride;
-        for (uint32_t j = 0; j < rwHeight; j++) {
+        for (uint32_t j = 0; j < uint32_t(rwHeight); j++) {
             DoReadPixelsAndConvert(srcFormat->format, readX, readY+j, rwWidth, 1,
                                    packFormat, packType, row, dataLen, rowStride);
             row += rowStride;

dom/canvas/WebGLTextureUpload.cpp

@@ -2011,11 +2011,15 @@ DoCopyTexOrSubImage(WebGLContext* webgl, const char* funcName, bool isSubImage,
 
     ////
 
-    uint32_t readX, readY;
+    int32_t readX, readY;
-    uint32_t writeX, writeY;
+    int32_t writeX, writeY;
-    uint32_t rwWidth, rwHeight;
+    int32_t rwWidth, rwHeight;
-    Intersect(srcTotalWidth, xWithinSrc, dstWidth, &readX, &writeX, &rwWidth);
+    if (!Intersect(srcTotalWidth, xWithinSrc, dstWidth, &readX, &writeX, &rwWidth) ||
-    Intersect(srcTotalHeight, yWithinSrc, dstHeight, &readY, &writeY, &rwHeight);
+        !Intersect(srcTotalHeight, yWithinSrc, dstHeight, &readY, &writeY, &rwHeight))
+    {
+        webgl->ErrorOutOfMemory("%s: Bad subrect selection.", funcName);
+        return false;
+    }
 
     writeX += xOffset;
     writeY += yOffset;