Bug 1273712 (part 2) - Add a new overloading of Base64Encode(). r=erahm.

This will be used in bug 1273711 to avoid an OOM.

This also tweaks one of the existing overloadings of Base64Encode to return
NS_ERROR_OUT_OF_MEMORY on OOM instead of NS_ERROR_INVALID_ARG.

--HG--
extra : rebase_source : a2ad472b11ac2c858487bf5fdae84d183084773b

xpcom/io/Base64.cpp

@@ -281,6 +281,43 @@ Base64EncodeInputStream(nsIInputStream* aInputStream,
   return EncodeInputStream<nsAString>(aInputStream, aDest, aCount, aOffset);
 }
 
+nsresult
+Base64Encode(const char* aBinary, uint32_t aBinaryLen, char** aBase64)
+{
+  // Check for overflow.
+  if (aBinaryLen > (UINT32_MAX / 4) * 3) {
+    return NS_ERROR_FAILURE;
+  }
+
+  // Don't ask PR_Base64Encode to encode empty strings.
+  if (aBinaryLen == 0) {
+    char* base64 = (char*)moz_xmalloc(1);
+    base64[0] = '\0';
+    *aBase64 = base64;
+    return NS_OK;
+  }
+
+  uint32_t base64Len = ((aBinaryLen + 2) / 3) * 4;
+
+  // Add one byte for null termination.
+  char* base64 = (char*)malloc(base64Len + 1);
+  if (!base64) {
+    return NS_ERROR_OUT_OF_MEMORY;
+  }
+
+  if (!PL_Base64Encode(aBinary, aBinaryLen, base64)) {
+    free(base64);
+    return NS_ERROR_INVALID_ARG;
+  }
+
+  // PL_Base64Encode doesn't null terminate the buffer for us when we pass
+  // the buffer in. Do that manually.
+  base64[base64Len] = '\0';
+
+  *aBase64 = base64;
+  return NS_OK;
+}
+
 nsresult
 Base64Encode(const nsACString& aBinary, nsACString& aBase64)
 {
@@ -289,7 +326,7 @@ Base64Encode(const nsACString& aBinary, nsACString& aBase64)
     return NS_ERROR_FAILURE;
   }
 
-  // Don't ask PR_Base64Encode to encode empty strings
+  // Don't ask PR_Base64Encode to encode empty strings.
   if (aBinary.IsEmpty()) {
     aBase64.Truncate();
     return NS_OK;
@@ -297,22 +334,24 @@ Base64Encode(const nsACString& aBinary, nsACString& aBase64)
 
   uint32_t base64Len = ((aBinary.Length() + 2) / 3) * 4;
 
-  char* base64;
-
   // Add one byte for null termination.
-  if (aBase64.SetCapacity(base64Len + 1, fallible) &&
+  if (!aBase64.SetCapacity(base64Len + 1, fallible)) {
-      (base64 = aBase64.BeginWriting()) &&
+    return NS_ERROR_OUT_OF_MEMORY;
-      PL_Base64Encode(aBinary.BeginReading(), aBinary.Length(), base64)) {
-    // PL_Base64Encode doesn't null terminate the buffer for us when we pass
-    // the buffer in. Do that manually.
-    base64[base64Len] = '\0';
-
-    aBase64.SetLength(base64Len);
-    return NS_OK;
   }
 
-  aBase64.Truncate();
+  char* base64 = aBase64.BeginWriting();
-  return NS_ERROR_INVALID_ARG;
+  if (!base64 ||
+      !PL_Base64Encode(aBinary.BeginReading(), aBinary.Length(), base64)) {
+    aBase64.Truncate();
+    return NS_ERROR_INVALID_ARG;
+  }
+
+  // PL_Base64Encode doesn't null terminate the buffer for us when we pass
+  // the buffer in. Do that manually.
+  base64[base64Len] = '\0';
+
+  aBase64.SetLength(base64Len);
+  return NS_OK;
 }
 
 nsresult

xpcom/io/Base64.h

@@ -24,6 +24,8 @@ Base64EncodeInputStream(nsIInputStream* aInputStream,
                         uint32_t aCount,
                         uint32_t aOffset = 0);
 
+nsresult
+Base64Encode(const char* aBinary, uint32_t aBinaryLen, char** aBase64);
 nsresult
 Base64Encode(const nsACString& aBinary, nsACString& aBase64);
 nsresult