Bug 1120985 - Allow nsMultiMixedConv to compute its boundary if content-type=application/package r=honzab

Also makes nsMultiMixedConv/nsPartChannel save and return individual headers for each part of the resource file.

netwerk/base/moz.build

@@ -87,6 +87,7 @@ XPIDL_SOURCES += [
     'nsIRequest.idl',
     'nsIRequestObserver.idl',
     'nsIRequestObserverProxy.idl',
+    'nsIResponseHeadProvider.idl',
     'nsIResumableChannel.idl',
     'nsISecretDecoderRing.idl',
     'nsISecureBrowserUI.idl',

netwerk/base/nsIResponseHeadProvider.idl

@@ -0,0 +1,35 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+
+interface nsIHttpHeaderVisitor;
+
+%{C++
+namespace mozilla {
+namespace net {
+class nsHttpResponseHead;
+}
+}
+%}
+
+[ptr] native nsHttpResponseHeadPtr(mozilla::net::nsHttpResponseHead);
+
+/**
+ * nsIResponseHeadProvider
+ */
+[scriptable, builtinclass, uuid(cd0d0804-2e0c-4bff-aa0a-78a3e3159b69)]
+interface nsIResponseHeadProvider : nsISupports
+{
+  /**
+   * Returns a pointer to a nsHttpResponseHead. May return null.
+   */
+  [notxpcom] nsHttpResponseHeadPtr GetResponseHead();
+
+  /**
+   * May be used to iterate through the response headers
+   */
+  void visitResponseHeaders(in nsIHttpHeaderVisitor aVisitor);
+};

netwerk/protocol/http/nsHttpHeaderArray.h

@@ -13,6 +13,12 @@
 
 class nsIHttpHeaderVisitor;
 
+// This needs to be forward declared here so we can include only this header
+// without also including PHttpChannelParams.h
+namespace IPC {
+    template <typename> struct ParamTraits;
+}
+
 namespace mozilla { namespace net {
 
 class nsHttpHeaderArray

netwerk/protocol/http/nsHttpResponseHead.h

@@ -10,6 +10,12 @@
 #include "nsHttp.h"
 #include "nsString.h"
 
+// This needs to be forward declared here so we can include only this header
+// without also including PHttpChannelParams.h
+namespace IPC {
+    template <typename> struct ParamTraits;
+}
+
 namespace mozilla { namespace net {
 
 //-----------------------------------------------------------------------------

netwerk/streamconv/converters/nsMultiMixedConv.cpp

@@ -109,6 +109,7 @@ NS_INTERFACE_MAP_BEGIN(nsPartChannel)
     NS_INTERFACE_MAP_ENTRY(nsIChannel)
     NS_INTERFACE_MAP_ENTRY(nsIByteRangeRequest)
     NS_INTERFACE_MAP_ENTRY(nsIMultiPartChannel)
+    NS_INTERFACE_MAP_ENTRY(nsIResponseHeadProvider)
 NS_INTERFACE_MAP_END
 
 //
@@ -381,6 +382,24 @@ nsPartChannel::GetIsLastPart(bool *aIsLastPart)
     return NS_OK;
 }
 
+//
+// nsIResponseHeadProvider
+//
+
+NS_IMETHODIMP_(mozilla::net::nsHttpResponseHead *)
+nsPartChannel::GetResponseHead()
+{
+    return mResponseHead;
+}
+
+NS_IMETHODIMP
+nsPartChannel::VisitResponseHeaders(nsIHttpHeaderVisitor *visitor)
+{
+    if (!mResponseHead)
+        return NS_ERROR_NOT_AVAILABLE;
+    return mResponseHead->Headers().VisitHeaders(visitor);
+}
+
 //
 // nsIByteRangeRequest implementation...
 //
@@ -483,10 +502,6 @@ NS_IMETHODIMP
 nsMultiMixedConv::OnDataAvailable(nsIRequest *request, nsISupports *context,
                                   nsIInputStream *inStr, uint64_t sourceOffset,
                                   uint32_t count) {
-
-    if (mToken.IsEmpty()) // no token, no love.
-        return NS_ERROR_FAILURE;
-
     nsresult rv = NS_OK;
     AutoFree buffer(nullptr);
     uint32_t bufLen = 0, read = 0;
@@ -529,16 +544,40 @@ nsMultiMixedConv::OnDataAvailable(nsIRequest *request, nsISupports *context,
         mFirstOnData = false;
         NS_ASSERTION(!mBufLen, "this is our first time through, we can't have buffered data");
         const char * token = mToken.get();
-           
+
         PushOverLine(cursor, bufLen);
 
-        if (bufLen < mTokenLen+2) {
+        bool needMoreChars = bufLen < mTokenLen + 2;
+        nsAutoCString firstBuffer(buffer, bufLen);
+        int32_t posCR = firstBuffer.Find("\r");
+
+        if (needMoreChars || (posCR == kNotFound)) {
             // we don't have enough data yet to make this comparison.
             // skip this check, and try again the next time OnData()
             // is called.
             mFirstOnData = true;
-        }
-        else if (!PL_strnstr(cursor, token, mTokenLen+2)) {
+        } else if (mPackagedApp) {
+            // We need to check the line starts with --
+            if (!StringBeginsWith(firstBuffer, NS_LITERAL_CSTRING("--"))) {
+                return NS_ERROR_FAILURE;
+            }
+
+            // If the boundary was set in the header,
+            // we need to check it matches with the one in the file.
+            if (mTokenLen &&
+                !StringBeginsWith(Substring(firstBuffer, 2), mToken)) {
+                return NS_ERROR_FAILURE;
+            }
+
+            // Save the token.
+            if (!mTokenLen) {
+                mToken = nsCString(Substring(firstBuffer, 2).BeginReading(),
+                                   posCR - 2);
+                mTokenLen = mToken.Length();
+            }
+
+            cursor = buffer;
+        } else if (!PL_strnstr(cursor, token, mTokenLen + 2)) {
             char *newBuffer = (char *) realloc(buffer, bufLen + mTokenLen + 1);
             if (!newBuffer)
                 return NS_ERROR_OUT_OF_MEMORY;
@@ -557,6 +596,9 @@ nsMultiMixedConv::OnDataAvailable(nsIRequest *request, nsISupports *context,
 
     char *token = nullptr;
 
+    // This may get initialized by ParseHeaders and the resulting
+    // HttpResponseHead will be passed to nsPartChannel by SendStart
+
     if (mProcessingHeaders) {
         // we were not able to process all the headers
         // for this "part" given the previous buffer given to 
@@ -694,8 +736,25 @@ nsMultiMixedConv::OnStartRequest(nsIRequest *request, nsISupports *ctxt) {
         if (NS_FAILED(rv)) return NS_ERROR_FAILURE;
     }
 
+    // http://www.w3.org/TR/web-packaging/#streamable-package-format
+    // Although it is compatible with multipart/* this format does not require
+    // the boundary to be included in the header, as it can be ascertained from
+    // the content of the file.
+    if (delimiter.Find("application/package") != kNotFound) {
+        mPackagedApp = true;
+        mToken.Truncate();
+        mTokenLen = 0;
+    }
+
     bndry = strstr(delimiter.BeginWriting(), "boundary");
-    if (!bndry) return NS_ERROR_FAILURE;
+
+    if (!bndry && mPackagedApp) {
+        return NS_OK;
+    }
+
+    if (!bndry) {
+        return NS_ERROR_FAILURE;
+    }
 
     bndry = strchr(bndry, '=');
     if (!bndry) return NS_ERROR_FAILURE;
@@ -712,9 +771,10 @@ nsMultiMixedConv::OnStartRequest(nsIRequest *request, nsISupports *ctxt) {
 
     mToken = boundaryString;
     mTokenLen = boundaryString.Length();
-    
-    if (mTokenLen == 0)
-        return NS_ERROR_FAILURE;
+
+   if (mTokenLen == 0 && !mPackagedApp) {
+       return NS_ERROR_FAILURE;
+   }
 
     return NS_OK;
 }
@@ -723,8 +783,14 @@ NS_IMETHODIMP
 nsMultiMixedConv::OnStopRequest(nsIRequest *request, nsISupports *ctxt,
                                 nsresult aStatus) {
 
-    if (mToken.IsEmpty())  // no token, no love.
-        return NS_ERROR_FAILURE;
+    nsresult rv = NS_OK;
+
+    // We should definitely have found a token at this point. Not having one
+    // is clearly an error, so we need to pass it to the listener.
+    if (mToken.IsEmpty()) {
+        aStatus = NS_ERROR_FAILURE;
+        rv = NS_ERROR_FAILURE;
+    }
 
     if (mPartChannel) {
         mPartChannel->SetIsLastPart();
@@ -753,7 +819,7 @@ nsMultiMixedConv::OnStopRequest(nsIRequest *request, nsISupports *ctxt,
         (void) mFinalListener->OnStopRequest(request, ctxt, aStatus);
     }
 
-    return NS_OK;
+    return rv;
 }
 
 
@@ -771,6 +837,7 @@ nsMultiMixedConv::nsMultiMixedConv() :
     mByteRangeEnd       = 0;
     mTotalSent          = 0;
     mIsByteRangeRequest = false;
+    mPackagedApp        = false;
 }
 
 nsMultiMixedConv::~nsMultiMixedConv() {
@@ -835,6 +902,9 @@ nsMultiMixedConv::SendStart(nsIChannel *aChannel) {
     // Set up the new part channel...
     mPartChannel = newChannel;
 
+    // We pass the headers to the nsPartChannel
+    mPartChannel->SetResponseHead(mResponseHead.forget());
+
     rv = mPartChannel->SetContentType(mContentType);
     if (NS_FAILED(rv)) return rv;
 
@@ -941,7 +1011,14 @@ nsMultiMixedConv::ParseHeaders(nsIChannel *aChannel, char *&aPtr,
     uint32_t cursorLen = aLen;
     bool done = false;
     uint32_t lineFeedIncrement = 1;
-    
+
+    // We only create an nsHttpResponseHead for packaged app channels
+    // It may already be initialized, from a previous call of ParseHeaders
+    // since the headers for a single part may come in more then one chunk
+    if (mPackagedApp && !mResponseHead) {
+        mResponseHead = new nsHttpResponseHead();
+    }
+
     mContentLength = UINT64_MAX; // XXX what if we were already called?
     while (cursorLen && (newLine = (char *) memchr(cursor, nsCRT::LF, cursorLen))) {
         // adjust for linefeeds
@@ -965,6 +1042,13 @@ nsMultiMixedConv::ParseHeaders(nsIChannel *aChannel, char *&aPtr,
 
         char tmpChar = *newLine;
         *newLine = '\0'; // cursor is now null terminated
+
+        if (mResponseHead) {
+            // ParseHeaderLine is destructive. We create a copy
+            nsAutoCString tmpHeader(cursor);
+            mResponseHead->ParseHeaderLine(tmpHeader.get());
+        }
+
         char *colon = (char *) strchr(cursor, ':');
         if (colon) {
             *colon = '\0';

netwerk/streamconv/converters/nsMultiMixedConv.h

@@ -14,6 +14,10 @@
 #include "nsIMultiPartChannel.h"
 #include "nsAutoPtr.h"
 #include "mozilla/Attributes.h"
+#include "nsIResponseHeadProvider.h"
+#include "nsHttpResponseHead.h"
+
+using mozilla::net::nsHttpResponseHead;
 
 #define NS_MULTIMIXEDCONVERTER_CID                         \
 { /* 7584CE90-5B25-11d3-A175-0050041CAF44 */         \
@@ -32,6 +36,7 @@
 //
 class nsPartChannel MOZ_FINAL : public nsIChannel,
                                 public nsIByteRangeRequest,
+                                public nsIResponseHeadProvider,
                                 public nsIMultiPartChannel
 {
 public:
@@ -47,11 +52,13 @@ public:
   /* SetContentDisposition expects the full value of the Content-Disposition
    * header */
   void SetContentDisposition(const nsACString& aContentDispositionHeader);
+  void SetResponseHead(nsHttpResponseHead * head) { mResponseHead = head; }
 
   NS_DECL_ISUPPORTS
   NS_DECL_NSIREQUEST
   NS_DECL_NSICHANNEL
   NS_DECL_NSIBYTERANGEREQUEST
+  NS_DECL_NSIRESPONSEHEADPROVIDER
   NS_DECL_NSIMULTIPARTCHANNEL
 
 protected:
@@ -60,7 +67,8 @@ protected:
 protected:
   nsCOMPtr<nsIChannel>    mMultipartChannel;
   nsCOMPtr<nsIStreamListener> mListener;
-  
+  nsAutoPtr<nsHttpResponseHead> mResponseHead;
+
   nsresult                mStatus;
   nsLoadFlags             mLoadFlags;
 
@@ -168,6 +176,12 @@ protected:
     bool                mIsByteRangeRequest;
 
     uint32_t            mCurrentPartID;
+
+    // This is true if the content-type is application/package
+    // Streamable packages don't require the boundary in the header
+    // as it can be ascertained from the package file.
+    bool                mPackagedApp;
+    nsAutoPtr<nsHttpResponseHead> mResponseHead;
 };
 
 #endif /* __nsmultimixedconv__h__ */

netwerk/test/unit/test_multipart_streamconv_application_package.js

@@ -0,0 +1,211 @@
+// Tests:
+//   test_multipart
+//     Loads the multipart file returned by contentHandler()
+//     The boundary is ascertained from the first line in the multipart file
+//   test_multipart_with_boundary
+//     Loads the multipart file returned by contentHandler_with_boundary()
+//     The boundary is given in the Content-Type headers, and is also present
+//     in the first line of the file.
+//   test_multipart_chunked_headers
+//     Tests that the headers are properly passed even when they come in multiple
+//     chunks (several calls to OnDataAvailable). It first passes the first 60
+//     characters, then the rest of the response.
+
+// testData.token - the multipart file's boundary
+// Call testData.getData() to get the file contents as a string
+
+// multipartListener
+//   - a listener that checks that the multipart file is correctly split into multiple parts
+
+// headerListener
+//   - checks that the headers for each part is set correctly
+
+Cu.import("resource://testing-common/httpd.js");
+Cu.import("resource://gre/modules/Services.jsm");
+
+var httpserver = null;
+
+XPCOMUtils.defineLazyGetter(this, "uri", function() {
+  return "http://localhost:" + httpserver.identity.primaryPort;
+});
+
+function make_channel(url) {
+  var ios = Cc["@mozilla.org/network/io-service;1"].
+            getService(Ci.nsIIOService);
+  return ios.newChannel2(url,
+                         "",
+                         null,
+                         null,      // aLoadingNode
+                         Services.scriptSecurityManager.getSystemPrincipal(),
+                         null,      // aTriggeringPrincipal
+                         Ci.nsILoadInfo.SEC_NORMAL,
+                         Ci.nsIContentPolicy.TYPE_OTHER);
+}
+
+function contentHandler(metadata, response)
+{
+  response.setHeader("Content-Type", 'application/package');
+  var body = testData.getData();
+  response.bodyOutputStream.write(body, body.length);
+}
+
+function contentHandler_with_boundary(metadata, response)
+{
+  response.setHeader("Content-Type", 'application/package; boundary="'+testData.token+'"');
+  var body = testData.getData();
+  response.bodyOutputStream.write(body, body.length);
+}
+
+function contentHandler_chunked_headers(metadata, response)
+{
+  response.setHeader("Content-Type", 'application/package');
+  var body = testData.getData();
+
+  response.bodyOutputStream.write(body.substring(0,60), 60);
+  response.processAsync();
+  do_timeout(5, function() {
+    response.bodyOutputStream.write(body.substring(60), body.length-60);
+    response.finish();
+  });
+}
+
+var testData = {
+  content: [
+   { headers: ["Content-Location: /index.html", "Content-Type: text/html"], data: "<html>\r\n  <head>\r\n    <script src=\"/scripts/app.js\"></script>\r\n    ...\r\n  </head>\r\n  ...\r\n</html>\r\n", type: "text/html" },
+   { headers: ["Content-Location: /scripts/app.js", "Content-Type: text/javascript"], data: "module Math from '/scripts/helpers/math.js';\r\n...\r\n", type: "text/javascript" },
+   { headers: ["Content-Location: /scripts/helpers/math.js", "Content-Type: text/javascript"], data: "export function sum(nums) { ... }\r\n...\r\n", type: "text/javascript" }
+  ],
+  token : "gc0pJq0M:08jU534c0p",
+  getData: function() {
+    var str = "";
+    for (var i in this.content) {
+      str += "--" + this.token + "\r\n";
+      for (var j in this.content[i].headers) {
+        str += this.content[i].headers[j] + "\r\n";
+      }
+      str += "\r\n";
+      str += this.content[i].data + "\r\n";
+    }
+
+    str += "--" + this.token + "--";
+    return str;
+  }
+}
+
+function multipartListener(test) {
+  this._buffer = "";
+  this.testNum = 0;
+  this.test = test;
+  this.numTests = this.test.content.length;
+}
+
+multipartListener.prototype.responseHandler = function(request, buffer) {
+    equal(buffer, this.test.content[this.testNum].data);
+    equal(request.QueryInterface(Ci.nsIChannel).contentType, this.test.content[this.testNum].type);
+    if (++this.testNum == this.numTests) {
+      run_next_test();
+    }
+}
+
+multipartListener.prototype.QueryInterface = function(iid) {
+  if (iid.equals(Components.interfaces.nsIStreamListener) ||
+      iid.equals(Components.interfaces.nsIRequestObserver) ||
+      iid.equals(Components.interfaces.nsISupports))
+    return this;
+  throw Components.results.NS_ERROR_NO_INTERFACE;
+}
+
+multipartListener.prototype.onStartRequest = function(request, context) {
+  this._buffer = "";
+  this.headerListener = new headerListener(this.test.content[this.testNum].headers);
+  let headerProvider = request.QueryInterface(Ci.nsIResponseHeadProvider);
+  if (headerProvider) {
+    headerProvider.visitResponseHeaders(this.headerListener);
+  }
+}
+
+multipartListener.prototype.onDataAvailable = function(request, context, stream, offset, count) {
+  try {
+    this._buffer = this._buffer.concat(read_stream(stream, count));
+  } catch (ex) {
+    do_throw("Error in onDataAvailable: " + ex);
+  }
+}
+
+multipartListener.prototype.onStopRequest = function(request, context, status) {
+  try {
+    equal(this.headerListener.index, this.test.content[this.testNum].headers.length);
+    this.responseHandler(request, this._buffer);
+  } catch (ex) {
+    do_throw("Error in closure function: " + ex);
+  }
+}
+
+function headerListener(headers) {
+  this.expectedHeaders = headers;
+  this.index = 0;
+}
+
+headerListener.prototype.QueryInterface = function(iid) {
+  if (iid.equals(Components.interfaces.nsIHttpHeaderVisitor) ||
+      iid.equals(Components.interfaces.nsISupports))
+    return this;
+  throw Components.results.NS_ERROR_NO_INTERFACE;
+}
+
+headerListener.prototype.visitHeader = function(header, value) {
+  ok(this.index <= this.expectedHeaders.length);
+  equal(header + ": " + value, this.expectedHeaders[this.index]);
+  this.index++;
+}
+
+function test_multipart() {
+  var streamConv = Cc["@mozilla.org/streamConverters;1"]
+                     .getService(Ci.nsIStreamConverterService);
+  var conv = streamConv.asyncConvertData("multipart/mixed",
+           "*/*",
+           new multipartListener(testData),
+           null);
+
+  var chan = make_channel(uri + "/multipart");
+  chan.asyncOpen(conv, null);
+}
+
+function test_multipart_with_boundary() {
+  var streamConv = Cc["@mozilla.org/streamConverters;1"]
+                     .getService(Ci.nsIStreamConverterService);
+  var conv = streamConv.asyncConvertData("multipart/mixed",
+           "*/*",
+           new multipartListener(testData),
+           null);
+
+  var chan = make_channel(uri + "/multipart2");
+  chan.asyncOpen(conv, null);
+}
+
+function test_multipart_chunked_headers() {
+  var streamConv = Cc["@mozilla.org/streamConverters;1"]
+                     .getService(Ci.nsIStreamConverterService);
+  var conv = streamConv.asyncConvertData("multipart/mixed",
+           "*/*",
+           new multipartListener(testData),
+           null);
+
+  var chan = make_channel(uri + "/multipart3");
+  chan.asyncOpen(conv, null);
+}
+
+function run_test()
+{
+  httpserver = new HttpServer();
+  httpserver.registerPathHandler("/multipart", contentHandler);
+  httpserver.registerPathHandler("/multipart2", contentHandler_with_boundary);
+  httpserver.registerPathHandler("/multipart3", contentHandler_chunked_headers);
+  httpserver.start(-1);
+
+  run_next_test();
+}
+
+add_test(test_multipart);
+add_test(test_multipart_with_boundary);
+add_test(test_multipart_chunked_headers);

netwerk/test/unit/xpcshell.ini

@@ -309,4 +309,5 @@ skip-if = os != "win"
 # The local cert service used by this test is not currently shipped on Android
 skip-if = os == "android"
 [test_1073747.js]
+[test_multipart_streamconv_application_package.js]
 [test_safeoutputstream_append.js]