Bug 1675207 - Fail the websocket connection when there is no enough memory r=necko-reviewers,valentin

Differential Revision: https://phabricator.services.mozilla.com/D99145

dom/websocket/WebSocket.cpp

@@ -860,6 +860,23 @@ WebSocketImpl::OnServerClose(nsISupports* aContext, uint16_t aCode,
   return NS_OK;
 }
 
+NS_IMETHODIMP
+WebSocketImpl::OnError() {
+  if (!IsTargetThread()) {
+    return Dispatch(
+        NS_NewRunnableFunction("dom::FailConnectionRunnable",
+                               [self = RefPtr{this}]() {
+                                 self->FailConnection(
+                                     nsIWebSocketChannel::CLOSE_ABNORMAL);
+                               }),
+        NS_DISPATCH_NORMAL);
+  }
+
+  AssertIsOnTargetThread();
+  FailConnection(nsIWebSocketChannel::CLOSE_ABNORMAL);
+  return NS_OK;
+}
+
 //-----------------------------------------------------------------------------
 // WebSocketImpl::nsIInterfaceRequestor
 //-----------------------------------------------------------------------------

netwerk/protocol/websocket/WebSocketChannelChild.cpp

@@ -294,22 +294,44 @@ class MessageEvent : public WebSocketEvent {
   bool mBinary;
 };
 
-void WebSocketChannelChild::RecvOnMessageAvailableInternal(
+bool WebSocketChannelChild::RecvOnMessageAvailableInternal(
     const nsDependentCSubstring& aMsg, bool aMoreData, bool aBinary) {
   if (aMoreData) {
-    mReceivedMsgBuffer.Append(aMsg);
-    return;
+    return mReceivedMsgBuffer.Append(aMsg, fallible);
+  }
+
+  if (!mReceivedMsgBuffer.Append(aMsg, fallible)) {
+    return false;
   }
 
-  mReceivedMsgBuffer.Append(aMsg);
   mEventQ->RunOrEnqueue(new EventTargetDispatcher(
       this, new MessageEvent(mReceivedMsgBuffer, aBinary), mTargetThread));
   mReceivedMsgBuffer.Truncate();
+  return true;
+}
+
+class OnErrorEvent : public WebSocketEvent {
+ public:
+  OnErrorEvent() = default;
+
+  void Run(WebSocketChannelChild* aChild) override { aChild->OnError(); }
+};
+
+void WebSocketChannelChild::OnError() {
+  LOG(("WebSocketChannelChild::OnError() %p", this));
+  if (mListenerMT) {
+    AutoEventEnqueuer ensureSerialDispatch(mEventQ);
+    Unused << mListenerMT->mListener->OnError();
+  }
 }
 
 mozilla::ipc::IPCResult WebSocketChannelChild::RecvOnMessageAvailable(
     const nsDependentCSubstring& aMsg, const bool& aMoreData) {
-  RecvOnMessageAvailableInternal(aMsg, aMoreData, false);
+  if (!RecvOnMessageAvailableInternal(aMsg, aMoreData, false)) {
+    LOG(("WebSocketChannelChild %p append message failed", this));
+    mEventQ->RunOrEnqueue(
+        new EventTargetDispatcher(this, new OnErrorEvent(), mTargetThread));
+  }
   return IPC_OK();
 }
 
@@ -331,7 +353,11 @@ void WebSocketChannelChild::OnMessageAvailable(const nsCString& aMsg) {
 
 mozilla::ipc::IPCResult WebSocketChannelChild::RecvOnBinaryMessageAvailable(
     const nsDependentCSubstring& aMsg, const bool& aMoreData) {
-  RecvOnMessageAvailableInternal(aMsg, aMoreData, true);
+  if (!RecvOnMessageAvailableInternal(aMsg, aMoreData, true)) {
+    LOG(("WebSocketChannelChild %p append message failed", this));
+    mEventQ->RunOrEnqueue(
+        new EventTargetDispatcher(this, new OnErrorEvent(), mTargetThread));
+  }
   return IPC_OK();
 }
 

netwerk/protocol/websocket/WebSocketChannelChild.h

@@ -82,9 +82,11 @@ class WebSocketChannelChild final : public BaseWebSocketChannel,
   // This function tries to get a labeled event target for |mNeckoTarget|.
   void SetupNeckoTarget();
 
-  void RecvOnMessageAvailableInternal(const nsDependentCSubstring& aMsg,
+  bool RecvOnMessageAvailableInternal(const nsDependentCSubstring& aMsg,
                                       bool aMoreData, bool aBinary);
 
+  void OnError();
+
   RefPtr<ChannelEventQueue> mEventQ;
   nsString mEffectiveURL;
   nsCString mReceivedMsgBuffer;
@@ -100,6 +102,7 @@ class WebSocketChannelChild final : public BaseWebSocketChannel,
   friend class AcknowledgeEvent;
   friend class ServerCloseEvent;
   friend class AsyncOpenFailedEvent;
+  friend class OnErrorEvent;
 };
 
 }  // namespace net

netwerk/protocol/websocket/WebSocketChannelParent.cpp

@@ -307,6 +307,9 @@ WebSocketChannelParent::OnServerClose(nsISupports* aContext, uint16_t code,
   return NS_OK;
 }
 
+NS_IMETHODIMP
+WebSocketChannelParent::OnError() { return NS_OK; }
+
 void WebSocketChannelParent::ActorDestroy(ActorDestroyReason why) {
   LOG(("WebSocketChannelParent::ActorDestroy() %p\n", this));
 

netwerk/protocol/websocket/nsIWebSocketListener.idl

@@ -85,6 +85,12 @@ interface nsIWebSocketListener : nsISupports
                                   in unsigned short aCode,
                                   in AUTF8String aReason);
 
+    /**
+     * Called to inform an error is happened. The connection will be closed
+     * when this is called.
+     */
+    [must_use] void OnError();
+
 };
 
 

netwerk/test/fuzz/TestWebsocketFuzzing.cpp

@@ -96,6 +96,12 @@ FuzzingWebSocketListener::OnBinaryMessageAvailable(nsISupports* aContext,
   return NS_OK;
 }
 
+NS_IMETHODIMP
+FuzzingWebSocketListener::OnError() {
+  FUZZING_LOG(("FuzzingWebSocketListener::OnError"));
+  return NS_OK;
+}
+
 static int FuzzingInitNetworkWebsocket(int* argc, char*** argv) {
   Preferences::SetBool("network.dns.native-is-localhost", true);
   Preferences::SetBool("fuzzing.necko.enabled", true);