Bug 1159489: WebRTC bitrate limits for video depend on input resolution and framerate r=pkerr

media/webrtc/signaling/src/media-conduit/VideoConduit.cpp

@@ -80,6 +80,7 @@ WebrtcVideoConduit::WebrtcVideoConduit():
   mReceivingWidth(640),
   mReceivingHeight(480),
   mSendingFramerate(DEFAULT_VIDEO_MAX_FRAMERATE),
+  mLastFramerateTenths(DEFAULT_VIDEO_MAX_FRAMERATE*10),
   mNumReceivingStreams(1),
   mVideoLatencyTestEnable(false),
   mVideoLatencyAvg(0),
@@ -221,6 +222,23 @@ bool WebrtcVideoConduit::GetVideoEncoderStats(double* framerateMean,
   mVideoCodecStat->GetEncoderStats(framerateMean, framerateStdDev,
                                    bitrateMean, bitrateStdDev,
                                    droppedFrames);
+
+  // See if we need to adjust bandwidth.
+  // Avoid changing bandwidth constantly; use hysteresis.
+
+  // Note: mLastFramerate is a relaxed Atomic because we're setting it here, and
+  // reading it on whatever thread calls DeliverFrame/SendVideoFrame.  Alternately
+  // we could use a lock.  Note that we don't change it often, and read it once per frame.
+  // We scale by *10 because mozilla::Atomic<> doesn't do 'double' or 'float'.
+  double framerate = mLastFramerateTenths/10.0; // fetch once
+  if (std::abs(*framerateMean - framerate)/framerate > 0.1 &&
+      *framerateMean >= 0.5) {
+    // unchanged resolution, but adjust bandwidth limits to match camera fps
+    CSFLogDebug(logTag, "Encoder frame rate changed from %f to %f",
+                (mLastFramerateTenths/10.0), *framerateMean);
+    mLastFramerateTenths = *framerateMean * 10;
+    SelectSendResolution(mSendingWidth, mSendingHeight, true);
+  }
   return true;
 }
 
@@ -911,11 +929,78 @@ WebrtcVideoConduit::ConfigureRecvMediaCodecs(
   return kMediaConduitNoError;
 }
 
+void
+WebrtcVideoConduit::SelectBandwidth(webrtc::VideoCodec& vie_codec,
+                                    unsigned short width,
+                                    unsigned short height)
+{
+  // max bandwidth should be proportional (not linearly!) to resolution, and
+  // proportional (perhaps linearly, or close) to current frame rate.
+  unsigned int fs, mb_width, mb_height;
+
+  mb_width = (width + 15) >> 4;
+  mb_height = (height + 15) >> 4;
+  fs = mb_width * mb_height;
+
+  // For now, try to set the max rates well above the knee in the curve.
+  // Chosen somewhat arbitrarily; it's hard to find good data oriented for
+  // realtime interactive/talking-head recording.  These rates assume
+  // 30fps.
+#define MB_OF(w,h) ((unsigned int)((((w)>>4))*((unsigned int)((h)>>4))))
+
+  // XXX replace this with parsing a config var with roughly a format
+  // of "max_fs,min_bw,max_bw," repeated to populate a table (which we
+  // should consider sorting because people won't assume they need to).
+  // Then iterate through the sorted array comparing fs.
+  if (fs > MB_OF(1920, 1200)) {
+    // >HD (3K, 4K, etc)
+    vie_codec.minBitrate = 1500;
+    vie_codec.maxBitrate = 10000;
+  } else if (fs > MB_OF(1280, 720)) {
+    // HD ~1080-1200
+    vie_codec.minBitrate = 1200;
+    vie_codec.maxBitrate = 5000;
+  } else if (fs > MB_OF(800, 480)) {
+    // HD ~720
+    vie_codec.minBitrate = 600;
+    vie_codec.maxBitrate = 2500;
+  } else if (fs > std::max(MB_OF(400, 240), MB_OF(352, 288))) {
+    // WVGA
+    // VGA
+    vie_codec.minBitrate = 200;
+    vie_codec.maxBitrate = 1300;
+  } else if (fs > MB_OF(176, 144)) {
+    // WQVGA
+    // CIF
+    vie_codec.minBitrate = 100;
+    vie_codec.maxBitrate = 500;
+  } else {
+    // QCIF and below
+    vie_codec.minBitrate = 40;
+    vie_codec.maxBitrate = 250;
+  }
+
+  // mLastFramerateTenths is an atomic, and scaled by *10
+  double framerate = std::min((mLastFramerateTenths/10.),60.0);
+  MOZ_ASSERT(framerate > 0);
+  // Now linear reduction/increase based on fps (max 60fps i.e. doubling)
+  if (framerate >= 10) {
+    vie_codec.minBitrate = vie_codec.minBitrate * (framerate/30);
+    vie_codec.maxBitrate = vie_codec.maxBitrate * (framerate/30);
+  } else {
+    // At low framerates, don't reduce bandwidth as much - cut slope to 1/2.
+    // Mostly this would be ultra-low-light situations/mobile or screensharing.
+    vie_codec.minBitrate = vie_codec.minBitrate * ((10-(framerate/2))/30);
+    vie_codec.maxBitrate = vie_codec.maxBitrate * ((10-(framerate/2))/30);
+  }
+}
+
 // XXX we need to figure out how to feed back changes in preferred capture
 // resolution to the getUserMedia source
 bool
 WebrtcVideoConduit::SelectSendResolution(unsigned short width,
-                                         unsigned short height)
+                                         unsigned short height,
+                                         bool force)
 {
   // XXX This will do bandwidth-resolution adaptation as well - bug 877954
 
@@ -985,8 +1070,9 @@ WebrtcVideoConduit::SelectSendResolution(unsigned short width,
   }
 
   // Adapt to getUserMedia resolution changes
-  // check if we need to reconfigure the sending resolution
-  if (mSendingWidth != width || mSendingHeight != height)
+  // check if we need to reconfigure the sending resolution.
+  // force tells us to do it regardless, such as when the FPS changes
+  if (mSendingWidth != width || mSendingHeight != height || force)
   {
     // This will avoid us continually retrying this operation if it fails.
     // If the resolution changes, we'll try again.  In the meantime, we'll
@@ -1003,10 +1089,11 @@ WebrtcVideoConduit::SelectSendResolution(unsigned short width,
       CSFLogError(logTag, "%s: GetSendCodec failed, err %d", __FUNCTION__, err);
       return false;
     }
-    if (vie_codec.width != width || vie_codec.height != height)
+    if (vie_codec.width != width || vie_codec.height != height || force)
     {
       vie_codec.width = width;
       vie_codec.height = height;
+      SelectBandwidth(vie_codec, width, height);
 
       if ((err = mPtrViECodec->SetSendCodec(mChannel, vie_codec)) != 0)
       {
@@ -1014,8 +1101,9 @@ WebrtcVideoConduit::SelectSendResolution(unsigned short width,
                     __FUNCTION__, width, height, err);
         return false;
       }
-      CSFLogDebug(logTag, "%s: Encoder resolution changed to %ux%u",
-                  __FUNCTION__, width, height);
+      CSFLogDebug(logTag, "%s: Encoder resolution changed to %ux%u, bitrate %u:%u",
+                  __FUNCTION__, width, height,
+                  vie_codec.minBitrate, vie_codec.maxBitrate);
     } // else no change; mSendingWidth likely was 0
   }
   return true;
@@ -1126,14 +1214,14 @@ WebrtcVideoConduit::SendVideoFrame(unsigned char* video_frame,
   // RawVideoType == VideoType
   webrtc::RawVideoType type = static_cast<webrtc::RawVideoType>((int)video_type);
 
-  //Transmission should be enabled before we insert any frames.
+  // Transmission should be enabled before we insert any frames.
   if(!mEngineTransmitting)
   {
     CSFLogError(logTag, "%s Engine not transmitting ", __FUNCTION__);
     return kMediaConduitSessionNotInited;
   }
 
-  if (!SelectSendResolution(width, height))
+  if (!SelectSendResolution(width, height, false))
   {
     return kMediaConduitCaptureError;
   }
@@ -1141,7 +1229,7 @@ WebrtcVideoConduit::SendVideoFrame(unsigned char* video_frame,
   {
     return kMediaConduitCaptureError;
   }
-  //insert the frame to video engine in I420 format only
+  // insert the frame to video engine in I420 format only
   MOZ_ASSERT(mPtrExtCapture);
   if(mPtrExtCapture->IncomingFrame(video_frame,
                                    video_frame_length,

media/webrtc/signaling/src/media-conduit/VideoConduit.h

@@ -7,6 +7,7 @@
 
 #include "nsAutoPtr.h"
 #include "mozilla/Attributes.h"
+#include "mozilla/Atomics.h"
 
 #include "MediaConduitInterface.h"
 #include "MediaEngineWrapper.h"
@@ -134,13 +135,18 @@ public:
 
   virtual MediaConduitErrorCode SetReceiverTransport(mozilla::RefPtr<TransportInterface> aTransport) override;
 
+  void SelectBandwidth(webrtc::VideoCodec& vie_codec,
+                       unsigned short width,
+                       unsigned short height);
   /**
    * Function to select and change the encoding resolution based on incoming frame size
    * and current available bandwidth.
    * @param width, height: dimensions of the frame
+   * @param force: force setting the codec config if framerate may require a bandwidth change
    */
   bool SelectSendResolution(unsigned short width,
-                            unsigned short height);
+                            unsigned short height,
+                            bool force);
 
   /**
    * Function to select and change the encoding frame rate based on incoming frame rate
@@ -337,6 +343,8 @@ private:
   unsigned short mReceivingWidth;
   unsigned short mReceivingHeight;
   unsigned int   mSendingFramerate;
+  // scaled by *10 because Atomic<double/float> isn't supported
+  mozilla::Atomic<int32_t, mozilla::Relaxed> mLastFramerateTenths;
   unsigned short mNumReceivingStreams;
   bool mVideoLatencyTestEnable;
   uint64_t mVideoLatencyAvg;