Bug 1410096 - Move labeled queues to SchedulerGroup. r=smaug

This improves the performance of GetEvent and PutEvent in
LabeledEventQueue by removing a hash table mapping groups to queues.

xpcom/threads/AbstractEventQueue.h

@@ -19,7 +19,9 @@ enum class EventPriority
   High,
   Input,
   Normal,
-  Idle
+  Idle,
+
+  Count
 };
 
 // AbstractEventQueue is an abstract base class for all our unsynchronized event

xpcom/threads/EventQueue.cpp

@@ -9,7 +9,7 @@
 
 using namespace mozilla;
 
-EventQueue::EventQueue()
+EventQueue::EventQueue(EventPriority aPriority)
 {
 }
 

xpcom/threads/EventQueue.h

@@ -18,7 +18,8 @@ namespace mozilla {
 class EventQueue final : public AbstractEventQueue
 {
 public:
-  EventQueue();
+  EventQueue() {}
+  explicit EventQueue(EventPriority aPriority);
 
   void PutEvent(already_AddRefed<nsIRunnable>&& aEvent,
                 EventPriority aPriority,

xpcom/threads/LabeledEventQueue.cpp

@@ -13,11 +13,14 @@
 
 using namespace mozilla::dom;
 
+using EpochQueueEntry = SchedulerGroup::EpochQueueEntry;
+
 LinkedList<SchedulerGroup>* LabeledEventQueue::sSchedulerGroups;
 size_t LabeledEventQueue::sLabeledEventQueueCount;
 SchedulerGroup* LabeledEventQueue::sCurrentSchedulerGroup;
 
-LabeledEventQueue::LabeledEventQueue()
+LabeledEventQueue::LabeledEventQueue(EventPriority aPriority)
+  : mPriority(aPriority)
 {
   // LabeledEventQueue should only be used by one consumer since it uses a
   // single static sSchedulerGroups field. It's hard to assert this, though, so
@@ -77,6 +80,8 @@ LabeledEventQueue::PutEvent(already_AddRefed<nsIRunnable>&& aEvent,
                             EventPriority aPriority,
                             const MutexAutoLock& aProofOfLock)
 {
+  MOZ_ASSERT(aPriority == mPriority);
+
   nsCOMPtr<nsIRunnable> event(aEvent);
 
   MOZ_ASSERT(event.get());
@@ -100,8 +105,8 @@ LabeledEventQueue::PutEvent(already_AddRefed<nsIRunnable>&& aEvent,
   mNumEvents++;
   epoch->mNumEvents++;
 
-  RunnableEpochQueue* queue = isLabeled ? mLabeled.LookupOrAdd(group) : &mUnlabeled;
-  queue->Push(QueueEntry(event.forget(), epoch->mEpochNumber));
+  RunnableEpochQueue& queue = isLabeled ? group->GetQueue(aPriority) : mUnlabeled;
+  queue.Push(EpochQueueEntry(event.forget(), epoch->mEpochNumber));
 
   if (group && group->EnqueueEvent() == SchedulerGroup::NewlyQueued) {
     // This group didn't have any events before. Add it to the
@@ -150,13 +155,13 @@ LabeledEventQueue::GetEvent(EventPriority* aPriority,
 
   Epoch epoch = mEpochs.FirstElement();
   if (!epoch.IsLabeled()) {
-    QueueEntry& first = mUnlabeled.FirstElement();
+    EpochQueueEntry& first = mUnlabeled.FirstElement();
     if (!IsReadyToRun(first.mRunnable, nullptr)) {
       return nullptr;
     }
 
     PopEpoch();
-    QueueEntry entry = mUnlabeled.Pop();
+    EpochQueueEntry entry = mUnlabeled.Pop();
     MOZ_ASSERT(entry.mEpochNumber == epoch.mEpochNumber);
     MOZ_ASSERT(entry.mRunnable.get());
     return entry.mRunnable.forget();
@@ -197,17 +202,15 @@ LabeledEventQueue::GetEvent(EventPriority* aPriority,
   do {
     mAvoidActiveTabCount--;
 
-    auto queueEntry = mLabeled.Lookup(group);
-    if (!queueEntry) {
+    RunnableEpochQueue& queue = group->GetQueue(mPriority);
+
+    if (queue.IsEmpty()) {
       // This can happen if |group| is in a different LabeledEventQueue than |this|.
       group = NextSchedulerGroup(group);
       continue;
     }
 
-    RunnableEpochQueue* queue = queueEntry.Data();
-    MOZ_ASSERT(!queue->IsEmpty());
-
-    QueueEntry& first = queue->FirstElement();
+    EpochQueueEntry& first = queue.FirstElement();
     if (first.mEpochNumber == epoch.mEpochNumber &&
         IsReadyToRun(first.mRunnable, group)) {
       sCurrentSchedulerGroup = NextSchedulerGroup(group);
@@ -226,10 +229,7 @@ LabeledEventQueue::GetEvent(EventPriority* aPriority,
         }
         group->removeFrom(*sSchedulerGroups);
       }
-      QueueEntry entry = queue->Pop();
-      if (queue->IsEmpty()) {
-        queueEntry.Remove();
-      }
+      EpochQueueEntry entry = queue.Pop();
       return entry.mRunnable.forget();
     }
 
@@ -261,24 +261,26 @@ LabeledEventQueue::HasReadyEvent(const MutexAutoLock& aProofOfLock)
   Epoch& frontEpoch = mEpochs.FirstElement();
 
   if (!frontEpoch.IsLabeled()) {
-    QueueEntry& entry = mUnlabeled.FirstElement();
+    EpochQueueEntry& entry = mUnlabeled.FirstElement();
     return IsReadyToRun(entry.mRunnable, nullptr);
   }
 
-  // Go through the labeled queues and look for one whose head is from the
-  // current epoch and is allowed to run.
+  // Go through the scheduler groups and look for one that has events
+  // for the priority of this labeled queue that is in the current
+  // epoch and is allowed to run.
   uintptr_t currentEpoch = frontEpoch.mEpochNumber;
-  for (auto iter = mLabeled.Iter(); !iter.Done(); iter.Next()) {
-    SchedulerGroup* key = iter.Key();
-    RunnableEpochQueue* queue = iter.Data();
-    MOZ_ASSERT(!queue->IsEmpty());
+  for (SchedulerGroup* group : *sSchedulerGroups) {
+    RunnableEpochQueue& queue = group->GetQueue(mPriority);
+    if (queue.IsEmpty()) {
+      continue;
+    }
 
-    QueueEntry& entry = queue->FirstElement();
+    EpochQueueEntry& entry = queue.FirstElement();
     if (entry.mEpochNumber != currentEpoch) {
       continue;
     }
 
-    if (IsReadyToRun(entry.mRunnable, key)) {
+    if (IsReadyToRun(entry.mRunnable, group)) {
       return true;
     }
   }

xpcom/threads/LabeledEventQueue.h

@@ -10,6 +10,7 @@
 #include <stdint.h>
 #include "mozilla/AbstractEventQueue.h"
 #include "mozilla/LinkedList.h"
+#include "mozilla/SchedulerGroup.h"
 #include "mozilla/Queue.h"
 #include "nsClassHashtable.h"
 #include "nsHashKeys.h"
@@ -30,7 +31,7 @@ class SchedulerGroup;
 class LabeledEventQueue final : public AbstractEventQueue
 {
 public:
-  LabeledEventQueue();
+  explicit LabeledEventQueue(EventPriority aPriority);
   ~LabeledEventQueue();
 
   void PutEvent(already_AddRefed<nsIRunnable>&& aEvent,
@@ -83,17 +84,6 @@ private:
   // decrement the number of events in the current epoch. If this number reaches
   // zero, we pop from the epoch queue.
 
-  struct QueueEntry
-  {
-    nsCOMPtr<nsIRunnable> mRunnable;
-    uintptr_t mEpochNumber;
-
-    QueueEntry(already_AddRefed<nsIRunnable> aRunnable, uintptr_t aEpoch)
-      : mRunnable(aRunnable)
-      , mEpochNumber(aEpoch)
-    {}
-  };
-
   struct Epoch
   {
     static Epoch First(bool aIsLabeled)
@@ -131,8 +121,7 @@ private:
   void PopEpoch();
   static SchedulerGroup* NextSchedulerGroup(SchedulerGroup* aGroup);
 
-  using RunnableEpochQueue = Queue<QueueEntry, 32>;
-  using LabeledMap = nsClassHashtable<nsRefPtrHashKey<SchedulerGroup>, RunnableEpochQueue>;
+  using RunnableEpochQueue = SchedulerGroup::RunnableEpochQueue;
   using EpochQueue = Queue<Epoch, 8>;
 
   // List of SchedulerGroups that might have events. This is static, so it
@@ -144,7 +133,6 @@ private:
   static size_t sLabeledEventQueueCount;
   static SchedulerGroup* sCurrentSchedulerGroup;
 
-  LabeledMap mLabeled;
   RunnableEpochQueue mUnlabeled;
   EpochQueue mEpochs;
   size_t mNumEvents = 0;
@@ -154,6 +142,7 @@ private:
   // foreground and background SchedulerGroups. For details, see its usage in
   // LabeledEventQueue.cpp.
   int64_t mAvoidActiveTabCount = 0;
+  EventPriority mPriority;
 };
 
 } // namespace mozilla

xpcom/threads/MainThreadQueue.h

@@ -23,10 +23,10 @@ CreateMainThread(nsIIdlePeriod* aIdlePeriod, SynchronizedQueueT** aSynchronizedQ
   using MainThreadQueueT = PrioritizedEventQueue<InnerQueueT>;
 
   auto queue = MakeUnique<MainThreadQueueT>(
-    MakeUnique<InnerQueueT>(),
-    MakeUnique<InnerQueueT>(),
-    MakeUnique<InnerQueueT>(),
-    MakeUnique<InnerQueueT>(),
+    MakeUnique<InnerQueueT>(EventPriority::High),
+    MakeUnique<InnerQueueT>(EventPriority::Input),
+    MakeUnique<InnerQueueT>(EventPriority::Normal),
+    MakeUnique<InnerQueueT>(EventPriority::Idle),
     do_AddRef(aIdlePeriod));
 
   MainThreadQueueT* prioritized = queue.get();

xpcom/threads/PrioritizedEventQueue.cpp

@@ -65,6 +65,9 @@ PrioritizedEventQueue<InnerQueueT>::PutEvent(already_AddRefed<nsIRunnable>&& aEv
   case EventPriority::Idle:
     mIdleQueue->PutEvent(event.forget(), priority, aProofOfLock);
     break;
+  case EventPriority::Count:
+    MOZ_CRASH("EventPriority::Count isn't a valid priority");
+    break;
   }
 }
 

xpcom/threads/SchedulerGroup.h

@@ -7,8 +7,10 @@
 #ifndef mozilla_SchedulerGroup_h
 #define mozilla_SchedulerGroup_h
 
+#include "mozilla/AbstractEventQueue.h"
 #include "mozilla/AlreadyAddRefed.h"
 #include "mozilla/LinkedList.h"
+#include "mozilla/Queue.h"
 #include "mozilla/TaskCategory.h"
 #include "mozilla/ThreadLocal.h"
 #include "mozilla/TimeStamp.h"
@@ -168,6 +170,25 @@ public:
   };
   static void SetValidatingAccess(ValidationType aType);
 
+  struct EpochQueueEntry
+  {
+    nsCOMPtr<nsIRunnable> mRunnable;
+    uintptr_t mEpochNumber;
+
+    EpochQueueEntry(already_AddRefed<nsIRunnable> aRunnable, uintptr_t aEpoch)
+      : mRunnable(aRunnable)
+      , mEpochNumber(aEpoch)
+    {
+    }
+  };
+
+  using RunnableEpochQueue = Queue<EpochQueueEntry, 32>;
+
+  RunnableEpochQueue& GetQueue(mozilla::EventPriority aPriority)
+  {
+    return mEventQueues[size_t(aPriority)];
+  }
+
 protected:
   static nsresult InternalUnlabeledDispatch(TaskCategory aCategory,
                                             already_AddRefed<Runnable>&& aRunnable);
@@ -203,6 +224,7 @@ protected:
 
   nsCOMPtr<nsISerialEventTarget> mEventTargets[size_t(TaskCategory::Count)];
   RefPtr<AbstractThread> mAbstractThreads[size_t(TaskCategory::Count)];
+  RunnableEpochQueue mEventQueues[size_t(mozilla::EventPriority::Count)];
 };
 
 NS_DEFINE_STATIC_IID_ACCESSOR(SchedulerGroup::Runnable, NS_SCHEDULERGROUPRUNNABLE_IID);