Optimize SelectionDAG's AssignTopologicalOrder even further.

Completely eliminate the TopOrder std::vector. Instead, sort
the AllNodes list in place. This also eliminates the need to
call AllNodes.size(), a linear-time operation, before
performing the sort.

Also, eliminate the Sources temporary std::vector, since it
essentially duplicates the sorted result as it is being
built.

This also changes the direction of the topological sort
from bottom-up to top-down. The AllNodes list starts out in
roughly top-down order, so this reduces the amount of
reordering needed. Top-down is also more convenient for
Legalize, and ISel needed only minor adjustments.

llvm-svn: 56867

include/llvm/CodeGen/DAGISelHeader.h

@@ -18,7 +18,7 @@
 #define LLVM_CODEGEN_DAGISEL_HEADER_H
 
 /// ISelQueue - Instruction selector priority queue sorted 
-/// in the order of increasing NodeId() values.
+/// in the order of decreasing NodeId() values.
 std::vector<SDNode*> ISelQueue;
 
 /// Keep track of nodes which have already been added to queue.
@@ -43,10 +43,10 @@ static bool IsChainCompatible(SDNode *Chain, SDNode *Op) {
 }
 
 /// isel_sort - Sorting functions for the selection queue in the
-/// increasing NodeId order.
+/// decreasing NodeId order.
 struct isel_sort : public std::binary_function<SDNode*, SDNode*, bool> {
   bool operator()(const SDNode* left, const SDNode* right) const {
-    return (left->getNodeId() > right->getNodeId());
+    return left->getNodeId() < right->getNodeId();
   }
 };
 
@@ -108,7 +108,7 @@ class VISIBILITY_HIDDEN ISelQueueUpdater :
   };
 
 /// UpdateQueue - update the instruction selction queue to maintain 
-/// the increasing NodeId() ordering property.
+/// the decreasing NodeId() ordering property.
 inline void UpdateQueue(const ISelQueueUpdater &ISQU) {
   if (ISQU.hadDelete())
     std::make_heap(ISelQueue.begin(), ISelQueue.end(),isel_sort());

include/llvm/CodeGen/SelectionDAG.h

@@ -663,10 +663,10 @@ public:
                                   unsigned Num,
                                   DAGUpdateListener *UpdateListener = 0);
 
-  /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
-  /// based on their topological order. It returns the maximum id and a vector
-  /// of the SDNodes* in assigned order by reference.
-  unsigned AssignTopologicalOrder(std::vector<SDNode*> &TopOrder);
+  /// AssignTopologicalOrder - Topological-sort the AllNodes list and a
+  /// assign a unique node id for each node in the DAG based on their
+  /// topological order. Returns the number of nodes.
+  unsigned AssignTopologicalOrder();
 
   /// isCommutativeBinOp - Returns true if the opcode is a commutative binary
   /// operation.

include/llvm/CodeGen/SelectionDAGISel.h

@@ -48,7 +48,6 @@ public:
   AliasAnalysis *AA;
   GCFunctionInfo *GFI;
   bool Fast;
-  std::vector<SDNode*> TopOrder;
   static char ID;
 
   explicit SelectionDAGISel(TargetLowering &tli, bool fast = false);
@@ -67,7 +66,7 @@ public:
   virtual void InstructionSelectPostProcessing() {}
   
   void SelectRootInit() {
-    DAGSize = CurDAG->AssignTopologicalOrder(TopOrder);
+    DAGSize = CurDAG->AssignTopologicalOrder();
   }
 
   /// SelectInlineAsmMemoryOperand - Select the specified address as a target

lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

@@ -280,11 +280,10 @@ void SelectionDAGLegalize::LegalizeDAG() {
   // practice however, this causes us to run out of stack space on large basic
   // blocks.  To avoid this problem, compute an ordering of the nodes where each
   // node is only legalized after all of its operands are legalized.
-  std::vector<SDNode *> TopOrder;
-  unsigned N = DAG.AssignTopologicalOrder(TopOrder);
-  for (unsigned i = N; i != 0; --i)
-    HandleOp(SDValue(TopOrder[i-1], 0));
-  TopOrder.clear();
+  DAG.AssignTopologicalOrder();
+  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
+       E = prior(DAG.allnodes_end()); I != next(E); ++I)
+    HandleOp(SDValue(I, 0));
 
   // Finally, it's possible the root changed.  Get the new root.
   SDValue OldRoot = DAG.getRoot();

lib/CodeGen/SelectionDAG/SelectionDAG.cpp

@@ -587,7 +587,7 @@ void SelectionDAG::DeleteNode(SDNode *N) {
 
 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
 
-  // Drop all of the operands and decrement used nodes use counts.
+  // Drop all of the operands and decrement used node's use counts.
   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I)
     I->getVal()->removeUser(std::distance(N->op_begin(), I), N);
   if (N->OperandsNeedDelete)
@@ -4569,38 +4569,74 @@ void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
 /// based on their topological order. It returns the maximum id and a vector
 /// of the SDNodes* in assigned order by reference.
-unsigned SelectionDAG::AssignTopologicalOrder(std::vector<SDNode*> &TopOrder) {
-  unsigned DAGSize = AllNodes.size();
-  std::vector<SDNode*> Sources;
+unsigned SelectionDAG::AssignTopologicalOrder() {
 
-  for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I){
-    SDNode *N = I;
-    unsigned Degree = N->use_size();
-    // Temporarily use the Node Id as scratch space for the degree count.
-    N->setNodeId(Degree);
-    if (Degree == 0)
-      Sources.push_back(N);
-  }
+  unsigned DAGSize = 0;
 
-  TopOrder.clear();
-  TopOrder.reserve(DAGSize);
-  int Id = 0;
-  while (!Sources.empty()) {
-    SDNode *N = Sources.back();
-    Sources.pop_back();
-    TopOrder.push_back(N);
-    N->setNodeId(Id++);
-    for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
-      SDNode *P = I->getVal();
-      unsigned Degree = P->getNodeId();
-      --Degree;
-      P->setNodeId(Degree);
-      if (Degree == 0)
-        Sources.push_back(P);
+  // SortedPos tracks the progress of the algorithm. Nodes before it are
+  // sorted, nodes after it are unsorted. When the algorithm completes
+  // it is at the end of the list.
+  allnodes_iterator SortedPos = allnodes_begin();
+
+  // Visit all the nodes. Add nodes with no operands to the TopOrder result
+  // array immediately. Annotate nodes that do have operands with their
+  // operand count. Before we do this, the Node Id fields of the nodes
+  // may contain arbitrary values. After, the Node Id fields for nodes
+  // before SortedPos will contain the topological sort index, and the
+  // Node Id fields for nodes At SortedPos and after will contain the
+  // count of outstanding operands.
+  for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
+    SDNode *N = I++;
+    unsigned Degree = N->getNumOperands();
+    if (Degree == 0) {
+      // A node with no uses, add it to the result array immediately.
+      N->setNodeId(DAGSize++);
+      allnodes_iterator Q = N;
+      if (Q != SortedPos)
+        SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
+      ++SortedPos;
+    } else {
+      // Temporarily use the Node Id as scratch space for the degree count.
+      N->setNodeId(Degree);
     }
   }
 
-  return Id;
+  // Visit all the nodes. As we iterate, moves nodes into sorted order,
+  // such that by the time the end is reached all nodes will be sorted.
+  for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
+    SDNode *N = I;
+    for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
+         UI != UE; ++UI) {
+      SDNode *P = *UI;
+      unsigned Degree = P->getNodeId();
+      --Degree;
+      if (Degree == 0) {
+        // All of P's operands are sorted, so P may sorted now.
+        P->setNodeId(DAGSize++);
+        if (P != SortedPos)
+          SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
+        ++SortedPos;
+      } else {
+        // Update P's outstanding operand count.
+        P->setNodeId(Degree);
+      }
+    }
+  }
+
+  assert(SortedPos == AllNodes.end() &&
+         "Topological sort incomplete!");
+  assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
+         "First node in topological sort is not the entry token!");
+  assert(AllNodes.front().getNodeId() == 0 &&
+         "First node in topological sort has non-zero id!");
+  assert(AllNodes.front().getNumOperands() == 0 &&
+         "First node in topological sort has operands!");
+  assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
+         "Last node in topologic sort has unexpected id!");
+  assert(AllNodes.back().use_empty() &&
+         "Last node in topologic sort has users!");
+  assert(DAGSize == allnodes_size() && "TopOrder result count mismatch!");
+  return DAGSize;
 }
 
 

lib/Target/X86/X86ISelDAGToDAG.cpp

@@ -269,7 +269,7 @@ static void findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse,
                           SDNode *Root, bool &found,
                           SmallPtrSet<SDNode*, 16> &Visited) {
   if (found ||
-      Use->getNodeId() > Def->getNodeId() ||
+      Use->getNodeId() < Def->getNodeId() ||
       !Visited.insert(Use))
     return;