File indexing completed on 2026-05-10 08:43:34
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015 #ifndef LLVM_CODEGEN_SCHEDULEDAG_H
0016 #define LLVM_CODEGEN_SCHEDULEDAG_H
0017
0018 #include "llvm/ADT/BitVector.h"
0019 #include "llvm/ADT/PointerIntPair.h"
0020 #include "llvm/ADT/SmallVector.h"
0021 #include "llvm/ADT/iterator.h"
0022 #include "llvm/CodeGen/MachineInstr.h"
0023 #include "llvm/CodeGen/TargetLowering.h"
0024 #include "llvm/Support/ErrorHandling.h"
0025 #include <cassert>
0026 #include <cstddef>
0027 #include <iterator>
0028 #include <string>
0029 #include <vector>
0030
0031 namespace llvm {
0032
0033 template <class GraphType> struct GraphTraits;
0034 template<class Graph> class GraphWriter;
0035 class TargetMachine;
0036 class MachineFunction;
0037 class MachineRegisterInfo;
0038 class MCInstrDesc;
0039 struct MCSchedClassDesc;
0040 class SDNode;
0041 class SUnit;
0042 class ScheduleDAG;
0043 class TargetInstrInfo;
0044 class TargetRegisterClass;
0045 class TargetRegisterInfo;
0046
0047
0048
0049 class SDep {
0050 public:
0051
0052 enum Kind {
0053 Data,
0054 Anti,
0055 Output,
0056 Order
0057 };
0058
0059
0060
0061
0062
0063
0064
0065
0066
0067
0068 enum OrderKind {
0069 Barrier,
0070 MayAliasMem,
0071 MustAliasMem,
0072 Artificial,
0073 Weak,
0074 Cluster
0075 };
0076
0077 private:
0078
0079
0080 PointerIntPair<SUnit *, 2, Kind> Dep;
0081
0082
0083 union {
0084
0085
0086
0087 unsigned Reg;
0088
0089
0090 unsigned OrdKind;
0091 } Contents;
0092
0093
0094
0095
0096 unsigned Latency = 0u;
0097
0098 public:
0099
0100
0101 SDep() : Dep(nullptr, Data) {}
0102
0103
0104 SDep(SUnit *S, Kind kind, unsigned Reg)
0105 : Dep(S, kind), Contents() {
0106 switch (kind) {
0107 default:
0108 llvm_unreachable("Reg given for non-register dependence!");
0109 case Anti:
0110 case Output:
0111 assert(Reg != 0 &&
0112 "SDep::Anti and SDep::Output must use a non-zero Reg!");
0113 Contents.Reg = Reg;
0114 Latency = 0;
0115 break;
0116 case Data:
0117 Contents.Reg = Reg;
0118 Latency = 1;
0119 break;
0120 }
0121 }
0122
0123 SDep(SUnit *S, OrderKind kind)
0124 : Dep(S, Order), Contents(), Latency(0) {
0125 Contents.OrdKind = kind;
0126 }
0127
0128
0129 bool overlaps(const SDep &Other) const;
0130
0131 bool operator==(const SDep &Other) const {
0132 return overlaps(Other) && Latency == Other.Latency;
0133 }
0134
0135 bool operator!=(const SDep &Other) const {
0136 return !operator==(Other);
0137 }
0138
0139
0140
0141
0142 unsigned getLatency() const {
0143 return Latency;
0144 }
0145
0146
0147 void setLatency(unsigned Lat) {
0148 Latency = Lat;
0149 }
0150
0151
0152 SUnit *getSUnit() const;
0153
0154
0155 void setSUnit(SUnit *SU);
0156
0157
0158 Kind getKind() const;
0159
0160
0161 bool isCtrl() const {
0162 return getKind() != Data;
0163 }
0164
0165
0166
0167
0168 bool isNormalMemory() const {
0169 return getKind() == Order && (Contents.OrdKind == MayAliasMem
0170 || Contents.OrdKind == MustAliasMem);
0171 }
0172
0173
0174 bool isBarrier() const {
0175 return getKind() == Order && Contents.OrdKind == Barrier;
0176 }
0177
0178
0179 bool isNormalMemoryOrBarrier() const {
0180 return (isNormalMemory() || isBarrier());
0181 }
0182
0183
0184
0185
0186 bool isMustAlias() const {
0187 return getKind() == Order && Contents.OrdKind == MustAliasMem;
0188 }
0189
0190
0191
0192
0193
0194 bool isWeak() const {
0195 return getKind() == Order && Contents.OrdKind >= Weak;
0196 }
0197
0198
0199
0200 bool isArtificial() const {
0201 return getKind() == Order && Contents.OrdKind == Artificial;
0202 }
0203
0204
0205
0206 bool isCluster() const {
0207 return getKind() == Order && Contents.OrdKind == Cluster;
0208 }
0209
0210
0211 bool isAssignedRegDep() const {
0212 return getKind() == Data && Contents.Reg != 0;
0213 }
0214
0215
0216
0217
0218 unsigned getReg() const {
0219 assert((getKind() == Data || getKind() == Anti || getKind() == Output) &&
0220 "getReg called on non-register dependence edge!");
0221 return Contents.Reg;
0222 }
0223
0224
0225
0226
0227
0228 void setReg(unsigned Reg) {
0229 assert((getKind() == Data || getKind() == Anti || getKind() == Output) &&
0230 "setReg called on non-register dependence edge!");
0231 assert((getKind() != Anti || Reg != 0) &&
0232 "SDep::Anti edge cannot use the zero register!");
0233 assert((getKind() != Output || Reg != 0) &&
0234 "SDep::Output edge cannot use the zero register!");
0235 Contents.Reg = Reg;
0236 }
0237
0238 void dump(const TargetRegisterInfo *TRI = nullptr) const;
0239 };
0240
0241
0242 class SUnit {
0243 private:
0244 enum : unsigned { BoundaryID = ~0u };
0245
0246 union {
0247 SDNode *Node;
0248 MachineInstr *Instr;
0249 };
0250
0251 public:
0252 SUnit *OrigNode = nullptr;
0253
0254
0255 const MCSchedClassDesc *SchedClass =
0256 nullptr;
0257
0258 const TargetRegisterClass *CopyDstRC =
0259 nullptr;
0260 const TargetRegisterClass *CopySrcRC = nullptr;
0261
0262 SmallVector<SDep, 4> Preds;
0263 SmallVector<SDep, 4> Succs;
0264
0265 typedef SmallVectorImpl<SDep>::iterator pred_iterator;
0266 typedef SmallVectorImpl<SDep>::iterator succ_iterator;
0267 typedef SmallVectorImpl<SDep>::const_iterator const_pred_iterator;
0268 typedef SmallVectorImpl<SDep>::const_iterator const_succ_iterator;
0269
0270 unsigned NodeNum = BoundaryID;
0271 unsigned NodeQueueId = 0;
0272 unsigned NumPreds = 0;
0273 unsigned NumSuccs = 0;
0274 unsigned NumPredsLeft = 0;
0275 unsigned NumSuccsLeft = 0;
0276 unsigned WeakPredsLeft = 0;
0277 unsigned WeakSuccsLeft = 0;
0278 unsigned TopReadyCycle = 0;
0279 unsigned BotReadyCycle = 0;
0280
0281 private:
0282 unsigned Depth = 0;
0283 unsigned Height = 0;
0284
0285 public:
0286 bool isVRegCycle : 1;
0287 bool isCall : 1;
0288 bool isCallOp : 1;
0289 bool isTwoAddress : 1;
0290 bool isCommutable : 1;
0291 bool hasPhysRegUses : 1;
0292 bool hasPhysRegDefs : 1;
0293 bool hasPhysRegClobbers : 1;
0294 bool isPending : 1;
0295 bool isAvailable : 1;
0296 bool isScheduled : 1;
0297 bool isScheduleHigh : 1;
0298 bool isScheduleLow : 1;
0299 bool isCloned : 1;
0300 bool isUnbuffered : 1;
0301 bool hasReservedResource : 1;
0302 unsigned short NumRegDefsLeft = 0;
0303 unsigned short Latency = 0;
0304
0305 private:
0306 bool isDepthCurrent : 1;
0307 bool isHeightCurrent : 1;
0308 bool isNode : 1;
0309 bool isInst : 1;
0310
0311 public:
0312 Sched::Preference SchedulingPref : 4;
0313 static_assert(Sched::Preference::Last <= (1 << 4),
0314 "not enough bits in bitfield");
0315
0316
0317
0318 SUnit(SDNode *node, unsigned nodenum)
0319 : Node(node), NodeNum(nodenum), isVRegCycle(false), isCall(false),
0320 isCallOp(false), isTwoAddress(false), isCommutable(false),
0321 hasPhysRegUses(false), hasPhysRegDefs(false),
0322 hasPhysRegClobbers(false), isPending(false), isAvailable(false),
0323 isScheduled(false), isScheduleHigh(false), isScheduleLow(false),
0324 isCloned(false), isUnbuffered(false), hasReservedResource(false),
0325 isDepthCurrent(false), isHeightCurrent(false), isNode(true),
0326 isInst(false), SchedulingPref(Sched::None) {}
0327
0328
0329
0330 SUnit(MachineInstr *instr, unsigned nodenum)
0331 : Instr(instr), NodeNum(nodenum), isVRegCycle(false), isCall(false),
0332 isCallOp(false), isTwoAddress(false), isCommutable(false),
0333 hasPhysRegUses(false), hasPhysRegDefs(false),
0334 hasPhysRegClobbers(false), isPending(false), isAvailable(false),
0335 isScheduled(false), isScheduleHigh(false), isScheduleLow(false),
0336 isCloned(false), isUnbuffered(false), hasReservedResource(false),
0337 isDepthCurrent(false), isHeightCurrent(false), isNode(false),
0338 isInst(true), SchedulingPref(Sched::None) {}
0339
0340
0341 SUnit()
0342 : Node(nullptr), isVRegCycle(false), isCall(false), isCallOp(false),
0343 isTwoAddress(false), isCommutable(false), hasPhysRegUses(false),
0344 hasPhysRegDefs(false), hasPhysRegClobbers(false), isPending(false),
0345 isAvailable(false), isScheduled(false), isScheduleHigh(false),
0346 isScheduleLow(false), isCloned(false), isUnbuffered(false),
0347 hasReservedResource(false), isDepthCurrent(false),
0348 isHeightCurrent(false), isNode(false), isInst(false),
0349 SchedulingPref(Sched::None) {}
0350
0351
0352
0353
0354
0355
0356
0357
0358 bool isBoundaryNode() const { return NodeNum == BoundaryID; }
0359
0360
0361
0362 void setNode(SDNode *N) {
0363 assert(!isInst && "Setting SDNode of SUnit with MachineInstr!");
0364 Node = N;
0365 isNode = true;
0366 }
0367
0368
0369
0370 SDNode *getNode() const {
0371 assert(!isInst && (isNode || !Instr) &&
0372 "Reading SDNode of SUnit without SDNode!");
0373 return Node;
0374 }
0375
0376
0377
0378 bool isInstr() const { return isInst && Instr; }
0379
0380
0381
0382 void setInstr(MachineInstr *MI) {
0383 assert(!isNode && "Setting MachineInstr of SUnit with SDNode!");
0384 Instr = MI;
0385 isInst = true;
0386 }
0387
0388
0389
0390 MachineInstr *getInstr() const {
0391 assert(!isNode && (isInst || !Node) &&
0392 "Reading MachineInstr of SUnit without MachineInstr!");
0393 return Instr;
0394 }
0395
0396
0397
0398 bool addPred(const SDep &D, bool Required = true);
0399
0400
0401
0402 bool addPredBarrier(SUnit *SU) {
0403 SDep Dep(SU, SDep::Barrier);
0404 unsigned TrueMemOrderLatency =
0405 ((SU->getInstr()->mayStore() && this->getInstr()->mayLoad()) ? 1 : 0);
0406 Dep.setLatency(TrueMemOrderLatency);
0407 return addPred(Dep);
0408 }
0409
0410
0411
0412 void removePred(const SDep &D);
0413
0414
0415
0416 unsigned getDepth() const {
0417 if (!isDepthCurrent)
0418 const_cast<SUnit *>(this)->ComputeDepth();
0419 return Depth;
0420 }
0421
0422
0423
0424 unsigned getHeight() const {
0425 if (!isHeightCurrent)
0426 const_cast<SUnit *>(this)->ComputeHeight();
0427 return Height;
0428 }
0429
0430
0431
0432
0433 void setDepthToAtLeast(unsigned NewDepth);
0434
0435
0436
0437
0438 void setHeightToAtLeast(unsigned NewHeight);
0439
0440
0441
0442 void setDepthDirty();
0443
0444
0445
0446 void setHeightDirty();
0447
0448
0449 bool isPred(const SUnit *N) const {
0450 for (const SDep &Pred : Preds)
0451 if (Pred.getSUnit() == N)
0452 return true;
0453 return false;
0454 }
0455
0456
0457 bool isSucc(const SUnit *N) const {
0458 for (const SDep &Succ : Succs)
0459 if (Succ.getSUnit() == N)
0460 return true;
0461 return false;
0462 }
0463
0464 bool isTopReady() const {
0465 return NumPredsLeft == 0;
0466 }
0467 bool isBottomReady() const {
0468 return NumSuccsLeft == 0;
0469 }
0470
0471
0472
0473 void biasCriticalPath();
0474
0475 void dumpAttributes() const;
0476
0477 private:
0478 void ComputeDepth();
0479 void ComputeHeight();
0480 };
0481
0482
0483 inline bool SDep::overlaps(const SDep &Other) const {
0484 if (Dep != Other.Dep)
0485 return false;
0486 switch (Dep.getInt()) {
0487 case Data:
0488 case Anti:
0489 case Output:
0490 return Contents.Reg == Other.Contents.Reg;
0491 case Order:
0492 return Contents.OrdKind == Other.Contents.OrdKind;
0493 }
0494 llvm_unreachable("Invalid dependency kind!");
0495 }
0496
0497
0498 inline SUnit *SDep::getSUnit() const { return Dep.getPointer(); }
0499
0500
0501 inline void SDep::setSUnit(SUnit *SU) { Dep.setPointer(SU); }
0502
0503
0504 inline SDep::Kind SDep::getKind() const { return Dep.getInt(); }
0505
0506
0507
0508
0509
0510
0511
0512
0513
0514 class SchedulingPriorityQueue {
0515 virtual void anchor();
0516
0517 unsigned CurCycle = 0;
0518 bool HasReadyFilter;
0519
0520 public:
0521 SchedulingPriorityQueue(bool rf = false) : HasReadyFilter(rf) {}
0522
0523 virtual ~SchedulingPriorityQueue() = default;
0524
0525 virtual bool isBottomUp() const = 0;
0526
0527 virtual void initNodes(std::vector<SUnit> &SUnits) = 0;
0528 virtual void addNode(const SUnit *SU) = 0;
0529 virtual void updateNode(const SUnit *SU) = 0;
0530 virtual void releaseState() = 0;
0531
0532 virtual bool empty() const = 0;
0533
0534 bool hasReadyFilter() const { return HasReadyFilter; }
0535
0536 virtual bool tracksRegPressure() const { return false; }
0537
0538 virtual bool isReady(SUnit *) const {
0539 assert(!HasReadyFilter && "The ready filter must override isReady()");
0540 return true;
0541 }
0542
0543 virtual void push(SUnit *U) = 0;
0544
0545 void push_all(const std::vector<SUnit *> &Nodes) {
0546 for (SUnit *SU : Nodes)
0547 push(SU);
0548 }
0549
0550 virtual SUnit *pop() = 0;
0551
0552 virtual void remove(SUnit *SU) = 0;
0553
0554 virtual void dump(ScheduleDAG *) const {}
0555
0556
0557
0558
0559 virtual void scheduledNode(SUnit *) {}
0560
0561 virtual void unscheduledNode(SUnit *) {}
0562
0563 void setCurCycle(unsigned Cycle) {
0564 CurCycle = Cycle;
0565 }
0566
0567 unsigned getCurCycle() const {
0568 return CurCycle;
0569 }
0570 };
0571
0572 class ScheduleDAG {
0573 public:
0574 const TargetMachine &TM;
0575 const TargetInstrInfo *TII;
0576 const TargetRegisterInfo *TRI;
0577 MachineFunction &MF;
0578 MachineRegisterInfo &MRI;
0579 std::vector<SUnit> SUnits;
0580 SUnit EntrySU;
0581 SUnit ExitSU;
0582
0583 #ifdef NDEBUG
0584 static const bool StressSched = false;
0585 #else
0586 bool StressSched;
0587 #endif
0588
0589
0590
0591
0592
0593
0594
0595 ScheduleDAG(const ScheduleDAG &) = delete;
0596 ScheduleDAG &operator=(const ScheduleDAG &) = delete;
0597
0598 explicit ScheduleDAG(MachineFunction &mf);
0599
0600 virtual ~ScheduleDAG();
0601
0602
0603 void clearDAG();
0604
0605
0606
0607 const MCInstrDesc *getInstrDesc(const SUnit *SU) const {
0608 if (SU->isInstr()) return &SU->getInstr()->getDesc();
0609 return getNodeDesc(SU->getNode());
0610 }
0611
0612
0613 virtual void viewGraph(const Twine &Name, const Twine &Title);
0614 virtual void viewGraph();
0615
0616 virtual void dumpNode(const SUnit &SU) const = 0;
0617 virtual void dump() const = 0;
0618 void dumpNodeName(const SUnit &SU) const;
0619
0620
0621 virtual std::string getGraphNodeLabel(const SUnit *SU) const = 0;
0622
0623
0624 virtual std::string getDAGName() const = 0;
0625
0626
0627 virtual void addCustomGraphFeatures(GraphWriter<ScheduleDAG*> &) const {}
0628
0629 #ifndef NDEBUG
0630
0631
0632 unsigned VerifyScheduledDAG(bool isBottomUp);
0633 #endif
0634
0635 protected:
0636 void dumpNodeAll(const SUnit &SU) const;
0637
0638 private:
0639
0640 const MCInstrDesc *getNodeDesc(const SDNode *Node) const;
0641 };
0642
0643 class SUnitIterator {
0644 SUnit *Node;
0645 unsigned Operand;
0646
0647 SUnitIterator(SUnit *N, unsigned Op) : Node(N), Operand(Op) {}
0648
0649 public:
0650 using iterator_category = std::forward_iterator_tag;
0651 using value_type = SUnit;
0652 using difference_type = std::ptrdiff_t;
0653 using pointer = value_type *;
0654 using reference = value_type &;
0655
0656 bool operator==(const SUnitIterator& x) const {
0657 return Operand == x.Operand;
0658 }
0659 bool operator!=(const SUnitIterator& x) const { return !operator==(x); }
0660
0661 pointer operator*() const {
0662 return Node->Preds[Operand].getSUnit();
0663 }
0664 pointer operator->() const { return operator*(); }
0665
0666 SUnitIterator& operator++() {
0667 ++Operand;
0668 return *this;
0669 }
0670 SUnitIterator operator++(int) {
0671 SUnitIterator tmp = *this; ++*this; return tmp;
0672 }
0673
0674 static SUnitIterator begin(SUnit *N) { return SUnitIterator(N, 0); }
0675 static SUnitIterator end (SUnit *N) {
0676 return SUnitIterator(N, (unsigned)N->Preds.size());
0677 }
0678
0679 unsigned getOperand() const { return Operand; }
0680 const SUnit *getNode() const { return Node; }
0681
0682
0683 bool isCtrlDep() const {
0684 return getSDep().isCtrl();
0685 }
0686 bool isArtificialDep() const {
0687 return getSDep().isArtificial();
0688 }
0689 const SDep &getSDep() const {
0690 return Node->Preds[Operand];
0691 }
0692 };
0693
0694 template <> struct GraphTraits<SUnit*> {
0695 typedef SUnit *NodeRef;
0696 typedef SUnitIterator ChildIteratorType;
0697 static NodeRef getEntryNode(SUnit *N) { return N; }
0698 static ChildIteratorType child_begin(NodeRef N) {
0699 return SUnitIterator::begin(N);
0700 }
0701 static ChildIteratorType child_end(NodeRef N) {
0702 return SUnitIterator::end(N);
0703 }
0704 };
0705
0706 template <> struct GraphTraits<ScheduleDAG*> : public GraphTraits<SUnit*> {
0707 typedef pointer_iterator<std::vector<SUnit>::iterator> nodes_iterator;
0708 static nodes_iterator nodes_begin(ScheduleDAG *G) {
0709 return nodes_iterator(G->SUnits.begin());
0710 }
0711 static nodes_iterator nodes_end(ScheduleDAG *G) {
0712 return nodes_iterator(G->SUnits.end());
0713 }
0714 };
0715
0716
0717
0718
0719
0720 class ScheduleDAGTopologicalSort {
0721
0722 std::vector<SUnit> &SUnits;
0723 SUnit *ExitSU;
0724
0725
0726 bool Dirty = false;
0727
0728
0729 SmallVector<std::pair<SUnit *, SUnit *>, 16> Updates;
0730
0731
0732 std::vector<int> Index2Node;
0733
0734 std::vector<int> Node2Index;
0735
0736 BitVector Visited;
0737
0738
0739
0740
0741 void DFS(const SUnit *SU, int UpperBound, bool& HasLoop);
0742
0743
0744
0745 void Shift(BitVector& Visited, int LowerBound, int UpperBound);
0746
0747
0748 void Allocate(int n, int index);
0749
0750
0751
0752
0753 void FixOrder();
0754
0755 public:
0756 ScheduleDAGTopologicalSort(std::vector<SUnit> &SUnits, SUnit *ExitSU);
0757
0758
0759
0760 void AddSUnitWithoutPredecessors(const SUnit *SU);
0761
0762
0763 void InitDAGTopologicalSorting();
0764
0765
0766
0767
0768
0769
0770 std::vector<int> GetSubGraph(const SUnit &StartSU, const SUnit &TargetSU,
0771 bool &Success);
0772
0773
0774 bool IsReachable(const SUnit *SU, const SUnit *TargetSU);
0775
0776
0777 bool WillCreateCycle(SUnit *TargetSU, SUnit *SU);
0778
0779
0780
0781 void AddPred(SUnit *Y, SUnit *X);
0782
0783
0784
0785 void AddPredQueued(SUnit *Y, SUnit *X);
0786
0787
0788
0789
0790 void RemovePred(SUnit *M, SUnit *N);
0791
0792
0793
0794 void MarkDirty() { Dirty = true; }
0795
0796 typedef std::vector<int>::iterator iterator;
0797 typedef std::vector<int>::const_iterator const_iterator;
0798 iterator begin() { return Index2Node.begin(); }
0799 const_iterator begin() const { return Index2Node.begin(); }
0800 iterator end() { return Index2Node.end(); }
0801 const_iterator end() const { return Index2Node.end(); }
0802
0803 typedef std::vector<int>::reverse_iterator reverse_iterator;
0804 typedef std::vector<int>::const_reverse_iterator const_reverse_iterator;
0805 reverse_iterator rbegin() { return Index2Node.rbegin(); }
0806 const_reverse_iterator rbegin() const { return Index2Node.rbegin(); }
0807 reverse_iterator rend() { return Index2Node.rend(); }
0808 const_reverse_iterator rend() const { return Index2Node.rend(); }
0809 };
0810
0811 }
0812
0813 #endif