File indexing completed on 2026-05-10 08:43:36
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018 #ifndef LLVM_CODEGEN_SLOTINDEXES_H
0019 #define LLVM_CODEGEN_SLOTINDEXES_H
0020
0021 #include "llvm/ADT/DenseMap.h"
0022 #include "llvm/ADT/IntervalMap.h"
0023 #include "llvm/ADT/PointerIntPair.h"
0024 #include "llvm/ADT/SmallVector.h"
0025 #include "llvm/ADT/simple_ilist.h"
0026 #include "llvm/CodeGen/MachineBasicBlock.h"
0027 #include "llvm/CodeGen/MachineFunction.h"
0028 #include "llvm/CodeGen/MachineFunctionPass.h"
0029 #include "llvm/CodeGen/MachineInstr.h"
0030 #include "llvm/CodeGen/MachineInstrBundle.h"
0031 #include "llvm/CodeGen/MachinePassManager.h"
0032 #include "llvm/Support/Allocator.h"
0033 #include <algorithm>
0034 #include <cassert>
0035 #include <iterator>
0036 #include <utility>
0037
0038 namespace llvm {
0039
0040 class raw_ostream;
0041
0042
0043
0044
0045
0046 class IndexListEntry : public ilist_node<IndexListEntry> {
0047 MachineInstr *mi;
0048 unsigned index;
0049
0050 public:
0051 IndexListEntry(MachineInstr *mi, unsigned index) : mi(mi), index(index) {}
0052
0053 MachineInstr* getInstr() const { return mi; }
0054 void setInstr(MachineInstr *mi) {
0055 this->mi = mi;
0056 }
0057
0058 unsigned getIndex() const { return index; }
0059 void setIndex(unsigned index) {
0060 this->index = index;
0061 }
0062 };
0063
0064
0065 class SlotIndex {
0066 friend class SlotIndexes;
0067
0068 enum Slot {
0069
0070
0071
0072 Slot_Block,
0073
0074
0075
0076
0077
0078 Slot_EarlyClobber,
0079
0080
0081
0082 Slot_Register,
0083
0084
0085
0086
0087 Slot_Dead,
0088
0089 Slot_Count
0090 };
0091
0092 PointerIntPair<IndexListEntry*, 2, unsigned> lie;
0093
0094 IndexListEntry* listEntry() const {
0095 assert(isValid() && "Attempt to compare reserved index.");
0096 return lie.getPointer();
0097 }
0098
0099 unsigned getIndex() const {
0100 return listEntry()->getIndex() | getSlot();
0101 }
0102
0103
0104 Slot getSlot() const {
0105 return static_cast<Slot>(lie.getInt());
0106 }
0107
0108 public:
0109 enum {
0110
0111
0112 InstrDist = 4 * Slot_Count
0113 };
0114
0115
0116 SlotIndex() = default;
0117
0118
0119
0120
0121 SlotIndex(IndexListEntry *entry, unsigned slot) : lie(entry, slot) {}
0122
0123
0124 SlotIndex(const SlotIndex &li, Slot s) : lie(li.listEntry(), unsigned(s)) {
0125 assert(isValid() && "Attempt to construct index with 0 pointer.");
0126 }
0127
0128
0129
0130 bool isValid() const {
0131 return lie.getPointer();
0132 }
0133
0134
0135 explicit operator bool() const { return isValid(); }
0136
0137
0138 void print(raw_ostream &os) const;
0139
0140
0141 void dump() const;
0142
0143
0144 bool operator==(SlotIndex other) const {
0145 return lie == other.lie;
0146 }
0147
0148 bool operator!=(SlotIndex other) const {
0149 return lie != other.lie;
0150 }
0151
0152
0153
0154 bool operator<(SlotIndex other) const {
0155 return getIndex() < other.getIndex();
0156 }
0157
0158
0159 bool operator<=(SlotIndex other) const {
0160 return getIndex() <= other.getIndex();
0161 }
0162
0163
0164
0165 bool operator>(SlotIndex other) const {
0166 return getIndex() > other.getIndex();
0167 }
0168
0169
0170
0171 bool operator>=(SlotIndex other) const {
0172 return getIndex() >= other.getIndex();
0173 }
0174
0175
0176 static bool isSameInstr(SlotIndex A, SlotIndex B) {
0177 return A.listEntry() == B.listEntry();
0178 }
0179
0180
0181
0182 static bool isEarlierInstr(SlotIndex A, SlotIndex B) {
0183 return A.listEntry()->getIndex() < B.listEntry()->getIndex();
0184 }
0185
0186
0187
0188 static bool isEarlierEqualInstr(SlotIndex A, SlotIndex B) {
0189 return !isEarlierInstr(B, A);
0190 }
0191
0192
0193 int distance(SlotIndex other) const {
0194 return other.getIndex() - getIndex();
0195 }
0196
0197
0198
0199
0200
0201
0202
0203 int getApproxInstrDistance(SlotIndex other) const {
0204 return (other.listEntry()->getIndex() - listEntry()->getIndex())
0205 / Slot_Count;
0206 }
0207
0208
0209 bool isBlock() const { return getSlot() == Slot_Block; }
0210
0211
0212 bool isEarlyClobber() const { return getSlot() == Slot_EarlyClobber; }
0213
0214
0215
0216 bool isRegister() const { return getSlot() == Slot_Register; }
0217
0218
0219 bool isDead() const { return getSlot() == Slot_Dead; }
0220
0221
0222
0223
0224 SlotIndex getBaseIndex() const {
0225 return SlotIndex(listEntry(), Slot_Block);
0226 }
0227
0228
0229
0230
0231 SlotIndex getBoundaryIndex() const {
0232 return SlotIndex(listEntry(), Slot_Dead);
0233 }
0234
0235
0236
0237 SlotIndex getRegSlot(bool EC = false) const {
0238 return SlotIndex(listEntry(), EC ? Slot_EarlyClobber : Slot_Register);
0239 }
0240
0241
0242 SlotIndex getDeadSlot() const {
0243 return SlotIndex(listEntry(), Slot_Dead);
0244 }
0245
0246
0247
0248
0249
0250
0251
0252 SlotIndex getNextSlot() const {
0253 Slot s = getSlot();
0254 if (s == Slot_Dead) {
0255 return SlotIndex(&*++listEntry()->getIterator(), Slot_Block);
0256 }
0257 return SlotIndex(listEntry(), s + 1);
0258 }
0259
0260
0261
0262 SlotIndex getNextIndex() const {
0263 return SlotIndex(&*++listEntry()->getIterator(), getSlot());
0264 }
0265
0266
0267
0268
0269
0270
0271
0272 SlotIndex getPrevSlot() const {
0273 Slot s = getSlot();
0274 if (s == Slot_Block) {
0275 return SlotIndex(&*--listEntry()->getIterator(), Slot_Dead);
0276 }
0277 return SlotIndex(listEntry(), s - 1);
0278 }
0279
0280
0281
0282 SlotIndex getPrevIndex() const {
0283 return SlotIndex(&*--listEntry()->getIterator(), getSlot());
0284 }
0285 };
0286
0287 inline raw_ostream& operator<<(raw_ostream &os, SlotIndex li) {
0288 li.print(os);
0289 return os;
0290 }
0291
0292 using IdxMBBPair = std::pair<SlotIndex, MachineBasicBlock *>;
0293
0294
0295
0296
0297 class SlotIndexes {
0298 friend class SlotIndexesWrapperPass;
0299
0300 private:
0301
0302 BumpPtrAllocator ileAllocator;
0303
0304 using IndexList = simple_ilist<IndexListEntry>;
0305 IndexList indexList;
0306
0307 MachineFunction *mf = nullptr;
0308
0309 using Mi2IndexMap = DenseMap<const MachineInstr *, SlotIndex>;
0310 Mi2IndexMap mi2iMap;
0311
0312
0313 SmallVector<std::pair<SlotIndex, SlotIndex>, 8> MBBRanges;
0314
0315
0316
0317 SmallVector<IdxMBBPair, 8> idx2MBBMap;
0318
0319
0320 SlotIndexes() = default;
0321
0322 void clear();
0323
0324 void analyze(MachineFunction &MF);
0325
0326 IndexListEntry* createEntry(MachineInstr *mi, unsigned index) {
0327 IndexListEntry *entry =
0328 static_cast<IndexListEntry *>(ileAllocator.Allocate(
0329 sizeof(IndexListEntry), alignof(IndexListEntry)));
0330
0331 new (entry) IndexListEntry(mi, index);
0332
0333 return entry;
0334 }
0335
0336
0337 void renumberIndexes(IndexList::iterator curItr);
0338
0339 public:
0340 SlotIndexes(SlotIndexes &&) = default;
0341
0342 SlotIndexes(MachineFunction &MF) { analyze(MF); }
0343
0344 ~SlotIndexes();
0345
0346 void reanalyze(MachineFunction &MF) {
0347 clear();
0348 analyze(MF);
0349 }
0350
0351 void print(raw_ostream &OS) const;
0352
0353
0354 void dump() const;
0355
0356
0357 void repairIndexesInRange(MachineBasicBlock *MBB,
0358 MachineBasicBlock::iterator Begin,
0359 MachineBasicBlock::iterator End);
0360
0361
0362 SlotIndex getZeroIndex() {
0363 assert(indexList.front().getIndex() == 0 && "First index is not 0?");
0364 return SlotIndex(&indexList.front(), 0);
0365 }
0366
0367
0368 SlotIndex getLastIndex() {
0369 return SlotIndex(&indexList.back(), 0);
0370 }
0371
0372
0373
0374 bool hasIndex(const MachineInstr &instr) const {
0375 return mi2iMap.count(&instr);
0376 }
0377
0378
0379 SlotIndex getInstructionIndex(const MachineInstr &MI,
0380 bool IgnoreBundle = false) const {
0381
0382 auto BundleStart = getBundleStart(MI.getIterator());
0383 auto BundleEnd = getBundleEnd(MI.getIterator());
0384
0385 const MachineInstr &BundleNonDebug =
0386 IgnoreBundle ? MI
0387 : *skipDebugInstructionsForward(BundleStart, BundleEnd);
0388 assert(!BundleNonDebug.isDebugInstr() &&
0389 "Could not use a debug instruction to query mi2iMap.");
0390 Mi2IndexMap::const_iterator itr = mi2iMap.find(&BundleNonDebug);
0391 assert(itr != mi2iMap.end() && "Instruction not found in maps.");
0392 return itr->second;
0393 }
0394
0395
0396
0397 MachineInstr* getInstructionFromIndex(SlotIndex index) const {
0398 return index.listEntry()->getInstr();
0399 }
0400
0401
0402
0403 SlotIndex getNextNonNullIndex(SlotIndex Index) {
0404 IndexList::iterator I = Index.listEntry()->getIterator();
0405 IndexList::iterator E = indexList.end();
0406 while (++I != E)
0407 if (I->getInstr())
0408 return SlotIndex(&*I, Index.getSlot());
0409
0410 return getLastIndex();
0411 }
0412
0413
0414
0415
0416 SlotIndex getIndexBefore(const MachineInstr &MI) const {
0417 const MachineBasicBlock *MBB = MI.getParent();
0418 assert(MBB && "MI must be inserted in a basic block");
0419 MachineBasicBlock::const_iterator I = MI, B = MBB->begin();
0420 while (true) {
0421 if (I == B)
0422 return getMBBStartIdx(MBB);
0423 --I;
0424 Mi2IndexMap::const_iterator MapItr = mi2iMap.find(&*I);
0425 if (MapItr != mi2iMap.end())
0426 return MapItr->second;
0427 }
0428 }
0429
0430
0431
0432
0433 SlotIndex getIndexAfter(const MachineInstr &MI) const {
0434 const MachineBasicBlock *MBB = MI.getParent();
0435 assert(MBB && "MI must be inserted in a basic block");
0436 MachineBasicBlock::const_iterator I = MI, E = MBB->end();
0437 while (true) {
0438 ++I;
0439 if (I == E)
0440 return getMBBEndIdx(MBB);
0441 Mi2IndexMap::const_iterator MapItr = mi2iMap.find(&*I);
0442 if (MapItr != mi2iMap.end())
0443 return MapItr->second;
0444 }
0445 }
0446
0447
0448 const std::pair<SlotIndex, SlotIndex> &
0449 getMBBRange(unsigned Num) const {
0450 return MBBRanges[Num];
0451 }
0452
0453
0454 const std::pair<SlotIndex, SlotIndex> &
0455 getMBBRange(const MachineBasicBlock *MBB) const {
0456 return getMBBRange(MBB->getNumber());
0457 }
0458
0459
0460 SlotIndex getMBBStartIdx(unsigned Num) const {
0461 return getMBBRange(Num).first;
0462 }
0463
0464
0465 SlotIndex getMBBStartIdx(const MachineBasicBlock *mbb) const {
0466 return getMBBRange(mbb).first;
0467 }
0468
0469
0470 SlotIndex getMBBEndIdx(unsigned Num) const {
0471 return getMBBRange(Num).second;
0472 }
0473
0474
0475 SlotIndex getMBBEndIdx(const MachineBasicBlock *mbb) const {
0476 return getMBBRange(mbb).second;
0477 }
0478
0479
0480
0481 using MBBIndexIterator = SmallVectorImpl<IdxMBBPair>::const_iterator;
0482
0483
0484
0485
0486 MBBIndexIterator getMBBLowerBound(MBBIndexIterator Start,
0487 SlotIndex Idx) const {
0488 return std::lower_bound(
0489 Start, MBBIndexEnd(), Idx,
0490 [](const IdxMBBPair &IM, SlotIndex Idx) { return IM.first < Idx; });
0491 }
0492 MBBIndexIterator getMBBLowerBound(SlotIndex Idx) const {
0493 return getMBBLowerBound(MBBIndexBegin(), Idx);
0494 }
0495
0496
0497
0498 MBBIndexIterator getMBBUpperBound(SlotIndex Idx) const {
0499 return std::upper_bound(
0500 MBBIndexBegin(), MBBIndexEnd(), Idx,
0501 [](SlotIndex Idx, const IdxMBBPair &IM) { return Idx < IM.first; });
0502 }
0503
0504
0505 MBBIndexIterator MBBIndexBegin() const {
0506 return idx2MBBMap.begin();
0507 }
0508
0509
0510 MBBIndexIterator MBBIndexEnd() const {
0511 return idx2MBBMap.end();
0512 }
0513
0514
0515 MachineBasicBlock* getMBBFromIndex(SlotIndex index) const {
0516 if (MachineInstr *MI = getInstructionFromIndex(index))
0517 return MI->getParent();
0518
0519 MBBIndexIterator I = std::prev(getMBBUpperBound(index));
0520 assert(I != MBBIndexEnd() && I->first <= index &&
0521 index < getMBBEndIdx(I->second) &&
0522 "index does not correspond to an MBB");
0523 return I->second;
0524 }
0525
0526
0527
0528
0529
0530
0531 SlotIndex insertMachineInstrInMaps(MachineInstr &MI, bool Late = false) {
0532 assert(!MI.isInsideBundle() &&
0533 "Instructions inside bundles should use bundle start's slot.");
0534 assert(!mi2iMap.contains(&MI) && "Instr already indexed.");
0535
0536
0537 assert(!MI.isDebugInstr() && "Cannot number debug instructions.");
0538
0539 assert(MI.getParent() != nullptr && "Instr must be added to function.");
0540
0541
0542 IndexList::iterator prevItr, nextItr;
0543 if (Late) {
0544
0545 nextItr = getIndexAfter(MI).listEntry()->getIterator();
0546 prevItr = std::prev(nextItr);
0547 } else {
0548
0549 prevItr = getIndexBefore(MI).listEntry()->getIterator();
0550 nextItr = std::next(prevItr);
0551 }
0552
0553
0554
0555 unsigned dist = ((nextItr->getIndex() - prevItr->getIndex())/2) & ~3u;
0556 unsigned newNumber = prevItr->getIndex() + dist;
0557
0558
0559 IndexList::iterator newItr =
0560 indexList.insert(nextItr, *createEntry(&MI, newNumber));
0561
0562
0563 if (dist == 0)
0564 renumberIndexes(newItr);
0565
0566 SlotIndex newIndex(&*newItr, SlotIndex::Slot_Block);
0567 mi2iMap.insert(std::make_pair(&MI, newIndex));
0568 return newIndex;
0569 }
0570
0571
0572
0573
0574
0575
0576
0577 void removeMachineInstrFromMaps(MachineInstr &MI,
0578 bool AllowBundled = false);
0579
0580
0581
0582
0583 void removeSingleMachineInstrFromMaps(MachineInstr &MI);
0584
0585
0586
0587
0588 SlotIndex replaceMachineInstrInMaps(MachineInstr &MI, MachineInstr &NewMI) {
0589 Mi2IndexMap::iterator mi2iItr = mi2iMap.find(&MI);
0590 if (mi2iItr == mi2iMap.end())
0591 return SlotIndex();
0592 SlotIndex replaceBaseIndex = mi2iItr->second;
0593 IndexListEntry *miEntry(replaceBaseIndex.listEntry());
0594 assert(miEntry->getInstr() == &MI &&
0595 "Mismatched instruction in index tables.");
0596 miEntry->setInstr(&NewMI);
0597 mi2iMap.erase(mi2iItr);
0598 mi2iMap.insert(std::make_pair(&NewMI, replaceBaseIndex));
0599 return replaceBaseIndex;
0600 }
0601
0602
0603
0604
0605
0606 void insertMBBInMaps(MachineBasicBlock *mbb) {
0607 assert(mbb != &mbb->getParent()->front() &&
0608 "Can't insert a new block at the beginning of a function.");
0609 auto prevMBB = std::prev(MachineFunction::iterator(mbb));
0610
0611
0612
0613 IndexListEntry *startEntry = createEntry(nullptr, 0);
0614 IndexListEntry *endEntry = getMBBEndIdx(&*prevMBB).listEntry();
0615 IndexListEntry *insEntry =
0616 mbb->empty() ? endEntry
0617 : getInstructionIndex(mbb->front()).listEntry();
0618 IndexList::iterator newItr =
0619 indexList.insert(insEntry->getIterator(), *startEntry);
0620
0621 SlotIndex startIdx(startEntry, SlotIndex::Slot_Block);
0622 SlotIndex endIdx(endEntry, SlotIndex::Slot_Block);
0623
0624 MBBRanges[prevMBB->getNumber()].second = startIdx;
0625
0626 assert(unsigned(mbb->getNumber()) == MBBRanges.size() &&
0627 "Blocks must be added in order");
0628 MBBRanges.push_back(std::make_pair(startIdx, endIdx));
0629 idx2MBBMap.push_back(IdxMBBPair(startIdx, mbb));
0630
0631 renumberIndexes(newItr);
0632 llvm::sort(idx2MBBMap, less_first());
0633 }
0634
0635
0636 void packIndexes();
0637 };
0638
0639
0640 template <>
0641 struct IntervalMapInfo<SlotIndex> : IntervalMapHalfOpenInfo<SlotIndex> {
0642 };
0643
0644 class SlotIndexesAnalysis : public AnalysisInfoMixin<SlotIndexesAnalysis> {
0645 friend AnalysisInfoMixin<SlotIndexesAnalysis>;
0646 static AnalysisKey Key;
0647
0648 public:
0649 using Result = SlotIndexes;
0650 Result run(MachineFunction &MF, MachineFunctionAnalysisManager &);
0651 };
0652
0653 class SlotIndexesPrinterPass : public PassInfoMixin<SlotIndexesPrinterPass> {
0654 raw_ostream &OS;
0655
0656 public:
0657 explicit SlotIndexesPrinterPass(raw_ostream &OS) : OS(OS) {}
0658 PreservedAnalyses run(MachineFunction &MF,
0659 MachineFunctionAnalysisManager &MFAM);
0660 static bool isRequired() { return true; }
0661 };
0662
0663 class SlotIndexesWrapperPass : public MachineFunctionPass {
0664 SlotIndexes SI;
0665
0666 public:
0667 static char ID;
0668
0669 SlotIndexesWrapperPass();
0670
0671 void getAnalysisUsage(AnalysisUsage &au) const override;
0672 void releaseMemory() override { SI.clear(); }
0673
0674 bool runOnMachineFunction(MachineFunction &fn) override {
0675 SI.analyze(fn);
0676 return false;
0677 }
0678
0679 SlotIndexes &getSI() { return SI; }
0680 };
0681
0682 }
0683
0684 #endif