File indexing completed on 2026-05-10 08:44:22
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014 #ifndef LLVM_PASSES_CODEGENPASSBUILDER_H
0015 #define LLVM_PASSES_CODEGENPASSBUILDER_H
0016
0017 #include "llvm/ADT/SmallVector.h"
0018 #include "llvm/ADT/StringRef.h"
0019 #include "llvm/Analysis/AliasAnalysis.h"
0020 #include "llvm/Analysis/BasicAliasAnalysis.h"
0021 #include "llvm/Analysis/ProfileSummaryInfo.h"
0022 #include "llvm/Analysis/ScopedNoAliasAA.h"
0023 #include "llvm/Analysis/TargetTransformInfo.h"
0024 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
0025 #include "llvm/CodeGen/AssignmentTrackingAnalysis.h"
0026 #include "llvm/CodeGen/CallBrPrepare.h"
0027 #include "llvm/CodeGen/CodeGenPrepare.h"
0028 #include "llvm/CodeGen/DeadMachineInstructionElim.h"
0029 #include "llvm/CodeGen/DwarfEHPrepare.h"
0030 #include "llvm/CodeGen/EarlyIfConversion.h"
0031 #include "llvm/CodeGen/ExpandLargeDivRem.h"
0032 #include "llvm/CodeGen/ExpandLargeFpConvert.h"
0033 #include "llvm/CodeGen/ExpandMemCmp.h"
0034 #include "llvm/CodeGen/ExpandReductions.h"
0035 #include "llvm/CodeGen/FinalizeISel.h"
0036 #include "llvm/CodeGen/GCMetadata.h"
0037 #include "llvm/CodeGen/GlobalMerge.h"
0038 #include "llvm/CodeGen/GlobalMergeFunctions.h"
0039 #include "llvm/CodeGen/IndirectBrExpand.h"
0040 #include "llvm/CodeGen/InterleavedAccess.h"
0041 #include "llvm/CodeGen/InterleavedLoadCombine.h"
0042 #include "llvm/CodeGen/JMCInstrumenter.h"
0043 #include "llvm/CodeGen/LiveIntervals.h"
0044 #include "llvm/CodeGen/LocalStackSlotAllocation.h"
0045 #include "llvm/CodeGen/LowerEmuTLS.h"
0046 #include "llvm/CodeGen/MIRPrinter.h"
0047 #include "llvm/CodeGen/MachineCSE.h"
0048 #include "llvm/CodeGen/MachineFunctionAnalysis.h"
0049 #include "llvm/CodeGen/MachineLICM.h"
0050 #include "llvm/CodeGen/MachineModuleInfo.h"
0051 #include "llvm/CodeGen/MachinePassManager.h"
0052 #include "llvm/CodeGen/MachineVerifier.h"
0053 #include "llvm/CodeGen/OptimizePHIs.h"
0054 #include "llvm/CodeGen/PHIElimination.h"
0055 #include "llvm/CodeGen/PeepholeOptimizer.h"
0056 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
0057 #include "llvm/CodeGen/RegAllocFast.h"
0058 #include "llvm/CodeGen/RegUsageInfoCollector.h"
0059 #include "llvm/CodeGen/RegUsageInfoPropagate.h"
0060 #include "llvm/CodeGen/RegisterUsageInfo.h"
0061 #include "llvm/CodeGen/ReplaceWithVeclib.h"
0062 #include "llvm/CodeGen/SafeStack.h"
0063 #include "llvm/CodeGen/SelectOptimize.h"
0064 #include "llvm/CodeGen/ShadowStackGCLowering.h"
0065 #include "llvm/CodeGen/SjLjEHPrepare.h"
0066 #include "llvm/CodeGen/StackColoring.h"
0067 #include "llvm/CodeGen/StackProtector.h"
0068 #include "llvm/CodeGen/TailDuplication.h"
0069 #include "llvm/CodeGen/TargetPassConfig.h"
0070 #include "llvm/CodeGen/TwoAddressInstructionPass.h"
0071 #include "llvm/CodeGen/UnreachableBlockElim.h"
0072 #include "llvm/CodeGen/WasmEHPrepare.h"
0073 #include "llvm/CodeGen/WinEHPrepare.h"
0074 #include "llvm/IR/PassManager.h"
0075 #include "llvm/IR/Verifier.h"
0076 #include "llvm/IRPrinter/IRPrintingPasses.h"
0077 #include "llvm/MC/MCAsmInfo.h"
0078 #include "llvm/MC/MCTargetOptions.h"
0079 #include "llvm/Support/CodeGen.h"
0080 #include "llvm/Support/Debug.h"
0081 #include "llvm/Support/Error.h"
0082 #include "llvm/Support/ErrorHandling.h"
0083 #include "llvm/Target/CGPassBuilderOption.h"
0084 #include "llvm/Target/TargetMachine.h"
0085 #include "llvm/Transforms/CFGuard.h"
0086 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
0087 #include "llvm/Transforms/Scalar/LoopPassManager.h"
0088 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
0089 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
0090 #include "llvm/Transforms/Scalar/MergeICmps.h"
0091 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
0092 #include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
0093 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
0094 #include "llvm/Transforms/Utils/LowerInvoke.h"
0095 #include <cassert>
0096 #include <type_traits>
0097 #include <utility>
0098
0099 namespace llvm {
0100
0101
0102
0103 #define DUMMY_FUNCTION_PASS(NAME, PASS_NAME) \
0104 struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \
0105 template <typename... Ts> PASS_NAME(Ts &&...) {} \
0106 PreservedAnalyses run(Function &, FunctionAnalysisManager &) { \
0107 return PreservedAnalyses::all(); \
0108 } \
0109 };
0110 #define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME) \
0111 struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \
0112 template <typename... Ts> PASS_NAME(Ts &&...) {} \
0113 PreservedAnalyses run(Module &, ModuleAnalysisManager &) { \
0114 return PreservedAnalyses::all(); \
0115 } \
0116 };
0117 #define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME) \
0118 struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \
0119 template <typename... Ts> PASS_NAME(Ts &&...) {} \
0120 PreservedAnalyses run(MachineFunction &, \
0121 MachineFunctionAnalysisManager &) { \
0122 return PreservedAnalyses::all(); \
0123 } \
0124 };
0125 #include "llvm/Passes/MachinePassRegistry.def"
0126
0127
0128
0129
0130
0131
0132
0133 template <typename DerivedT, typename TargetMachineT> class CodeGenPassBuilder {
0134 public:
0135 explicit CodeGenPassBuilder(TargetMachineT &TM,
0136 const CGPassBuilderOption &Opts,
0137 PassInstrumentationCallbacks *PIC)
0138 : TM(TM), Opt(Opts), PIC(PIC) {
0139
0140
0141
0142
0143
0144 if (Opt.EnableIPRA)
0145 TM.Options.EnableIPRA = *Opt.EnableIPRA;
0146
0147 if (Opt.EnableGlobalISelAbort)
0148 TM.Options.GlobalISelAbort = *Opt.EnableGlobalISelAbort;
0149
0150 if (!Opt.OptimizeRegAlloc)
0151 Opt.OptimizeRegAlloc = getOptLevel() != CodeGenOptLevel::None;
0152 }
0153
0154 Error buildPipeline(ModulePassManager &MPM, raw_pwrite_stream &Out,
0155 raw_pwrite_stream *DwoOut,
0156 CodeGenFileType FileType) const;
0157
0158 PassInstrumentationCallbacks *getPassInstrumentationCallbacks() const {
0159 return PIC;
0160 }
0161
0162 protected:
0163 template <typename PassT>
0164 using has_required_t = decltype(std::declval<PassT &>().isRequired());
0165
0166 template <typename PassT>
0167 using is_module_pass_t = decltype(std::declval<PassT &>().run(
0168 std::declval<Module &>(), std::declval<ModuleAnalysisManager &>()));
0169
0170 template <typename PassT>
0171 using is_function_pass_t = decltype(std::declval<PassT &>().run(
0172 std::declval<Function &>(), std::declval<FunctionAnalysisManager &>()));
0173
0174 template <typename PassT>
0175 using is_machine_function_pass_t = decltype(std::declval<PassT &>().run(
0176 std::declval<MachineFunction &>(),
0177 std::declval<MachineFunctionAnalysisManager &>()));
0178
0179
0180
0181
0182
0183 class AddIRPass {
0184 public:
0185 AddIRPass(ModulePassManager &MPM, const DerivedT &PB) : MPM(MPM), PB(PB) {}
0186 ~AddIRPass() {
0187 if (!FPM.isEmpty())
0188 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
0189 }
0190
0191 template <typename PassT>
0192 void operator()(PassT &&Pass, StringRef Name = PassT::name()) {
0193 static_assert((is_detected<is_function_pass_t, PassT>::value ||
0194 is_detected<is_module_pass_t, PassT>::value) &&
0195 "Only module pass and function pass are supported.");
0196 bool Required = false;
0197 if constexpr (is_detected<has_required_t, PassT>::value)
0198 Required = PassT::isRequired();
0199 if (!PB.runBeforeAdding(Name) && !Required)
0200 return;
0201
0202
0203 if constexpr (is_detected<is_function_pass_t, PassT>::value) {
0204 FPM.addPass(std::forward<PassT>(Pass));
0205 } else {
0206
0207 if (!FPM.isEmpty()) {
0208 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
0209 FPM = FunctionPassManager();
0210 }
0211
0212 MPM.addPass(std::forward<PassT>(Pass));
0213 }
0214 }
0215
0216 private:
0217 ModulePassManager &MPM;
0218 FunctionPassManager FPM;
0219 const DerivedT &PB;
0220 };
0221
0222
0223 class AddMachinePass {
0224 public:
0225 AddMachinePass(ModulePassManager &MPM, const DerivedT &PB)
0226 : MPM(MPM), PB(PB) {}
0227 ~AddMachinePass() {
0228 if (!MFPM.isEmpty()) {
0229 FunctionPassManager FPM;
0230 FPM.addPass(
0231 createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
0232 FPM.addPass(InvalidateAnalysisPass<MachineFunctionAnalysis>());
0233 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
0234 }
0235 }
0236
0237 template <typename PassT>
0238 void operator()(PassT &&Pass, bool Force = false,
0239 StringRef Name = PassT::name()) {
0240 static_assert((is_detected<is_machine_function_pass_t, PassT>::value ||
0241 is_detected<is_module_pass_t, PassT>::value) &&
0242 "Only module pass and function pass are supported.");
0243
0244 if (!Force && !PB.runBeforeAdding(Name))
0245 return;
0246
0247
0248 if constexpr (is_detected<is_machine_function_pass_t, PassT>::value) {
0249 MFPM.addPass(std::forward<PassT>(Pass));
0250 } else {
0251
0252 if (!MFPM.isEmpty()) {
0253 MPM.addPass(createModuleToFunctionPassAdaptor(
0254 createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))));
0255 MFPM = MachineFunctionPassManager();
0256 }
0257
0258 MPM.addPass(std::forward<PassT>(Pass));
0259 }
0260
0261 for (auto &C : PB.AfterCallbacks)
0262 C(Name, MFPM);
0263 }
0264
0265 private:
0266 ModulePassManager &MPM;
0267 MachineFunctionPassManager MFPM;
0268 const DerivedT &PB;
0269 };
0270
0271 TargetMachineT &TM;
0272 CGPassBuilderOption Opt;
0273 PassInstrumentationCallbacks *PIC;
0274
0275 template <typename TMC> TMC &getTM() const { return static_cast<TMC &>(TM); }
0276 CodeGenOptLevel getOptLevel() const { return TM.getOptLevel(); }
0277
0278
0279
0280
0281 bool isGlobalISelAbortEnabled() const {
0282 return TM.Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
0283 }
0284
0285
0286
0287
0288 bool reportDiagnosticWhenGlobalISelFallback() const {
0289 return TM.Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
0290 }
0291
0292
0293
0294 Error addInstSelector(AddMachinePass &) const {
0295 return make_error<StringError>("addInstSelector is not overridden",
0296 inconvertibleErrorCode());
0297 }
0298
0299
0300 void addGlobalMergePass(AddIRPass &) const {}
0301
0302
0303
0304
0305
0306
0307
0308 void addILPOpts(AddMachinePass &) const {}
0309
0310
0311
0312 void addPreRegAlloc(AddMachinePass &) const {}
0313
0314
0315
0316
0317
0318
0319
0320
0321
0322
0323
0324
0325
0326 void addPreRewrite(AddMachinePass &) const {}
0327
0328
0329
0330 void addPostRewrite(AddMachinePass &) const {}
0331
0332
0333
0334 void addPostRegAlloc(AddMachinePass &) const {}
0335
0336
0337
0338 void addPreSched2(AddMachinePass &) const {}
0339
0340
0341
0342 void addPreEmitPass(AddMachinePass &) const {}
0343
0344
0345
0346
0347
0348
0349 void addPreEmitPass2(AddMachinePass &) const {}
0350
0351
0352
0353
0354
0355
0356 void addPreISel(AddIRPass &) const {
0357 llvm_unreachable("addPreISel is not overridden");
0358 }
0359
0360
0361
0362 Error addIRTranslator(AddMachinePass &) const {
0363 return make_error<StringError>("addIRTranslator is not overridden",
0364 inconvertibleErrorCode());
0365 }
0366
0367
0368
0369 void addPreLegalizeMachineIR(AddMachinePass &) const {}
0370
0371
0372
0373 Error addLegalizeMachineIR(AddMachinePass &) const {
0374 return make_error<StringError>("addLegalizeMachineIR is not overridden",
0375 inconvertibleErrorCode());
0376 }
0377
0378
0379
0380 void addPreRegBankSelect(AddMachinePass &) const {}
0381
0382
0383
0384
0385 Error addRegBankSelect(AddMachinePass &) const {
0386 return make_error<StringError>("addRegBankSelect is not overridden",
0387 inconvertibleErrorCode());
0388 }
0389
0390
0391
0392 void addPreGlobalInstructionSelect(AddMachinePass &) const {}
0393
0394
0395
0396
0397
0398 Error addGlobalInstructionSelect(AddMachinePass &) const {
0399 return make_error<StringError>(
0400 "addGlobalInstructionSelect is not overridden",
0401 inconvertibleErrorCode());
0402 }
0403
0404
0405
0406
0407
0408
0409 void addISelPasses(AddIRPass &) const;
0410
0411
0412
0413 Error addCoreISelPasses(AddMachinePass &) const;
0414
0415
0416
0417 Error addMachinePasses(AddMachinePass &) const;
0418
0419
0420 void addPassesToHandleExceptions(AddIRPass &) const;
0421
0422
0423
0424 void addIRPasses(AddIRPass &) const;
0425
0426
0427
0428 void addCodeGenPrepare(AddIRPass &) const;
0429
0430
0431
0432 void addISelPrepare(AddIRPass &) const;
0433
0434
0435
0436
0437
0438
0439
0440
0441
0442
0443 void addMachineSSAOptimization(AddMachinePass &) const;
0444
0445
0446
0447 Error addFastRegAlloc(AddMachinePass &) const;
0448
0449
0450
0451
0452 void addOptimizedRegAlloc(AddMachinePass &) const;
0453
0454
0455 void addMachineLateOptimization(AddMachinePass &) const;
0456
0457
0458
0459
0460 void addGCPasses(AddMachinePass &) const {}
0461
0462
0463 void addBlockPlacement(AddMachinePass &) const;
0464
0465 using CreateMCStreamer =
0466 std::function<Expected<std::unique_ptr<MCStreamer>>(MCContext &)>;
0467 void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const {
0468 llvm_unreachable("addAsmPrinter is not overridden");
0469 }
0470
0471
0472
0473
0474
0475
0476 void addTargetRegisterAllocator(AddMachinePass &, bool Optimized) const;
0477
0478
0479
0480 void addRegAllocPass(AddMachinePass &, bool Optimized) const;
0481
0482
0483
0484 Error addRegAssignmentFast(AddMachinePass &) const;
0485 Error addRegAssignmentOptimized(AddMachinePass &) const;
0486
0487
0488
0489 template <typename... PassTs> void disablePass() {
0490 BeforeCallbacks.emplace_back(
0491 [](StringRef Name) { return ((Name != PassTs::name()) && ...); });
0492 }
0493
0494
0495
0496 template <typename TargetPassT, typename InsertedPassT>
0497 void insertPass(InsertedPassT &&Pass) {
0498 AfterCallbacks.emplace_back(
0499 [&](StringRef Name, MachineFunctionPassManager &MFPM) mutable {
0500 if (Name == TargetPassT::name())
0501 MFPM.addPass(std::forward<InsertedPassT>(Pass));
0502 });
0503 }
0504
0505 private:
0506 DerivedT &derived() { return static_cast<DerivedT &>(*this); }
0507 const DerivedT &derived() const {
0508 return static_cast<const DerivedT &>(*this);
0509 }
0510
0511 bool runBeforeAdding(StringRef Name) const {
0512 bool ShouldAdd = true;
0513 for (auto &C : BeforeCallbacks)
0514 ShouldAdd &= C(Name);
0515 return ShouldAdd;
0516 }
0517
0518 void setStartStopPasses(const TargetPassConfig::StartStopInfo &Info) const;
0519
0520 Error verifyStartStop(const TargetPassConfig::StartStopInfo &Info) const;
0521
0522 mutable SmallVector<llvm::unique_function<bool(StringRef)>, 4>
0523 BeforeCallbacks;
0524 mutable SmallVector<
0525 llvm::unique_function<void(StringRef, MachineFunctionPassManager &)>, 4>
0526 AfterCallbacks;
0527
0528
0529 mutable bool Started = true;
0530 mutable bool Stopped = true;
0531 };
0532
0533 template <typename Derived, typename TargetMachineT>
0534 Error CodeGenPassBuilder<Derived, TargetMachineT>::buildPipeline(
0535 ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
0536 CodeGenFileType FileType) const {
0537 auto StartStopInfo = TargetPassConfig::getStartStopInfo(*PIC);
0538 if (!StartStopInfo)
0539 return StartStopInfo.takeError();
0540 setStartStopPasses(*StartStopInfo);
0541
0542 bool PrintAsm = TargetPassConfig::willCompleteCodeGenPipeline();
0543 bool PrintMIR = !PrintAsm && FileType != CodeGenFileType::Null;
0544
0545 {
0546 AddIRPass addIRPass(MPM, derived());
0547 addIRPass(RequireAnalysisPass<MachineModuleAnalysis, Module>());
0548 addIRPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
0549 addIRPass(RequireAnalysisPass<CollectorMetadataAnalysis, Module>());
0550 addISelPasses(addIRPass);
0551 }
0552
0553 AddMachinePass addPass(MPM, derived());
0554
0555 if (PrintMIR)
0556 addPass(PrintMIRPreparePass(Out), true);
0557
0558 if (auto Err = addCoreISelPasses(addPass))
0559 return std::move(Err);
0560
0561 if (auto Err = derived().addMachinePasses(addPass))
0562 return std::move(Err);
0563
0564 if (!Opt.DisableVerify)
0565 addPass(MachineVerifierPass());
0566
0567 if (PrintAsm) {
0568 derived().addAsmPrinter(
0569 addPass, [this, &Out, DwoOut, FileType](MCContext &Ctx) {
0570 return this->TM.createMCStreamer(Out, DwoOut, FileType, Ctx);
0571 });
0572 }
0573
0574 if (PrintMIR)
0575 addPass(PrintMIRPass(Out), true);
0576
0577 return verifyStartStop(*StartStopInfo);
0578 }
0579
0580 template <typename Derived, typename TargetMachineT>
0581 void CodeGenPassBuilder<Derived, TargetMachineT>::setStartStopPasses(
0582 const TargetPassConfig::StartStopInfo &Info) const {
0583 if (!Info.StartPass.empty()) {
0584 Started = false;
0585 BeforeCallbacks.emplace_back([this, &Info, AfterFlag = Info.StartAfter,
0586 Count = 0u](StringRef ClassName) mutable {
0587 if (Count == Info.StartInstanceNum) {
0588 if (AfterFlag) {
0589 AfterFlag = false;
0590 Started = true;
0591 }
0592 return Started;
0593 }
0594
0595 auto PassName = PIC->getPassNameForClassName(ClassName);
0596 if (Info.StartPass == PassName && ++Count == Info.StartInstanceNum)
0597 Started = !Info.StartAfter;
0598
0599 return Started;
0600 });
0601 }
0602
0603 if (!Info.StopPass.empty()) {
0604 Stopped = false;
0605 BeforeCallbacks.emplace_back([this, &Info, AfterFlag = Info.StopAfter,
0606 Count = 0u](StringRef ClassName) mutable {
0607 if (Count == Info.StopInstanceNum) {
0608 if (AfterFlag) {
0609 AfterFlag = false;
0610 Stopped = true;
0611 }
0612 return !Stopped;
0613 }
0614
0615 auto PassName = PIC->getPassNameForClassName(ClassName);
0616 if (Info.StopPass == PassName && ++Count == Info.StopInstanceNum)
0617 Stopped = !Info.StopAfter;
0618 return !Stopped;
0619 });
0620 }
0621 }
0622
0623 template <typename Derived, typename TargetMachineT>
0624 Error CodeGenPassBuilder<Derived, TargetMachineT>::verifyStartStop(
0625 const TargetPassConfig::StartStopInfo &Info) const {
0626 if (Started && Stopped)
0627 return Error::success();
0628
0629 if (!Started)
0630 return make_error<StringError>(
0631 "Can't find start pass \"" + Info.StartPass + "\".",
0632 std::make_error_code(std::errc::invalid_argument));
0633 if (!Stopped)
0634 return make_error<StringError>(
0635 "Can't find stop pass \"" + Info.StopPass + "\".",
0636 std::make_error_code(std::errc::invalid_argument));
0637 return Error::success();
0638 }
0639
0640 template <typename Derived, typename TargetMachineT>
0641 void CodeGenPassBuilder<Derived, TargetMachineT>::addISelPasses(
0642 AddIRPass &addPass) const {
0643 derived().addGlobalMergePass(addPass);
0644 if (TM.useEmulatedTLS())
0645 addPass(LowerEmuTLSPass());
0646
0647 addPass(PreISelIntrinsicLoweringPass(&TM));
0648 addPass(ExpandLargeDivRemPass(&TM));
0649 addPass(ExpandLargeFpConvertPass(&TM));
0650
0651 derived().addIRPasses(addPass);
0652 derived().addCodeGenPrepare(addPass);
0653 addPassesToHandleExceptions(addPass);
0654 derived().addISelPrepare(addPass);
0655 }
0656
0657
0658
0659 template <typename Derived, typename TargetMachineT>
0660 void CodeGenPassBuilder<Derived, TargetMachineT>::addIRPasses(
0661 AddIRPass &addPass) const {
0662
0663
0664 if (!Opt.DisableVerify)
0665 addPass(VerifierPass());
0666
0667
0668 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableLSR) {
0669 addPass(createFunctionToLoopPassAdaptor(LoopStrengthReducePass(),
0670 true));
0671 }
0672
0673 if (getOptLevel() != CodeGenOptLevel::None) {
0674
0675
0676
0677
0678 if (!Opt.DisableMergeICmps)
0679 addPass(MergeICmpsPass());
0680 addPass(ExpandMemCmpPass(&TM));
0681 }
0682
0683
0684
0685 addPass(GCLoweringPass());
0686 addPass(ShadowStackGCLoweringPass());
0687 addPass(LowerConstantIntrinsicsPass());
0688
0689
0690 addPass(UnreachableBlockElimPass());
0691
0692
0693 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableConstantHoisting)
0694 addPass(ConstantHoistingPass());
0695
0696
0697
0698 if (getOptLevel() != CodeGenOptLevel::None)
0699 addPass(ReplaceWithVeclib());
0700
0701 if (getOptLevel() != CodeGenOptLevel::None &&
0702 !Opt.DisablePartialLibcallInlining)
0703 addPass(PartiallyInlineLibCallsPass());
0704
0705
0706 addPass(EntryExitInstrumenterPass(true));
0707
0708
0709
0710
0711 addPass(ScalarizeMaskedMemIntrinPass());
0712
0713
0714 addPass(ExpandReductionsPass());
0715
0716
0717 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize)
0718 addPass(SelectOptimizePass(&TM));
0719
0720 if (Opt.EnableGlobalMergeFunc)
0721 addPass(GlobalMergeFuncPass());
0722 }
0723
0724
0725
0726 template <typename Derived, typename TargetMachineT>
0727 void CodeGenPassBuilder<Derived, TargetMachineT>::addPassesToHandleExceptions(
0728 AddIRPass &addPass) const {
0729 const MCAsmInfo *MCAI = TM.getMCAsmInfo();
0730 assert(MCAI && "No MCAsmInfo");
0731 switch (MCAI->getExceptionHandlingType()) {
0732 case ExceptionHandling::SjLj:
0733
0734
0735
0736
0737
0738
0739 addPass(SjLjEHPreparePass(&TM));
0740 [[fallthrough]];
0741 case ExceptionHandling::DwarfCFI:
0742 case ExceptionHandling::ARM:
0743 case ExceptionHandling::AIX:
0744 case ExceptionHandling::ZOS:
0745 addPass(DwarfEHPreparePass(&TM));
0746 break;
0747 case ExceptionHandling::WinEH:
0748
0749
0750
0751 addPass(WinEHPreparePass());
0752 addPass(DwarfEHPreparePass(&TM));
0753 break;
0754 case ExceptionHandling::Wasm:
0755
0756
0757
0758
0759 addPass(WinEHPreparePass(false));
0760 addPass(WasmEHPreparePass());
0761 break;
0762 case ExceptionHandling::None:
0763 addPass(LowerInvokePass());
0764
0765
0766 addPass(UnreachableBlockElimPass());
0767 break;
0768 }
0769 }
0770
0771
0772
0773 template <typename Derived, typename TargetMachineT>
0774 void CodeGenPassBuilder<Derived, TargetMachineT>::addCodeGenPrepare(
0775 AddIRPass &addPass) const {
0776 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP)
0777 addPass(CodeGenPreparePass(&TM));
0778
0779
0780 }
0781
0782
0783
0784 template <typename Derived, typename TargetMachineT>
0785 void CodeGenPassBuilder<Derived, TargetMachineT>::addISelPrepare(
0786 AddIRPass &addPass) const {
0787 derived().addPreISel(addPass);
0788
0789 addPass(CallBrPreparePass());
0790
0791
0792 addPass(SafeStackPass(&TM));
0793 addPass(StackProtectorPass(&TM));
0794
0795 if (Opt.PrintISelInput)
0796 addPass(PrintFunctionPass(dbgs(),
0797 "\n\n*** Final LLVM Code input to ISel ***\n"));
0798
0799
0800
0801 if (!Opt.DisableVerify)
0802 addPass(VerifierPass());
0803 }
0804
0805 template <typename Derived, typename TargetMachineT>
0806 Error CodeGenPassBuilder<Derived, TargetMachineT>::addCoreISelPasses(
0807 AddMachinePass &addPass) const {
0808
0809 TM.setO0WantsFastISel(Opt.EnableFastISelOption.value_or(true));
0810
0811
0812 enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
0813 SelectorType Selector;
0814
0815 if (Opt.EnableFastISelOption && *Opt.EnableFastISelOption == true)
0816 Selector = SelectorType::FastISel;
0817 else if ((Opt.EnableGlobalISelOption &&
0818 *Opt.EnableGlobalISelOption == true) ||
0819 (TM.Options.EnableGlobalISel &&
0820 (!Opt.EnableGlobalISelOption ||
0821 *Opt.EnableGlobalISelOption == false)))
0822 Selector = SelectorType::GlobalISel;
0823 else if (TM.getOptLevel() == CodeGenOptLevel::None && TM.getO0WantsFastISel())
0824 Selector = SelectorType::FastISel;
0825 else
0826 Selector = SelectorType::SelectionDAG;
0827
0828
0829 if (Selector == SelectorType::FastISel) {
0830 TM.setFastISel(true);
0831 TM.setGlobalISel(false);
0832 } else if (Selector == SelectorType::GlobalISel) {
0833 TM.setFastISel(false);
0834 TM.setGlobalISel(true);
0835 }
0836
0837
0838 if (Selector == SelectorType::GlobalISel) {
0839 if (auto Err = derived().addIRTranslator(addPass))
0840 return std::move(Err);
0841
0842 derived().addPreLegalizeMachineIR(addPass);
0843
0844 if (auto Err = derived().addLegalizeMachineIR(addPass))
0845 return std::move(Err);
0846
0847
0848
0849 derived().addPreRegBankSelect(addPass);
0850
0851 if (auto Err = derived().addRegBankSelect(addPass))
0852 return std::move(Err);
0853
0854 derived().addPreGlobalInstructionSelect(addPass);
0855
0856 if (auto Err = derived().addGlobalInstructionSelect(addPass))
0857 return std::move(Err);
0858
0859
0860 addPass(ResetMachineFunctionPass(reportDiagnosticWhenGlobalISelFallback(),
0861 isGlobalISelAbortEnabled()));
0862
0863
0864
0865 if (!isGlobalISelAbortEnabled())
0866 if (auto Err = derived().addInstSelector(addPass))
0867 return std::move(Err);
0868
0869 } else if (auto Err = derived().addInstSelector(addPass))
0870 return std::move(Err);
0871
0872
0873
0874 addPass(FinalizeISelPass());
0875
0876
0877
0878
0879 return Error::success();
0880 }
0881
0882
0883
0884
0885
0886
0887
0888
0889
0890
0891
0892
0893
0894
0895
0896
0897
0898 template <typename Derived, typename TargetMachineT>
0899 Error CodeGenPassBuilder<Derived, TargetMachineT>::addMachinePasses(
0900 AddMachinePass &addPass) const {
0901
0902 if (getOptLevel() != CodeGenOptLevel::None) {
0903 derived().addMachineSSAOptimization(addPass);
0904 } else {
0905
0906
0907 addPass(LocalStackSlotAllocationPass());
0908 }
0909
0910 if (TM.Options.EnableIPRA) {
0911 addPass(RequireAnalysisPass<PhysicalRegisterUsageAnalysis, Module>());
0912 addPass(RegUsageInfoPropagationPass());
0913 }
0914
0915 derived().addPreRegAlloc(addPass);
0916
0917
0918
0919 if (*Opt.OptimizeRegAlloc) {
0920 derived().addOptimizedRegAlloc(addPass);
0921 } else {
0922 if (auto Err = derived().addFastRegAlloc(addPass))
0923 return Err;
0924 }
0925
0926
0927 derived().addPostRegAlloc(addPass);
0928
0929 addPass(RemoveRedundantDebugValuesPass());
0930
0931
0932 if (getOptLevel() != CodeGenOptLevel::None) {
0933 addPass(PostRAMachineSinkingPass());
0934 addPass(ShrinkWrapPass());
0935 }
0936
0937 addPass(PrologEpilogInserterPass());
0938
0939
0940 if (getOptLevel() != CodeGenOptLevel::None)
0941 derived().addMachineLateOptimization(addPass);
0942
0943
0944 addPass(ExpandPostRAPseudosPass());
0945
0946
0947 derived().addPreSched2(addPass);
0948
0949 if (Opt.EnableImplicitNullChecks)
0950 addPass(ImplicitNullChecksPass());
0951
0952
0953
0954
0955 if (getOptLevel() != CodeGenOptLevel::None &&
0956 !TM.targetSchedulesPostRAScheduling()) {
0957 if (Opt.MISchedPostRA)
0958 addPass(PostMachineSchedulerPass());
0959 else
0960 addPass(PostRASchedulerPass());
0961 }
0962
0963
0964 derived().addGCPasses(addPass);
0965
0966
0967 if (getOptLevel() != CodeGenOptLevel::None)
0968 derived().addBlockPlacement(addPass);
0969
0970
0971 addPass(FEntryInserterPass());
0972
0973 addPass(XRayInstrumentationPass());
0974 addPass(PatchableFunctionPass());
0975
0976 derived().addPreEmitPass(addPass);
0977
0978 if (TM.Options.EnableIPRA)
0979
0980
0981 addPass(RegUsageInfoCollectorPass());
0982
0983 addPass(FuncletLayoutPass());
0984
0985 addPass(StackMapLivenessPass());
0986 addPass(LiveDebugValuesPass());
0987 addPass(MachineSanitizerBinaryMetadata());
0988
0989 if (TM.Options.EnableMachineOutliner &&
0990 getOptLevel() != CodeGenOptLevel::None &&
0991 Opt.EnableMachineOutliner != RunOutliner::NeverOutline) {
0992 bool RunOnAllFunctions =
0993 (Opt.EnableMachineOutliner == RunOutliner::AlwaysOutline);
0994 bool AddOutliner = RunOnAllFunctions || TM.Options.SupportsDefaultOutlining;
0995 if (AddOutliner)
0996 addPass(MachineOutlinerPass(RunOnAllFunctions));
0997 }
0998
0999
1000 derived().addPreEmitPass2(addPass);
1001
1002 return Error::success();
1003 }
1004
1005
1006 template <typename Derived, typename TargetMachineT>
1007 void CodeGenPassBuilder<Derived, TargetMachineT>::addMachineSSAOptimization(
1008 AddMachinePass &addPass) const {
1009
1010 addPass(EarlyTailDuplicatePass());
1011
1012
1013
1014 addPass(OptimizePHIsPass());
1015
1016
1017
1018 addPass(StackColoringPass());
1019
1020
1021
1022 addPass(LocalStackSlotAllocationPass());
1023
1024
1025
1026
1027
1028 addPass(DeadMachineInstructionElimPass());
1029
1030
1031
1032
1033 derived().addILPOpts(addPass);
1034
1035 addPass(EarlyMachineLICMPass());
1036 addPass(MachineCSEPass());
1037
1038 addPass(MachineSinkingPass());
1039
1040 addPass(PeepholeOptimizerPass());
1041
1042
1043 addPass(DeadMachineInstructionElimPass());
1044 }
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058 template <typename Derived, typename TargetMachineT>
1059 void CodeGenPassBuilder<Derived, TargetMachineT>::addTargetRegisterAllocator(
1060 AddMachinePass &addPass, bool Optimized) const {
1061 if (Optimized)
1062 addPass(RAGreedyPass());
1063 else
1064 addPass(RegAllocFastPass());
1065 }
1066
1067
1068
1069
1070 template <typename Derived, typename TargetMachineT>
1071 void CodeGenPassBuilder<Derived, TargetMachineT>::addRegAllocPass(
1072 AddMachinePass &addPass, bool Optimized) const {
1073
1074 }
1075
1076 template <typename Derived, typename TargetMachineT>
1077 Error CodeGenPassBuilder<Derived, TargetMachineT>::addRegAssignmentFast(
1078 AddMachinePass &addPass) const {
1079
1080 addRegAllocPass(addPass, false);
1081 return Error::success();
1082 }
1083
1084 template <typename Derived, typename TargetMachineT>
1085 Error CodeGenPassBuilder<Derived, TargetMachineT>::addRegAssignmentOptimized(
1086 AddMachinePass &addPass) const {
1087
1088 addRegAllocPass(addPass, true);
1089
1090
1091 derived().addPreRewrite(addPass);
1092
1093
1094 addPass(VirtRegRewriterPass());
1095
1096
1097
1098
1099 addPass(StackSlotColoringPass());
1100
1101 return Error::success();
1102 }
1103
1104
1105
1106 template <typename Derived, typename TargetMachineT>
1107 Error CodeGenPassBuilder<Derived, TargetMachineT>::addFastRegAlloc(
1108 AddMachinePass &addPass) const {
1109 addPass(PHIEliminationPass());
1110 addPass(TwoAddressInstructionPass());
1111 return derived().addRegAssignmentFast(addPass);
1112 }
1113
1114
1115
1116
1117 template <typename Derived, typename TargetMachineT>
1118 void CodeGenPassBuilder<Derived, TargetMachineT>::addOptimizedRegAlloc(
1119 AddMachinePass &addPass) const {
1120 addPass(DetectDeadLanesPass());
1121
1122 addPass(InitUndefPass());
1123
1124 addPass(ProcessImplicitDefsPass());
1125
1126
1127 addPass(PHIEliminationPass());
1128
1129
1130 if (Opt.EarlyLiveIntervals)
1131 addPass(RequireAnalysisPass<LiveIntervalsAnalysis, MachineFunction>());
1132
1133 addPass(TwoAddressInstructionPass());
1134 addPass(RegisterCoalescerPass());
1135
1136
1137
1138
1139 addPass(RenameIndependentSubregsPass());
1140
1141
1142 addPass(MachineSchedulerPass());
1143
1144 if (derived().addRegAssignmentOptimized(addPass)) {
1145
1146
1147 derived().addPostRewrite(addPass);
1148
1149
1150
1151 addPass(MachineCopyPropagationPass());
1152
1153
1154
1155
1156 addPass(MachineLICMPass());
1157 }
1158 }
1159
1160
1161
1162
1163
1164
1165 template <typename Derived, typename TargetMachineT>
1166 void CodeGenPassBuilder<Derived, TargetMachineT>::addMachineLateOptimization(
1167 AddMachinePass &addPass) const {
1168
1169 addPass(BranchFolderPass());
1170
1171
1172
1173
1174
1175 if (!TM.requiresStructuredCFG())
1176 addPass(TailDuplicatePass());
1177
1178
1179 addPass(MachineLateInstrsCleanupPass());
1180
1181
1182 addPass(MachineCopyPropagationPass());
1183 }
1184
1185
1186 template <typename Derived, typename TargetMachineT>
1187 void CodeGenPassBuilder<Derived, TargetMachineT>::addBlockPlacement(
1188 AddMachinePass &addPass) const {
1189 addPass(MachineBlockPlacementPass());
1190
1191 if (Opt.EnableBlockPlacementStats)
1192 addPass(MachineBlockPlacementStatsPass());
1193 }
1194
1195 }
1196
1197 #endif