EIC code displayed by LXR master/​include/​llvm/​CodeGen/​BasicTTIImpl.h	Source navigation Diff markup Identifier search General search
	Version: master test Revert   Change

File indexing completed on 2026-05-10 08:43:27

 //===- BasicTTIImpl.h -------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 /// \file
 /// This file provides a helper that implements much of the TTI interface in
 /// terms of the target-independent code generator and TargetLowering
 /// interfaces.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CODEGEN_BASICTTIIMPL_H
 #define LLVM_CODEGEN_BASICTTIIMPL_H
 
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/TargetTransformInfoImpl.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/CodeGenTypes/MachineValueType.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
 #include <limits>
 #include <optional>
 #include <utility>
 
 namespace llvm {
 
 class Function;
 class GlobalValue;
 class LLVMContext;
 class ScalarEvolution;
 class SCEV;
 class TargetMachine;
 
 extern cl::opt<unsigned> PartialUnrollingThreshold;
 
 /// Base class which can be used to help build a TTI implementation.
 ///
 /// This class provides as much implementation of the TTI interface as is
 /// possible using the target independent parts of the code generator.
 ///
 /// In order to subclass it, your class must implement a getST() method to
 /// return the subtarget, and a getTLI() method to return the target lowering.
 /// We need these methods implemented in the derived class so that this class
 /// doesn't have to duplicate storage for them.
 template <typename T>
 class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 private:
   using BaseT = TargetTransformInfoImplCRTPBase<T>;
   using TTI = TargetTransformInfo;
 
   /// Helper function to access this as a T.
   T *thisT() { return static_cast<T *>(this); }
 
   /// Estimate a cost of Broadcast as an extract and sequence of insert
   /// operations.
   InstructionCost getBroadcastShuffleOverhead(FixedVectorType *VTy,
                                               TTI::TargetCostKind CostKind) {
     InstructionCost Cost = 0;
     // Broadcast cost is equal to the cost of extracting the zero'th element
     // plus the cost of inserting it into every element of the result vector.
     Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, VTy,
                                         CostKind, 0, nullptr, nullptr);
 
     for (int i = 0, e = VTy->getNumElements(); i < e; ++i) {
       Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, VTy,
                                           CostKind, i, nullptr, nullptr);
     }
     return Cost;
   }
 
   /// Estimate a cost of shuffle as a sequence of extract and insert
   /// operations.
   InstructionCost getPermuteShuffleOverhead(FixedVectorType *VTy,
                                             TTI::TargetCostKind CostKind) {
     InstructionCost Cost = 0;
     // Shuffle cost is equal to the cost of extracting element from its argument
     // plus the cost of inserting them onto the result vector.
 
     // e.g. <4 x float> has a mask of <0,5,2,7> i.e we need to extract from
     // index 0 of first vector, index 1 of second vector,index 2 of first
     // vector and finally index 3 of second vector and insert them at index
     // <0,1,2,3> of result vector.
     for (int i = 0, e = VTy->getNumElements(); i < e; ++i) {
       Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, VTy,
                                           CostKind, i, nullptr, nullptr);
       Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, VTy,
                                           CostKind, i, nullptr, nullptr);
     }
     return Cost;
   }
 
   /// Estimate a cost of subvector extraction as a sequence of extract and
   /// insert operations.
   InstructionCost getExtractSubvectorOverhead(VectorType *VTy,
                                               TTI::TargetCostKind CostKind,
                                               int Index,
                                               FixedVectorType *SubVTy) {
     assert(VTy && SubVTy &&
            "Can only extract subvectors from vectors");
     int NumSubElts = SubVTy->getNumElements();
     assert((!isa<FixedVectorType>(VTy) ||
             (Index + NumSubElts) <=
                 (int)cast<FixedVectorType>(VTy)->getNumElements()) &&
            "SK_ExtractSubvector index out of range");
 
     InstructionCost Cost = 0;
     // Subvector extraction cost is equal to the cost of extracting element from
     // the source type plus the cost of inserting them into the result vector
     // type.
     for (int i = 0; i != NumSubElts; ++i) {
       Cost +=
           thisT()->getVectorInstrCost(Instruction::ExtractElement, VTy,
                                       CostKind, i + Index, nullptr, nullptr);
       Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, SubVTy,
                                           CostKind, i, nullptr, nullptr);
     }
     return Cost;
   }
 
   /// Estimate a cost of subvector insertion as a sequence of extract and
   /// insert operations.
   InstructionCost getInsertSubvectorOverhead(VectorType *VTy,
                                              TTI::TargetCostKind CostKind,
                                              int Index,
                                              FixedVectorType *SubVTy) {
     assert(VTy && SubVTy &&
            "Can only insert subvectors into vectors");
     int NumSubElts = SubVTy->getNumElements();
     assert((!isa<FixedVectorType>(VTy) ||
             (Index + NumSubElts) <=
                 (int)cast<FixedVectorType>(VTy)->getNumElements()) &&
            "SK_InsertSubvector index out of range");
 
     InstructionCost Cost = 0;
     // Subvector insertion cost is equal to the cost of extracting element from
     // the source type plus the cost of inserting them into the result vector
     // type.
     for (int i = 0; i != NumSubElts; ++i) {
       Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, SubVTy,
                                           CostKind, i, nullptr, nullptr);
       Cost +=
           thisT()->getVectorInstrCost(Instruction::InsertElement, VTy, CostKind,
                                       i + Index, nullptr, nullptr);
     }
     return Cost;
   }
 
   /// Local query method delegates up to T which *must* implement this!
   const TargetSubtargetInfo *getST() const {
     return static_cast<const T *>(this)->getST();
   }
 
   /// Local query method delegates up to T which *must* implement this!
   const TargetLoweringBase *getTLI() const {
     return static_cast<const T *>(this)->getTLI();
   }
 
   static ISD::MemIndexedMode getISDIndexedMode(TTI::MemIndexedMode M) {
     switch (M) {
       case TTI::MIM_Unindexed:
         return ISD::UNINDEXED;
       case TTI::MIM_PreInc:
         return ISD::PRE_INC;
       case TTI::MIM_PreDec:
         return ISD::PRE_DEC;
       case TTI::MIM_PostInc:
         return ISD::POST_INC;
       case TTI::MIM_PostDec:
         return ISD::POST_DEC;
     }
     llvm_unreachable("Unexpected MemIndexedMode");
   }
 
   InstructionCost getCommonMaskedMemoryOpCost(unsigned Opcode, Type *DataTy,
                                               Align Alignment,
                                               bool VariableMask,
                                               bool IsGatherScatter,
                                               TTI::TargetCostKind CostKind,
                                               unsigned AddressSpace = 0) {
     // We cannot scalarize scalable vectors, so return Invalid.
     if (isa<ScalableVectorType>(DataTy))
       return InstructionCost::getInvalid();
 
     auto *VT = cast<FixedVectorType>(DataTy);
     unsigned VF = VT->getNumElements();
 
     // Assume the target does not have support for gather/scatter operations
     // and provide a rough estimate.
     //
     // First, compute the cost of the individual memory operations.
     InstructionCost AddrExtractCost =
         IsGatherScatter ? getScalarizationOverhead(
                               FixedVectorType::get(
                                   PointerType::get(VT->getContext(), 0), VF),
                               /*Insert=*/false, /*Extract=*/true, CostKind)
                         : 0;
 
     // The cost of the scalar loads/stores.
     InstructionCost MemoryOpCost =
         VF * thisT()->getMemoryOpCost(Opcode, VT->getElementType(), Alignment,
                                       AddressSpace, CostKind);
 
     // Next, compute the cost of packing the result in a vector.
     InstructionCost PackingCost =
         getScalarizationOverhead(VT, Opcode != Instruction::Store,
                                  Opcode == Instruction::Store, CostKind);
 
     InstructionCost ConditionalCost = 0;
     if (VariableMask) {
       // Compute the cost of conditionally executing the memory operations with
       // variable masks. This includes extracting the individual conditions, a
       // branches and PHIs to combine the results.
       // NOTE: Estimating the cost of conditionally executing the memory
       // operations accurately is quite difficult and the current solution
       // provides a very rough estimate only.
       ConditionalCost =
           getScalarizationOverhead(
               FixedVectorType::get(Type::getInt1Ty(DataTy->getContext()), VF),
               /*Insert=*/false, /*Extract=*/true, CostKind) +
           VF * (thisT()->getCFInstrCost(Instruction::Br, CostKind) +
                 thisT()->getCFInstrCost(Instruction::PHI, CostKind));
     }
 
     return AddrExtractCost + MemoryOpCost + PackingCost + ConditionalCost;
   }
 
   /// Checks if the provided mask \p is a splat mask, i.e. it contains only -1
   /// or same non -1 index value and this index value contained at least twice.
   /// So, mask <0, -1,-1, -1> is not considered splat (it is just identity),
   /// same for <-1, 0, -1, -1> (just a slide), while <2, -1, 2, -1> is a splat
   /// with \p Index=2.
   static bool isSplatMask(ArrayRef<int> Mask, unsigned NumSrcElts, int &Index) {
     // Check that the broadcast index meets at least twice.
     bool IsCompared = false;
     if (int SplatIdx = PoisonMaskElem;
         all_of(enumerate(Mask), [&](const auto &P) {
           if (P.value() == PoisonMaskElem)
             return P.index() != Mask.size() - 1 || IsCompared;
           if (static_cast<unsigned>(P.value()) >= NumSrcElts * 2)
             return false;
           if (SplatIdx == PoisonMaskElem) {
             SplatIdx = P.value();
             return P.index() != Mask.size() - 1;
           }
           IsCompared = true;
           return SplatIdx == P.value();
         })) {
       Index = SplatIdx;
       return true;
     }
     return false;
   }
 
 protected:
   explicit BasicTTIImplBase(const TargetMachine *TM, const DataLayout &DL)
       : BaseT(DL) {}
   virtual ~BasicTTIImplBase() = default;
 
   using TargetTransformInfoImplBase::DL;
 
 public:
   /// \name Scalar TTI Implementations
   /// @{
   bool allowsMisalignedMemoryAccesses(LLVMContext &Context, unsigned BitWidth,
                                       unsigned AddressSpace, Align Alignment,
                                       unsigned *Fast) const {
     EVT E = EVT::getIntegerVT(Context, BitWidth);
     return getTLI()->allowsMisalignedMemoryAccesses(
         E, AddressSpace, Alignment, MachineMemOperand::MONone, Fast);
   }
 
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const {
     const TargetMachine &TM = getTLI()->getTargetMachine();
 
     const FeatureBitset &CallerBits =
         TM.getSubtargetImpl(*Caller)->getFeatureBits();
     const FeatureBitset &CalleeBits =
         TM.getSubtargetImpl(*Callee)->getFeatureBits();
 
     // Inline a callee if its target-features are a subset of the callers
     // target-features.
     return (CallerBits & CalleeBits) == CalleeBits;
   }
 
   bool hasBranchDivergence(const Function *F = nullptr) { return false; }
 
   bool isSourceOfDivergence(const Value *V) { return false; }
 
   bool isAlwaysUniform(const Value *V) { return false; }
 
   bool isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const {
     return false;
   }
 
   bool addrspacesMayAlias(unsigned AS0, unsigned AS1) const {
     return true;
   }
 
   unsigned getFlatAddressSpace() {
     // Return an invalid address space.
     return -1;
   }
 
   bool collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,
                                   Intrinsic::ID IID) const {
     return false;
   }
 
   bool isNoopAddrSpaceCast(unsigned FromAS, unsigned ToAS) const {
     return getTLI()->getTargetMachine().isNoopAddrSpaceCast(FromAS, ToAS);
   }
 
   unsigned getAssumedAddrSpace(const Value *V) const {
     return getTLI()->getTargetMachine().getAssumedAddrSpace(V);
   }
 
   bool isSingleThreaded() const {
     return getTLI()->getTargetMachine().Options.ThreadModel ==
            ThreadModel::Single;
   }
 
   std::pair<const Value *, unsigned>
   getPredicatedAddrSpace(const Value *V) const {
     return getTLI()->getTargetMachine().getPredicatedAddrSpace(V);
   }
 
   Value *rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, Value *OldV,
                                           Value *NewV) const {
     return nullptr;
   }
 
   bool isLegalAddImmediate(int64_t imm) {
     return getTLI()->isLegalAddImmediate(imm);
   }
 
   bool isLegalAddScalableImmediate(int64_t Imm) {
     return getTLI()->isLegalAddScalableImmediate(Imm);
   }
 
   bool isLegalICmpImmediate(int64_t imm) {
     return getTLI()->isLegalICmpImmediate(imm);
   }
 
   bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset,
                              bool HasBaseReg, int64_t Scale, unsigned AddrSpace,
                              Instruction *I = nullptr,
                              int64_t ScalableOffset = 0) {
     TargetLoweringBase::AddrMode AM;
     AM.BaseGV = BaseGV;
     AM.BaseOffs = BaseOffset;
     AM.HasBaseReg = HasBaseReg;
     AM.Scale = Scale;
     AM.ScalableOffset = ScalableOffset;
     return getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace, I);
   }
 
   int64_t getPreferredLargeGEPBaseOffset(int64_t MinOffset, int64_t MaxOffset) {
     return getTLI()->getPreferredLargeGEPBaseOffset(MinOffset, MaxOffset);
   }
 
   unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
                              Type *ScalarValTy) const {
     auto &&IsSupportedByTarget = [this, ScalarMemTy, ScalarValTy](unsigned VF) {
       auto *SrcTy = FixedVectorType::get(ScalarMemTy, VF / 2);
       EVT VT = getTLI()->getValueType(DL, SrcTy);
       if (getTLI()->isOperationLegal(ISD::STORE, VT) ||
           getTLI()->isOperationCustom(ISD::STORE, VT))
         return true;
 
       EVT ValVT =
           getTLI()->getValueType(DL, FixedVectorType::get(ScalarValTy, VF / 2));
       EVT LegalizedVT =
           getTLI()->getTypeToTransformTo(ScalarMemTy->getContext(), VT);
       return getTLI()->isTruncStoreLegal(LegalizedVT, ValVT);
     };
     while (VF > 2 && IsSupportedByTarget(VF))
       VF /= 2;
     return VF;
   }
 
   bool isIndexedLoadLegal(TTI::MemIndexedMode M, Type *Ty,
                           const DataLayout &DL) const {
     EVT VT = getTLI()->getValueType(DL, Ty);
     return getTLI()->isIndexedLoadLegal(getISDIndexedMode(M), VT);
   }
 
   bool isIndexedStoreLegal(TTI::MemIndexedMode M, Type *Ty,
                            const DataLayout &DL) const {
     EVT VT = getTLI()->getValueType(DL, Ty);
     return getTLI()->isIndexedStoreLegal(getISDIndexedMode(M), VT);
   }
 
   bool isLSRCostLess(TTI::LSRCost C1, TTI::LSRCost C2) {
     return TargetTransformInfoImplBase::isLSRCostLess(C1, C2);
   }
 
   bool isNumRegsMajorCostOfLSR() {
     return TargetTransformInfoImplBase::isNumRegsMajorCostOfLSR();
   }
 
   bool shouldDropLSRSolutionIfLessProfitable() const {
     return TargetTransformInfoImplBase::shouldDropLSRSolutionIfLessProfitable();
   }
 
   bool isProfitableLSRChainElement(Instruction *I) {
     return TargetTransformInfoImplBase::isProfitableLSRChainElement(I);
   }
 
   InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
                                        StackOffset BaseOffset, bool HasBaseReg,
                                        int64_t Scale, unsigned AddrSpace) {
     TargetLoweringBase::AddrMode AM;
     AM.BaseGV = BaseGV;
     AM.BaseOffs = BaseOffset.getFixed();
     AM.HasBaseReg = HasBaseReg;
     AM.Scale = Scale;
     AM.ScalableOffset = BaseOffset.getScalable();
     if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace))
       return 0;
     return -1;
   }
 
   bool isTruncateFree(Type *Ty1, Type *Ty2) {
     return getTLI()->isTruncateFree(Ty1, Ty2);
   }
 
   bool isProfitableToHoist(Instruction *I) {
     return getTLI()->isProfitableToHoist(I);
   }
 
   bool useAA() const { return getST()->useAA(); }
 
   bool isTypeLegal(Type *Ty) {
     EVT VT = getTLI()->getValueType(DL, Ty, /*AllowUnknown=*/true);
     return getTLI()->isTypeLegal(VT);
   }
 
   unsigned getRegUsageForType(Type *Ty) {
     EVT ETy = getTLI()->getValueType(DL, Ty);
     return getTLI()->getNumRegisters(Ty->getContext(), ETy);
   }
 
   InstructionCost getGEPCost(Type *PointeeType, const Value *Ptr,
                              ArrayRef<const Value *> Operands, Type *AccessType,
                              TTI::TargetCostKind CostKind) {
     return BaseT::getGEPCost(PointeeType, Ptr, Operands, AccessType, CostKind);
   }
 
   unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI,
                                             unsigned &JumpTableSize,
                                             ProfileSummaryInfo *PSI,
                                             BlockFrequencyInfo *BFI) {
     /// Try to find the estimated number of clusters. Note that the number of
     /// clusters identified in this function could be different from the actual
     /// numbers found in lowering. This function ignore switches that are
     /// lowered with a mix of jump table / bit test / BTree. This function was
     /// initially intended to be used when estimating the cost of switch in
     /// inline cost heuristic, but it's a generic cost model to be used in other
     /// places (e.g., in loop unrolling).
     unsigned N = SI.getNumCases();
     const TargetLoweringBase *TLI = getTLI();
     const DataLayout &DL = this->getDataLayout();
 
     JumpTableSize = 0;
     bool IsJTAllowed = TLI->areJTsAllowed(SI.getParent()->getParent());
 
     // Early exit if both a jump table and bit test are not allowed.
     if (N < 1 || (!IsJTAllowed && DL.getIndexSizeInBits(0u) < N))
       return N;
 
     APInt MaxCaseVal = SI.case_begin()->getCaseValue()->getValue();
     APInt MinCaseVal = MaxCaseVal;
     for (auto CI : SI.cases()) {
       const APInt &CaseVal = CI.getCaseValue()->getValue();
       if (CaseVal.sgt(MaxCaseVal))
         MaxCaseVal = CaseVal;
       if (CaseVal.slt(MinCaseVal))
         MinCaseVal = CaseVal;
     }
 
     // Check if suitable for a bit test
     if (N <= DL.getIndexSizeInBits(0u)) {
       SmallPtrSet<const BasicBlock *, 4> Dests;
       for (auto I : SI.cases())
         Dests.insert(I.getCaseSuccessor());
 
       if (TLI->isSuitableForBitTests(Dests.size(), N, MinCaseVal, MaxCaseVal,
                                      DL))
         return 1;
     }
 
     // Check if suitable for a jump table.
     if (IsJTAllowed) {
       if (N < 2 || N < TLI->getMinimumJumpTableEntries())
         return N;
       uint64_t Range =
           (MaxCaseVal - MinCaseVal)
               .getLimitedValue(std::numeric_limits<uint64_t>::max() - 1) + 1;
       // Check whether a range of clusters is dense enough for a jump table
       if (TLI->isSuitableForJumpTable(&SI, N, Range, PSI, BFI)) {
         JumpTableSize = Range;
         return 1;
       }
     }
     return N;
   }
 
   bool shouldBuildLookupTables() {
     const TargetLoweringBase *TLI = getTLI();
     return TLI->isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
            TLI->isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
   }
 
   bool shouldBuildRelLookupTables() const {
     const TargetMachine &TM = getTLI()->getTargetMachine();
     // If non-PIC mode, do not generate a relative lookup table.
     if (!TM.isPositionIndependent())
       return false;
 
     /// Relative lookup table entries consist of 32-bit offsets.
     /// Do not generate relative lookup tables for large code models
     /// in 64-bit achitectures where 32-bit offsets might not be enough.
     if (TM.getCodeModel() == CodeModel::Medium ||
         TM.getCodeModel() == CodeModel::Large)
       return false;
 
     const Triple &TargetTriple = TM.getTargetTriple();
     if (!TargetTriple.isArch64Bit())
       return false;
 
     // TODO: Triggers issues on aarch64 on darwin, so temporarily disable it
     // there.
     if (TargetTriple.getArch() == Triple::aarch64 && TargetTriple.isOSDarwin())
       return false;
 
     return true;
   }
 
   bool haveFastSqrt(Type *Ty) {
     const TargetLoweringBase *TLI = getTLI();
     EVT VT = TLI->getValueType(DL, Ty);
     return TLI->isTypeLegal(VT) &&
            TLI->isOperationLegalOrCustom(ISD::FSQRT, VT);
   }
 
   bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) {
     return true;
   }
 
   InstructionCost getFPOpCost(Type *Ty) {
     // Check whether FADD is available, as a proxy for floating-point in
     // general.
     const TargetLoweringBase *TLI = getTLI();
     EVT VT = TLI->getValueType(DL, Ty);
     if (TLI->isOperationLegalOrCustomOrPromote(ISD::FADD, VT))
       return TargetTransformInfo::TCC_Basic;
     return TargetTransformInfo::TCC_Expensive;
   }
 
   bool preferToKeepConstantsAttached(const Instruction &Inst,
                                      const Function &Fn) const {
     switch (Inst.getOpcode()) {
     default:
       break;
     case Instruction::SDiv:
     case Instruction::SRem:
     case Instruction::UDiv:
     case Instruction::URem: {
       if (!isa<ConstantInt>(Inst.getOperand(1)))
         return false;
       EVT VT = getTLI()->getValueType(DL, Inst.getType());
       return !getTLI()->isIntDivCheap(VT, Fn.getAttributes());
     }
     };
 
     return false;
   }
 
   unsigned getInliningThresholdMultiplier() const { return 1; }
   unsigned adjustInliningThreshold(const CallBase *CB) { return 0; }
   unsigned getCallerAllocaCost(const CallBase *CB, const AllocaInst *AI) const {
     return 0;
   }
 
   int getInlinerVectorBonusPercent() const { return 150; }
 
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP,
                                OptimizationRemarkEmitter *ORE) {
     // This unrolling functionality is target independent, but to provide some
     // motivation for its intended use, for x86:
 
     // According to the Intel 64 and IA-32 Architectures Optimization Reference
     // Manual, Intel Core models and later have a loop stream detector (and
     // associated uop queue) that can benefit from partial unrolling.
     // The relevant requirements are:
     //  - The loop must have no more than 4 (8 for Nehalem and later) branches
     //    taken, and none of them may be calls.
     //  - The loop can have no more than 18 (28 for Nehalem and later) uops.
 
     // According to the Software Optimization Guide for AMD Family 15h
     // Processors, models 30h-4fh (Steamroller and later) have a loop predictor
     // and loop buffer which can benefit from partial unrolling.
     // The relevant requirements are:
     //  - The loop must have fewer than 16 branches
     //  - The loop must have less than 40 uops in all executed loop branches
 
     // The number of taken branches in a loop is hard to estimate here, and
     // benchmarking has revealed that it is better not to be conservative when
     // estimating the branch count. As a result, we'll ignore the branch limits
     // until someone finds a case where it matters in practice.
 
     unsigned MaxOps;
     const TargetSubtargetInfo *ST = getST();
     if (PartialUnrollingThreshold.getNumOccurrences() > 0)
       MaxOps = PartialUnrollingThreshold;
     else if (ST->getSchedModel().LoopMicroOpBufferSize > 0)
       MaxOps = ST->getSchedModel().LoopMicroOpBufferSize;
     else
       return;
 
     // Scan the loop: don't unroll loops with calls.
     for (BasicBlock *BB : L->blocks()) {
       for (Instruction &I : *BB) {
         if (isa<CallInst>(I) || isa<InvokeInst>(I)) {
           if (const Function *F = cast<CallBase>(I).getCalledFunction()) {
             if (!thisT()->isLoweredToCall(F))
               continue;
           }
 
           if (ORE) {
             ORE->emit([&]() {
               return OptimizationRemark("TTI", "DontUnroll", L->getStartLoc(),
                                         L->getHeader())
                      << "advising against unrolling the loop because it "
                         "contains a "
                      << ore::NV("Call", &I);
             });
           }
           return;
         }
       }
     }
 
     // Enable runtime and partial unrolling up to the specified size.
     // Enable using trip count upper bound to unroll loops.
     UP.Partial = UP.Runtime = UP.UpperBound = true;
     UP.PartialThreshold = MaxOps;
 
     // Avoid unrolling when optimizing for size.
     UP.OptSizeThreshold = 0;
     UP.PartialOptSizeThreshold = 0;
 
     // Set number of instructions optimized when "back edge"
     // becomes "fall through" to default value of 2.
     UP.BEInsns = 2;
   }
 
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
                              TTI::PeelingPreferences &PP) {
     PP.PeelCount = 0;
     PP.AllowPeeling = true;
     PP.AllowLoopNestsPeeling = false;
     PP.PeelProfiledIterations = true;
   }
 
   bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
                                 AssumptionCache &AC,
                                 TargetLibraryInfo *LibInfo,
                                 HardwareLoopInfo &HWLoopInfo) {
     return BaseT::isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);
   }
 
   unsigned getEpilogueVectorizationMinVF() {
     return BaseT::getEpilogueVectorizationMinVF();
   }
 
   bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) {
     return BaseT::preferPredicateOverEpilogue(TFI);
   }
 
   TailFoldingStyle
   getPreferredTailFoldingStyle(bool IVUpdateMayOverflow = true) {
     return BaseT::getPreferredTailFoldingStyle(IVUpdateMayOverflow);
   }
 
   std::optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
                                                IntrinsicInst &II) {
     return BaseT::instCombineIntrinsic(IC, II);
   }
 
   std::optional<Value *>
   simplifyDemandedUseBitsIntrinsic(InstCombiner &IC, IntrinsicInst &II,
                                    APInt DemandedMask, KnownBits &Known,
                                    bool &KnownBitsComputed) {
     return BaseT::simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known,
                                                    KnownBitsComputed);
   }
 
   std::optional<Value *> simplifyDemandedVectorEltsIntrinsic(
       InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
       APInt &UndefElts2, APInt &UndefElts3,
       std::function<void(Instruction *, unsigned, APInt, APInt &)>
           SimplifyAndSetOp) {
     return BaseT::simplifyDemandedVectorEltsIntrinsic(
         IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3,
         SimplifyAndSetOp);
   }
 
   virtual std::optional<unsigned>
   getCacheSize(TargetTransformInfo::CacheLevel Level) const {
     return std::optional<unsigned>(
         getST()->getCacheSize(static_cast<unsigned>(Level)));
   }
 
   virtual std::optional<unsigned>
   getCacheAssociativity(TargetTransformInfo::CacheLevel Level) const {
     std::optional<unsigned> TargetResult =
         getST()->getCacheAssociativity(static_cast<unsigned>(Level));
 
     if (TargetResult)
       return TargetResult;
 
     return BaseT::getCacheAssociativity(Level);
   }
 
   virtual unsigned getCacheLineSize() const {
     return getST()->getCacheLineSize();
   }
 
   virtual unsigned getPrefetchDistance() const {
     return getST()->getPrefetchDistance();
   }
 
   virtual unsigned getMinPrefetchStride(unsigned NumMemAccesses,
                                         unsigned NumStridedMemAccesses,
                                         unsigned NumPrefetches,
                                         bool HasCall) const {
     return getST()->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses,
                                          NumPrefetches, HasCall);
   }
 
   virtual unsigned getMaxPrefetchIterationsAhead() const {
     return getST()->getMaxPrefetchIterationsAhead();
   }
 
   virtual bool enableWritePrefetching() const {
     return getST()->enableWritePrefetching();
   }
 
   virtual bool shouldPrefetchAddressSpace(unsigned AS) const {
     return getST()->shouldPrefetchAddressSpace(AS);
   }
 
   /// @}
 
   /// \name Vector TTI Implementations
   /// @{
 
   TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
     return TypeSize::getFixed(32);
   }
 
   std::optional<unsigned> getMaxVScale() const { return std::nullopt; }
   std::optional<unsigned> getVScaleForTuning() const { return std::nullopt; }
   bool isVScaleKnownToBeAPowerOfTwo() const { return false; }
 
   /// Estimate the overhead of scalarizing an instruction. Insert and Extract
   /// are set if the demanded result elements need to be inserted and/or
   /// extracted from vectors.
   InstructionCost getScalarizationOverhead(VectorType *InTy,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
                                            TTI::TargetCostKind CostKind,
                                            ArrayRef<Value *> VL = {}) {
     /// FIXME: a bitfield is not a reasonable abstraction for talking about
     /// which elements are needed from a scalable vector
     if (isa<ScalableVectorType>(InTy))
       return InstructionCost::getInvalid();
     auto *Ty = cast<FixedVectorType>(InTy);
 
     assert(DemandedElts.getBitWidth() == Ty->getNumElements() &&
            (VL.empty() || VL.size() == Ty->getNumElements()) &&
            "Vector size mismatch");
 
     InstructionCost Cost = 0;
 
     for (int i = 0, e = Ty->getNumElements(); i < e; ++i) {
       if (!DemandedElts[i])
         continue;
       if (Insert) {
         Value *InsertedVal = VL.empty() ? nullptr : VL[i];
         Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, Ty,
                                             CostKind, i, nullptr, InsertedVal);
       }
       if (Extract)
         Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, Ty,
                                             CostKind, i, nullptr, nullptr);
     }
 
     return Cost;
   }
 
   bool isTargetIntrinsicTriviallyScalarizable(Intrinsic::ID ID) const {
     return false;
   }
 
   bool isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
                                           unsigned ScalarOpdIdx) const {
     return false;
   }
 
   bool isTargetIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID,
                                               int OpdIdx) const {
     return OpdIdx == -1;
   }
 
   bool isTargetIntrinsicWithStructReturnOverloadAtField(Intrinsic::ID ID,
                                                         int RetIdx) const {
     return RetIdx == 0;
   }
 
   /// Helper wrapper for the DemandedElts variant of getScalarizationOverhead.
   InstructionCost getScalarizationOverhead(VectorType *InTy, bool Insert,
                                            bool Extract,
                                            TTI::TargetCostKind CostKind) {
     if (isa<ScalableVectorType>(InTy))
       return InstructionCost::getInvalid();
     auto *Ty = cast<FixedVectorType>(InTy);
 
     APInt DemandedElts = APInt::getAllOnes(Ty->getNumElements());
     return thisT()->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
                                              CostKind);
   }
 
   /// Estimate the overhead of scalarizing an instructions unique
   /// non-constant operands. The (potentially vector) types to use for each of
   /// argument are passes via Tys.
   InstructionCost
   getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,
                                    ArrayRef<Type *> Tys,
                                    TTI::TargetCostKind CostKind) {
     assert(Args.size() == Tys.size() && "Expected matching Args and Tys");
 
     InstructionCost Cost = 0;
     SmallPtrSet<const Value*, 4> UniqueOperands;
     for (int I = 0, E = Args.size(); I != E; I++) {
       // Disregard things like metadata arguments.
       const Value *A = Args[I];
       Type *Ty = Tys[I];
       if (!Ty->isIntOrIntVectorTy() && !Ty->isFPOrFPVectorTy() &&
           !Ty->isPtrOrPtrVectorTy())
         continue;
 
       if (!isa<Constant>(A) && UniqueOperands.insert(A).second) {
         if (auto *VecTy = dyn_cast<VectorType>(Ty))
           Cost += getScalarizationOverhead(VecTy, /*Insert*/ false,
                                            /*Extract*/ true, CostKind);
       }
     }
 
     return Cost;
   }
 
   /// Estimate the overhead of scalarizing the inputs and outputs of an
   /// instruction, with return type RetTy and arguments Args of type Tys. If
   /// Args are unknown (empty), then the cost associated with one argument is
   /// added as a heuristic.
   InstructionCost getScalarizationOverhead(VectorType *RetTy,
                                            ArrayRef<const Value *> Args,
                                            ArrayRef<Type *> Tys,
                                            TTI::TargetCostKind CostKind) {
     InstructionCost Cost = getScalarizationOverhead(
         RetTy, /*Insert*/ true, /*Extract*/ false, CostKind);
     if (!Args.empty())
       Cost += getOperandsScalarizationOverhead(Args, Tys, CostKind);
     else
       // When no information on arguments is provided, we add the cost
       // associated with one argument as a heuristic.
       Cost += getScalarizationOverhead(RetTy, /*Insert*/ false,
                                        /*Extract*/ true, CostKind);
 
     return Cost;
   }
 
   /// Estimate the cost of type-legalization and the legalized type.
   std::pair<InstructionCost, MVT> getTypeLegalizationCost(Type *Ty) const {
     LLVMContext &C = Ty->getContext();
     EVT MTy = getTLI()->getValueType(DL, Ty);
 
     InstructionCost Cost = 1;
     // We keep legalizing the type until we find a legal kind. We assume that
     // the only operation that costs anything is the split. After splitting
     // we need to handle two types.
     while (true) {
       TargetLoweringBase::LegalizeKind LK = getTLI()->getTypeConversion(C, MTy);
 
       if (LK.first == TargetLoweringBase::TypeScalarizeScalableVector) {
         // Ensure we return a sensible simple VT here, since many callers of
         // this function require it.
         MVT VT = MTy.isSimple() ? MTy.getSimpleVT() : MVT::i64;
         return std::make_pair(InstructionCost::getInvalid(), VT);
       }
 
       if (LK.first == TargetLoweringBase::TypeLegal)
         return std::make_pair(Cost, MTy.getSimpleVT());
 
       if (LK.first == TargetLoweringBase::TypeSplitVector ||
           LK.first == TargetLoweringBase::TypeExpandInteger)
         Cost *= 2;
 
       // Do not loop with f128 type.
       if (MTy == LK.second)
         return std::make_pair(Cost, MTy.getSimpleVT());
 
       // Keep legalizing the type.
       MTy = LK.second;
     }
   }
 
   unsigned getMaxInterleaveFactor(ElementCount VF) { return 1; }
 
   InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Opd1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Opd2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr) {
     // Check if any of the operands are vector operands.
     const TargetLoweringBase *TLI = getTLI();
     int ISD = TLI->InstructionOpcodeToISD(Opcode);
     assert(ISD && "Invalid opcode");
 
     // TODO: Handle more cost kinds.
     if (CostKind != TTI::TCK_RecipThroughput)
       return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind,
                                            Opd1Info, Opd2Info,
                                            Args, CxtI);
 
     std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
 
     bool IsFloat = Ty->isFPOrFPVectorTy();
     // Assume that floating point arithmetic operations cost twice as much as
     // integer operations.
     InstructionCost OpCost = (IsFloat ? 2 : 1);
 
     if (TLI->isOperationLegalOrPromote(ISD, LT.second)) {
       // The operation is legal. Assume it costs 1.
       // TODO: Once we have extract/insert subvector cost we need to use them.
       return LT.first * OpCost;
     }
 
     if (!TLI->isOperationExpand(ISD, LT.second)) {
       // If the operation is custom lowered, then assume that the code is twice
       // as expensive.
       return LT.first * 2 * OpCost;
     }
 
     // An 'Expand' of URem and SRem is special because it may default
     // to expanding the operation into a sequence of sub-operations
     // i.e. X % Y -> X-(X/Y)*Y.
     if (ISD == ISD::UREM || ISD == ISD::SREM) {
       bool IsSigned = ISD == ISD::SREM;
       if (TLI->isOperationLegalOrCustom(IsSigned ? ISD::SDIVREM : ISD::UDIVREM,
                                         LT.second) ||
           TLI->isOperationLegalOrCustom(IsSigned ? ISD::SDIV : ISD::UDIV,
                                         LT.second)) {
         unsigned DivOpc = IsSigned ? Instruction::SDiv : Instruction::UDiv;
         InstructionCost DivCost = thisT()->getArithmeticInstrCost(
             DivOpc, Ty, CostKind, Opd1Info, Opd2Info);
         InstructionCost MulCost =
             thisT()->getArithmeticInstrCost(Instruction::Mul, Ty, CostKind);
         InstructionCost SubCost =
             thisT()->getArithmeticInstrCost(Instruction::Sub, Ty, CostKind);
         return DivCost + MulCost + SubCost;
       }
     }
 
     // We cannot scalarize scalable vectors, so return Invalid.
     if (isa<ScalableVectorType>(Ty))
       return InstructionCost::getInvalid();
 
     // Else, assume that we need to scalarize this op.
     // TODO: If one of the types get legalized by splitting, handle this
     // similarly to what getCastInstrCost() does.
     if (auto *VTy = dyn_cast<FixedVectorType>(Ty)) {
       InstructionCost Cost = thisT()->getArithmeticInstrCost(
           Opcode, VTy->getScalarType(), CostKind, Opd1Info, Opd2Info,
           Args, CxtI);
       // Return the cost of multiple scalar invocation plus the cost of
       // inserting and extracting the values.
       SmallVector<Type *> Tys(Args.size(), Ty);
       return getScalarizationOverhead(VTy, Args, Tys, CostKind) +
              VTy->getNumElements() * Cost;
     }
 
     // We don't know anything about this scalar instruction.
     return OpCost;
   }
 
   TTI::ShuffleKind improveShuffleKindFromMask(TTI::ShuffleKind Kind,
                                               ArrayRef<int> Mask,
                                               VectorType *Ty, int &Index,
                                               VectorType *&SubTy) const {
     if (Mask.empty())
       return Kind;
     int NumSrcElts = Ty->getElementCount().getKnownMinValue();
     switch (Kind) {
     case TTI::SK_PermuteSingleSrc: {
       if (ShuffleVectorInst::isReverseMask(Mask, NumSrcElts))
         return TTI::SK_Reverse;
       if (ShuffleVectorInst::isZeroEltSplatMask(Mask, NumSrcElts))
         return TTI::SK_Broadcast;
       if (isSplatMask(Mask, NumSrcElts, Index))
         return TTI::SK_Broadcast;
       if (ShuffleVectorInst::isExtractSubvectorMask(Mask, NumSrcElts, Index) &&
           (Index + Mask.size()) <= (size_t)NumSrcElts) {
         SubTy = FixedVectorType::get(Ty->getElementType(), Mask.size());
         return TTI::SK_ExtractSubvector;
       }
       break;
     }
     case TTI::SK_PermuteTwoSrc: {
       int NumSubElts;
       if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask(
                                  Mask, NumSrcElts, NumSubElts, Index)) {
         if (Index + NumSubElts > NumSrcElts)
           return Kind;
         SubTy = FixedVectorType::get(Ty->getElementType(), NumSubElts);
         return TTI::SK_InsertSubvector;
       }
       if (ShuffleVectorInst::isSelectMask(Mask, NumSrcElts))
         return TTI::SK_Select;
       if (ShuffleVectorInst::isTransposeMask(Mask, NumSrcElts))
         return TTI::SK_Transpose;
       if (ShuffleVectorInst::isSpliceMask(Mask, NumSrcElts, Index))
         return TTI::SK_Splice;
       break;
     }
     case TTI::SK_Select:
     case TTI::SK_Reverse:
     case TTI::SK_Broadcast:
     case TTI::SK_Transpose:
     case TTI::SK_InsertSubvector:
     case TTI::SK_ExtractSubvector:
     case TTI::SK_Splice:
       break;
     }
     return Kind;
   }
 
   InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
                                  ArrayRef<int> Mask,
                                  TTI::TargetCostKind CostKind, int Index,
                                  VectorType *SubTp,
                                  ArrayRef<const Value *> Args = {},
                                  const Instruction *CxtI = nullptr) {
     switch (improveShuffleKindFromMask(Kind, Mask, Tp, Index, SubTp)) {
     case TTI::SK_Broadcast:
       if (auto *FVT = dyn_cast<FixedVectorType>(Tp))
         return getBroadcastShuffleOverhead(FVT, CostKind);
       return InstructionCost::getInvalid();
     case TTI::SK_Select:
     case TTI::SK_Splice:
     case TTI::SK_Reverse:
     case TTI::SK_Transpose:
     case TTI::SK_PermuteSingleSrc:
     case TTI::SK_PermuteTwoSrc:
       if (auto *FVT = dyn_cast<FixedVectorType>(Tp))
         return getPermuteShuffleOverhead(FVT, CostKind);
       return InstructionCost::getInvalid();
     case TTI::SK_ExtractSubvector:
       return getExtractSubvectorOverhead(Tp, CostKind, Index,
                                          cast<FixedVectorType>(SubTp));
     case TTI::SK_InsertSubvector:
       return getInsertSubvectorOverhead(Tp, CostKind, Index,
                                         cast<FixedVectorType>(SubTp));
     }
     llvm_unreachable("Unknown TTI::ShuffleKind");
   }
 
   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                    TTI::CastContextHint CCH,
                                    TTI::TargetCostKind CostKind,
                                    const Instruction *I = nullptr) {
     if (BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I) == 0)
       return 0;
 
     const TargetLoweringBase *TLI = getTLI();
     int ISD = TLI->InstructionOpcodeToISD(Opcode);
     assert(ISD && "Invalid opcode");
     std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Src);
     std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Dst);
 
     TypeSize SrcSize = SrcLT.second.getSizeInBits();
     TypeSize DstSize = DstLT.second.getSizeInBits();
     bool IntOrPtrSrc = Src->isIntegerTy() || Src->isPointerTy();
     bool IntOrPtrDst = Dst->isIntegerTy() || Dst->isPointerTy();
 
     switch (Opcode) {
     default:
       break;
     case Instruction::Trunc:
       // Check for NOOP conversions.
       if (TLI->isTruncateFree(SrcLT.second, DstLT.second))
         return 0;
       [[fallthrough]];
     case Instruction::BitCast:
       // Bitcast between types that are legalized to the same type are free and
       // assume int to/from ptr of the same size is also free.
       if (SrcLT.first == DstLT.first && IntOrPtrSrc == IntOrPtrDst &&
           SrcSize == DstSize)
         return 0;
       break;
     case Instruction::FPExt:
       if (I && getTLI()->isExtFree(I))
         return 0;
       break;
     case Instruction::ZExt:
       if (TLI->isZExtFree(SrcLT.second, DstLT.second))
         return 0;
       [[fallthrough]];
     case Instruction::SExt:
       if (I && getTLI()->isExtFree(I))
         return 0;
 
       // If this is a zext/sext of a load, return 0 if the corresponding
       // extending load exists on target and the result type is legal.
       if (CCH == TTI::CastContextHint::Normal) {
         EVT ExtVT = EVT::getEVT(Dst);
         EVT LoadVT = EVT::getEVT(Src);
         unsigned LType =
           ((Opcode == Instruction::ZExt) ? ISD::ZEXTLOAD : ISD::SEXTLOAD);
         if (DstLT.first == SrcLT.first &&
             TLI->isLoadExtLegal(LType, ExtVT, LoadVT))
           return 0;
       }
       break;
     case Instruction::AddrSpaceCast:
       if (TLI->isFreeAddrSpaceCast(Src->getPointerAddressSpace(),
                                    Dst->getPointerAddressSpace()))
         return 0;
       break;
     }
 
     auto *SrcVTy = dyn_cast<VectorType>(Src);
     auto *DstVTy = dyn_cast<VectorType>(Dst);
 
     // If the cast is marked as legal (or promote) then assume low cost.
     if (SrcLT.first == DstLT.first &&
         TLI->isOperationLegalOrPromote(ISD, DstLT.second))
       return SrcLT.first;
 
     // Handle scalar conversions.
     if (!SrcVTy && !DstVTy) {
       // Just check the op cost. If the operation is legal then assume it costs
       // 1.
       if (!TLI->isOperationExpand(ISD, DstLT.second))
         return 1;
 
       // Assume that illegal scalar instruction are expensive.
       return 4;
     }
 
     // Check vector-to-vector casts.
     if (DstVTy && SrcVTy) {
       // If the cast is between same-sized registers, then the check is simple.
       if (SrcLT.first == DstLT.first && SrcSize == DstSize) {
 
         // Assume that Zext is done using AND.
         if (Opcode == Instruction::ZExt)
           return SrcLT.first;
 
         // Assume that sext is done using SHL and SRA.
         if (Opcode == Instruction::SExt)
           return SrcLT.first * 2;
 
         // Just check the op cost. If the operation is legal then assume it
         // costs
         // 1 and multiply by the type-legalization overhead.
         if (!TLI->isOperationExpand(ISD, DstLT.second))
           return SrcLT.first * 1;
       }
 
       // If we are legalizing by splitting, query the concrete TTI for the cost
       // of casting the original vector twice. We also need to factor in the
       // cost of the split itself. Count that as 1, to be consistent with
       // getTypeLegalizationCost().
       bool SplitSrc =
           TLI->getTypeAction(Src->getContext(), TLI->getValueType(DL, Src)) ==
           TargetLowering::TypeSplitVector;
       bool SplitDst =
           TLI->getTypeAction(Dst->getContext(), TLI->getValueType(DL, Dst)) ==
           TargetLowering::TypeSplitVector;
       if ((SplitSrc || SplitDst) && SrcVTy->getElementCount().isVector() &&
           DstVTy->getElementCount().isVector()) {
         Type *SplitDstTy = VectorType::getHalfElementsVectorType(DstVTy);
         Type *SplitSrcTy = VectorType::getHalfElementsVectorType(SrcVTy);
         T *TTI = static_cast<T *>(this);
         // If both types need to be split then the split is free.
         InstructionCost SplitCost =
             (!SplitSrc || !SplitDst) ? TTI->getVectorSplitCost() : 0;
         return SplitCost +
                (2 * TTI->getCastInstrCost(Opcode, SplitDstTy, SplitSrcTy, CCH,
                                           CostKind, I));
       }
 
       // Scalarization cost is Invalid, can't assume any num elements.
       if (isa<ScalableVectorType>(DstVTy))
         return InstructionCost::getInvalid();
 
       // In other cases where the source or destination are illegal, assume
       // the operation will get scalarized.
       unsigned Num = cast<FixedVectorType>(DstVTy)->getNumElements();
       InstructionCost Cost = thisT()->getCastInstrCost(
           Opcode, Dst->getScalarType(), Src->getScalarType(), CCH, CostKind, I);
 
       // Return the cost of multiple scalar invocation plus the cost of
       // inserting and extracting the values.
       return getScalarizationOverhead(DstVTy, /*Insert*/ true, /*Extract*/ true,
                                       CostKind) +
              Num * Cost;
     }
 
     // We already handled vector-to-vector and scalar-to-scalar conversions.
     // This
     // is where we handle bitcast between vectors and scalars. We need to assume
     //  that the conversion is scalarized in one way or another.
     if (Opcode == Instruction::BitCast) {
       // Illegal bitcasts are done by storing and loading from a stack slot.
       return (SrcVTy ? getScalarizationOverhead(SrcVTy, /*Insert*/ false,
                                                 /*Extract*/ true, CostKind)
                      : 0) +
              (DstVTy ? getScalarizationOverhead(DstVTy, /*Insert*/ true,
                                                 /*Extract*/ false, CostKind)
                      : 0);
     }
 
     llvm_unreachable("Unhandled cast");
   }
 
   InstructionCost getExtractWithExtendCost(unsigned Opcode, Type *Dst,
                                            VectorType *VecTy, unsigned Index) {
     TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
     return thisT()->getVectorInstrCost(Instruction::ExtractElement, VecTy,
                                        CostKind, Index, nullptr, nullptr) +
            thisT()->getCastInstrCost(Opcode, Dst, VecTy->getElementType(),
                                      TTI::CastContextHint::None, CostKind);
   }
 
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
                                  const Instruction *I = nullptr) {
     return BaseT::getCFInstrCost(Opcode, CostKind, I);
   }
 
   InstructionCost getCmpSelInstrCost(
       unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       const Instruction *I = nullptr) {
     const TargetLoweringBase *TLI = getTLI();
     int ISD = TLI->InstructionOpcodeToISD(Opcode);
     assert(ISD && "Invalid opcode");
 
     // TODO: Handle other cost kinds.
     if (CostKind != TTI::TCK_RecipThroughput)
       return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
                                        Op1Info, Op2Info, I);
 
     // Selects on vectors are actually vector selects.
     if (ISD == ISD::SELECT) {
       assert(CondTy && "CondTy must exist");
       if (CondTy->isVectorTy())
         ISD = ISD::VSELECT;
     }
     std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
 
     if (!(ValTy->isVectorTy() && !LT.second.isVector()) &&
         !TLI->isOperationExpand(ISD, LT.second)) {
       // The operation is legal. Assume it costs 1. Multiply
       // by the type-legalization overhead.
       return LT.first * 1;
     }
 
     // Otherwise, assume that the cast is scalarized.
     // TODO: If one of the types get legalized by splitting, handle this
     // similarly to what getCastInstrCost() does.
     if (auto *ValVTy = dyn_cast<VectorType>(ValTy)) {
       if (isa<ScalableVectorType>(ValTy))
         return InstructionCost::getInvalid();
 
       unsigned Num = cast<FixedVectorType>(ValVTy)->getNumElements();
       if (CondTy)
         CondTy = CondTy->getScalarType();
       InstructionCost Cost =
           thisT()->getCmpSelInstrCost(Opcode, ValVTy->getScalarType(), CondTy,
                                       VecPred, CostKind, Op1Info, Op2Info, I);
 
       // Return the cost of multiple scalar invocation plus the cost of
       // inserting and extracting the values.
       return getScalarizationOverhead(ValVTy, /*Insert*/ true,
                                       /*Extract*/ false, CostKind) +
              Num * Cost;
     }
 
     // Unknown scalar opcode.
     return 1;
   }
 
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
                                      TTI::TargetCostKind CostKind,
                                      unsigned Index, Value *Op0, Value *Op1) {
     return getRegUsageForType(Val->getScalarType());
   }
 
   /// \param ScalarUserAndIdx encodes the information about extracts from a
   /// vector with 'Scalar' being the value being extracted,'User' being the user
   /// of the extract(nullptr if user is not known before vectorization) and
   /// 'Idx' being the extract lane.
   InstructionCost getVectorInstrCost(
       unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
       Value *Scalar,
       ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) {
     return thisT()->getVectorInstrCost(Opcode, Val, CostKind, Index, nullptr,
                                        nullptr);
   }
 
   InstructionCost getVectorInstrCost(const Instruction &I, Type *Val,
                                      TTI::TargetCostKind CostKind,
                                      unsigned Index) {
     Value *Op0 = nullptr;
     Value *Op1 = nullptr;
     if (auto *IE = dyn_cast<InsertElementInst>(&I)) {
       Op0 = IE->getOperand(0);
       Op1 = IE->getOperand(1);
     }
     return thisT()->getVectorInstrCost(I.getOpcode(), Val, CostKind, Index, Op0,
                                        Op1);
   }
 
   InstructionCost getReplicationShuffleCost(Type *EltTy, int ReplicationFactor,
                                             int VF,
                                             const APInt &DemandedDstElts,
                                             TTI::TargetCostKind CostKind) {
     assert(DemandedDstElts.getBitWidth() == (unsigned)VF * ReplicationFactor &&
            "Unexpected size of DemandedDstElts.");
 
     InstructionCost Cost;
 
     auto *SrcVT = FixedVectorType::get(EltTy, VF);
     auto *ReplicatedVT = FixedVectorType::get(EltTy, VF * ReplicationFactor);
 
     // The Mask shuffling cost is extract all the elements of the Mask
     // and insert each of them Factor times into the wide vector:
     //
     // E.g. an interleaved group with factor 3:
     //    %mask = icmp ult <8 x i32> %vec1, %vec2
     //    %interleaved.mask = shufflevector <8 x i1> %mask, <8 x i1> undef,
     //        <24 x i32> <0,0,0,1,1,1,2,2,2,3,3,3,4,4,4,5,5,5,6,6,6,7,7,7>
     // The cost is estimated as extract all mask elements from the <8xi1> mask
     // vector and insert them factor times into the <24xi1> shuffled mask
     // vector.
     APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedDstElts, VF);
     Cost += thisT()->getScalarizationOverhead(SrcVT, DemandedSrcElts,
                                               /*Insert*/ false,
                                               /*Extract*/ true, CostKind);
     Cost += thisT()->getScalarizationOverhead(ReplicatedVT, DemandedDstElts,
                                               /*Insert*/ true,
                                               /*Extract*/ false, CostKind);
 
     return Cost;
   }
 
   InstructionCost
   getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
                   unsigned AddressSpace, TTI::TargetCostKind CostKind,
                   TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
                   const Instruction *I = nullptr) {
     assert(!Src->isVoidTy() && "Invalid type");
     // Assume types, such as structs, are expensive.
     if (getTLI()->getValueType(DL, Src,  true) == MVT::Other)
       return 4;
     std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Src);
 
     // Assuming that all loads of legal types cost 1.
     InstructionCost Cost = LT.first;
     if (CostKind != TTI::TCK_RecipThroughput)
       return Cost;
 
     const DataLayout &DL = this->getDataLayout();
     if (Src->isVectorTy() &&
         // In practice it's not currently possible to have a change in lane
         // length for extending loads or truncating stores so both types should
         // have the same scalable property.
         TypeSize::isKnownLT(DL.getTypeStoreSizeInBits(Src),
                             LT.second.getSizeInBits())) {
       // This is a vector load that legalizes to a larger type than the vector
       // itself. Unless the corresponding extending load or truncating store is
       // legal, then this will scalarize.
       TargetLowering::LegalizeAction LA = TargetLowering::Expand;
       EVT MemVT = getTLI()->getValueType(DL, Src);
       if (Opcode == Instruction::Store)
         LA = getTLI()->getTruncStoreAction(LT.second, MemVT);
       else
         LA = getTLI()->getLoadExtAction(ISD::EXTLOAD, LT.second, MemVT);
 
       if (LA != TargetLowering::Legal && LA != TargetLowering::Custom) {
         // This is a vector load/store for some illegal type that is scalarized.
         // We must account for the cost of building or decomposing the vector.
         Cost += getScalarizationOverhead(
             cast<VectorType>(Src), Opcode != Instruction::Store,
             Opcode == Instruction::Store, CostKind);
       }
     }
 
     return Cost;
   }
 
   InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *DataTy,
                                         Align Alignment, unsigned AddressSpace,
                                         TTI::TargetCostKind CostKind) {
     // TODO: Pass on AddressSpace when we have test coverage.
     return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, true, false,
                                        CostKind);
   }
 
   InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
                                          const Value *Ptr, bool VariableMask,
                                          Align Alignment,
                                          TTI::TargetCostKind CostKind,
                                          const Instruction *I = nullptr) {
     return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask,
                                        true, CostKind);
   }
 
   InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy,
                                          const Value *Ptr, bool VariableMask,
                                          Align Alignment,
                                          TTI::TargetCostKind CostKind,
                                          const Instruction *I) {
     // For a target without strided memory operations (or for an illegal
     // operation type on one which does), assume we lower to a gather/scatter
     // operation.  (Which may in turn be scalarized.)
     return thisT()->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
                                            Alignment, CostKind, I);
   }
 
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
       bool UseMaskForCond = false, bool UseMaskForGaps = false) {
 
     // We cannot scalarize scalable vectors, so return Invalid.
     if (isa<ScalableVectorType>(VecTy))
       return InstructionCost::getInvalid();
 
     auto *VT = cast<FixedVectorType>(VecTy);
 
     unsigned NumElts = VT->getNumElements();
     assert(Factor > 1 && NumElts % Factor == 0 && "Invalid interleave factor");
 
     unsigned NumSubElts = NumElts / Factor;
     auto *SubVT = FixedVectorType::get(VT->getElementType(), NumSubElts);
 
     // Firstly, the cost of load/store operation.
     InstructionCost Cost;
     if (UseMaskForCond || UseMaskForGaps)
       Cost = thisT()->getMaskedMemoryOpCost(Opcode, VecTy, Alignment,
                                             AddressSpace, CostKind);
     else
       Cost = thisT()->getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace,
                                       CostKind);
 
     // Legalize the vector type, and get the legalized and unlegalized type
     // sizes.
     MVT VecTyLT = getTypeLegalizationCost(VecTy).second;
     unsigned VecTySize = thisT()->getDataLayout().getTypeStoreSize(VecTy);
     unsigned VecTyLTSize = VecTyLT.getStoreSize();
 
     // Scale the cost of the memory operation by the fraction of legalized
     // instructions that will actually be used. We shouldn't account for the
     // cost of dead instructions since they will be removed.
     //
     // E.g., An interleaved load of factor 8:
     //       %vec = load <16 x i64>, <16 x i64>* %ptr
     //       %v0 = shufflevector %vec, undef, <0, 8>
     //
     // If <16 x i64> is legalized to 8 v2i64 loads, only 2 of the loads will be
     // used (those corresponding to elements [0:1] and [8:9] of the unlegalized
     // type). The other loads are unused.
     //
     // TODO: Note that legalization can turn masked loads/stores into unmasked
     // (legalized) loads/stores. This can be reflected in the cost.
     if (Cost.isValid() && VecTySize > VecTyLTSize) {
       // The number of loads of a legal type it will take to represent a load
       // of the unlegalized vector type.
       unsigned NumLegalInsts = divideCeil(VecTySize, VecTyLTSize);
 
       // The number of elements of the unlegalized type that correspond to a
       // single legal instruction.
       unsigned NumEltsPerLegalInst = divideCeil(NumElts, NumLegalInsts);
 
       // Determine which legal instructions will be used.
       BitVector UsedInsts(NumLegalInsts, false);
       for (unsigned Index : Indices)
         for (unsigned Elt = 0; Elt < NumSubElts; ++Elt)
           UsedInsts.set((Index + Elt * Factor) / NumEltsPerLegalInst);
 
       // Scale the cost of the load by the fraction of legal instructions that
       // will be used.
       Cost = divideCeil(UsedInsts.count() * *Cost.getValue(), NumLegalInsts);
     }
 
     // Then plus the cost of interleave operation.
     assert(Indices.size() <= Factor &&
            "Interleaved memory op has too many members");
 
     const APInt DemandedAllSubElts = APInt::getAllOnes(NumSubElts);
     const APInt DemandedAllResultElts = APInt::getAllOnes(NumElts);
 
     APInt DemandedLoadStoreElts = APInt::getZero(NumElts);
     for (unsigned Index : Indices) {
       assert(Index < Factor && "Invalid index for interleaved memory op");
       for (unsigned Elm = 0; Elm < NumSubElts; Elm++)
         DemandedLoadStoreElts.setBit(Index + Elm * Factor);
     }
 
     if (Opcode == Instruction::Load) {
       // The interleave cost is similar to extract sub vectors' elements
       // from the wide vector, and insert them into sub vectors.
       //
       // E.g. An interleaved load of factor 2 (with one member of index 0):
       //      %vec = load <8 x i32>, <8 x i32>* %ptr
       //      %v0 = shuffle %vec, undef, <0, 2, 4, 6>         ; Index 0
       // The cost is estimated as extract elements at 0, 2, 4, 6 from the
       // <8 x i32> vector and insert them into a <4 x i32> vector.
       InstructionCost InsSubCost = thisT()->getScalarizationOverhead(
           SubVT, DemandedAllSubElts,
           /*Insert*/ true, /*Extract*/ false, CostKind);
       Cost += Indices.size() * InsSubCost;
       Cost += thisT()->getScalarizationOverhead(VT, DemandedLoadStoreElts,
                                                 /*Insert*/ false,
                                                 /*Extract*/ true, CostKind);
     } else {
       // The interleave cost is extract elements from sub vectors, and
       // insert them into the wide vector.
       //
       // E.g. An interleaved store of factor 3 with 2 members at indices 0,1:
       // (using VF=4):
       //    %v0_v1 = shuffle %v0, %v1, <0,4,undef,1,5,undef,2,6,undef,3,7,undef>
       //    %gaps.mask = <true, true, false, true, true, false,
       //                  true, true, false, true, true, false>
       //    call llvm.masked.store <12 x i32> %v0_v1, <12 x i32>* %ptr,
       //                           i32 Align, <12 x i1> %gaps.mask
       // The cost is estimated as extract all elements (of actual members,
       // excluding gaps) from both <4 x i32> vectors and insert into the <12 x
       // i32> vector.
       InstructionCost ExtSubCost = thisT()->getScalarizationOverhead(
           SubVT, DemandedAllSubElts,
           /*Insert*/ false, /*Extract*/ true, CostKind);
       Cost += ExtSubCost * Indices.size();
       Cost += thisT()->getScalarizationOverhead(VT, DemandedLoadStoreElts,
                                                 /*Insert*/ true,
                                                 /*Extract*/ false, CostKind);
     }
 
     if (!UseMaskForCond)
       return Cost;
 
     Type *I8Type = Type::getInt8Ty(VT->getContext());
 
     Cost += thisT()->getReplicationShuffleCost(
         I8Type, Factor, NumSubElts,
         UseMaskForGaps ? DemandedLoadStoreElts : DemandedAllResultElts,
         CostKind);
 
     // The Gaps mask is invariant and created outside the loop, therefore the
     // cost of creating it is not accounted for here. However if we have both
     // a MaskForGaps and some other mask that guards the execution of the
     // memory access, we need to account for the cost of And-ing the two masks
     // inside the loop.
     if (UseMaskForGaps) {
       auto *MaskVT = FixedVectorType::get(I8Type, NumElts);
       Cost += thisT()->getArithmeticInstrCost(BinaryOperator::And, MaskVT,
                                               CostKind);
     }
 
     return Cost;
   }
 
   /// Get intrinsic cost based on arguments.
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                         TTI::TargetCostKind CostKind) {
     // Check for generically free intrinsics.
     if (BaseT::getIntrinsicInstrCost(ICA, CostKind) == 0)
       return 0;
 
     // Assume that target intrinsics are cheap.
     Intrinsic::ID IID = ICA.getID();
     if (Intrinsic::isTargetIntrinsic(IID))
       return TargetTransformInfo::TCC_Basic;
 
     // VP Intrinsics should have the same cost as their non-vp counterpart.
     // TODO: Adjust the cost to make the vp intrinsic cheaper than its non-vp
     // counterpart when the vector length argument is smaller than the maximum
     // vector length.
     // TODO: Support other kinds of VPIntrinsics
     if (VPIntrinsic::isVPIntrinsic(ICA.getID())) {
       std::optional<unsigned> FOp =
           VPIntrinsic::getFunctionalOpcodeForVP(ICA.getID());
       if (FOp) {
         if (ICA.getID() == Intrinsic::vp_load) {
           Align Alignment;
           if (auto *VPI = dyn_cast_or_null<VPIntrinsic>(ICA.getInst()))
             Alignment = VPI->getPointerAlignment().valueOrOne();
           unsigned AS = 0;
           if (ICA.getArgTypes().size() > 1)
             if (auto *PtrTy = dyn_cast<PointerType>(ICA.getArgTypes()[0]))
               AS = PtrTy->getAddressSpace();
           return thisT()->getMemoryOpCost(*FOp, ICA.getReturnType(), Alignment,
                                           AS, CostKind);
         }
         if (ICA.getID() == Intrinsic::vp_store) {
           Align Alignment;
           if (auto *VPI = dyn_cast_or_null<VPIntrinsic>(ICA.getInst()))
             Alignment = VPI->getPointerAlignment().valueOrOne();
           unsigned AS = 0;
           if (ICA.getArgTypes().size() >= 2)
             if (auto *PtrTy = dyn_cast<PointerType>(ICA.getArgTypes()[1]))
               AS = PtrTy->getAddressSpace();
           return thisT()->getMemoryOpCost(*FOp, ICA.getArgTypes()[0], Alignment,
                                           AS, CostKind);
         }
         if (VPBinOpIntrinsic::isVPBinOp(ICA.getID())) {
           return thisT()->getArithmeticInstrCost(*FOp, ICA.getReturnType(),
                                                  CostKind);
         }
         if (VPCastIntrinsic::isVPCast(ICA.getID())) {
           return thisT()->getCastInstrCost(
               *FOp, ICA.getReturnType(), ICA.getArgTypes()[0],
               TTI::CastContextHint::None, CostKind);
         }
         if (VPCmpIntrinsic::isVPCmp(ICA.getID())) {
           // We can only handle vp_cmp intrinsics with underlying instructions.
           if (ICA.getInst()) {
             assert(FOp);
             auto *UI = cast<VPCmpIntrinsic>(ICA.getInst());
             return thisT()->getCmpSelInstrCost(*FOp, ICA.getArgTypes()[0],
                                                ICA.getReturnType(),
                                                UI->getPredicate(), CostKind);
           }
         }
       }
 
       std::optional<Intrinsic::ID> FID =
           VPIntrinsic::getFunctionalIntrinsicIDForVP(ICA.getID());
       if (FID) {
         // Non-vp version will have same arg types except mask and vector
         // length.
         assert(ICA.getArgTypes().size() >= 2 &&
                "Expected VPIntrinsic to have Mask and Vector Length args and "
                "types");
         ArrayRef<Type *> NewTys = ArrayRef(ICA.getArgTypes()).drop_back(2);
 
         // VPReduction intrinsics have a start value argument that their non-vp
         // counterparts do not have, except for the fadd and fmul non-vp
         // counterpart.
         if (VPReductionIntrinsic::isVPReduction(ICA.getID()) &&
             *FID != Intrinsic::vector_reduce_fadd &&
             *FID != Intrinsic::vector_reduce_fmul)
           NewTys = NewTys.drop_front();
 
         IntrinsicCostAttributes NewICA(*FID, ICA.getReturnType(), NewTys,
                                        ICA.getFlags());
         return thisT()->getIntrinsicInstrCost(NewICA, CostKind);
       }
     }
 
     if (ICA.isTypeBasedOnly())
       return getTypeBasedIntrinsicInstrCost(ICA, CostKind);
 
     Type *RetTy = ICA.getReturnType();
 
     ElementCount RetVF =
         (RetTy->isVectorTy() ? cast<VectorType>(RetTy)->getElementCount()
                              : ElementCount::getFixed(1));
     const IntrinsicInst *I = ICA.getInst();
     const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
     FastMathFlags FMF = ICA.getFlags();
     switch (IID) {
     default:
       break;
 
     case Intrinsic::powi:
       if (auto *RHSC = dyn_cast<ConstantInt>(Args[1])) {
         bool ShouldOptForSize = I->getParent()->getParent()->hasOptSize();
         if (getTLI()->isBeneficialToExpandPowI(RHSC->getSExtValue(),
                                                ShouldOptForSize)) {
           // The cost is modeled on the expansion performed by ExpandPowI in
           // SelectionDAGBuilder.
           APInt Exponent = RHSC->getValue().abs();
           unsigned ActiveBits = Exponent.getActiveBits();
           unsigned PopCount = Exponent.popcount();
           InstructionCost Cost = (ActiveBits + PopCount - 2) *
                                  thisT()->getArithmeticInstrCost(
                                      Instruction::FMul, RetTy, CostKind);
           if (RHSC->isNegative())
             Cost += thisT()->getArithmeticInstrCost(Instruction::FDiv, RetTy,
                                                     CostKind);
           return Cost;
         }
       }
       break;
     case Intrinsic::cttz:
       // FIXME: If necessary, this should go in target-specific overrides.
       if (RetVF.isScalar() && getTLI()->isCheapToSpeculateCttz(RetTy))
         return TargetTransformInfo::TCC_Basic;
       break;
 
     case Intrinsic::ctlz:
       // FIXME: If necessary, this should go in target-specific overrides.
       if (RetVF.isScalar() && getTLI()->isCheapToSpeculateCtlz(RetTy))
         return TargetTransformInfo::TCC_Basic;
       break;
 
     case Intrinsic::memcpy:
       return thisT()->getMemcpyCost(ICA.getInst());
 
     case Intrinsic::masked_scatter: {
       const Value *Mask = Args[3];
       bool VarMask = !isa<Constant>(Mask);
       Align Alignment = cast<ConstantInt>(Args[2])->getAlignValue();
       return thisT()->getGatherScatterOpCost(Instruction::Store,
                                              ICA.getArgTypes()[0], Args[1],
                                              VarMask, Alignment, CostKind, I);
     }
     case Intrinsic::masked_gather: {
       const Value *Mask = Args[2];
       bool VarMask = !isa<Constant>(Mask);
       Align Alignment = cast<ConstantInt>(Args[1])->getAlignValue();
       return thisT()->getGatherScatterOpCost(Instruction::Load, RetTy, Args[0],
                                              VarMask, Alignment, CostKind, I);
     }
     case Intrinsic::experimental_vp_strided_store: {
       const Value *Data = Args[0];
       const Value *Ptr = Args[1];
       const Value *Mask = Args[3];
       const Value *EVL = Args[4];
       bool VarMask = !isa<Constant>(Mask) || !isa<Constant>(EVL);
       Type *EltTy = cast<VectorType>(Data->getType())->getElementType();
       Align Alignment =
           I->getParamAlign(1).value_or(thisT()->DL.getABITypeAlign(EltTy));
       return thisT()->getStridedMemoryOpCost(Instruction::Store,
                                              Data->getType(), Ptr, VarMask,
                                              Alignment, CostKind, I);
     }
     case Intrinsic::experimental_vp_strided_load: {
       const Value *Ptr = Args[0];
       const Value *Mask = Args[2];
       const Value *EVL = Args[3];
       bool VarMask = !isa<Constant>(Mask) || !isa<Constant>(EVL);
       Type *EltTy = cast<VectorType>(RetTy)->getElementType();
       Align Alignment =
           I->getParamAlign(0).value_or(thisT()->DL.getABITypeAlign(EltTy));
       return thisT()->getStridedMemoryOpCost(Instruction::Load, RetTy, Ptr,
                                              VarMask, Alignment, CostKind, I);
     }
     case Intrinsic::stepvector: {
       if (isa<ScalableVectorType>(RetTy))
         return BaseT::getIntrinsicInstrCost(ICA, CostKind);
       // The cost of materialising a constant integer vector.
       return TargetTransformInfo::TCC_Basic;
     }
     case Intrinsic::vector_extract: {
       // FIXME: Handle case where a scalable vector is extracted from a scalable
       // vector
       if (isa<ScalableVectorType>(RetTy))
         return BaseT::getIntrinsicInstrCost(ICA, CostKind);
       unsigned Index = cast<ConstantInt>(Args[1])->getZExtValue();
       return thisT()->getShuffleCost(TTI::SK_ExtractSubvector,
                                      cast<VectorType>(Args[0]->getType()), {},
                                      CostKind, Index, cast<VectorType>(RetTy));
     }
     case Intrinsic::vector_insert: {
       // FIXME: Handle case where a scalable vector is inserted into a scalable
       // vector
       if (isa<ScalableVectorType>(Args[1]->getType()))
         return BaseT::getIntrinsicInstrCost(ICA, CostKind);
       unsigned Index = cast<ConstantInt>(Args[2])->getZExtValue();
       return thisT()->getShuffleCost(
           TTI::SK_InsertSubvector, cast<VectorType>(Args[0]->getType()), {},
           CostKind, Index, cast<VectorType>(Args[1]->getType()));
     }
     case Intrinsic::vector_reverse: {
       return thisT()->getShuffleCost(TTI::SK_Reverse,
                                      cast<VectorType>(Args[0]->getType()), {},
                                      CostKind, 0, cast<VectorType>(RetTy));
     }
     case Intrinsic::vector_splice: {
       unsigned Index = cast<ConstantInt>(Args[2])->getZExtValue();
       return thisT()->getShuffleCost(TTI::SK_Splice,
                                      cast<VectorType>(Args[0]->getType()), {},
                                      CostKind, Index, cast<VectorType>(RetTy));
     }
     case Intrinsic::vector_reduce_add:
     case Intrinsic::vector_reduce_mul:
     case Intrinsic::vector_reduce_and:
     case Intrinsic::vector_reduce_or:
     case Intrinsic::vector_reduce_xor:
     case Intrinsic::vector_reduce_smax:
     case Intrinsic::vector_reduce_smin:
     case Intrinsic::vector_reduce_fmax:
     case Intrinsic::vector_reduce_fmin:
     case Intrinsic::vector_reduce_fmaximum:
     case Intrinsic::vector_reduce_fminimum:
     case Intrinsic::vector_reduce_umax:
     case Intrinsic::vector_reduce_umin: {
       IntrinsicCostAttributes Attrs(IID, RetTy, Args[0]->getType(), FMF, I, 1);
       return getTypeBasedIntrinsicInstrCost(Attrs, CostKind);
     }
     case Intrinsic::vector_reduce_fadd:
     case Intrinsic::vector_reduce_fmul: {
       IntrinsicCostAttributes Attrs(
           IID, RetTy, {Args[0]->getType(), Args[1]->getType()}, FMF, I, 1);
       return getTypeBasedIntrinsicInstrCost(Attrs, CostKind);
     }
     case Intrinsic::fshl:
     case Intrinsic::fshr: {
       const Value *X = Args[0];
       const Value *Y = Args[1];
       const Value *Z = Args[2];
       const TTI::OperandValueInfo OpInfoX = TTI::getOperandInfo(X);
       const TTI::OperandValueInfo OpInfoY = TTI::getOperandInfo(Y);
       const TTI::OperandValueInfo OpInfoZ = TTI::getOperandInfo(Z);
       const TTI::OperandValueInfo OpInfoBW =
         {TTI::OK_UniformConstantValue,
          isPowerOf2_32(RetTy->getScalarSizeInBits()) ? TTI::OP_PowerOf2
          : TTI::OP_None};
 
       // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
       // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
       InstructionCost Cost = 0;
       Cost +=
           thisT()->getArithmeticInstrCost(BinaryOperator::Or, RetTy, CostKind);
       Cost +=
           thisT()->getArithmeticInstrCost(BinaryOperator::Sub, RetTy, CostKind);
       Cost += thisT()->getArithmeticInstrCost(
           BinaryOperator::Shl, RetTy, CostKind, OpInfoX,
           {OpInfoZ.Kind, TTI::OP_None});
       Cost += thisT()->getArithmeticInstrCost(
           BinaryOperator::LShr, RetTy, CostKind, OpInfoY,
           {OpInfoZ.Kind, TTI::OP_None});
       // Non-constant shift amounts requires a modulo.
       if (!OpInfoZ.isConstant())
         Cost += thisT()->getArithmeticInstrCost(BinaryOperator::URem, RetTy,
                                                 CostKind, OpInfoZ, OpInfoBW);
       // For non-rotates (X != Y) we must add shift-by-zero handling costs.
       if (X != Y) {
         Type *CondTy = RetTy->getWithNewBitWidth(1);
         Cost +=
             thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, CondTy,
                                         CmpInst::ICMP_EQ, CostKind);
         Cost +=
             thisT()->getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,
                                         CmpInst::ICMP_EQ, CostKind);
       }
       return Cost;
     }
     case Intrinsic::get_active_lane_mask: {
       EVT ResVT = getTLI()->getValueType(DL, RetTy, true);
       EVT ArgType = getTLI()->getValueType(DL, ICA.getArgTypes()[0], true);
 
       // If we're not expanding the intrinsic then we assume this is cheap
       // to implement.
       if (!getTLI()->shouldExpandGetActiveLaneMask(ResVT, ArgType)) {
         return getTypeLegalizationCost(RetTy).first;
       }
 
       // Create the expanded types that will be used to calculate the uadd_sat
       // operation.
       Type *ExpRetTy = VectorType::get(
           ICA.getArgTypes()[0], cast<VectorType>(RetTy)->getElementCount());
       IntrinsicCostAttributes Attrs(Intrinsic::uadd_sat, ExpRetTy, {}, FMF);
       InstructionCost Cost =
           thisT()->getTypeBasedIntrinsicInstrCost(Attrs, CostKind);
       Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, ExpRetTy, RetTy,
                                           CmpInst::ICMP_ULT, CostKind);
       return Cost;
     }
     case Intrinsic::experimental_cttz_elts: {
       EVT ArgType = getTLI()->getValueType(DL, ICA.getArgTypes()[0], true);
 
       // If we're not expanding the intrinsic then we assume this is cheap
       // to implement.
       if (!getTLI()->shouldExpandCttzElements(ArgType))
         return getTypeLegalizationCost(RetTy).first;
 
       // TODO: The costs below reflect the expansion code in
       // SelectionDAGBuilder, but we may want to sacrifice some accuracy in
       // favour of compile time.
 
       // Find the smallest "sensible" element type to use for the expansion.
       bool ZeroIsPoison = !cast<ConstantInt>(Args[1])->isZero();
       ConstantRange VScaleRange(APInt(64, 1), APInt::getZero(64));
       if (isa<ScalableVectorType>(ICA.getArgTypes()[0]) && I && I->getCaller())
         VScaleRange = getVScaleRange(I->getCaller(), 64);
 
       unsigned EltWidth = getTLI()->getBitWidthForCttzElements(
           RetTy, ArgType.getVectorElementCount(), ZeroIsPoison, &VScaleRange);
       Type *NewEltTy = IntegerType::getIntNTy(RetTy->getContext(), EltWidth);
 
       // Create the new vector type & get the vector length
       Type *NewVecTy = VectorType::get(
           NewEltTy, cast<VectorType>(Args[0]->getType())->getElementCount());
 
       IntrinsicCostAttributes StepVecAttrs(Intrinsic::stepvector, NewVecTy, {},
                                            FMF);
       InstructionCost Cost =
           thisT()->getIntrinsicInstrCost(StepVecAttrs, CostKind);
 
       Cost +=
           thisT()->getArithmeticInstrCost(Instruction::Sub, NewVecTy, CostKind);
       Cost += thisT()->getCastInstrCost(Instruction::SExt, NewVecTy,
                                         Args[0]->getType(),
                                         TTI::CastContextHint::None, CostKind);
       Cost +=
           thisT()->getArithmeticInstrCost(Instruction::And, NewVecTy, CostKind);
 
       IntrinsicCostAttributes ReducAttrs(Intrinsic::vector_reduce_umax,
                                          NewEltTy, NewVecTy, FMF, I, 1);
       Cost += thisT()->getTypeBasedIntrinsicInstrCost(ReducAttrs, CostKind);
       Cost +=
           thisT()->getArithmeticInstrCost(Instruction::Sub, NewEltTy, CostKind);
 
       return Cost;
     }
     case Intrinsic::experimental_vector_match:
       return thisT()->getTypeBasedIntrinsicInstrCost(ICA, CostKind);
     }
 
     // Assume that we need to scalarize this intrinsic.)
     // Compute the scalarization overhead based on Args for a vector
     // intrinsic.
     InstructionCost ScalarizationCost = InstructionCost::getInvalid();
     if (RetVF.isVector() && !RetVF.isScalable()) {
       ScalarizationCost = 0;
       if (!RetTy->isVoidTy())
         ScalarizationCost += getScalarizationOverhead(
             cast<VectorType>(RetTy),
             /*Insert*/ true, /*Extract*/ false, CostKind);
       ScalarizationCost +=
           getOperandsScalarizationOverhead(Args, ICA.getArgTypes(), CostKind);
     }
 
     IntrinsicCostAttributes Attrs(IID, RetTy, ICA.getArgTypes(), FMF, I,
                                   ScalarizationCost);
     return thisT()->getTypeBasedIntrinsicInstrCost(Attrs, CostKind);
   }
 
   /// Get intrinsic cost based on argument types.
   /// If ScalarizationCostPassed is std::numeric_limits<unsigned>::max(), the
   /// cost of scalarizing the arguments and the return value will be computed
   /// based on types.
   InstructionCost
   getTypeBasedIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                  TTI::TargetCostKind CostKind) {
     Intrinsic::ID IID = ICA.getID();
     Type *RetTy = ICA.getReturnType();
     const SmallVectorImpl<Type *> &Tys = ICA.getArgTypes();
     FastMathFlags FMF = ICA.getFlags();
     InstructionCost ScalarizationCostPassed = ICA.getScalarizationCost();
     bool SkipScalarizationCost = ICA.skipScalarizationCost();
 
     VectorType *VecOpTy = nullptr;
     if (!Tys.empty()) {
       // The vector reduction operand is operand 0 except for fadd/fmul.
       // Their operand 0 is a scalar start value, so the vector op is operand 1.
       unsigned VecTyIndex = 0;
       if (IID == Intrinsic::vector_reduce_fadd ||
           IID == Intrinsic::vector_reduce_fmul)
         VecTyIndex = 1;
       assert(Tys.size() > VecTyIndex && "Unexpected IntrinsicCostAttributes");
       VecOpTy = dyn_cast<VectorType>(Tys[VecTyIndex]);
     }
 
     // Library call cost - other than size, make it expensive.
     unsigned SingleCallCost = CostKind == TTI::TCK_CodeSize ? 1 : 10;
     unsigned ISD = 0;
     switch (IID) {
     default: {
       // Scalable vectors cannot be scalarized, so return Invalid.
       if (isa<ScalableVectorType>(RetTy) || any_of(Tys, [](const Type *Ty) {
             return isa<ScalableVectorType>(Ty);
           }))
         return InstructionCost::getInvalid();
 
       // Assume that we need to scalarize this intrinsic.
       InstructionCost ScalarizationCost =
           SkipScalarizationCost ? ScalarizationCostPassed : 0;
       unsigned ScalarCalls = 1;
       Type *ScalarRetTy = RetTy;
       if (auto *RetVTy = dyn_cast<VectorType>(RetTy)) {
         if (!SkipScalarizationCost)
           ScalarizationCost = getScalarizationOverhead(
               RetVTy, /*Insert*/ true, /*Extract*/ false, CostKind);
         ScalarCalls = std::max(ScalarCalls,
                                cast<FixedVectorType>(RetVTy)->getNumElements());
         ScalarRetTy = RetTy->getScalarType();
       }
       SmallVector<Type *, 4> ScalarTys;
       for (Type *Ty : Tys) {
         if (auto *VTy = dyn_cast<VectorType>(Ty)) {
           if (!SkipScalarizationCost)
             ScalarizationCost += getScalarizationOverhead(
                 VTy, /*Insert*/ false, /*Extract*/ true, CostKind);
           ScalarCalls = std::max(ScalarCalls,
                                  cast<FixedVectorType>(VTy)->getNumElements());
           Ty = Ty->getScalarType();
         }
         ScalarTys.push_back(Ty);
       }
       if (ScalarCalls == 1)
         return 1; // Return cost of a scalar intrinsic. Assume it to be cheap.
 
       IntrinsicCostAttributes ScalarAttrs(IID, ScalarRetTy, ScalarTys, FMF);
       InstructionCost ScalarCost =
           thisT()->getIntrinsicInstrCost(ScalarAttrs, CostKind);
 
       return ScalarCalls * ScalarCost + ScalarizationCost;
     }
     // Look for intrinsics that can be lowered directly or turned into a scalar
     // intrinsic call.
     case Intrinsic::sqrt:
       ISD = ISD::FSQRT;
       break;
     case Intrinsic::sin:
       ISD = ISD::FSIN;
       break;
     case Intrinsic::cos:
       ISD = ISD::FCOS;
       break;
     case Intrinsic::sincos:
       ISD = ISD::FSINCOS;
       break;
     case Intrinsic::tan:
       ISD = ISD::FTAN;
       break;
     case Intrinsic::asin:
       ISD = ISD::FASIN;
       break;
     case Intrinsic::acos:
       ISD = ISD::FACOS;
       break;
     case Intrinsic::atan:
       ISD = ISD::FATAN;
       break;
     case Intrinsic::atan2:
       ISD = ISD::FATAN2;
       break;
     case Intrinsic::sinh:
       ISD = ISD::FSINH;
       break;
     case Intrinsic::cosh:
       ISD = ISD::FCOSH;
       break;
     case Intrinsic::tanh:
       ISD = ISD::FTANH;
       break;
     case Intrinsic::exp:
       ISD = ISD::FEXP;
       break;
     case Intrinsic::exp2:
       ISD = ISD::FEXP2;
       break;
     case Intrinsic::exp10:
       ISD = ISD::FEXP10;
       break;
     case Intrinsic::log:
       ISD = ISD::FLOG;
       break;
     case Intrinsic::log10:
       ISD = ISD::FLOG10;
       break;
     case Intrinsic::log2:
       ISD = ISD::FLOG2;
       break;
     case Intrinsic::fabs:
       ISD = ISD::FABS;
       break;
     case Intrinsic::canonicalize:
       ISD = ISD::FCANONICALIZE;
       break;
     case Intrinsic::minnum:
       ISD = ISD::FMINNUM;
       break;
     case Intrinsic::maxnum:
       ISD = ISD::FMAXNUM;
       break;
     case Intrinsic::minimum:
       ISD = ISD::FMINIMUM;
       break;
     case Intrinsic::maximum:
       ISD = ISD::FMAXIMUM;
       break;
     case Intrinsic::minimumnum:
       ISD = ISD::FMINIMUMNUM;
       break;
     case Intrinsic::maximumnum:
       ISD = ISD::FMAXIMUMNUM;
       break;
     case Intrinsic::copysign:
       ISD = ISD::FCOPYSIGN;
       break;
     case Intrinsic::floor:
       ISD = ISD::FFLOOR;
       break;
     case Intrinsic::ceil:
       ISD = ISD::FCEIL;
       break;
     case Intrinsic::trunc:
       ISD = ISD::FTRUNC;
       break;
     case Intrinsic::nearbyint:
       ISD = ISD::FNEARBYINT;
       break;
     case Intrinsic::rint:
       ISD = ISD::FRINT;
       break;
     case Intrinsic::lrint:
       ISD = ISD::LRINT;
       break;
     case Intrinsic::llrint:
       ISD = ISD::LLRINT;
       break;
     case Intrinsic::round:
       ISD = ISD::FROUND;
       break;
     case Intrinsic::roundeven:
       ISD = ISD::FROUNDEVEN;
       break;
     case Intrinsic::pow:
       ISD = ISD::FPOW;
       break;
     case Intrinsic::fma:
       ISD = ISD::FMA;
       break;
     case Intrinsic::fmuladd:
       ISD = ISD::FMA;
       break;
     case Intrinsic::experimental_constrained_fmuladd:
       ISD = ISD::STRICT_FMA;
       break;
     // FIXME: We should return 0 whenever getIntrinsicCost == TCC_Free.
     case Intrinsic::lifetime_start:
     case Intrinsic::lifetime_end:
     case Intrinsic::sideeffect:
     case Intrinsic::pseudoprobe:
     case Intrinsic::arithmetic_fence:
       return 0;
     case Intrinsic::masked_store: {
       Type *Ty = Tys[0];
       Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
       return thisT()->getMaskedMemoryOpCost(Instruction::Store, Ty, TyAlign, 0,
                                             CostKind);
     }
     case Intrinsic::masked_load: {
       Type *Ty = RetTy;
       Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
       return thisT()->getMaskedMemoryOpCost(Instruction::Load, Ty, TyAlign, 0,
                                             CostKind);
     }
     case Intrinsic::vector_reduce_add:
     case Intrinsic::vector_reduce_mul:
     case Intrinsic::vector_reduce_and:
     case Intrinsic::vector_reduce_or:
     case Intrinsic::vector_reduce_xor:
       return thisT()->getArithmeticReductionCost(
           getArithmeticReductionInstruction(IID), VecOpTy, std::nullopt,
           CostKind);
     case Intrinsic::vector_reduce_fadd:
     case Intrinsic::vector_reduce_fmul:
       return thisT()->getArithmeticReductionCost(
           getArithmeticReductionInstruction(IID), VecOpTy, FMF, CostKind);
     case Intrinsic::vector_reduce_smax:
     case Intrinsic::vector_reduce_smin:
     case Intrinsic::vector_reduce_umax:
     case Intrinsic::vector_reduce_umin:
     case Intrinsic::vector_reduce_fmax:
     case Intrinsic::vector_reduce_fmin:
     case Intrinsic::vector_reduce_fmaximum:
     case Intrinsic::vector_reduce_fminimum:
       return thisT()->getMinMaxReductionCost(getMinMaxReductionIntrinsicOp(IID),
                                              VecOpTy, ICA.getFlags(), CostKind);
     case Intrinsic::experimental_vector_match: {
       auto *SearchTy = cast<VectorType>(ICA.getArgTypes()[0]);
       auto *NeedleTy = cast<FixedVectorType>(ICA.getArgTypes()[1]);
       unsigned SearchSize = NeedleTy->getNumElements();
 
       // If we're not expanding the intrinsic then we assume this is cheap to
       // implement.
       EVT SearchVT = getTLI()->getValueType(DL, SearchTy);
       if (!getTLI()->shouldExpandVectorMatch(SearchVT, SearchSize))
         return getTypeLegalizationCost(RetTy).first;
 
       // Approximate the cost based on the expansion code in
       // SelectionDAGBuilder.
       InstructionCost Cost = 0;
       Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, NeedleTy,
                                           CostKind, 1, nullptr, nullptr);
       Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, SearchTy,
                                           CostKind, 0, nullptr, nullptr);
       Cost += thisT()->getShuffleCost(TTI::SK_Broadcast, SearchTy, std::nullopt,
                                       CostKind, 0, nullptr);
       Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, SearchTy, RetTy,
                                           CmpInst::ICMP_EQ, CostKind);
       Cost +=
           thisT()->getArithmeticInstrCost(BinaryOperator::Or, RetTy, CostKind);
       Cost *= SearchSize;
       Cost +=
           thisT()->getArithmeticInstrCost(BinaryOperator::And, RetTy, CostKind);
       return Cost;
     }
     case Intrinsic::abs:
       ISD = ISD::ABS;
       break;
     case Intrinsic::fshl:
       ISD = ISD::FSHL;
       break;
     case Intrinsic::fshr:
       ISD = ISD::FSHR;
       break;
     case Intrinsic::smax:
       ISD = ISD::SMAX;
       break;
     case Intrinsic::smin:
       ISD = ISD::SMIN;
       break;
     case Intrinsic::umax:
       ISD = ISD::UMAX;
       break;
     case Intrinsic::umin:
       ISD = ISD::UMIN;
       break;
     case Intrinsic::sadd_sat:
       ISD = ISD::SADDSAT;
       break;
     case Intrinsic::ssub_sat:
       ISD = ISD::SSUBSAT;
       break;
     case Intrinsic::uadd_sat:
       ISD = ISD::UADDSAT;
       break;
     case Intrinsic::usub_sat:
       ISD = ISD::USUBSAT;
       break;
     case Intrinsic::smul_fix:
       ISD = ISD::SMULFIX;
       break;
     case Intrinsic::umul_fix:
       ISD = ISD::UMULFIX;
       break;
     case Intrinsic::sadd_with_overflow:
       ISD = ISD::SADDO;
       break;
     case Intrinsic::ssub_with_overflow:
       ISD = ISD::SSUBO;
       break;
     case Intrinsic::uadd_with_overflow:
       ISD = ISD::UADDO;
       break;
     case Intrinsic::usub_with_overflow:
       ISD = ISD::USUBO;
       break;
     case Intrinsic::smul_with_overflow:
       ISD = ISD::SMULO;
       break;
     case Intrinsic::umul_with_overflow:
       ISD = ISD::UMULO;
       break;
     case Intrinsic::fptosi_sat:
       ISD = ISD::FP_TO_SINT_SAT;
       break;
     case Intrinsic::fptoui_sat:
       ISD = ISD::FP_TO_UINT_SAT;
       break;
     case Intrinsic::ctpop:
       ISD = ISD::CTPOP;
       // In case of legalization use TCC_Expensive. This is cheaper than a
       // library call but still not a cheap instruction.
       SingleCallCost = TargetTransformInfo::TCC_Expensive;
       break;
     case Intrinsic::ctlz:
       ISD = ISD::CTLZ;
       break;
     case Intrinsic::cttz:
       ISD = ISD::CTTZ;
       break;
     case Intrinsic::bswap:
       ISD = ISD::BSWAP;
       break;
     case Intrinsic::bitreverse:
       ISD = ISD::BITREVERSE;
       break;
     case Intrinsic::ucmp:
       ISD = ISD::UCMP;
       break;
     case Intrinsic::scmp:
       ISD = ISD::SCMP;
       break;
     }
 
     auto *ST = dyn_cast<StructType>(RetTy);
     Type *LegalizeTy = ST ? ST->getContainedType(0) : RetTy;
     std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(LegalizeTy);
 
     const TargetLoweringBase *TLI = getTLI();
 
     if (TLI->isOperationLegalOrPromote(ISD, LT.second)) {
       if (IID == Intrinsic::fabs && LT.second.isFloatingPoint() &&
           TLI->isFAbsFree(LT.second)) {
         return 0;
       }
 
       // The operation is legal. Assume it costs 1.
       // If the type is split to multiple registers, assume that there is some
       // overhead to this.
       // TODO: Once we have extract/insert subvector cost we need to use them.
       if (LT.first > 1)
         return (LT.first * 2);
       else
         return (LT.first * 1);
     } else if (!TLI->isOperationExpand(ISD, LT.second)) {
       // If the operation is custom lowered then assume
       // that the code is twice as expensive.
       return (LT.first * 2);
     }
 
     switch (IID) {
     case Intrinsic::fmuladd: {
       // If we can't lower fmuladd into an FMA estimate the cost as a floating
       // point mul followed by an add.
 
       return thisT()->getArithmeticInstrCost(BinaryOperator::FMul, RetTy,
                                              CostKind) +
              thisT()->getArithmeticInstrCost(BinaryOperator::FAdd, RetTy,
                                              CostKind);
     }
     case Intrinsic::experimental_constrained_fmuladd: {
       IntrinsicCostAttributes FMulAttrs(
         Intrinsic::experimental_constrained_fmul, RetTy, Tys);
       IntrinsicCostAttributes FAddAttrs(
         Intrinsic::experimental_constrained_fadd, RetTy, Tys);
       return thisT()->getIntrinsicInstrCost(FMulAttrs, CostKind) +
              thisT()->getIntrinsicInstrCost(FAddAttrs, CostKind);
     }
     case Intrinsic::smin:
     case Intrinsic::smax:
     case Intrinsic::umin:
     case Intrinsic::umax: {
       // minmax(X,Y) = select(icmp(X,Y),X,Y)
       Type *CondTy = RetTy->getWithNewBitWidth(1);
       bool IsUnsigned = IID == Intrinsic::umax || IID == Intrinsic::umin;
       CmpInst::Predicate Pred =
           IsUnsigned ? CmpInst::ICMP_UGT : CmpInst::ICMP_SGT;
       InstructionCost Cost = 0;
       Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, CondTy,
                                           Pred, CostKind);
       Cost += thisT()->getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,
                                           Pred, CostKind);
       return Cost;
     }
     case Intrinsic::sadd_with_overflow:
     case Intrinsic::ssub_with_overflow: {
       Type *SumTy = RetTy->getContainedType(0);
       Type *OverflowTy = RetTy->getContainedType(1);
       unsigned Opcode = IID == Intrinsic::sadd_with_overflow
                             ? BinaryOperator::Add
                             : BinaryOperator::Sub;
 
       //   Add:
       //   Overflow -> (Result < LHS) ^ (RHS < 0)
       //   Sub:
       //   Overflow -> (Result < LHS) ^ (RHS > 0)
       InstructionCost Cost = 0;
       Cost += thisT()->getArithmeticInstrCost(Opcode, SumTy, CostKind);
       Cost +=
           2 * thisT()->getCmpSelInstrCost(Instruction::ICmp, SumTy, OverflowTy,
                                           CmpInst::ICMP_SGT, CostKind);
       Cost += thisT()->getArithmeticInstrCost(BinaryOperator::Xor, OverflowTy,
                                               CostKind);
       return Cost;
     }
     case Intrinsic::uadd_with_overflow:
     case Intrinsic::usub_with_overflow: {
       Type *SumTy = RetTy->getContainedType(0);
       Type *OverflowTy = RetTy->getContainedType(1);
       unsigned Opcode = IID == Intrinsic::uadd_with_overflow
                             ? BinaryOperator::Add
                             : BinaryOperator::Sub;
       CmpInst::Predicate Pred = IID == Intrinsic::uadd_with_overflow
                                     ? CmpInst::ICMP_ULT
                                     : CmpInst::ICMP_UGT;
 
       InstructionCost Cost = 0;
       Cost += thisT()->getArithmeticInstrCost(Opcode, SumTy, CostKind);
       Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, SumTy,
                                           OverflowTy, Pred, CostKind);
       return Cost;
     }
     case Intrinsic::smul_with_overflow:
     case Intrinsic::umul_with_overflow: {
       Type *MulTy = RetTy->getContainedType(0);
       Type *OverflowTy = RetTy->getContainedType(1);
       unsigned ExtSize = MulTy->getScalarSizeInBits() * 2;
       Type *ExtTy = MulTy->getWithNewBitWidth(ExtSize);
       bool IsSigned = IID == Intrinsic::smul_with_overflow;
 
       unsigned ExtOp = IsSigned ? Instruction::SExt : Instruction::ZExt;
       TTI::CastContextHint CCH = TTI::CastContextHint::None;
 
       InstructionCost Cost = 0;
       Cost += 2 * thisT()->getCastInstrCost(ExtOp, ExtTy, MulTy, CCH, CostKind);
       Cost +=
           thisT()->getArithmeticInstrCost(Instruction::Mul, ExtTy, CostKind);
       Cost += 2 * thisT()->getCastInstrCost(Instruction::Trunc, MulTy, ExtTy,
                                             CCH, CostKind);
       Cost += thisT()->getArithmeticInstrCost(
           Instruction::LShr, ExtTy, CostKind, {TTI::OK_AnyValue, TTI::OP_None},
           {TTI::OK_UniformConstantValue, TTI::OP_None});
 
       if (IsSigned)
         Cost += thisT()->getArithmeticInstrCost(
             Instruction::AShr, MulTy, CostKind,
             {TTI::OK_AnyValue, TTI::OP_None},
             {TTI::OK_UniformConstantValue, TTI::OP_None});
 
       Cost += thisT()->getCmpSelInstrCost(
           BinaryOperator::ICmp, MulTy, OverflowTy, CmpInst::ICMP_NE, CostKind);
       return Cost;
     }
     case Intrinsic::sadd_sat:
     case Intrinsic::ssub_sat: {
       // Assume a default expansion.
       Type *CondTy = RetTy->getWithNewBitWidth(1);
 
       Type *OpTy = StructType::create({RetTy, CondTy});
       Intrinsic::ID OverflowOp = IID == Intrinsic::sadd_sat
                                      ? Intrinsic::sadd_with_overflow
                                      : Intrinsic::ssub_with_overflow;
       CmpInst::Predicate Pred = CmpInst::ICMP_SGT;
 
       // SatMax -> Overflow && SumDiff < 0
       // SatMin -> Overflow && SumDiff >= 0
       InstructionCost Cost = 0;
       IntrinsicCostAttributes Attrs(OverflowOp, OpTy, {RetTy, RetTy}, FMF,
                                     nullptr, ScalarizationCostPassed);
       Cost += thisT()->getIntrinsicInstrCost(Attrs, CostKind);
       Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, CondTy,
                                           Pred, CostKind);
       Cost += 2 * thisT()->getCmpSelInstrCost(BinaryOperator::Select, RetTy,
                                               CondTy, Pred, CostKind);
       return Cost;
     }
     case Intrinsic::uadd_sat:
     case Intrinsic::usub_sat: {
       Type *CondTy = RetTy->getWithNewBitWidth(1);
 
       Type *OpTy = StructType::create({RetTy, CondTy});
       Intrinsic::ID OverflowOp = IID == Intrinsic::uadd_sat
                                      ? Intrinsic::uadd_with_overflow
                                      : Intrinsic::usub_with_overflow;
 
       InstructionCost Cost = 0;
       IntrinsicCostAttributes Attrs(OverflowOp, OpTy, {RetTy, RetTy}, FMF,
                                     nullptr, ScalarizationCostPassed);
       Cost += thisT()->getIntrinsicInstrCost(Attrs, CostKind);
       Cost +=
           thisT()->getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,
                                       CmpInst::BAD_ICMP_PREDICATE, CostKind);
       return Cost;
     }
     case Intrinsic::smul_fix:
     case Intrinsic::umul_fix: {
       unsigned ExtSize = RetTy->getScalarSizeInBits() * 2;
       Type *ExtTy = RetTy->getWithNewBitWidth(ExtSize);
 
       unsigned ExtOp =
           IID == Intrinsic::smul_fix ? Instruction::SExt : Instruction::ZExt;
       TTI::CastContextHint CCH = TTI::CastContextHint::None;
 
       InstructionCost Cost = 0;
       Cost += 2 * thisT()->getCastInstrCost(ExtOp, ExtTy, RetTy, CCH, CostKind);
       Cost +=
           thisT()->getArithmeticInstrCost(Instruction::Mul, ExtTy, CostKind);
       Cost += 2 * thisT()->getCastInstrCost(Instruction::Trunc, RetTy, ExtTy,
                                             CCH, CostKind);
       Cost += thisT()->getArithmeticInstrCost(
           Instruction::LShr, RetTy, CostKind, {TTI::OK_AnyValue, TTI::OP_None},
           {TTI::OK_UniformConstantValue, TTI::OP_None});
       Cost += thisT()->getArithmeticInstrCost(
           Instruction::Shl, RetTy, CostKind, {TTI::OK_AnyValue, TTI::OP_None},
           {TTI::OK_UniformConstantValue, TTI::OP_None});
       Cost += thisT()->getArithmeticInstrCost(Instruction::Or, RetTy, CostKind);
       return Cost;
     }
     case Intrinsic::abs: {
       // abs(X) = select(icmp(X,0),X,sub(0,X))
       Type *CondTy = RetTy->getWithNewBitWidth(1);
       CmpInst::Predicate Pred = CmpInst::ICMP_SGT;
       InstructionCost Cost = 0;
       Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, CondTy,
                                           Pred, CostKind);
       Cost += thisT()->getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,
                                           Pred, CostKind);
       // TODO: Should we add an OperandValueProperties::OP_Zero property?
       Cost += thisT()->getArithmeticInstrCost(
           BinaryOperator::Sub, RetTy, CostKind,
           {TTI::OK_UniformConstantValue, TTI::OP_None});
       return Cost;
     }
     case Intrinsic::fshl:
     case Intrinsic::fshr: {
       // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
       // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
       Type *CondTy = RetTy->getWithNewBitWidth(1);
       InstructionCost Cost = 0;
       Cost +=
           thisT()->getArithmeticInstrCost(BinaryOperator::Or, RetTy, CostKind);
       Cost +=
           thisT()->getArithmeticInstrCost(BinaryOperator::Sub, RetTy, CostKind);
       Cost +=
           thisT()->getArithmeticInstrCost(BinaryOperator::Shl, RetTy, CostKind);
       Cost += thisT()->getArithmeticInstrCost(BinaryOperator::LShr, RetTy,
                                               CostKind);
       Cost += thisT()->getArithmeticInstrCost(BinaryOperator::URem, RetTy,
                                               CostKind);
       // Shift-by-zero handling.
       Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, CondTy,
                                           CmpInst::ICMP_EQ, CostKind);
       Cost += thisT()->getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,
                                           CmpInst::ICMP_EQ, CostKind);
       return Cost;
     }
     case Intrinsic::fptosi_sat:
     case Intrinsic::fptoui_sat: {
       if (Tys.empty())
         break;
       Type *FromTy = Tys[0];
       bool IsSigned = IID == Intrinsic::fptosi_sat;
 
       InstructionCost Cost = 0;
       IntrinsicCostAttributes Attrs1(Intrinsic::minnum, FromTy,
                                      {FromTy, FromTy});
       Cost += thisT()->getIntrinsicInstrCost(Attrs1, CostKind);
       IntrinsicCostAttributes Attrs2(Intrinsic::maxnum, FromTy,
                                      {FromTy, FromTy});
       Cost += thisT()->getIntrinsicInstrCost(Attrs2, CostKind);
       Cost += thisT()->getCastInstrCost(
           IsSigned ? Instruction::FPToSI : Instruction::FPToUI, RetTy, FromTy,
           TTI::CastContextHint::None, CostKind);
       if (IsSigned) {
         Type *CondTy = RetTy->getWithNewBitWidth(1);
         Cost += thisT()->getCmpSelInstrCost(
             BinaryOperator::FCmp, FromTy, CondTy, CmpInst::FCMP_UNO, CostKind);
         Cost += thisT()->getCmpSelInstrCost(
             BinaryOperator::Select, RetTy, CondTy, CmpInst::FCMP_UNO, CostKind);
       }
       return Cost;
     }
     case Intrinsic::ucmp:
     case Intrinsic::scmp: {
       Type *CmpTy = Tys[0];
       Type *CondTy = RetTy->getWithNewBitWidth(1);
       InstructionCost Cost =
           thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, CmpTy, CondTy,
                                       CmpIntrinsic::getGTPredicate(IID),
                                       CostKind) +
           thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, CmpTy, CondTy,
                                       CmpIntrinsic::getLTPredicate(IID),
                                       CostKind);
 
       EVT VT = TLI->getValueType(DL, CmpTy, true);
       if (TLI->shouldExpandCmpUsingSelects(VT)) {
         // x < y ? -1 : (x > y ? 1 : 0)
         Cost += 2 * thisT()->getCmpSelInstrCost(
                         BinaryOperator::Select, RetTy, CondTy,
                         ICmpInst::BAD_ICMP_PREDICATE, CostKind);
       } else {
         // zext(x > y) - zext(x < y)
         Cost +=
             2 * thisT()->getCastInstrCost(CastInst::ZExt, RetTy, CondTy,
                                           TTI::CastContextHint::None, CostKind);
         Cost += thisT()->getArithmeticInstrCost(BinaryOperator::Sub, RetTy,
                                                 CostKind);
       }
       return Cost;
     }
     default:
       break;
     }
 
     // Else, assume that we need to scalarize this intrinsic. For math builtins
     // this will emit a costly libcall, adding call overhead and spills. Make it
     // very expensive.
     if (auto *RetVTy = dyn_cast<VectorType>(RetTy)) {
       // Scalable vectors cannot be scalarized, so return Invalid.
       if (isa<ScalableVectorType>(RetTy) || any_of(Tys, [](const Type *Ty) {
             return isa<ScalableVectorType>(Ty);
           }))
         return InstructionCost::getInvalid();
 
       InstructionCost ScalarizationCost =
           SkipScalarizationCost
               ? ScalarizationCostPassed
               : getScalarizationOverhead(RetVTy, /*Insert*/ true,
                                          /*Extract*/ false, CostKind);
 
       unsigned ScalarCalls = cast<FixedVectorType>(RetVTy)->getNumElements();
       SmallVector<Type *, 4> ScalarTys;
       for (Type *Ty : Tys) {
         if (Ty->isVectorTy())
           Ty = Ty->getScalarType();
         ScalarTys.push_back(Ty);
       }
       IntrinsicCostAttributes Attrs(IID, RetTy->getScalarType(), ScalarTys, FMF);
       InstructionCost ScalarCost =
           thisT()->getIntrinsicInstrCost(Attrs, CostKind);
       for (Type *Ty : Tys) {
         if (auto *VTy = dyn_cast<VectorType>(Ty)) {
           if (!ICA.skipScalarizationCost())
             ScalarizationCost += getScalarizationOverhead(
                 VTy, /*Insert*/ false, /*Extract*/ true, CostKind);
           ScalarCalls = std::max(ScalarCalls,
                                  cast<FixedVectorType>(VTy)->getNumElements());
         }
       }
       return ScalarCalls * ScalarCost + ScalarizationCost;
     }
 
     // This is going to be turned into a library call, make it expensive.
     return SingleCallCost;
   }
 
   /// Compute a cost of the given call instruction.
   ///
   /// Compute the cost of calling function F with return type RetTy and
   /// argument types Tys. F might be nullptr, in this case the cost of an
   /// arbitrary call with the specified signature will be returned.
   /// This is used, for instance,  when we estimate call of a vector
   /// counterpart of the given function.
   /// \param F Called function, might be nullptr.
   /// \param RetTy Return value types.
   /// \param Tys Argument types.
   /// \returns The cost of Call instruction.
   InstructionCost getCallInstrCost(Function *F, Type *RetTy,
                                    ArrayRef<Type *> Tys,
                                    TTI::TargetCostKind CostKind) {
     return 10;
   }
 
   unsigned getNumberOfParts(Type *Tp) {
     std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp);
     if (!LT.first.isValid())
       return 0;
     // Try to find actual number of parts for non-power-of-2 elements as
     // ceil(num-of-elements/num-of-subtype-elements).
     if (auto *FTp = dyn_cast<FixedVectorType>(Tp);
         Tp && LT.second.isFixedLengthVector() &&
         !has_single_bit(FTp->getNumElements())) {
       if (auto *SubTp = dyn_cast_if_present<FixedVectorType>(
               EVT(LT.second).getTypeForEVT(Tp->getContext()));
           SubTp && SubTp->getElementType() == FTp->getElementType())
         return divideCeil(FTp->getNumElements(), SubTp->getNumElements());
     }
     return *LT.first.getValue();
   }
 
   InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *,
                                             const SCEV *) {
     return 0;
   }
 
   /// Try to calculate arithmetic and shuffle op costs for reduction intrinsics.
   /// We're assuming that reduction operation are performing the following way:
   ///
   /// %val1 = shufflevector<n x t> %val, <n x t> %undef,
   /// <n x i32> <i32 n/2, i32 n/2 + 1, ..., i32 n, i32 undef, ..., i32 undef>
   ///            \----------------v-------------/  \----------v------------/
   ///                            n/2 elements               n/2 elements
   /// %red1 = op <n x t> %val, <n x t> val1
   /// After this operation we have a vector %red1 where only the first n/2
   /// elements are meaningful, the second n/2 elements are undefined and can be
   /// dropped. All other operations are actually working with the vector of
   /// length n/2, not n, though the real vector length is still n.
   /// %val2 = shufflevector<n x t> %red1, <n x t> %undef,
   /// <n x i32> <i32 n/4, i32 n/4 + 1, ..., i32 n/2, i32 undef, ..., i32 undef>
   ///            \----------------v-------------/  \----------v------------/
   ///                            n/4 elements               3*n/4 elements
   /// %red2 = op <n x t> %red1, <n x t> val2  - working with the vector of
   /// length n/2, the resulting vector has length n/4 etc.
   ///
   /// The cost model should take into account that the actual length of the
   /// vector is reduced on each iteration.
   InstructionCost getTreeReductionCost(unsigned Opcode, VectorType *Ty,
                                        TTI::TargetCostKind CostKind) {
     // Targets must implement a default value for the scalable case, since
     // we don't know how many lanes the vector has.
     if (isa<ScalableVectorType>(Ty))
       return InstructionCost::getInvalid();
 
     Type *ScalarTy = Ty->getElementType();
     unsigned NumVecElts = cast<FixedVectorType>(Ty)->getNumElements();
     if ((Opcode == Instruction::Or || Opcode == Instruction::And) &&
         ScalarTy == IntegerType::getInt1Ty(Ty->getContext()) &&
         NumVecElts >= 2) {
       // Or reduction for i1 is represented as:
       // %val = bitcast <ReduxWidth x i1> to iReduxWidth
       // %res = cmp ne iReduxWidth %val, 0
       // And reduction for i1 is represented as:
       // %val = bitcast <ReduxWidth x i1> to iReduxWidth
       // %res = cmp eq iReduxWidth %val, 11111
       Type *ValTy = IntegerType::get(Ty->getContext(), NumVecElts);
       return thisT()->getCastInstrCost(Instruction::BitCast, ValTy, Ty,
                                        TTI::CastContextHint::None, CostKind) +
              thisT()->getCmpSelInstrCost(Instruction::ICmp, ValTy,
                                          CmpInst::makeCmpResultType(ValTy),
                                          CmpInst::BAD_ICMP_PREDICATE, CostKind);
     }
     unsigned NumReduxLevels = Log2_32(NumVecElts);
     InstructionCost ArithCost = 0;
     InstructionCost ShuffleCost = 0;
     std::pair<InstructionCost, MVT> LT = thisT()->getTypeLegalizationCost(Ty);
     unsigned LongVectorCount = 0;
     unsigned MVTLen =
         LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
     while (NumVecElts > MVTLen) {
       NumVecElts /= 2;
       VectorType *SubTy = FixedVectorType::get(ScalarTy, NumVecElts);
       ShuffleCost += thisT()->getShuffleCost(TTI::SK_ExtractSubvector, Ty, {},
                                              CostKind, NumVecElts, SubTy);
       ArithCost += thisT()->getArithmeticInstrCost(Opcode, SubTy, CostKind);
       Ty = SubTy;
       ++LongVectorCount;
     }
 
     NumReduxLevels -= LongVectorCount;
 
     // The minimal length of the vector is limited by the real length of vector
     // operations performed on the current platform. That's why several final
     // reduction operations are performed on the vectors with the same
     // architecture-dependent length.
 
     // By default reductions need one shuffle per reduction level.
     ShuffleCost +=
         NumReduxLevels * thisT()->getShuffleCost(TTI::SK_PermuteSingleSrc, Ty,
                                                  {}, CostKind, 0, Ty);
     ArithCost +=
         NumReduxLevels * thisT()->getArithmeticInstrCost(Opcode, Ty, CostKind);
     return ShuffleCost + ArithCost +
            thisT()->getVectorInstrCost(Instruction::ExtractElement, Ty,
                                        CostKind, 0, nullptr, nullptr);
   }
 
   /// Try to calculate the cost of performing strict (in-order) reductions,
   /// which involves doing a sequence of floating point additions in lane
   /// order, starting with an initial value. For example, consider a scalar
   /// initial value 'InitVal' of type float and a vector of type <4 x float>:
   ///
   ///   Vector = <float %v0, float %v1, float %v2, float %v3>
   ///
   ///   %add1 = %InitVal + %v0
   ///   %add2 = %add1 + %v1
   ///   %add3 = %add2 + %v2
   ///   %add4 = %add3 + %v3
   ///
   /// As a simple estimate we can say the cost of such a reduction is 4 times
   /// the cost of a scalar FP addition. We can only estimate the costs for
   /// fixed-width vectors here because for scalable vectors we do not know the
   /// runtime number of operations.
   InstructionCost getOrderedReductionCost(unsigned Opcode, VectorType *Ty,
                                           TTI::TargetCostKind CostKind) {
     // Targets must implement a default value for the scalable case, since
     // we don't know how many lanes the vector has.
     if (isa<ScalableVectorType>(Ty))
       return InstructionCost::getInvalid();
 
     auto *VTy = cast<FixedVectorType>(Ty);
     InstructionCost ExtractCost = getScalarizationOverhead(
         VTy, /*Insert=*/false, /*Extract=*/true, CostKind);
     InstructionCost ArithCost = thisT()->getArithmeticInstrCost(
         Opcode, VTy->getElementType(), CostKind);
     ArithCost *= VTy->getNumElements();
 
     return ExtractCost + ArithCost;
   }
 
   InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
                                              std::optional<FastMathFlags> FMF,
                                              TTI::TargetCostKind CostKind) {
     assert(Ty && "Unknown reduction vector type");
     if (TTI::requiresOrderedReduction(FMF))
       return getOrderedReductionCost(Opcode, Ty, CostKind);
     return getTreeReductionCost(Opcode, Ty, CostKind);
   }
 
   /// Try to calculate op costs for min/max reduction operations.
   /// \param CondTy Conditional type for the Select instruction.
   InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                          FastMathFlags FMF,
                                          TTI::TargetCostKind CostKind) {
     // Targets must implement a default value for the scalable case, since
     // we don't know how many lanes the vector has.
     if (isa<ScalableVectorType>(Ty))
       return InstructionCost::getInvalid();
 
     Type *ScalarTy = Ty->getElementType();
     unsigned NumVecElts = cast<FixedVectorType>(Ty)->getNumElements();
     unsigned NumReduxLevels = Log2_32(NumVecElts);
     InstructionCost MinMaxCost = 0;
     InstructionCost ShuffleCost = 0;
     std::pair<InstructionCost, MVT> LT = thisT()->getTypeLegalizationCost(Ty);
     unsigned LongVectorCount = 0;
     unsigned MVTLen =
         LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
     while (NumVecElts > MVTLen) {
       NumVecElts /= 2;
       auto *SubTy = FixedVectorType::get(ScalarTy, NumVecElts);
 
       ShuffleCost += thisT()->getShuffleCost(TTI::SK_ExtractSubvector, Ty, {},
                                              CostKind, NumVecElts, SubTy);
 
       IntrinsicCostAttributes Attrs(IID, SubTy, {SubTy, SubTy}, FMF);
       MinMaxCost += getIntrinsicInstrCost(Attrs, CostKind);
       Ty = SubTy;
       ++LongVectorCount;
     }
 
     NumReduxLevels -= LongVectorCount;
 
     // The minimal length of the vector is limited by the real length of vector
     // operations performed on the current platform. That's why several final
     // reduction opertions are perfomed on the vectors with the same
     // architecture-dependent length.
     ShuffleCost +=
         NumReduxLevels * thisT()->getShuffleCost(TTI::SK_PermuteSingleSrc, Ty,
                                                  {}, CostKind, 0, Ty);
     IntrinsicCostAttributes Attrs(IID, Ty, {Ty, Ty}, FMF);
     MinMaxCost += NumReduxLevels * getIntrinsicInstrCost(Attrs, CostKind);
     // The last min/max should be in vector registers and we counted it above.
     // So just need a single extractelement.
     return ShuffleCost + MinMaxCost +
            thisT()->getVectorInstrCost(Instruction::ExtractElement, Ty,
                                        CostKind, 0, nullptr, nullptr);
   }
 
   InstructionCost getExtendedReductionCost(unsigned Opcode, bool IsUnsigned,
                                            Type *ResTy, VectorType *Ty,
                                            FastMathFlags FMF,
                                            TTI::TargetCostKind CostKind) {
     if (auto *FTy = dyn_cast<FixedVectorType>(Ty);
         FTy && IsUnsigned && Opcode == Instruction::Add &&
         FTy->getElementType() == IntegerType::getInt1Ty(Ty->getContext())) {
       // Represent vector_reduce_add(ZExt(<n x i1>)) as
       // ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).
       auto *IntTy =
           IntegerType::get(ResTy->getContext(), FTy->getNumElements());
       IntrinsicCostAttributes ICA(Intrinsic::ctpop, IntTy, {IntTy}, FMF);
       return thisT()->getCastInstrCost(Instruction::BitCast, IntTy, FTy,
                                        TTI::CastContextHint::None, CostKind) +
              thisT()->getIntrinsicInstrCost(ICA, CostKind);
     }
     // Without any native support, this is equivalent to the cost of
     // vecreduce.opcode(ext(Ty A)).
     VectorType *ExtTy = VectorType::get(ResTy, Ty);
     InstructionCost RedCost =
         thisT()->getArithmeticReductionCost(Opcode, ExtTy, FMF, CostKind);
     InstructionCost ExtCost = thisT()->getCastInstrCost(
         IsUnsigned ? Instruction::ZExt : Instruction::SExt, ExtTy, Ty,
         TTI::CastContextHint::None, CostKind);
 
     return RedCost + ExtCost;
   }
 
   InstructionCost getMulAccReductionCost(bool IsUnsigned, Type *ResTy,
                                          VectorType *Ty,
                                          TTI::TargetCostKind CostKind) {
     // Without any native support, this is equivalent to the cost of
     // vecreduce.add(mul(ext(Ty A), ext(Ty B))) or
     // vecreduce.add(mul(A, B)).
     VectorType *ExtTy = VectorType::get(ResTy, Ty);
     InstructionCost RedCost = thisT()->getArithmeticReductionCost(
         Instruction::Add, ExtTy, std::nullopt, CostKind);
     InstructionCost ExtCost = thisT()->getCastInstrCost(
         IsUnsigned ? Instruction::ZExt : Instruction::SExt, ExtTy, Ty,
         TTI::CastContextHint::None, CostKind);
 
     InstructionCost MulCost =
         thisT()->getArithmeticInstrCost(Instruction::Mul, ExtTy, CostKind);
 
     return RedCost + MulCost + 2 * ExtCost;
   }
 
   InstructionCost getVectorSplitCost() { return 1; }
 
   /// @}
 };
 
 /// Concrete BasicTTIImpl that can be used if no further customization
 /// is needed.
 class BasicTTIImpl : public BasicTTIImplBase<BasicTTIImpl> {
   using BaseT = BasicTTIImplBase<BasicTTIImpl>;
 
   friend class BasicTTIImplBase<BasicTTIImpl>;
 
   const TargetSubtargetInfo *ST;
   const TargetLoweringBase *TLI;
 
   const TargetSubtargetInfo *getST() const { return ST; }
   const TargetLoweringBase *getTLI() const { return TLI; }
 
 public:
   explicit BasicTTIImpl(const TargetMachine *TM, const Function &F);
 };
 
 } // end namespace llvm
 
 #endif // LLVM_CODEGEN_BASICTTIIMPL_H