clang/Support/RISCVVIntrinsicUtils.h

0001 //===--- RISCVVIntrinsicUtils.h - RISC-V Vector Intrinsic Utils -*- C++ -*-===//
0002 //
0003 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
0004 // See https://llvm.org/LICENSE.txt for license information.
0005 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
0006 //
0007 //===----------------------------------------------------------------------===//
0008
0009 #ifndef CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
0010 #define CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
0011
0012 #include "llvm/ADT/ArrayRef.h"
0013 #include "llvm/ADT/BitmaskEnum.h"
0014 #include "llvm/ADT/SmallVector.h"
0015 #include "llvm/ADT/StringRef.h"
0016 #include <cstdint>
0017 #include <optional>
0018 #include <set>
0019 #include <string>
0020 #include <unordered_map>
0021 #include <vector>
0022
0023 namespace llvm {
0024 class raw_ostream;
0025 } // end namespace llvm
0026
0027 namespace clang {
0028 namespace RISCV {
0029
0030 using VScaleVal = std::optional<unsigned>;
0031
0032 // Modifier for vector type.
0033 enum class VectorTypeModifier : uint8_t {
0034   NoModifier,
0035   Widening2XVector,
0036   Widening4XVector,
0037   Widening8XVector,
0038   MaskVector,
0039   Log2EEW3,
0040   Log2EEW4,
0041   Log2EEW5,
0042   Log2EEW6,
0043   FixedSEW8,
0044   FixedSEW16,
0045   FixedSEW32,
0046   FixedSEW64,
0047   LFixedLog2LMULN3,
0048   LFixedLog2LMULN2,
0049   LFixedLog2LMULN1,
0050   LFixedLog2LMUL0,
0051   LFixedLog2LMUL1,
0052   LFixedLog2LMUL2,
0053   LFixedLog2LMUL3,
0054   SFixedLog2LMULN3,
0055   SFixedLog2LMULN2,
0056   SFixedLog2LMULN1,
0057   SFixedLog2LMUL0,
0058   SFixedLog2LMUL1,
0059   SFixedLog2LMUL2,
0060   SFixedLog2LMUL3,
0061   SEFixedLog2LMULN3,
0062   SEFixedLog2LMULN2,
0063   SEFixedLog2LMULN1,
0064   SEFixedLog2LMUL0,
0065   SEFixedLog2LMUL1,
0066   SEFixedLog2LMUL2,
0067   SEFixedLog2LMUL3,
0068   Tuple2,
0069   Tuple3,
0070   Tuple4,
0071   Tuple5,
0072   Tuple6,
0073   Tuple7,
0074   Tuple8,
0075 };
0076
0077 // Similar to basic type but used to describe what's kind of type related to
0078 // basic vector type, used to compute type info of arguments.
0079 enum class BaseTypeModifier : uint8_t {
0080   Invalid,
0081   Scalar,
0082   Vector,
0083   Void,
0084   SizeT,
0085   Ptrdiff,
0086   UnsignedLong,
0087   SignedLong,
0088   Float32
0089 };
0090
0091 // Modifier for type, used for both scalar and vector types.
0092 enum class TypeModifier : uint8_t {
0093   NoModifier = 0,
0094   Pointer = 1 << 0,
0095   Const = 1 << 1,
0096   Immediate = 1 << 2,
0097   UnsignedInteger = 1 << 3,
0098   SignedInteger = 1 << 4,
0099   Float = 1 << 5,
0100   BFloat = 1 << 6,
0101   // LMUL1 should be kind of VectorTypeModifier, but that might come with
0102   // Widening2XVector for widening reduction.
0103   // However that might require VectorTypeModifier become bitmask rather than
0104   // simple enum, so we decide keek LMUL1 in TypeModifier for code size
0105   // optimization of clang binary size.
0106   LMUL1 = 1 << 7,
0107   MaxOffset = 7,
0108   LLVM_MARK_AS_BITMASK_ENUM(LMUL1),
0109 };
0110
0111 class Policy {
0112 public:
0113   enum PolicyType {
0114     Undisturbed,
0115     Agnostic,
0116   };
0117
0118 private:
0119   // The default assumption for an RVV instruction is TAMA, as an undisturbed
0120   // policy generally will affect the performance of an out-of-order core.
0121   const PolicyType TailPolicy = Agnostic;
0122   const PolicyType MaskPolicy = Agnostic;
0123
0124 public:
0125   Policy() = default;
0126   Policy(PolicyType TailPolicy) : TailPolicy(TailPolicy) {}
0127   Policy(PolicyType TailPolicy, PolicyType MaskPolicy)
0128       : TailPolicy(TailPolicy), MaskPolicy(MaskPolicy) {}
0129
0130   bool isTAMAPolicy() const {
0131     return TailPolicy == Agnostic && MaskPolicy == Agnostic;
0132   }
0133
0134   bool isTAMUPolicy() const {
0135     return TailPolicy == Agnostic && MaskPolicy == Undisturbed;
0136   }
0137
0138   bool isTUMAPolicy() const {
0139     return TailPolicy == Undisturbed && MaskPolicy == Agnostic;
0140   }
0141
0142   bool isTUMUPolicy() const {
0143     return TailPolicy == Undisturbed && MaskPolicy == Undisturbed;
0144   }
0145
0146   bool isTAPolicy() const { return TailPolicy == Agnostic; }
0147
0148   bool isTUPolicy() const { return TailPolicy == Undisturbed; }
0149
0150   bool isMAPolicy() const { return MaskPolicy == Agnostic; }
0151
0152   bool isMUPolicy() const { return MaskPolicy == Undisturbed; }
0153
0154   bool operator==(const Policy &Other) const {
0155     return TailPolicy == Other.TailPolicy && MaskPolicy == Other.MaskPolicy;
0156   }
0157
0158   bool operator!=(const Policy &Other) const { return !(*this == Other); }
0159
0160   bool operator<(const Policy &Other) const {
0161     // Just for maintain the old order for quick test.
0162     if (MaskPolicy != Other.MaskPolicy)
0163       return Other.MaskPolicy < MaskPolicy;
0164     return TailPolicy < Other.TailPolicy;
0165   }
0166 };
0167
0168 // PrototypeDescriptor is used to compute type info of arguments or return
0169 // value.
0170 struct PrototypeDescriptor {
0171   constexpr PrototypeDescriptor() = default;
0172   constexpr PrototypeDescriptor(
0173       BaseTypeModifier PT,
0174       VectorTypeModifier VTM = VectorTypeModifier::NoModifier,
0175       TypeModifier TM = TypeModifier::NoModifier)
0176       : PT(static_cast<uint8_t>(PT)), VTM(static_cast<uint8_t>(VTM)),
0177         TM(static_cast<uint8_t>(TM)) {}
0178   constexpr PrototypeDescriptor(uint8_t PT, uint8_t VTM, uint8_t TM)
0179       : PT(PT), VTM(VTM), TM(TM) {}
0180
0181   uint8_t PT = static_cast<uint8_t>(BaseTypeModifier::Invalid);
0182   uint8_t VTM = static_cast<uint8_t>(VectorTypeModifier::NoModifier);
0183   uint8_t TM = static_cast<uint8_t>(TypeModifier::NoModifier);
0184
0185   bool operator!=(const PrototypeDescriptor &PD) const {
0186     return !(*this == PD);
0187   }
0188   bool operator==(const PrototypeDescriptor &PD) const {
0189     return PD.PT == PT && PD.VTM == VTM && PD.TM == TM;
0190   }
0191   bool operator<(const PrototypeDescriptor &PD) const {
0192     return std::tie(PT, VTM, TM) < std::tie(PD.PT, PD.VTM, PD.TM);
0193   }
0194   static const PrototypeDescriptor Mask;
0195   static const PrototypeDescriptor Vector;
0196   static const PrototypeDescriptor VL;
0197   static std::optional<PrototypeDescriptor>
0198   parsePrototypeDescriptor(llvm::StringRef PrototypeStr);
0199 };
0200
0201 llvm::SmallVector<PrototypeDescriptor>
0202 parsePrototypes(llvm::StringRef Prototypes);
0203
0204 // Basic type of vector type.
0205 enum class BasicType : uint8_t {
0206   Unknown = 0,
0207   Int8 = 1 << 0,
0208   Int16 = 1 << 1,
0209   Int32 = 1 << 2,
0210   Int64 = 1 << 3,
0211   BFloat16 = 1 << 4,
0212   Float16 = 1 << 5,
0213   Float32 = 1 << 6,
0214   Float64 = 1 << 7,
0215   MaxOffset = 7,
0216   LLVM_MARK_AS_BITMASK_ENUM(Float64),
0217 };
0218
0219 // Type of vector type.
0220 enum ScalarTypeKind : uint8_t {
0221   Void,
0222   Size_t,
0223   Ptrdiff_t,
0224   UnsignedLong,
0225   SignedLong,
0226   Boolean,
0227   SignedInteger,
0228   UnsignedInteger,
0229   Float,
0230   BFloat,
0231   Invalid,
0232   Undefined,
0233 };
0234
0235 // Exponential LMUL
0236 struct LMULType {
0237   int Log2LMUL;
0238   LMULType(int Log2LMUL);
0239   // Return the C/C++ string representation of LMUL
0240   std::string str() const;
0241   std::optional<unsigned> getScale(unsigned ElementBitwidth) const;
0242   void MulLog2LMUL(int Log2LMUL);
0243 };
0244
0245 class RVVType;
0246 using RVVTypePtr = RVVType *;
0247 using RVVTypes = std::vector<RVVTypePtr>;
0248 class RVVTypeCache;
0249
0250 // This class is compact representation of a valid and invalid RVVType.
0251 class RVVType {
0252   friend class RVVTypeCache;
0253
0254   BasicType BT;
0255   ScalarTypeKind ScalarType = Undefined;
0256   LMULType LMUL;
0257   bool IsPointer = false;
0258   // IsConstant indices are "int", but have the constant expression.
0259   bool IsImmediate = false;
0260   // Const qualifier for pointer to const object or object of const type.
0261   bool IsConstant = false;
0262   unsigned ElementBitwidth = 0;
0263   VScaleVal Scale = 0;
0264   bool Valid;
0265   bool IsTuple = false;
0266   unsigned NF = 0;
0267
0268   std::string BuiltinStr;
0269   std::string ClangBuiltinStr;
0270   std::string Str;
0271   std::string ShortStr;
0272
0273   enum class FixedLMULType { LargerThan, SmallerThan, SmallerOrEqual };
0274
0275   RVVType(BasicType BT, int Log2LMUL, const PrototypeDescriptor &Profile);
0276
0277 public:
0278   // Return the string representation of a type, which is an encoded string for
0279   // passing to the BUILTIN() macro in Builtins.def.
0280   const std::string &getBuiltinStr() const { return BuiltinStr; }
0281
0282   // Return the clang builtin type for RVV vector type which are used in the
0283   // riscv_vector.h header file.
0284   const std::string &getClangBuiltinStr() const { return ClangBuiltinStr; }
0285
0286   // Return the C/C++ string representation of a type for use in the
0287   // riscv_vector.h header file.
0288   const std::string &getTypeStr() const { return Str; }
0289
0290   // Return the short name of a type for C/C++ name suffix.
0291   const std::string &getShortStr() {
0292     // Not all types are used in short name, so compute the short name by
0293     // demanded.
0294     if (ShortStr.empty())
0295       initShortStr();
0296     return ShortStr;
0297   }
0298
0299   bool isValid() const { return Valid; }
0300   bool isScalar() const { return Scale && *Scale == 0; }
0301   bool isVector() const { return Scale && *Scale != 0; }
0302   bool isVector(unsigned Width) const {
0303     return isVector() && ElementBitwidth == Width;
0304   }
0305   bool isFloat() const { return ScalarType == ScalarTypeKind::Float; }
0306   bool isBFloat() const { return ScalarType == ScalarTypeKind::BFloat; }
0307   bool isSignedInteger() const {
0308     return ScalarType == ScalarTypeKind::SignedInteger;
0309   }
0310   bool isFloatVector(unsigned Width) const {
0311     return isVector() && isFloat() && ElementBitwidth == Width;
0312   }
0313   bool isFloat(unsigned Width) const {
0314     return isFloat() && ElementBitwidth == Width;
0315   }
0316   bool isConstant() const { return IsConstant; }
0317   bool isPointer() const { return IsPointer; }
0318   bool isTuple() const { return IsTuple; }
0319   unsigned getElementBitwidth() const { return ElementBitwidth; }
0320
0321   ScalarTypeKind getScalarType() const { return ScalarType; }
0322   VScaleVal getScale() const { return Scale; }
0323   unsigned getNF() const {
0324     assert(NF > 1 && NF <= 8 && "Only legal NF should be fetched");
0325     return NF;
0326   }
0327
0328 private:
0329   // Verify RVV vector type and set Valid.
0330   bool verifyType() const;
0331
0332   // Creates a type based on basic types of TypeRange
0333   void applyBasicType();
0334
0335   // Applies a prototype modifier to the current type. The result maybe an
0336   // invalid type.
0337   void applyModifier(const PrototypeDescriptor &prototype);
0338
0339   void applyLog2EEW(unsigned Log2EEW);
0340   void applyFixedSEW(unsigned NewSEW);
0341   void applyFixedLog2LMUL(int Log2LMUL, enum FixedLMULType Type);
0342
0343   // Compute and record a string for legal type.
0344   void initBuiltinStr();
0345   // Compute and record a builtin RVV vector type string.
0346   void initClangBuiltinStr();
0347   // Compute and record a type string for used in the header.
0348   void initTypeStr();
0349   // Compute and record a short name of a type for C/C++ name suffix.
0350   void initShortStr();
0351 };
0352
0353 // This class is used to manage RVVType, RVVType should only created by this
0354 // class, also provided thread-safe cache capability.
0355 class RVVTypeCache {
0356 private:
0357   std::unordered_map<uint64_t, RVVType> LegalTypes;
0358   std::set<uint64_t> IllegalTypes;
0359
0360 public:
0361   /// Compute output and input types by applying different config (basic type
0362   /// and LMUL with type transformers). It also record result of type in legal
0363   /// or illegal set to avoid compute the same config again. The result maybe
0364   /// have illegal RVVType.
0365   std::optional<RVVTypes>
0366   computeTypes(BasicType BT, int Log2LMUL, unsigned NF,
0367                llvm::ArrayRef<PrototypeDescriptor> Prototype);
0368   std::optional<RVVTypePtr> computeType(BasicType BT, int Log2LMUL,
0369                                         PrototypeDescriptor Proto);
0370 };
0371
0372 enum PolicyScheme : uint8_t {
0373   SchemeNone,
0374   // Passthru operand is at first parameter in C builtin.
0375   HasPassthruOperand,
0376   HasPolicyOperand,
0377 };
0378
0379 // TODO refactor RVVIntrinsic class design after support all intrinsic
0380 // combination. This represents an instantiation of an intrinsic with a
0381 // particular type and prototype
0382 class RVVIntrinsic {
0383
0384 private:
0385   std::string BuiltinName; // Builtin name
0386   std::string Name;        // C intrinsic name.
0387   std::string OverloadedName;
0388   std::string IRName;
0389   bool IsMasked;
0390   bool HasMaskedOffOperand;
0391   bool HasVL;
0392   PolicyScheme Scheme;
0393   bool SupportOverloading;
0394   bool HasBuiltinAlias;
0395   std::string ManualCodegen;
0396   RVVTypePtr OutputType; // Builtin output type
0397   RVVTypes InputTypes;   // Builtin input types
0398   // The types we use to obtain the specific LLVM intrinsic. They are index of
0399   // InputTypes. -1 means the return type.
0400   std::vector<int64_t> IntrinsicTypes;
0401   unsigned NF = 1;
0402   Policy PolicyAttrs;
0403
0404 public:
0405   RVVIntrinsic(llvm::StringRef Name, llvm::StringRef Suffix,
0406                llvm::StringRef OverloadedName, llvm::StringRef OverloadedSuffix,
0407                llvm::StringRef IRName, bool IsMasked, bool HasMaskedOffOperand,
0408                bool HasVL, PolicyScheme Scheme, bool SupportOverloading,
0409                bool HasBuiltinAlias, llvm::StringRef ManualCodegen,
0410                const RVVTypes &Types,
0411                const std::vector<int64_t> &IntrinsicTypes,
0412                unsigned NF, Policy PolicyAttrs, bool HasFRMRoundModeOp);
0413   ~RVVIntrinsic() = default;
0414
0415   RVVTypePtr getOutputType() const { return OutputType; }
0416   const RVVTypes &getInputTypes() const { return InputTypes; }
0417   llvm::StringRef getBuiltinName() const { return BuiltinName; }
0418   bool hasMaskedOffOperand() const { return HasMaskedOffOperand; }
0419   bool hasVL() const { return HasVL; }
0420   bool hasPolicy() const { return Scheme != PolicyScheme::SchemeNone; }
0421   bool hasPassthruOperand() const {
0422     return Scheme == PolicyScheme::HasPassthruOperand;
0423   }
0424   bool hasPolicyOperand() const {
0425     return Scheme == PolicyScheme::HasPolicyOperand;
0426   }
0427   bool supportOverloading() const { return SupportOverloading; }
0428   bool hasBuiltinAlias() const { return HasBuiltinAlias; }
0429   bool hasManualCodegen() const { return !ManualCodegen.empty(); }
0430   bool isMasked() const { return IsMasked; }
0431   llvm::StringRef getOverloadedName() const { return OverloadedName; }
0432   llvm::StringRef getIRName() const { return IRName; }
0433   llvm::StringRef getManualCodegen() const { return ManualCodegen; }
0434   PolicyScheme getPolicyScheme() const { return Scheme; }
0435   unsigned getNF() const { return NF; }
0436   const std::vector<int64_t> &getIntrinsicTypes() const {
0437     return IntrinsicTypes;
0438   }
0439   Policy getPolicyAttrs() const {
0440     return PolicyAttrs;
0441   }
0442   unsigned getPolicyAttrsBits() const {
0443     // CGBuiltin.cpp
0444     // The 0th bit simulates the `vta` of RVV
0445     // The 1st bit simulates the `vma` of RVV
0446     // int PolicyAttrs = 0;
0447
0448     if (PolicyAttrs.isTUMAPolicy())
0449       return 2;
0450     if (PolicyAttrs.isTAMAPolicy())
0451       return 3;
0452     if (PolicyAttrs.isTUMUPolicy())
0453       return 0;
0454     if (PolicyAttrs.isTAMUPolicy())
0455       return 1;
0456
0457     llvm_unreachable("unsupport policy");
0458     return 0;
0459   }
0460
0461   // Return the type string for a BUILTIN() macro in Builtins.def.
0462   std::string getBuiltinTypeStr() const;
0463
0464   static std::string
0465   getSuffixStr(RVVTypeCache &TypeCache, BasicType Type, int Log2LMUL,
0466                llvm::ArrayRef<PrototypeDescriptor> PrototypeDescriptors);
0467
0468   static llvm::SmallVector<PrototypeDescriptor>
0469   computeBuiltinTypes(llvm::ArrayRef<PrototypeDescriptor> Prototype,
0470                       bool IsMasked, bool HasMaskedOffOperand, bool HasVL,
0471                       unsigned NF, PolicyScheme DefaultScheme,
0472                       Policy PolicyAttrs, bool IsTuple);
0473
0474   static llvm::SmallVector<Policy> getSupportedUnMaskedPolicies();
0475   static llvm::SmallVector<Policy>
0476       getSupportedMaskedPolicies(bool HasTailPolicy, bool HasMaskPolicy);
0477
0478   static void updateNamesAndPolicy(bool IsMasked, bool HasPolicy,
0479                                    std::string &Name, std::string &BuiltinName,
0480                                    std::string &OverloadedName,
0481                                    Policy &PolicyAttrs, bool HasFRMRoundModeOp);
0482 };
0483
0484 // RVVRequire should be sync'ed with target features, but only
0485 // required features used in riscv_vector.td.
0486 enum RVVRequire : uint32_t {
0487   RVV_REQ_None = 0,
0488   RVV_REQ_RV64 = 1 << 0,
0489   RVV_REQ_Zvfhmin = 1 << 1,
0490   RVV_REQ_Xsfvcp = 1 << 2,
0491   RVV_REQ_Xsfvfnrclipxfqf = 1 << 3,
0492   RVV_REQ_Xsfvfwmaccqqq = 1 << 4,
0493   RVV_REQ_Xsfvqmaccdod = 1 << 5,
0494   RVV_REQ_Xsfvqmaccqoq = 1 << 6,
0495   RVV_REQ_Zvbb = 1 << 7,
0496   RVV_REQ_Zvbc = 1 << 8,
0497   RVV_REQ_Zvkb = 1 << 9,
0498   RVV_REQ_Zvkg = 1 << 10,
0499   RVV_REQ_Zvkned = 1 << 11,
0500   RVV_REQ_Zvknha = 1 << 12,
0501   RVV_REQ_Zvknhb = 1 << 13,
0502   RVV_REQ_Zvksed = 1 << 14,
0503   RVV_REQ_Zvksh = 1 << 15,
0504   RVV_REQ_Zvfbfwma = 1 << 16,
0505   RVV_REQ_Zvfbfmin = 1 << 17,
0506   RVV_REQ_Zvfh = 1 << 18,
0507   RVV_REQ_Experimental = 1 << 19,
0508
0509   LLVM_MARK_AS_BITMASK_ENUM(RVV_REQ_Experimental)
0510 };
0511
0512 // Raw RVV intrinsic info, used to expand later.
0513 // This struct is highly compact for minimized code size.
0514 struct RVVIntrinsicRecord {
0515   // Intrinsic name, e.g. vadd_vv
0516   const char *Name;
0517
0518   // Overloaded intrinsic name, could be empty if it can be computed from Name.
0519   // e.g. vadd
0520   const char *OverloadedName;
0521
0522   // Prototype for this intrinsic, index of RVVSignatureTable.
0523   uint16_t PrototypeIndex;
0524
0525   // Suffix of intrinsic name, index of RVVSignatureTable.
0526   uint16_t SuffixIndex;
0527
0528   // Suffix of overloaded intrinsic name, index of RVVSignatureTable.
0529   uint16_t OverloadedSuffixIndex;
0530
0531   // Length of the prototype.
0532   uint8_t PrototypeLength;
0533
0534   // Length of intrinsic name suffix.
0535   uint8_t SuffixLength;
0536
0537   // Length of overloaded intrinsic suffix.
0538   uint8_t OverloadedSuffixSize;
0539
0540   // Required target features for this intrinsic.
0541   uint32_t RequiredExtensions;
0542
0543   // Supported type, mask of BasicType.
0544   uint8_t TypeRangeMask;
0545
0546   // Supported LMUL.
0547   uint8_t Log2LMULMask;
0548
0549   // Number of fields, greater than 1 if it's segment load/store.
0550   uint8_t NF;
0551
0552   bool HasMasked : 1;
0553   bool HasVL : 1;
0554   bool HasMaskedOffOperand : 1;
0555   bool HasTailPolicy : 1;
0556   bool HasMaskPolicy : 1;
0557   bool HasFRMRoundModeOp : 1;
0558   bool IsTuple : 1;
0559   LLVM_PREFERRED_TYPE(PolicyScheme)
0560   uint8_t UnMaskedPolicyScheme : 2;
0561   LLVM_PREFERRED_TYPE(PolicyScheme)
0562   uint8_t MaskedPolicyScheme : 2;
0563 };
0564
0565 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
0566                               const RVVIntrinsicRecord &RVVInstrRecord);
0567
0568 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
0569 } // end namespace RISCV
0570
0571 } // end namespace clang
0572
0573 #endif // CLANG_SUPPORT_RISCVVINTRINSICUTILS_H