Target/GlobalISel/Combine.td

0001 //===- Combine.td - Combine rule definitions ---------------*- tablegen -*-===//
0002 //
0003 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
0004 // See https://llvm.org/LICENSE.txt for license information.
0005 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
0006 //
0007 //===----------------------------------------------------------------------===//
0008 //
0009 // Declare GlobalISel combine rules and provide mechanisms to opt-out.
0010 //
0011 //===----------------------------------------------------------------------===//
0012
0013
0014 //===----------------------------------------------------------------------===//
0015 // Base Classes
0016 //
0017 // These are the core classes that the combiner backend relies on.
0018 //===----------------------------------------------------------------------===//
0019
0020 /// All arguments of the defs operator must be subclasses of GIDefKind or
0021 /// sub-dags whose operator is GIDefKindWithArgs.
0022 class GIDefKind;
0023 class GIDefKindWithArgs;
0024
0025 /// Declare a root node. There must be at least one of these in every combine
0026 /// rule.
0027 def root : GIDefKind;
0028
0029 def defs;
0030
0031 def pattern;
0032 def match;
0033 def apply;
0034
0035 def wip_match_opcode;
0036
0037 // Common base class for GICombineRule and GICombineGroup.
0038 class GICombine {
0039   // See GICombineGroup. We only declare it here to make the tablegen pass
0040   // simpler.
0041   list<GICombine> Rules = ?;
0042 }
0043
0044 // A group of combine rules that can be added to a GICombiner or another group.
0045 class GICombineGroup<list<GICombine> rules> : GICombine {
0046   // The rules contained in this group. The rules in a group are flattened into
0047   // a single list and sorted into whatever order is most efficient. However,
0048   // they will never be re-ordered such that behaviour differs from the
0049   // specified order. It is therefore possible to use the order of rules in this
0050   // list to describe priorities.
0051   let Rules = rules;
0052 }
0053
0054 // Declares a combiner implementation class
0055 class GICombiner<string classname, list<GICombine> rules>
0056     : GICombineGroup<rules> {
0057   // The class name to use in the generated output.
0058   string Classname = classname;
0059   // Combiners can use this so they're free to define tryCombineAll themselves
0060   // and do extra work before/after calling the TableGen-erated code.
0061   string CombineAllMethodName = "tryCombineAll";
0062 }
0063
0064 /// Declares data that is passed from the match stage to the apply stage.
0065 class GIDefMatchData<string type>  {
0066   /// A C++ type name indicating the storage type.
0067   string Type = type;
0068 }
0069
0070 class GICombineRule<dag defs, dag match, dag apply> : GICombine {
0071   /// Defines the external interface of the match rule. This includes:
0072   /// * The names of the root nodes (requires at least one)
0073   /// See GIDefKind for details.
0074   dag Defs = defs;
0075
0076   /// Defines the things which must be true for the pattern to match
0077   dag Match = match;
0078
0079   /// Defines the things which happen after the decision is made to apply a
0080   /// combine rule.
0081   dag Apply = apply;
0082
0083   /// Defines the predicates that are checked before the match function
0084   /// is called. Targets can use this to, for instance, check Subtarget
0085   /// features.
0086   list<Predicate> Predicates = [];
0087
0088   // Maximum number of permutations of this rule that can be emitted.
0089   // Set to -1 to disable the limit.
0090   int MaxPermutations = 16;
0091 }
0092
0093 def gi_mo;
0094 def gi_imm;
0095
0096 // This is an equivalent of PatFrags but for MIR Patterns.
0097 //
0098 // GICombinePatFrags can be used in place of instructions for 'match' patterns.
0099 // Much like normal instructions, the defs (outs) come first, and the ins second
0100 //
0101 // Out operands can only be of type "root" or "gi_mo", and they must be defined
0102 // by an instruction pattern in all alternatives.
0103 //
0104 // In operands can be gi_imm or gi_mo. They cannot be redefined in any alternative
0105 // pattern and may only appear in the C++ code, or in the output operand of an
0106 // instruction pattern.
0107 class GICombinePatFrag<dag outs, dag ins, list<dag> alts> {
0108   dag InOperands = ins;
0109   dag OutOperands = outs;
0110   list<dag> Alternatives = alts;
0111 }
0112
0113 //===----------------------------------------------------------------------===//
0114 // Pattern Special Types
0115 //===----------------------------------------------------------------------===//
0116
0117 class GISpecialType;
0118
0119 // In an apply pattern, GITypeOf can be used to set the type of a new temporary
0120 // register to match the type of a matched register.
0121 //
0122 // This can only be used on temporary registers defined by the apply pattern.
0123 //
0124 // TODO: Make this work in matchers as well?
0125 //
0126 // FIXME: Syntax is very ugly.
0127 class GITypeOf<string opName> : GISpecialType {
0128   string OpName = opName;
0129 }
0130
0131 // The type of an operand that can match a variable amount of operands.
0132 // This type contains a minimum and maximum number of operands to match.
0133 // The minimum must be 1 or more, as we cannot have an operand representing
0134 // zero operands, and the max can be zero (which means "unlimited") or a value
0135 // greater than the minimum.
0136 class GIVariadic<int min = 1, int max = 0> : GISpecialType {
0137   int MinArgs = min;
0138   int MaxArgs = max;
0139 }
0140
0141 //===----------------------------------------------------------------------===//
0142 // Pattern Builtins
0143 //===----------------------------------------------------------------------===//
0144
0145 // "Magic" Builtin instructions for MIR patterns.
0146 // The definitions that implement
0147 class GIBuiltinInst;
0148
0149 // Replace all references to a register with another one.
0150 //
0151 // Usage:
0152 //    (apply (GIReplaceReg $old, $new))
0153 //
0154 // Operands:
0155 // - $old (out) register defined by a matched instruction
0156 // - $new (in)  register
0157 //
0158 // Semantics:
0159 // - Can only appear in an 'apply' pattern.
0160 // - If both old/new are operands of matched instructions,
0161 //   "canReplaceReg" is checked before applying the rule.
0162 def GIReplaceReg : GIBuiltinInst;
0163
0164 // Apply action that erases the match root.
0165 //
0166 // Usage:
0167 //    (apply (GIEraseRoot))
0168 //
0169 // Semantics:
0170 // - Can only appear as the only pattern of an 'apply' pattern list.
0171 // - The root cannot have any output operands.
0172 // - The root must be a CodeGenInstruction
0173 //
0174 // TODO: Allow using this directly, like (apply GIEraseRoot)
0175 def GIEraseRoot : GIBuiltinInst;
0176
0177 //===----------------------------------------------------------------------===//
0178 // Pattern MIFlags
0179 //===----------------------------------------------------------------------===//
0180
0181 class MIFlagEnum<string enumName> {
0182   string EnumName = "MachineInstr::" # enumName;
0183 }
0184
0185 def FmNoNans    : MIFlagEnum<"FmNoNans">;
0186 def FmNoInfs    : MIFlagEnum<"FmNoInfs">;
0187 def FmNsz       : MIFlagEnum<"FmNsz">;
0188 def FmArcp      : MIFlagEnum<"FmArcp">;
0189 def FmContract  : MIFlagEnum<"FmContract">;
0190 def FmAfn       : MIFlagEnum<"FmAfn">;
0191 def FmReassoc   : MIFlagEnum<"FmReassoc">;
0192 def IsExact     : MIFlagEnum<"IsExact">;
0193 def NoSWrap     : MIFlagEnum<"NoSWrap">;
0194 def NoUWrap     : MIFlagEnum<"NoUWrap">;
0195 def NonNeg      : MIFlagEnum<"NonNeg">;
0196
0197 def MIFlags;
0198 // def not; -> Already defined as a SDNode
0199
0200 //===----------------------------------------------------------------------===//
0201
0202 def extending_load_matchdata : GIDefMatchData<"PreferredTuple">;
0203 def indexed_load_store_matchdata : GIDefMatchData<"IndexedLoadStoreMatchInfo">;
0204 def instruction_steps_matchdata: GIDefMatchData<"InstructionStepsMatchInfo">;
0205
0206 def register_matchinfo: GIDefMatchData<"Register">;
0207 def int64_matchinfo: GIDefMatchData<"int64_t">;
0208 def apint_matchinfo : GIDefMatchData<"APInt">;
0209 def constantfp_matchinfo : GIDefMatchData<"ConstantFP*">;
0210 def build_fn_matchinfo :
0211 GIDefMatchData<"std::function<void(MachineIRBuilder &)>">;
0212 def unsigned_matchinfo: GIDefMatchData<"unsigned">;
0213
0214 def copy_prop : GICombineRule<
0215   (defs root:$d),
0216   (match (COPY $d, $s):$mi,
0217          [{ return Helper.matchCombineCopy(*${mi}); }]),
0218   (apply [{ Helper.applyCombineCopy(*${mi}); }])>;
0219
0220 // idempotent operations
0221 // Fold (freeze (freeze x)) -> (freeze x).
0222 // Fold (fabs (fabs x)) -> (fabs x).
0223 // Fold (fcanonicalize (fcanonicalize x)) -> (fcanonicalize x).
0224 def idempotent_prop_frags : GICombinePatFrag<
0225   (outs root:$dst, $src), (ins),
0226   !foreach(op, [G_FREEZE, G_FABS, G_FCANONICALIZE],
0227            (pattern (op $dst, $src), (op $src, $x)))>;
0228
0229 def idempotent_prop : GICombineRule<
0230    (defs root:$dst),
0231    (match (idempotent_prop_frags $dst, $src)),
0232    (apply (GIReplaceReg $dst, $src))>;
0233
0234 // Convert freeze(Op(Op0, NonPoisonOps...)) to Op(freeze(Op0), NonPoisonOps...)
0235 // when Op0 is not guaranteed non-poison
0236 def push_freeze_to_prevent_poison_from_propagating : GICombineRule<
0237   (defs root:$root, build_fn_matchinfo:$matchinfo),
0238   (match (G_FREEZE $dst, $src):$root,
0239          [{ return !isGuaranteedNotToBePoison(${src}.getReg(), MRI) && Helper.matchFreezeOfSingleMaybePoisonOperand(*${root}, ${matchinfo}); }]),
0240   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
0241
0242 def extending_loads : GICombineRule<
0243   (defs root:$root, extending_load_matchdata:$matchinfo),
0244   (match (wip_match_opcode G_LOAD, G_SEXTLOAD, G_ZEXTLOAD):$root,
0245          [{ return Helper.matchCombineExtendingLoads(*${root}, ${matchinfo}); }]),
0246   (apply [{ Helper.applyCombineExtendingLoads(*${root}, ${matchinfo}); }])>;
0247
0248 def load_and_mask : GICombineRule<
0249   (defs root:$root, build_fn_matchinfo:$matchinfo),
0250   (match (wip_match_opcode G_AND):$root,
0251         [{ return Helper.matchCombineLoadWithAndMask(*${root}, ${matchinfo}); }]),
0252   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
0253 def combines_for_extload: GICombineGroup<[extending_loads, load_and_mask]>;
0254
0255 def sext_trunc_sextload : GICombineRule<
0256   (defs root:$d),
0257   (match (wip_match_opcode G_SEXT_INREG):$d,
0258          [{ return Helper.matchSextTruncSextLoad(*${d}); }]),
0259   (apply [{ Helper.applySextTruncSextLoad(*${d}); }])>;
0260
0261 def sext_inreg_of_load_matchdata : GIDefMatchData<"std::tuple<Register, unsigned>">;
0262 def sext_inreg_of_load : GICombineRule<
0263   (defs root:$root, sext_inreg_of_load_matchdata:$matchinfo),
0264   (match (wip_match_opcode G_SEXT_INREG):$root,
0265          [{ return Helper.matchSextInRegOfLoad(*${root}, ${matchinfo}); }]),
0266   (apply [{ Helper.applySextInRegOfLoad(*${root}, ${matchinfo}); }])>;
0267
0268 def sext_inreg_to_zext_inreg : GICombineRule<
0269   (defs root:$dst),
0270   (match
0271     (G_SEXT_INREG $dst, $src, $imm):$root,
0272       [{
0273         unsigned BitWidth = MRI.getType(${src}.getReg()).getScalarSizeInBits();
0274         return Helper.getKnownBits()->maskedValueIsZero(${src}.getReg(),
0275                  APInt::getOneBitSet(BitWidth, ${imm}.getImm() - 1)); }]),
0276     (apply [{
0277       Helper.getBuilder().setInstrAndDebugLoc(*${root});
0278       Helper.getBuilder().buildZExtInReg(${dst}, ${src}, ${imm}.getImm());
0279       ${root}->eraseFromParent();
0280   }])
0281 >;
0282
0283 def combine_extracted_vector_load : GICombineRule<
0284   (defs root:$root, build_fn_matchinfo:$matchinfo),
0285   (match (wip_match_opcode G_EXTRACT_VECTOR_ELT):$root,
0286         [{ return Helper.matchCombineExtractedVectorLoad(*${root}, ${matchinfo}); }]),
0287   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
0288
0289 def combine_indexed_load_store : GICombineRule<
0290   (defs root:$root, indexed_load_store_matchdata:$matchinfo),
0291   (match (wip_match_opcode G_LOAD, G_SEXTLOAD, G_ZEXTLOAD, G_STORE):$root,
0292          [{ return Helper.matchCombineIndexedLoadStore(*${root}, ${matchinfo}); }]),
0293   (apply [{ Helper.applyCombineIndexedLoadStore(*${root}, ${matchinfo}); }])>;
0294
0295 def opt_brcond_by_inverting_cond_matchdata : GIDefMatchData<"MachineInstr *">;
0296 def opt_brcond_by_inverting_cond : GICombineRule<
0297   (defs root:$root, opt_brcond_by_inverting_cond_matchdata:$matchinfo),
0298   (match (wip_match_opcode G_BR):$root,
0299          [{ return Helper.matchOptBrCondByInvertingCond(*${root}, ${matchinfo}); }]),
0300   (apply [{ Helper.applyOptBrCondByInvertingCond(*${root}, ${matchinfo}); }])>;
0301
0302 def ptr_add_immed_matchdata : GIDefMatchData<"PtrAddChain">;
0303 def ptr_add_immed_chain : GICombineRule<
0304   (defs root:$d, ptr_add_immed_matchdata:$matchinfo),
0305   (match (wip_match_opcode G_PTR_ADD):$d,
0306          [{ return Helper.matchPtrAddImmedChain(*${d}, ${matchinfo}); }]),
0307   (apply [{ Helper.applyPtrAddImmedChain(*${d}, ${matchinfo}); }])>;
0308
0309 def shift_const_op : GICombinePatFrag<
0310   (outs root:$dst), (ins),
0311   !foreach(op,
0312            [G_SHL, G_ASHR, G_LSHR],
0313            (pattern (op $dst, $shifted, $amt)))>;
0314 def shift_result_matchdata : GIDefMatchData<"std::optional<int64_t>">;
0315 def shifts_too_big : GICombineRule<
0316   (defs root:$root, shift_result_matchdata:$matchinfo),
0317   (match (shift_const_op $root):$mi,
0318          [{ return Helper.matchShiftsTooBig(*${mi}, ${matchinfo}); }]),
0319   (apply [{
0320     if (${matchinfo}) {
0321       Helper.replaceInstWithConstant(*${mi}, *${matchinfo});
0322     } else {
0323       Helper.replaceInstWithUndef(*${mi});
0324     }
0325   }])>;
0326
0327 // Fold shift (shift base x), y -> shift base, (x+y), if shifts are same
0328 def shift_immed_matchdata : GIDefMatchData<"RegisterImmPair">;
0329 def shift_immed_chain : GICombineRule<
0330   (defs root:$d, shift_immed_matchdata:$matchinfo),
0331   (match (wip_match_opcode G_SHL, G_ASHR, G_LSHR, G_SSHLSAT, G_USHLSAT):$d,
0332          [{ return Helper.matchShiftImmedChain(*${d}, ${matchinfo}); }]),
0333   (apply [{ Helper.applyShiftImmedChain(*${d}, ${matchinfo}); }])>;
0334
0335 // Transform shift (logic (shift X, C0), Y), C1
0336 //        -> logic (shift X, (C0+C1)), (shift Y, C1), if shifts are same
0337 def shift_of_shifted_logic_matchdata : GIDefMatchData<"ShiftOfShiftedLogic">;
0338 def shift_of_shifted_logic_chain : GICombineRule<
0339   (defs root:$d, shift_of_shifted_logic_matchdata:$matchinfo),
0340   (match (wip_match_opcode G_SHL, G_ASHR, G_LSHR, G_USHLSAT, G_SSHLSAT):$d,
0341          [{ return Helper.matchShiftOfShiftedLogic(*${d}, ${matchinfo}); }]),
0342   (apply [{ Helper.applyShiftOfShiftedLogic(*${d}, ${matchinfo}); }])>;
0343
0344 def mul_to_shl : GICombineRule<
0345   (defs root:$d, unsigned_matchinfo:$matchinfo),
0346   (match (G_MUL $d, $op1, $op2):$mi,
0347          [{ return Helper.matchCombineMulToShl(*${mi}, ${matchinfo}); }]),
0348   (apply [{ Helper.applyCombineMulToShl(*${mi}, ${matchinfo}); }])>;
0349
0350 // (sub x, C) -> (add x, -C)
0351 def sub_to_add : GICombineRule<
0352   (defs root:$d, build_fn_matchinfo:$matchinfo),
0353   (match (G_CONSTANT $c, $imm),
0354          (G_SUB $d, $op1, $c):$mi,
0355          [{ return Helper.matchCombineSubToAdd(*${mi}, ${matchinfo}); }]),
0356   (apply [{ Helper.applyBuildFnNoErase(*${mi}, ${matchinfo}); }])>;
0357
0358 // shl ([asz]ext x), y => zext (shl x, y), if shift does not overflow int
0359 def reduce_shl_of_extend_matchdata : GIDefMatchData<"RegisterImmPair">;
0360 def reduce_shl_of_extend : GICombineRule<
0361   (defs root:$dst, reduce_shl_of_extend_matchdata:$matchinfo),
0362   (match (G_SHL $dst, $src0, $src1):$mi,
0363          [{ return Helper.matchCombineShlOfExtend(*${mi}, ${matchinfo}); }]),
0364   (apply [{ Helper.applyCombineShlOfExtend(*${mi}, ${matchinfo}); }])>;
0365
0366 // Combine bitreverse(shl (bitreverse x), y)) -> (lshr x, y)
0367 def bitreverse_shl : GICombineRule<
0368   (defs root:$d),
0369   (match (G_BITREVERSE $rev, $val),
0370          (G_SHL $src, $rev, $amt):$mi,
0371          (G_BITREVERSE $d, $src),
0372          [{ return Helper.isLegalOrBeforeLegalizer({TargetOpcode::G_LSHR,
0373                                                    {MRI.getType(${val}.getReg()),
0374                                                     MRI.getType(${amt}.getReg())}}); }]),
0375   (apply (G_LSHR $d, $val, $amt))>;
0376
0377 // Combine bitreverse(lshr (bitreverse x), y)) -> (shl x, y)
0378 def bitreverse_lshr : GICombineRule<
0379   (defs root:$d),
0380   (match (G_BITREVERSE $rev, $val),
0381          (G_LSHR $src, $rev, $amt):$mi,
0382          (G_BITREVERSE $d, $src),
0383          [{ return Helper.isLegalOrBeforeLegalizer({TargetOpcode::G_SHL,
0384                                                    {MRI.getType(${val}.getReg()),
0385                                                     MRI.getType(${amt}.getReg())}}); }]),
0386   (apply (G_SHL $d, $val, $amt))>;
0387
0388 // Combine (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
0389 // Combine (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
0390 def commute_shift : GICombineRule<
0391   (defs root:$d, build_fn_matchinfo:$matchinfo),
0392   (match (wip_match_opcode G_SHL):$d,
0393          [{ return Helper.matchCommuteShift(*${d}, ${matchinfo}); }]),
0394   (apply [{ Helper.applyBuildFn(*${d}, ${matchinfo}); }])>;
0395
0396 def narrow_binop_feeding_and : GICombineRule<
0397   (defs root:$root, build_fn_matchinfo:$matchinfo),
0398   (match (wip_match_opcode G_AND):$root,
0399          [{ return Helper.matchNarrowBinopFeedingAnd(*${root}, ${matchinfo}); }]),
0400   (apply [{ Helper.applyBuildFnNoErase(*${root}, ${matchinfo}); }])>;
0401
0402 // [us]itofp(undef) = 0, because the result value is bounded.
0403 def undef_to_fp_zero : GICombineRule<
0404   (defs root:$root),
0405   (match (wip_match_opcode G_UITOFP, G_SITOFP):$root,
0406          [{ return Helper.matchAnyExplicitUseIsUndef(*${root}); }]),
0407   (apply [{ Helper.replaceInstWithFConstant(*${root}, 0.0); }])>;
0408
0409 def undef_to_int_zero: GICombineRule<
0410   (defs root:$root),
0411   (match (wip_match_opcode G_AND, G_MUL):$root,
0412          [{ return Helper.matchAnyExplicitUseIsUndef(*${root}); }]),
0413   (apply [{ Helper.replaceInstWithConstant(*${root}, 0); }])>;
0414
0415 def undef_to_negative_one: GICombineRule<
0416   (defs root:$root),
0417   (match (wip_match_opcode G_OR):$root,
0418          [{ return Helper.matchAnyExplicitUseIsUndef(*${root}); }]),
0419   (apply [{ Helper.replaceInstWithConstant(*${root}, -1); }])>;
0420
0421 def binop_left_undef_to_zero: GICombineRule<
0422   (defs root:$root),
0423   (match (wip_match_opcode G_SHL, G_UDIV, G_UREM):$root,
0424          [{ return Helper.matchOperandIsUndef(*${root}, 1); }]),
0425   (apply [{ Helper.replaceInstWithConstant(*${root}, 0); }])>;
0426
0427 def binop_right_undef_to_undef: GICombineRule<
0428   (defs root:$root),
0429   (match (wip_match_opcode G_SHL, G_ASHR, G_LSHR):$root,
0430          [{ return Helper.matchOperandIsUndef(*${root}, 2); }]),
0431   (apply [{ Helper.replaceInstWithUndef(*${root}); }])>;
0432
0433 def unary_undef_to_zero: GICombineRule<
0434   (defs root:$root),
0435   (match (wip_match_opcode G_ABS):$root,
0436          [{ return Helper.matchOperandIsUndef(*${root}, 1); }]),
0437   (apply [{ Helper.replaceInstWithConstant(*${root}, 0); }])>;
0438
0439 def unary_undef_to_undef_frags : GICombinePatFrag<
0440   (outs root:$dst), (ins),
0441   !foreach(op,
0442            [G_TRUNC, G_BITCAST, G_ANYEXT, G_PTRTOINT, G_INTTOPTR, G_FPTOSI,
0443             G_FPTOUI],
0444            (pattern (op $dst, $x), (G_IMPLICIT_DEF $x)))>;
0445 def unary_undef_to_undef : GICombineRule<
0446   (defs root:$dst),
0447   (match (unary_undef_to_undef_frags $dst)),
0448   (apply [{ Helper.replaceInstWithUndef(*${dst}.getParent()); }])>;
0449
0450 // Instructions where if any source operand is undef, the instruction can be
0451 // replaced with undef.
0452 def propagate_undef_any_op: GICombineRule<
0453   (defs root:$root),
0454   (match (wip_match_opcode G_ADD, G_SUB, G_XOR):$root,
0455          [{ return Helper.matchAnyExplicitUseIsUndef(*${root}); }]),
0456   (apply [{ Helper.replaceInstWithUndef(*${root}); }])>;
0457
0458 // Instructions where if all source operands are undef, the instruction can be
0459 // replaced with undef.
0460 def propagate_undef_all_ops: GICombineRule<
0461   (defs root:$root),
0462   (match (wip_match_opcode G_SHUFFLE_VECTOR, G_BUILD_VECTOR):$root,
0463           [{ return Helper.matchAllExplicitUsesAreUndef(*${root}); }]),
0464   (apply [{ Helper.replaceInstWithUndef(*${root}); }])>;
0465
0466 // Replace a G_SHUFFLE_VECTOR with an undef mask with a G_IMPLICIT_DEF.
0467 def propagate_undef_shuffle_mask: GICombineRule<
0468   (defs root:$root),
0469   (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
0470          [{ return Helper.matchUndefShuffleVectorMask(*${root}); }]),
0471   (apply [{ Helper.replaceInstWithUndef(*${root}); }])>;
0472
0473 // Replace a G_SHUFFLE_VECTOR with a G_EXTRACT_VECTOR_ELT.
0474 def shuffle_to_extract: GICombineRule<
0475   (defs root:$root),
0476   (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
0477          [{ return Helper.matchShuffleToExtract(*${root}); }]),
0478   (apply [{ Helper.applyShuffleToExtract(*${root}); }])>;
0479
0480   // Replace an insert/extract element of an out of bounds index with undef.
0481   def insert_extract_vec_elt_out_of_bounds : GICombineRule<
0482   (defs root:$root),
0483   (match (wip_match_opcode G_INSERT_VECTOR_ELT, G_EXTRACT_VECTOR_ELT):$root,
0484          [{ return Helper.matchInsertExtractVecEltOutOfBounds(*${root}); }]),
0485   (apply [{ Helper.replaceInstWithUndef(*${root}); }])>;
0486
0487 // Fold (cond ? x : x) -> x
0488 def select_same_val: GICombineRule<
0489   (defs root:$root),
0490   (match (wip_match_opcode G_SELECT):$root,
0491     [{ return Helper.matchSelectSameVal(*${root}); }]),
0492   (apply [{ Helper.replaceSingleDefInstWithOperand(*${root}, 2); }])
0493 >;
0494
0495 // Fold (undef ? x : y) -> y
0496 def select_undef_cmp: GICombineRule<
0497   (defs root:$dst),
0498   (match (G_IMPLICIT_DEF $undef),
0499          (G_SELECT $dst, $undef, $x, $y)),
0500   (apply (GIReplaceReg $dst, $y))
0501 >;
0502
0503 // Fold (true ? x : y) -> x
0504 // Fold (false ? x : y) -> y
0505 def select_constant_cmp: GICombineRule<
0506   (defs root:$root, unsigned_matchinfo:$matchinfo),
0507   (match (wip_match_opcode G_SELECT):$root,
0508     [{ return Helper.matchConstantSelectCmp(*${root}, ${matchinfo}); }]),
0509   (apply [{ Helper.replaceSingleDefInstWithOperand(*${root}, ${matchinfo}); }])
0510 >;
0511
0512 // Fold (C op x) -> (x op C)
0513 // TODO: handle more isCommutable opcodes
0514 // TODO: handle compares (currently not marked as isCommutable)
0515 def commute_int_constant_to_rhs : GICombineRule<
0516   (defs root:$root),
0517   (match (wip_match_opcode G_ADD, G_MUL, G_AND, G_OR, G_XOR,
0518                            G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_UADDO, G_SADDO,
0519                            G_UMULO, G_SMULO, G_UMULH, G_SMULH,
0520                            G_UADDSAT, G_SADDSAT, G_SMULFIX, G_UMULFIX,
0521                            G_SMULFIXSAT, G_UMULFIXSAT):$root,
0522     [{ return Helper.matchCommuteConstantToRHS(*${root}); }]),
0523   (apply [{ Helper.applyCommuteBinOpOperands(*${root}); }])
0524 >;
0525
0526 def commute_fp_constant_to_rhs : GICombineRule<
0527   (defs root:$root),
0528   (match (wip_match_opcode G_FADD, G_FMUL, G_FMINNUM, G_FMAXNUM,
0529                            G_FMINNUM_IEEE, G_FMAXNUM_IEEE,
0530                            G_FMINIMUM, G_FMAXIMUM):$root,
0531     [{ return Helper.matchCommuteFPConstantToRHS(*${root}); }]),
0532   (apply [{ Helper.applyCommuteBinOpOperands(*${root}); }])
0533 >;
0534
0535 def commute_constant_to_rhs : GICombineGroup<[
0536   commute_int_constant_to_rhs,
0537   commute_fp_constant_to_rhs
0538 ]>;
0539
0540 // Fold x op 0 -> x
0541 def right_identity_zero_frags : GICombinePatFrag<
0542   (outs root:$dst), (ins $x),
0543   !foreach(op,
0544            [G_SUB, G_ADD, G_OR, G_XOR, G_SHL, G_ASHR,
0545             G_LSHR, G_PTR_ADD, G_ROTL, G_ROTR],
0546            (pattern (op $dst, $x, 0)))>;
0547 def right_identity_zero: GICombineRule<
0548   (defs root:$dst),
0549   (match (right_identity_zero_frags $dst, $lhs)),
0550   (apply (GIReplaceReg $dst, $lhs))
0551 >;
0552
0553 def right_identity_neg_zero_fp: GICombineRule<
0554   (defs root:$dst),
0555   (match (G_FADD $dst, $x, $y):$root,
0556     [{ return Helper.matchConstantFPOp(${y}, -0.0); }]),
0557   (apply (GIReplaceReg $dst, $x))
0558 >;
0559
0560 // Fold x op 1 -> x
0561 def right_identity_one_int: GICombineRule<
0562   (defs root:$dst),
0563   (match (G_MUL $dst, $x, 1)),
0564   (apply (GIReplaceReg $dst, $x))
0565 >;
0566
0567 def right_identity_one_fp: GICombineRule<
0568   (defs root:$dst),
0569   (match (G_FMUL $dst, $x, $y):$root,
0570     [{ return Helper.matchConstantFPOp(${y}, 1.0); }]),
0571   (apply (GIReplaceReg $dst, $x))
0572 >;
0573
0574 def right_identity_neg_one_fp: GICombineRule<
0575   (defs root:$dst),
0576   (match (G_FMUL $dst, $x, $y):$root,
0577     [{ return Helper.matchConstantFPOp(${y}, -1.0); }]),
0578   (apply (G_FNEG $dst, $x))
0579 >;
0580
0581 def right_identity_one : GICombineGroup<[right_identity_one_int, right_identity_one_fp]>;
0582
0583 // Fold (x op x) - > x
0584 def binop_same_val_frags : GICombinePatFrag<
0585   (outs root:$dst), (ins $x),
0586   [
0587     (pattern (G_AND $dst, $x, $x)),
0588     (pattern (G_OR $dst, $x, $x)),
0589   ]
0590 >;
0591 def binop_same_val: GICombineRule<
0592   (defs root:$dst),
0593   (match (binop_same_val_frags $dst, $src)),
0594   (apply (GIReplaceReg $dst, $src))
0595 >;
0596
0597 // Fold (0 op x) - > 0
0598 def binop_left_to_zero: GICombineRule<
0599   (defs root:$root),
0600   (match (wip_match_opcode G_SHL, G_LSHR, G_ASHR, G_SDIV, G_UDIV, G_SREM,
0601                            G_UREM):$root,
0602     [{ return Helper.matchOperandIsZero(*${root}, 1); }]),
0603   (apply [{ Helper.replaceSingleDefInstWithOperand(*${root}, 1); }])
0604 >;
0605
0606 def urem_pow2_to_mask : GICombineRule<
0607   (defs root:$root),
0608   (match (wip_match_opcode G_UREM):$root,
0609     [{ return Helper.matchOperandIsKnownToBeAPowerOfTwo(*${root}, 2); }]),
0610   (apply [{ Helper.applySimplifyURemByPow2(*${root}); }])
0611 >;
0612
0613 // Push a binary operator through a select on constants.
0614 //
0615 // binop (select cond, K0, K1), K2 ->
0616 //   select cond, (binop K0, K2), (binop K1, K2)
0617
0618 // Every binary operator that has constant folding. We currently do
0619 // not have constant folding for G_FPOW, G_FMAXNUM_IEEE or
0620 // G_FMINNUM_IEEE.
0621 def fold_binop_into_select : GICombineRule<
0622   (defs root:$root, unsigned_matchinfo:$select_op_no),
0623   (match (wip_match_opcode
0624     G_ADD, G_SUB, G_PTR_ADD, G_AND, G_OR, G_XOR,
0625     G_SDIV, G_SREM, G_UDIV, G_UREM, G_LSHR, G_ASHR, G_SHL,
0626     G_SMIN, G_SMAX, G_UMIN, G_UMAX,
0627     G_FMUL, G_FADD, G_FSUB, G_FDIV, G_FREM,
0628     G_FMINNUM, G_FMAXNUM, G_FMINIMUM, G_FMAXIMUM):$root,
0629     [{ return Helper.matchFoldBinOpIntoSelect(*${root}, ${select_op_no}); }]),
0630   (apply [{ Helper.applyFoldBinOpIntoSelect(*${root}, ${select_op_no}); }])
0631 >;
0632
0633 // Transform d = [su]div(x, y) and r = [su]rem(x, y) - > d, r = [su]divrem(x, y)
0634 def div_rem_to_divrem_matchdata : GIDefMatchData<"MachineInstr *">;
0635 def div_rem_to_divrem : GICombineRule<
0636   (defs root:$root, div_rem_to_divrem_matchdata:$matchinfo),
0637   (match (wip_match_opcode G_SDIV, G_UDIV, G_SREM, G_UREM):$root,
0638     [{ return Helper.matchCombineDivRem(*${root}, ${matchinfo}); }]),
0639   (apply [{ Helper.applyCombineDivRem(*${root}, ${matchinfo}); }])
0640 >;
0641
0642 // Fold (x op 0) - > 0
0643 def binop_right_to_zero: GICombineRule<
0644   (defs root:$dst),
0645   (match (G_MUL $dst, $lhs, 0:$zero)),
0646   (apply (GIReplaceReg $dst, $zero))
0647 >;
0648
0649 // Erase stores of undef values.
0650 def erase_undef_store : GICombineRule<
0651   (defs root:$root),
0652   (match (wip_match_opcode G_STORE):$root,
0653     [{ return Helper.matchUndefStore(*${root}); }]),
0654   (apply [{ Helper.eraseInst(*${root}); }])
0655 >;
0656
0657 def simplify_add_to_sub_matchinfo: GIDefMatchData<"std::tuple<Register, Register>">;
0658 def simplify_add_to_sub: GICombineRule <
0659   (defs root:$root, simplify_add_to_sub_matchinfo:$info),
0660   (match (wip_match_opcode G_ADD):$root,
0661     [{ return Helper.matchSimplifyAddToSub(*${root}, ${info}); }]),
0662   (apply [{ Helper.applySimplifyAddToSub(*${root}, ${info});}])
0663 >;
0664
0665 // Fold fp_op(cst) to the constant result of the floating point operation.
0666 class constant_fold_unary_fp_op_rule<Instruction opcode> : GICombineRule <
0667   (defs root:$dst),
0668   (match (opcode $dst, $src0):$root, (G_FCONSTANT $src0, $cst)),
0669   (apply [{ Helper.applyCombineConstantFoldFpUnary(*${root}, ${cst}.getFPImm()); }])
0670 >;
0671
0672 def constant_fold_fneg : constant_fold_unary_fp_op_rule<G_FNEG>;
0673 def constant_fold_fabs : constant_fold_unary_fp_op_rule<G_FABS>;
0674 def constant_fold_fsqrt : constant_fold_unary_fp_op_rule<G_FSQRT>;
0675 def constant_fold_flog2 : constant_fold_unary_fp_op_rule<G_FLOG2>;
0676 def constant_fold_fptrunc : constant_fold_unary_fp_op_rule<G_FPTRUNC>;
0677
0678 // Fold constant zero int to fp conversions.
0679 class itof_const_zero_fold_rule<Instruction opcode> : GICombineRule <
0680   (defs root:$dst),
0681   (match (opcode $dst, 0)),
0682   // Can't use COPY $dst, 0 here because the 0 operand may be a smaller type
0683   // than the destination for itofp.
0684   (apply [{ Helper.replaceInstWithFConstant(*${dst}.getParent(), 0.0); }])
0685 >;
0686 def itof_const_zero_fold_si : itof_const_zero_fold_rule<G_SITOFP>;
0687 def itof_const_zero_fold_ui : itof_const_zero_fold_rule<G_UITOFP>;
0688
0689 def constant_fold_fp_ops : GICombineGroup<[
0690   constant_fold_fneg,
0691   constant_fold_fabs,
0692   constant_fold_fsqrt,
0693   constant_fold_flog2,
0694   constant_fold_fptrunc,
0695   itof_const_zero_fold_si,
0696   itof_const_zero_fold_ui
0697 ]>;
0698
0699 // Fold int2ptr(ptr2int(x)) -> x
0700 def p2i_to_i2p: GICombineRule<
0701   (defs root:$root, register_matchinfo:$info),
0702   (match (wip_match_opcode G_INTTOPTR):$root,
0703     [{ return Helper.matchCombineI2PToP2I(*${root}, ${info}); }]),
0704   (apply [{ Helper.applyCombineI2PToP2I(*${root}, ${info}); }])
0705 >;
0706
0707 // Fold ptr2int(int2ptr(x)) -> x
0708 def i2p_to_p2i: GICombineRule<
0709   (defs root:$dst, register_matchinfo:$info),
0710   (match (G_INTTOPTR $t, $ptr),
0711          (G_PTRTOINT $dst, $t):$mi,
0712     [{ ${info} = ${ptr}.getReg(); return true; }]),
0713   (apply [{ Helper.applyCombineP2IToI2P(*${mi}, ${info}); }])
0714 >;
0715
0716 // Fold add ptrtoint(x), y -> ptrtoint (ptr_add x), y
0717 def add_p2i_to_ptradd_matchinfo : GIDefMatchData<"std::pair<Register, bool>">;
0718 def add_p2i_to_ptradd : GICombineRule<
0719   (defs root:$root, add_p2i_to_ptradd_matchinfo:$info),
0720   (match (wip_match_opcode G_ADD):$root,
0721     [{ return Helper.matchCombineAddP2IToPtrAdd(*${root}, ${info}); }]),
0722   (apply [{ Helper.applyCombineAddP2IToPtrAdd(*${root}, ${info}); }])
0723 >;
0724
0725 // Fold (ptr_add (int2ptr C1), C2) -> C1 + C2
0726 def const_ptradd_to_i2p: GICombineRule<
0727   (defs root:$root, apint_matchinfo:$info),
0728   (match (wip_match_opcode G_PTR_ADD):$root,
0729     [{ return Helper.matchCombineConstPtrAddToI2P(*${root}, ${info}); }]),
0730   (apply [{ Helper.applyCombineConstPtrAddToI2P(*${root}, ${info}); }])
0731 >;
0732
0733 // Simplify: (logic_op (op x...), (op y...)) -> (op (logic_op x, y))
0734 def hoist_logic_op_with_same_opcode_hands: GICombineRule <
0735   (defs root:$root, instruction_steps_matchdata:$info),
0736   (match (wip_match_opcode G_AND, G_OR, G_XOR):$root,
0737     [{ return Helper.matchHoistLogicOpWithSameOpcodeHands(*${root}, ${info}); }]),
0738   (apply [{ Helper.applyBuildInstructionSteps(*${root}, ${info});}])
0739 >;
0740
0741 // Fold ashr (shl x, C), C -> sext_inreg (C)
0742 def shl_ashr_to_sext_inreg_matchinfo : GIDefMatchData<"std::tuple<Register, int64_t>">;
0743 def shl_ashr_to_sext_inreg : GICombineRule<
0744   (defs root:$root, shl_ashr_to_sext_inreg_matchinfo:$info),
0745   (match (wip_match_opcode G_ASHR): $root,
0746     [{ return Helper.matchAshrShlToSextInreg(*${root}, ${info}); }]),
0747   (apply [{ Helper.applyAshShlToSextInreg(*${root}, ${info});}])
0748 >;
0749
0750 // Fold and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0
0751 def overlapping_and: GICombineRule <
0752   (defs root:$root, build_fn_matchinfo:$info),
0753   (match (wip_match_opcode G_AND):$root,
0754          [{ return Helper.matchOverlappingAnd(*${root}, ${info}); }]),
0755   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])
0756 >;
0757
0758 // Fold (x & y) -> x or (x & y) -> y when (x & y) is known to equal x or equal y.
0759 def redundant_and: GICombineRule <
0760   (defs root:$root, register_matchinfo:$matchinfo),
0761   (match (wip_match_opcode G_AND):$root,
0762          [{ return Helper.matchRedundantAnd(*${root}, ${matchinfo}); }]),
0763   (apply [{ Helper.replaceSingleDefInstWithReg(*${root}, ${matchinfo}); }])
0764 >;
0765
0766 // Fold (x | y) -> x or (x | y) -> y when (x | y) is known to equal x or equal y.
0767 def redundant_or: GICombineRule <
0768   (defs root:$root, register_matchinfo:$matchinfo),
0769   (match (wip_match_opcode G_OR):$root,
0770          [{ return Helper.matchRedundantOr(*${root}, ${matchinfo}); }]),
0771   (apply [{ Helper.replaceSingleDefInstWithReg(*${root}, ${matchinfo}); }])
0772 >;
0773
0774 // If the input is already sign extended, just drop the extension.
0775 // sext_inreg x, K ->
0776 //   if computeNumSignBits(x) >= (x.getScalarSizeInBits() - K + 1)
0777 def redundant_sext_inreg: GICombineRule <
0778   (defs root:$root),
0779   (match (wip_match_opcode G_SEXT_INREG):$root,
0780          [{ return Helper.matchRedundantSExtInReg(*${root}); }]),
0781      (apply [{ Helper.replaceSingleDefInstWithOperand(*${root}, 1); }])
0782 >;
0783
0784 // Fold (anyext (trunc x)) -> x if the source type is same as
0785 // the destination type.
0786 def anyext_trunc_fold: GICombineRule <
0787   (defs root:$root, register_matchinfo:$matchinfo),
0788   (match (wip_match_opcode G_ANYEXT):$root,
0789          [{ return Helper.matchCombineAnyExtTrunc(*${root}, ${matchinfo}); }]),
0790   (apply [{ Helper.replaceSingleDefInstWithReg(*${root}, ${matchinfo}); }])
0791 >;
0792
0793 // Fold (zext (trunc x)) -> x if the source type is same as the destination type
0794 // and truncated bits are known to be zero.
0795 def zext_trunc_fold: GICombineRule <
0796   (defs root:$root, register_matchinfo:$matchinfo),
0797   (match (wip_match_opcode G_ZEXT):$root,
0798          [{ return Helper.matchCombineZextTrunc(*${root}, ${matchinfo}); }]),
0799   (apply [{ Helper.replaceSingleDefInstWithReg(*${root}, ${matchinfo}); }])
0800 >;
0801
0802 def not_cmp_fold_matchinfo : GIDefMatchData<"SmallVector<Register, 4>">;
0803 def not_cmp_fold : GICombineRule<
0804   (defs root:$d, not_cmp_fold_matchinfo:$info),
0805   (match (wip_match_opcode G_XOR): $d,
0806   [{ return Helper.matchNotCmp(*${d}, ${info}); }]),
0807   (apply [{ Helper.applyNotCmp(*${d}, ${info}); }])
0808 >;
0809
0810 // Fold (fneg (fneg x)) -> x.
0811 def fneg_fneg_fold: GICombineRule <
0812   (defs root:$dst),
0813   (match (G_FNEG $t, $src),
0814          (G_FNEG $dst, $t)),
0815   (apply (GIReplaceReg $dst, $src))
0816 >;
0817
0818 // Fold (unmerge(merge x, y, z)) -> z, y, z.
0819 def unmerge_merge_matchinfo : GIDefMatchData<"SmallVector<Register, 8>">;
0820 def unmerge_merge : GICombineRule<
0821   (defs root:$d, unmerge_merge_matchinfo:$info),
0822   (match (wip_match_opcode G_UNMERGE_VALUES): $d,
0823   [{ return Helper.matchCombineUnmergeMergeToPlainValues(*${d}, ${info}); }]),
0824   (apply [{ Helper.applyCombineUnmergeMergeToPlainValues(*${d}, ${info}); }])
0825 >;
0826
0827 // Fold merge(unmerge).
0828 def merge_unmerge : GICombineRule<
0829   (defs root:$d, register_matchinfo:$matchinfo),
0830   (match (wip_match_opcode G_MERGE_VALUES):$d,
0831   [{ return Helper.matchCombineMergeUnmerge(*${d}, ${matchinfo}); }]),
0832   (apply [{ Helper.replaceSingleDefInstWithReg(*${d}, ${matchinfo}); }])
0833 >;
0834
0835 // Fold (fabs (fneg x)) -> (fabs x).
0836 def fabs_fneg_fold: GICombineRule <
0837   (defs root:$dst),
0838   (match  (G_FNEG $tmp, $x),
0839           (G_FABS $dst, $tmp)),
0840   (apply (G_FABS $dst, $x))>;
0841
0842 // Fold (unmerge cst) -> cst1, cst2, ...
0843 def unmerge_cst_matchinfo : GIDefMatchData<"SmallVector<APInt, 8>">;
0844 def unmerge_cst : GICombineRule<
0845   (defs root:$d, unmerge_cst_matchinfo:$info),
0846   (match (wip_match_opcode G_UNMERGE_VALUES): $d,
0847   [{ return Helper.matchCombineUnmergeConstant(*${d}, ${info}); }]),
0848   (apply [{ Helper.applyCombineUnmergeConstant(*${d}, ${info}); }])
0849 >;
0850
0851 // Fold (unmerge undef) -> undef, undef, ...
0852 def unmerge_undef : GICombineRule<
0853   (defs root:$root, build_fn_matchinfo:$info),
0854   (match (wip_match_opcode G_UNMERGE_VALUES): $root,
0855          [{ return Helper.matchCombineUnmergeUndef(*${root}, ${info}); }]),
0856   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])
0857 >;
0858
0859 // Transform x,y<dead> = unmerge z -> x = trunc z.
0860 def unmerge_dead_to_trunc : GICombineRule<
0861   (defs root:$d),
0862   (match (wip_match_opcode G_UNMERGE_VALUES): $d,
0863   [{ return Helper.matchCombineUnmergeWithDeadLanesToTrunc(*${d}); }]),
0864   (apply [{ Helper.applyCombineUnmergeWithDeadLanesToTrunc(*${d}); }])
0865 >;
0866
0867 // Transform unmerge any build vector -> build vector anyext
0868 def unmerge_anyext_build_vector : GICombineRule<
0869   (defs root:$root, build_fn_matchinfo:$matchinfo),
0870   (match (wip_match_opcode G_UNMERGE_VALUES): $root,
0871   [{ return Helper.matchUnmergeValuesAnyExtBuildVector(*${root}, ${matchinfo}); }]),
0872   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])
0873 >;
0874
0875 // Transform x,y = unmerge(zext(z)) -> x = zext z; y = 0.
0876 def unmerge_zext_to_zext : GICombineRule<
0877   (defs root:$d),
0878   (match (wip_match_opcode G_UNMERGE_VALUES): $d,
0879   [{ return Helper.matchCombineUnmergeZExtToZExt(*${d}); }]),
0880   (apply [{ Helper.applyCombineUnmergeZExtToZExt(*${d}); }])
0881 >;
0882
0883 /// Transform merge_x_undef -> anyext.
0884 def merge_of_x_and_undef : GICombineRule <
0885   (defs root:$root, build_fn_matchinfo:$matchinfo),
0886   (match (G_IMPLICIT_DEF $undef),
0887          (G_MERGE_VALUES $root, $x, $undef):$MI,
0888          [{ return Helper.matchMergeXAndUndef(*${MI}, ${matchinfo}); }]),
0889   (apply [{ Helper.applyBuildFn(*${MI}, ${matchinfo}); }])>;
0890
0891 /// Transform merge_x_zero -> zext.
0892 def merge_of_x_and_zero : GICombineRule <
0893   (defs root:$root, build_fn_matchinfo:$matchinfo),
0894   (match (G_CONSTANT $zero, 0),
0895          (G_MERGE_VALUES $root, $x, $zero):$MI,
0896          [{ return Helper.matchMergeXAndZero(*${MI}, ${matchinfo}); }]),
0897   (apply [{ Helper.applyBuildFn(*${MI}, ${matchinfo}); }])>;
0898
0899 def merge_combines: GICombineGroup<[
0900   unmerge_anyext_build_vector,
0901   unmerge_merge,
0902   merge_unmerge,
0903   unmerge_cst,
0904   unmerge_undef,
0905   unmerge_dead_to_trunc,
0906   unmerge_zext_to_zext,
0907   merge_of_x_and_undef,
0908   merge_of_x_and_zero
0909 ]>;
0910
0911 // Under certain conditions, transform:
0912 //  trunc (shl x, K)     -> shl (trunc x), K//
0913 //  trunc ([al]shr x, K) -> (trunc ([al]shr (trunc x), K))
0914 def trunc_shift_matchinfo : GIDefMatchData<"std::pair<MachineInstr*, LLT>">;
0915 def trunc_shift: GICombineRule <
0916   (defs root:$root, trunc_shift_matchinfo:$matchinfo),
0917   (match (wip_match_opcode G_TRUNC):$root,
0918          [{ return Helper.matchCombineTruncOfShift(*${root}, ${matchinfo}); }]),
0919   (apply [{ Helper.applyCombineTruncOfShift(*${root}, ${matchinfo}); }])
0920 >;
0921
0922 // Transform (mul x, -1) -> (sub 0, x)
0923 def mul_by_neg_one: GICombineRule <
0924   (defs root:$dst),
0925   (match (G_MUL $dst, $x, -1)),
0926   (apply (G_SUB $dst, 0, $x))
0927 >;
0928
0929 // Fold (xor (and x, y), y) -> (and (not x), y)
0930 def xor_of_and_with_same_reg_matchinfo :
0931     GIDefMatchData<"std::pair<Register, Register>">;
0932 def xor_of_and_with_same_reg: GICombineRule <
0933   (defs root:$root, xor_of_and_with_same_reg_matchinfo:$matchinfo),
0934   (match (wip_match_opcode G_XOR):$root,
0935          [{ return Helper.matchXorOfAndWithSameReg(*${root}, ${matchinfo}); }]),
0936   (apply [{ Helper.applyXorOfAndWithSameReg(*${root}, ${matchinfo}); }])
0937 >;
0938
0939 // Transform (ptr_add 0, x) -> (int_to_ptr x)
0940 def ptr_add_with_zero: GICombineRule<
0941   (defs root:$root),
0942   (match (wip_match_opcode G_PTR_ADD):$root,
0943          [{ return Helper.matchPtrAddZero(*${root}); }]),
0944   (apply [{ Helper.applyPtrAddZero(*${root}); }])>;
0945
0946 def regs_small_vec : GIDefMatchData<"SmallVector<Register, 4>">;
0947 def combine_insert_vec_elts_build_vector : GICombineRule<
0948   (defs root:$root, regs_small_vec:$info),
0949   (match (wip_match_opcode G_INSERT_VECTOR_ELT):$root,
0950     [{ return Helper.matchCombineInsertVecElts(*${root}, ${info}); }]),
0951   (apply [{ Helper.applyCombineInsertVecElts(*${root}, ${info}); }])>;
0952
0953 def load_or_combine : GICombineRule<
0954   (defs root:$root, build_fn_matchinfo:$info),
0955   (match (wip_match_opcode G_OR):$root,
0956     [{ return Helper.matchLoadOrCombine(*${root}, ${info}); }]),
0957   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
0958
0959 def extend_through_phis_matchdata: GIDefMatchData<"MachineInstr*">;
0960 def extend_through_phis : GICombineRule<
0961   (defs root:$root, extend_through_phis_matchdata:$matchinfo),
0962   (match (wip_match_opcode G_PHI):$root,
0963     [{ return Helper.matchExtendThroughPhis(*${root}, ${matchinfo}); }]),
0964   (apply [{ Helper.applyExtendThroughPhis(*${root}, ${matchinfo}); }])>;
0965
0966 // Currently only the one combine above.
0967 def insert_vec_elt_combines : GICombineGroup<
0968                             [combine_insert_vec_elts_build_vector]>;
0969
0970 def extract_vec_elt_build_vec : GICombineRule<
0971   (defs root:$root, register_matchinfo:$matchinfo),
0972   (match (wip_match_opcode G_EXTRACT_VECTOR_ELT):$root,
0973     [{ return Helper.matchExtractVecEltBuildVec(*${root}, ${matchinfo}); }]),
0974   (apply [{ Helper.applyExtractVecEltBuildVec(*${root}, ${matchinfo}); }])>;
0975
0976 // Fold away full elt extracts from a build_vector.
0977 def extract_all_elts_from_build_vector_matchinfo :
0978   GIDefMatchData<"SmallVector<std::pair<Register, MachineInstr*>>">;
0979 def extract_all_elts_from_build_vector : GICombineRule<
0980   (defs root:$root, extract_all_elts_from_build_vector_matchinfo:$matchinfo),
0981   (match (wip_match_opcode G_BUILD_VECTOR):$root,
0982     [{ return Helper.matchExtractAllEltsFromBuildVector(*${root}, ${matchinfo}); }]),
0983   (apply [{ Helper.applyExtractAllEltsFromBuildVector(*${root}, ${matchinfo}); }])>;
0984
0985 def extract_vec_elt_combines : GICombineGroup<[
0986   extract_vec_elt_build_vec,
0987   extract_all_elts_from_build_vector]>;
0988
0989 def funnel_shift_from_or_shift : GICombineRule<
0990   (defs root:$root, build_fn_matchinfo:$info),
0991   (match (wip_match_opcode G_OR):$root,
0992     [{ return Helper.matchOrShiftToFunnelShift(*${root}, ${info}); }]),
0993   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])
0994 >;
0995
0996 def funnel_shift_to_rotate : GICombineRule<
0997   (defs root:$root),
0998   (match (wip_match_opcode G_FSHL, G_FSHR):$root,
0999     [{ return Helper.matchFunnelShiftToRotate(*${root}); }]),
1000   (apply [{ Helper.applyFunnelShiftToRotate(*${root}); }])
1001 >;
1002
1003 // Fold fshr x, y, 0 -> y
1004 def funnel_shift_right_zero: GICombineRule<
1005   (defs root:$root),
1006   (match (G_FSHR $x, $y, $z, 0):$root),
1007   (apply (COPY $x, $z))
1008 >;
1009
1010 // Fold fshl x, y, 0 -> x
1011 def funnel_shift_left_zero: GICombineRule<
1012   (defs root:$root),
1013   (match (G_FSHL $x, $y, $z, 0):$root),
1014   (apply (COPY $x, $y))
1015 >;
1016
1017 // Fold fsh(l/r) x, y, C -> fsh(l/r) x, y, C % bw
1018 def funnel_shift_overshift: GICombineRule<
1019   (defs root:$root),
1020   (match (wip_match_opcode G_FSHL, G_FSHR):$root,
1021     [{ return Helper.matchConstantLargerBitWidth(*${root}, 3); }]),
1022   (apply [{ Helper.applyFunnelShiftConstantModulo(*${root}); }])
1023 >;
1024
1025 def rotate_out_of_range : GICombineRule<
1026   (defs root:$root),
1027   (match (wip_match_opcode G_ROTR, G_ROTL):$root,
1028     [{ return Helper.matchRotateOutOfRange(*${root}); }]),
1029   (apply [{ Helper.applyRotateOutOfRange(*${root}); }])
1030 >;
1031
1032 def icmp_to_true_false_known_bits : GICombineRule<
1033   (defs root:$d, int64_matchinfo:$matchinfo),
1034   (match (wip_match_opcode G_ICMP):$d,
1035          [{ return Helper.matchICmpToTrueFalseKnownBits(*${d}, ${matchinfo}); }]),
1036   (apply [{ Helper.replaceInstWithConstant(*${d}, ${matchinfo}); }])>;
1037
1038 def icmp_to_lhs_known_bits : GICombineRule<
1039   (defs root:$root, build_fn_matchinfo:$info),
1040   (match (wip_match_opcode G_ICMP):$root,
1041          [{ return Helper.matchICmpToLHSKnownBits(*${root}, ${info}); }]),
1042   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1043
1044 def redundant_binop_in_equality : GICombineRule<
1045   (defs root:$root, build_fn_matchinfo:$info),
1046   (match (wip_match_opcode G_ICMP):$root,
1047          [{ return Helper.matchRedundantBinOpInEquality(*${root}, ${info}); }]),
1048   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1049
1050 // Transform: (X == 0 & Y == 0) -> (X | Y) == 0
1051 def double_icmp_zero_and_combine: GICombineRule<
1052   (defs root:$root),
1053   (match (G_ICMP $d1, $p, $s1, 0),
1054          (G_ICMP $d2, $p, $s2, 0),
1055          (G_AND $root, $d1, $d2),
1056          [{ return ${p}.getPredicate() == CmpInst::ICMP_EQ &&
1057                        !MRI.getType(${s1}.getReg()).getScalarType().isPointer() &&
1058                        (MRI.getType(${s1}.getReg()) ==
1059                            MRI.getType(${s2}.getReg())); }]),
1060   (apply (G_OR $ordst, $s1, $s2),
1061          (G_ICMP $root, $p, $ordst, 0))
1062 >;
1063
1064 // Transform: (X != 0 | Y != 0) -> (X | Y) != 0
1065 def double_icmp_zero_or_combine: GICombineRule<
1066   (defs root:$root),
1067   (match (G_ICMP $d1, $p, $s1, 0),
1068          (G_ICMP $d2, $p, $s2, 0),
1069          (G_OR $root, $d1, $d2),
1070          [{ return ${p}.getPredicate() == CmpInst::ICMP_NE &&
1071                        !MRI.getType(${s1}.getReg()).getScalarType().isPointer() &&
1072                        (MRI.getType(${s1}.getReg()) ==
1073                            MRI.getType(${s2}.getReg())); }]),
1074   (apply (G_OR $ordst, $s1, $s2),
1075          (G_ICMP $root, $p, $ordst, 0))
1076 >;
1077
1078 def and_or_disjoint_mask : GICombineRule<
1079   (defs root:$root, build_fn_matchinfo:$info),
1080   (match (wip_match_opcode G_AND):$root,
1081          [{ return Helper.matchAndOrDisjointMask(*${root}, ${info}); }]),
1082   (apply [{ Helper.applyBuildFnNoErase(*${root}, ${info}); }])>;
1083
1084 def bitfield_extract_from_and : GICombineRule<
1085   (defs root:$root, build_fn_matchinfo:$info),
1086   (match (G_CONSTANT $mask, $imm2),
1087          (G_CONSTANT $lsb, $imm1),
1088          (G_LSHR $shift, $x, $lsb),
1089          (G_AND $root, $shift, $mask):$root,
1090     [{ return Helper.matchBitfieldExtractFromAnd(*${root}, ${info}); }]),
1091   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1092
1093 def funnel_shift_combines : GICombineGroup<[funnel_shift_from_or_shift,
1094                                             funnel_shift_to_rotate,
1095                                             funnel_shift_right_zero,
1096                                             funnel_shift_left_zero,
1097                                             funnel_shift_overshift]>;
1098
1099 def bitfield_extract_from_sext_inreg : GICombineRule<
1100   (defs root:$root, build_fn_matchinfo:$info),
1101   (match (wip_match_opcode G_SEXT_INREG):$root,
1102     [{ return Helper.matchBitfieldExtractFromSExtInReg(*${root}, ${info}); }]),
1103   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1104
1105 def bitfield_extract_from_shr : GICombineRule<
1106   (defs root:$root, build_fn_matchinfo:$info),
1107   (match (wip_match_opcode G_ASHR, G_LSHR):$root,
1108     [{ return Helper.matchBitfieldExtractFromShr(*${root}, ${info}); }]),
1109   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1110
1111 def bitfield_extract_from_shr_and : GICombineRule<
1112   (defs root:$root, build_fn_matchinfo:$info),
1113   (match (wip_match_opcode G_ASHR, G_LSHR):$root,
1114     [{ return Helper.matchBitfieldExtractFromShrAnd(*${root}, ${info}); }]),
1115   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1116
1117 def form_bitfield_extract : GICombineGroup<[bitfield_extract_from_sext_inreg,
1118                                             bitfield_extract_from_and,
1119                                             bitfield_extract_from_shr,
1120                                             bitfield_extract_from_shr_and]>;
1121
1122 def udiv_by_const : GICombineRule<
1123   (defs root:$root),
1124   (match (wip_match_opcode G_UDIV):$root,
1125    [{ return Helper.matchUDivByConst(*${root}); }]),
1126   (apply [{ Helper.applyUDivByConst(*${root}); }])>;
1127
1128 def sdiv_by_const : GICombineRule<
1129   (defs root:$root),
1130   (match (wip_match_opcode G_SDIV):$root,
1131    [{ return Helper.matchSDivByConst(*${root}); }]),
1132   (apply [{ Helper.applySDivByConst(*${root}); }])>;
1133
1134 def sdiv_by_pow2 : GICombineRule<
1135   (defs root:$root),
1136   (match (G_SDIV $dst, $x, $y, (MIFlags (not IsExact))):$root,
1137    [{ return Helper.matchDivByPow2(*${root}, /*IsSigned=*/true); }]),
1138   (apply [{ Helper.applySDivByPow2(*${root}); }])>;
1139
1140 def udiv_by_pow2 : GICombineRule<
1141   (defs root:$root),
1142   (match (G_UDIV $dst, $x, $y, (MIFlags (not IsExact))):$root,
1143    [{ return Helper.matchDivByPow2(*${root}, /*IsSigned=*/false); }]),
1144   (apply [{ Helper.applyUDivByPow2(*${root}); }])>;
1145
1146 def intdiv_combines : GICombineGroup<[udiv_by_const, sdiv_by_const,
1147                                       sdiv_by_pow2, udiv_by_pow2]>;
1148
1149 def reassoc_ptradd : GICombineRule<
1150   (defs root:$root, build_fn_matchinfo:$matchinfo),
1151   (match (wip_match_opcode G_PTR_ADD):$root,
1152     [{ return Helper.matchReassocPtrAdd(*${root}, ${matchinfo}); }]),
1153   (apply [{ Helper.applyBuildFnNoErase(*${root}, ${matchinfo}); }])>;
1154
1155 def reassoc_comm_binops : GICombineRule<
1156   (defs root:$root, build_fn_matchinfo:$matchinfo),
1157   (match (G_ADD $root, $src1, $src2):$root,
1158     [{ return Helper.matchReassocCommBinOp(*${root}, ${matchinfo}); }]),
1159   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1160
1161 def reassocs : GICombineGroup<[reassoc_ptradd, reassoc_comm_binops]>;
1162
1163 // Constant fold operations.
1164 def constant_fold_binop : GICombineRule<
1165   (defs root:$d, apint_matchinfo:$matchinfo),
1166   (match (wip_match_opcode G_ADD, G_PTR_ADD, G_AND, G_ASHR, G_LSHR, G_MUL, G_OR,
1167                            G_SHL, G_SUB, G_XOR, G_UDIV, G_SDIV, G_UREM, G_SREM,
1168                            G_SMIN, G_SMAX, G_UMIN, G_UMAX):$d,
1169    [{ return Helper.matchConstantFoldBinOp(*${d}, ${matchinfo}); }]),
1170   (apply [{ Helper.replaceInstWithConstant(*${d}, ${matchinfo}); }])>;
1171
1172 def constant_fold_fp_binop : GICombineRule<
1173   (defs root:$d, constantfp_matchinfo:$matchinfo),
1174   (match (wip_match_opcode G_FADD, G_FSUB, G_FMUL, G_FDIV):$d,
1175    [{ return Helper.matchConstantFoldFPBinOp(*${d}, ${matchinfo}); }]),
1176   (apply [{ Helper.replaceInstWithFConstant(*${d}, ${matchinfo}); }])>;
1177
1178
1179 def constant_fold_fma : GICombineRule<
1180   (defs root:$d, constantfp_matchinfo:$matchinfo),
1181   (match (wip_match_opcode G_FMAD, G_FMA):$d,
1182    [{ return Helper.matchConstantFoldFMA(*${d}, ${matchinfo}); }]),
1183   (apply [{ Helper.replaceInstWithFConstant(*${d}, ${matchinfo}); }])>;
1184
1185 def constant_fold_cast_op : GICombineRule<
1186   (defs root:$d, apint_matchinfo:$matchinfo),
1187   (match (wip_match_opcode G_ZEXT, G_SEXT, G_ANYEXT):$d,
1188    [{ return Helper.matchConstantFoldCastOp(*${d}, ${matchinfo}); }]),
1189   (apply [{ Helper.replaceInstWithConstant(*${d}, ${matchinfo}); }])>;
1190
1191 def mulo_by_2: GICombineRule<
1192   (defs root:$root, build_fn_matchinfo:$matchinfo),
1193   (match (wip_match_opcode G_UMULO, G_SMULO):$root,
1194          [{ return Helper.matchMulOBy2(*${root}, ${matchinfo}); }]),
1195   (apply [{ Helper.applyBuildFnNoErase(*${root}, ${matchinfo}); }])>;
1196
1197 def mulo_by_0: GICombineRule<
1198   (defs root:$root, build_fn_matchinfo:$matchinfo),
1199   (match (wip_match_opcode G_UMULO, G_SMULO):$root,
1200          [{ return Helper.matchMulOBy0(*${root}, ${matchinfo}); }]),
1201   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1202
1203 // Transform (uadde x, y, 0) -> (uaddo x, y)
1204 //           (sadde x, y, 0) -> (saddo x, y)
1205 //           (usube x, y, 0) -> (usubo x, y)
1206 //           (ssube x, y, 0) -> (ssubo x, y)
1207 def adde_to_addo: GICombineRule<
1208   (defs root:$root, build_fn_matchinfo:$matchinfo),
1209   (match (wip_match_opcode G_UADDE, G_SADDE, G_USUBE, G_SSUBE):$root,
1210          [{ return Helper.matchAddEToAddO(*${root}, ${matchinfo}); }]),
1211   (apply [{ Helper.applyBuildFnNoErase(*${root}, ${matchinfo}); }])>;
1212
1213 def mulh_to_lshr : GICombineRule<
1214   (defs root:$root),
1215   (match (wip_match_opcode G_UMULH):$root,
1216          [{ return Helper.matchUMulHToLShr(*${root}); }]),
1217   (apply [{ Helper.applyUMulHToLShr(*${root}); }])>;
1218
1219 def mulh_combines : GICombineGroup<[mulh_to_lshr]>;
1220
1221 def redundant_neg_operands: GICombineRule<
1222   (defs root:$root, build_fn_matchinfo:$matchinfo),
1223   (match (wip_match_opcode G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMAD, G_FMA):$root,
1224     [{ return Helper.matchRedundantNegOperands(*${root}, ${matchinfo}); }]),
1225   (apply [{ Helper.applyBuildFnNoErase(*${root}, ${matchinfo}); }])>;
1226
1227 // Transform (fsub +-0.0, X) -> (fneg X)
1228 def fsub_to_fneg: GICombineRule<
1229   (defs root:$root, register_matchinfo:$matchinfo),
1230   (match (wip_match_opcode G_FSUB):$root,
1231     [{ return Helper.matchFsubToFneg(*${root}, ${matchinfo}); }]),
1232   (apply [{ Helper.applyFsubToFneg(*${root}, ${matchinfo}); }])>;
1233
1234 // Transform (fadd x, (fmul y, z)) -> (fma y, z, x)
1235 //           (fadd x, (fmul y, z)) -> (fmad y, z, x)
1236 // Transform (fadd (fmul x, y), z) -> (fma x, y, z)
1237 //           (fadd (fmul x, y), z) -> (fmad x, y, z)
1238 def combine_fadd_fmul_to_fmad_or_fma: GICombineRule<
1239   (defs root:$root, build_fn_matchinfo:$info),
1240   (match (wip_match_opcode G_FADD):$root,
1241          [{ return Helper.matchCombineFAddFMulToFMadOrFMA(*${root},
1242                                                           ${info}); }]),
1243   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1244
1245 // Transform (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
1246 //                                         -> (fmad (fpext x), (fpext y), z)
1247 // Transform (fadd x, (fpext (fmul y, z))) -> (fma (fpext y), (fpext z), x)
1248 //                                         -> (fmad (fpext y), (fpext z), x)
1249 def combine_fadd_fpext_fmul_to_fmad_or_fma: GICombineRule<
1250   (defs root:$root, build_fn_matchinfo:$info),
1251   (match (wip_match_opcode G_FADD):$root,
1252          [{ return Helper.matchCombineFAddFpExtFMulToFMadOrFMA(*${root},
1253                                                                ${info}); }]),
1254   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1255
1256 // Transform (fadd (fma x, y, (fmul z, u)), v)  -> (fma x, y, (fma z, u, v))
1257 //           (fadd (fmad x, y, (fmul z, u)), v) -> (fmad x, y, (fmad z, u, v))
1258 // Transform (fadd v, (fma x, y, (fmul z, u)))  -> (fma x, y, (fma z, u, v))
1259 //           (fadd v, (fmad x, y, (fmul z, u))) -> (fmad x, y, (fmad z, u, v))
1260 def combine_fadd_fma_fmul_to_fmad_or_fma: GICombineRule<
1261   (defs root:$root, build_fn_matchinfo:$info),
1262   (match (wip_match_opcode G_FADD):$root,
1263          [{ return Helper.matchCombineFAddFMAFMulToFMadOrFMA(*${root},
1264                                                              ${info}); }]),
1265   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1266
1267 // Transform (fadd (fma x, y, (fpext (fmul u, v))), z) ->
1268 //           (fma x, y, (fma (fpext u), (fpext v), z))
1269 def combine_fadd_fpext_fma_fmul_to_fmad_or_fma: GICombineRule<
1270   (defs root:$root, build_fn_matchinfo:$info),
1271   (match (wip_match_opcode G_FADD):$root,
1272          [{ return Helper.matchCombineFAddFpExtFMulToFMadOrFMAAggressive(
1273                                                   *${root}, ${info}); }]),
1274   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1275
1276 // Transform (fsub (fmul x, y), z) -> (fma x, y, -z)
1277 //                                 -> (fmad x, y, -z)
1278 def combine_fsub_fmul_to_fmad_or_fma: GICombineRule<
1279   (defs root:$root, build_fn_matchinfo:$info),
1280   (match (wip_match_opcode G_FSUB):$root,
1281          [{ return Helper.matchCombineFSubFMulToFMadOrFMA(*${root},
1282                                                           ${info}); }]),
1283   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1284
1285 // Transform (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z))
1286 //           (fsub x, (fneg (fmul, y, z))) -> (fma y, z, x)
1287 def combine_fsub_fneg_fmul_to_fmad_or_fma: GICombineRule<
1288   (defs root:$root, build_fn_matchinfo:$info),
1289   (match (wip_match_opcode G_FSUB):$root,
1290          [{ return Helper.matchCombineFSubFNegFMulToFMadOrFMA(*${root},
1291                                                               ${info}); }]),
1292   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1293
1294 // Transform (fsub (fpext (fmul x, y)), z) ->
1295 //           (fma (fpext x), (fpext y), (fneg z))
1296 def combine_fsub_fpext_fmul_to_fmad_or_fma: GICombineRule<
1297   (defs root:$root, build_fn_matchinfo:$info),
1298   (match (wip_match_opcode G_FSUB):$root,
1299          [{ return Helper.matchCombineFSubFpExtFMulToFMadOrFMA(*${root},
1300                                                                ${info}); }]),
1301   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1302
1303 // Transform (fsub (fneg (fpext (fmul x, y))), z) ->
1304 //           (fneg (fma (fpext x), (fpext y), z))
1305 def combine_fsub_fpext_fneg_fmul_to_fmad_or_fma: GICombineRule<
1306   (defs root:$root, build_fn_matchinfo:$info),
1307   (match (wip_match_opcode G_FSUB):$root,
1308          [{ return Helper.matchCombineFSubFpExtFNegFMulToFMadOrFMA(
1309                                             *${root}, ${info}); }]),
1310   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1311
1312 def combine_minmax_nan: GICombineRule<
1313   (defs root:$root, unsigned_matchinfo:$info),
1314   (match (wip_match_opcode G_FMINNUM, G_FMAXNUM, G_FMINIMUM, G_FMAXIMUM):$root,
1315          [{ return Helper.matchCombineFMinMaxNaN(*${root}, ${info}); }]),
1316   (apply [{ Helper.replaceSingleDefInstWithOperand(*${root}, ${info}); }])>;
1317
1318 // Transform (add x, (sub y, x)) -> y
1319 // Transform (add (sub y, x), x) -> y
1320 def add_sub_reg_frags : GICombinePatFrag<
1321   (outs root:$dst), (ins $src),
1322   [
1323     (pattern (G_ADD $dst, $x, $tmp), (G_SUB $tmp, $src, $x)),
1324     (pattern (G_ADD $dst, $tmp, $x), (G_SUB $tmp, $src, $x))
1325   ]>;
1326 def add_sub_reg: GICombineRule <
1327   (defs root:$dst),
1328   (match (add_sub_reg_frags $dst, $src)),
1329   (apply (GIReplaceReg $dst, $src))>;
1330
1331 def buildvector_identity_fold : GICombineRule<
1332   (defs root:$build_vector, register_matchinfo:$matchinfo),
1333   (match (wip_match_opcode G_BUILD_VECTOR_TRUNC, G_BUILD_VECTOR):$build_vector,
1334          [{ return Helper.matchBuildVectorIdentityFold(*${build_vector}, ${matchinfo}); }]),
1335   (apply [{ Helper.replaceSingleDefInstWithReg(*${build_vector}, ${matchinfo}); }])>;
1336
1337 def trunc_buildvector_fold : GICombineRule<
1338   (defs root:$op, register_matchinfo:$matchinfo),
1339   (match (wip_match_opcode G_TRUNC):$op,
1340       [{ return Helper.matchTruncBuildVectorFold(*${op}, ${matchinfo}); }]),
1341   (apply [{ Helper.replaceSingleDefInstWithReg(*${op}, ${matchinfo}); }])>;
1342
1343 def trunc_lshr_buildvector_fold : GICombineRule<
1344   (defs root:$op, register_matchinfo:$matchinfo),
1345   (match (wip_match_opcode G_TRUNC):$op,
1346       [{ return Helper.matchTruncLshrBuildVectorFold(*${op}, ${matchinfo}); }]),
1347   (apply [{ Helper.replaceSingleDefInstWithReg(*${op}, ${matchinfo}); }])>;
1348
1349 // Transform:
1350 //   (x + y) - y -> x
1351 //   (x + y) - x -> y
1352 //   x - (y + x) -> 0 - y
1353 //   x - (x + z) -> 0 - z
1354 def sub_add_reg: GICombineRule <
1355   (defs root:$root, build_fn_matchinfo:$matchinfo),
1356   (match (wip_match_opcode G_SUB):$root,
1357          [{ return Helper.matchSubAddSameReg(*${root}, ${matchinfo}); }]),
1358   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1359
1360 def bitcast_bitcast_fold : GICombineRule<
1361   (defs root:$dst),
1362   (match (G_BITCAST $dst, $src1):$op, (G_BITCAST $src1, $src0),
1363       [{ return MRI.getType(${src0}.getReg()) == MRI.getType(${dst}.getReg()); }]),
1364   (apply [{ Helper.replaceSingleDefInstWithReg(*${op}, ${src0}.getReg()); }])>;
1365
1366
1367 def fptrunc_fpext_fold : GICombineRule<
1368   (defs root:$dst),
1369   (match (G_FPTRUNC $dst, $src1):$op, (G_FPEXT $src1, $src0),
1370       [{ return MRI.getType(${src0}.getReg()) == MRI.getType(${dst}.getReg()); }]),
1371   (apply [{ Helper.replaceSingleDefInstWithReg(*${op}, ${src0}.getReg()); }])>;
1372
1373
1374 def select_to_minmax: GICombineRule<
1375   (defs root:$root, build_fn_matchinfo:$info),
1376   (match (wip_match_opcode G_SELECT):$root,
1377          [{ return Helper.matchSimplifySelectToMinMax(*${root}, ${info}); }]),
1378   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
1379
1380 def select_to_iminmax: GICombineRule<
1381   (defs root:$root, build_fn_matchinfo:$info),
1382   (match (G_ICMP $tst, $tst1, $a, $b),
1383          (G_SELECT $root, $tst, $x, $y),
1384          [{ return Helper.matchSelectIMinMax(${root}, ${info}); }]),
1385   (apply [{ Helper.applyBuildFnMO(${root}, ${info}); }])>;
1386
1387 def simplify_neg_minmax : GICombineRule<
1388   (defs root:$root, build_fn_matchinfo:$matchinfo),
1389   (match (wip_match_opcode G_SUB):$root,
1390          [{ return Helper.matchSimplifyNegMinMax(*${root}, ${matchinfo}); }]),
1391   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1392
1393 def match_selects : GICombineRule<
1394   (defs root:$root, build_fn_matchinfo:$matchinfo),
1395   (match (wip_match_opcode G_SELECT):$root,
1396         [{ return Helper.matchSelect(*${root}, ${matchinfo}); }]),
1397   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1398
1399 def match_ands : GICombineRule<
1400   (defs root:$root, build_fn_matchinfo:$matchinfo),
1401   (match (wip_match_opcode G_AND):$root,
1402         [{ return Helper.matchAnd(*${root}, ${matchinfo}); }]),
1403   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1404
1405 def match_ors : GICombineRule<
1406   (defs root:$root, build_fn_matchinfo:$matchinfo),
1407   (match (wip_match_opcode G_OR):$root,
1408         [{ return Helper.matchOr(*${root}, ${matchinfo}); }]),
1409   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1410
1411 def match_addos : GICombineRule<
1412   (defs root:$root, build_fn_matchinfo:$matchinfo),
1413   (match (wip_match_opcode G_SADDO, G_UADDO):$root,
1414         [{ return Helper.matchAddOverflow(*${root}, ${matchinfo}); }]),
1415   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1416
1417 def match_subo_no_overflow : GICombineRule<
1418   (defs root:$root, build_fn_matchinfo:$matchinfo),
1419   (match (wip_match_opcode G_SSUBO, G_USUBO):$root,
1420         [{ return Helper.matchSuboCarryOut(*${root}, ${matchinfo}); }]),
1421   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1422
1423 def match_extract_of_element_undef_vector: GICombineRule <
1424   (defs root:$root),
1425   (match (G_IMPLICIT_DEF $vector),
1426          (G_EXTRACT_VECTOR_ELT $root, $vector, $idx)),
1427   (apply (G_IMPLICIT_DEF $root))
1428 >;
1429
1430 def match_extract_of_element_undef_index: GICombineRule <
1431   (defs root:$root),
1432   (match (G_IMPLICIT_DEF $idx),
1433          (G_EXTRACT_VECTOR_ELT $root, $vector, $idx)),
1434   (apply (G_IMPLICIT_DEF $root))
1435 >;
1436
1437 def match_extract_of_element : GICombineRule<
1438   (defs root:$root, build_fn_matchinfo:$matchinfo),
1439   (match (wip_match_opcode G_EXTRACT_VECTOR_ELT):$root,
1440         [{ return Helper.matchExtractVectorElement(*${root}, ${matchinfo}); }]),
1441   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1442
1443 def extract_vector_element_not_const : GICombineRule<
1444    (defs root:$root),
1445    (match (G_INSERT_VECTOR_ELT $src, $x, $value, $idx),
1446           (G_EXTRACT_VECTOR_ELT $root, $src, $idx)),
1447    (apply (GIReplaceReg $root, $value))>;
1448
1449 def extract_vector_element_different_indices : GICombineRule<
1450    (defs root:$root, build_fn_matchinfo:$matchinfo),
1451    (match (G_INSERT_VECTOR_ELT $src, $x, $value, $idx2),
1452           (G_EXTRACT_VECTOR_ELT $root, $src, $idx1),
1453    [{ return Helper.matchExtractVectorElementWithDifferentIndices(${root}, ${matchinfo}); }]),
1454    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1455
1456 def extract_vector_element_build_vector : GICombineRule<
1457    (defs root:$root, build_fn_matchinfo:$matchinfo),
1458    (match (G_CONSTANT $idx, $imm),
1459           (G_BUILD_VECTOR $src, GIVariadic<>:$unused):$Build,
1460           (G_EXTRACT_VECTOR_ELT $root, $src, $idx):$Extract,
1461    [{ return Helper.matchExtractVectorElementWithBuildVector(*${Extract}, *${Build},
1462       ${matchinfo}); }]),
1463    (apply [{ Helper.applyBuildFn(*${Extract}, ${matchinfo}); }])>;
1464
1465 def extract_vector_element_shuffle_vector : GICombineRule<
1466    (defs root:$root, build_fn_matchinfo:$matchinfo),
1467    (match (G_CONSTANT $idx, $imm),
1468           (G_SHUFFLE_VECTOR $src, $src1, $src2, $mask):$Shuffle,
1469           (G_EXTRACT_VECTOR_ELT $root, $src, $idx):$Extract,
1470    [{ return Helper.matchExtractVectorElementWithShuffleVector(*${Extract}, *${Shuffle},
1471       ${matchinfo}); }]),
1472    (apply [{ Helper.applyBuildFn(*${Extract}, ${matchinfo}); }])>;
1473
1474 def extract_vector_element_build_vector_trunc2 : GICombineRule<
1475    (defs root:$root, build_fn_matchinfo:$matchinfo),
1476    (match (G_BUILD_VECTOR_TRUNC $src, $x, $y),
1477           (G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1478    [{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]),
1479    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1480
1481 def extract_vector_element_build_vector_trunc3 : GICombineRule<
1482    (defs root:$root, build_fn_matchinfo:$matchinfo),
1483    (match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z),
1484           (G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1485    [{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]),
1486    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1487
1488 def extract_vector_element_build_vector_trunc4 : GICombineRule<
1489    (defs root:$root, build_fn_matchinfo:$matchinfo),
1490    (match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a),
1491           (G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1492    [{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]),
1493    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1494
1495 def extract_vector_element_build_vector_trunc5 : GICombineRule<
1496    (defs root:$root, build_fn_matchinfo:$matchinfo),
1497    (match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a, $b),
1498           (G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1499    [{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]),
1500    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1501
1502 def extract_vector_element_build_vector_trunc6 : GICombineRule<
1503    (defs root:$root, build_fn_matchinfo:$matchinfo),
1504    (match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a, $b, $c),
1505           (G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1506    [{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]),
1507    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1508
1509 def extract_vector_element_build_vector_trunc7 : GICombineRule<
1510    (defs root:$root, build_fn_matchinfo:$matchinfo),
1511    (match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a, $b, $c, $d),
1512           (G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1513    [{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]),
1514    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1515
1516 def extract_vector_element_build_vector_trunc8 : GICombineRule<
1517    (defs root:$root, build_fn_matchinfo:$matchinfo),
1518    (match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a, $b, $c, $d, $e),
1519           (G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1520    [{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]),
1521    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1522
1523 def sext_trunc : GICombineRule<
1524    (defs root:$root, build_fn_matchinfo:$matchinfo),
1525    (match (G_TRUNC $src, $x, (MIFlags NoSWrap)),
1526           (G_SEXT $root, $src),
1527    [{ return Helper.matchSextOfTrunc(${root}, ${matchinfo}); }]),
1528    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1529
1530 def zext_trunc : GICombineRule<
1531    (defs root:$root, build_fn_matchinfo:$matchinfo),
1532    (match (G_TRUNC $src, $x, (MIFlags NoUWrap)),
1533           (G_ZEXT $root, $src),
1534    [{ return Helper.matchZextOfTrunc(${root}, ${matchinfo}); }]),
1535    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1536
1537 def nneg_zext : GICombineRule<
1538    (defs root:$root, build_fn_matchinfo:$matchinfo),
1539    (match (G_ZEXT $root, $x, (MIFlags NonNeg)),
1540    [{ return Helper.matchNonNegZext(${root}, ${matchinfo}); }]),
1541    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1542
1543 // Combines concat operations
1544 def concat_matchinfo : GIDefMatchData<"SmallVector<Register>">;
1545 def combine_concat_vector : GICombineRule<
1546   (defs root:$root, concat_matchinfo:$matchinfo),
1547   (match (wip_match_opcode G_CONCAT_VECTORS):$root,
1548         [{ return Helper.matchCombineConcatVectors(*${root}, ${matchinfo}); }]),
1549   (apply [{ Helper.applyCombineConcatVectors(*${root}, ${matchinfo}); }])>;
1550
1551 // Combines Shuffles of Concats
1552 // a = G_CONCAT_VECTORS x, y, undef, undef
1553 // b = G_CONCAT_VECTORS z, undef, undef, undef
1554 // c = G_SHUFFLE_VECTORS a, b, <0, 1, 4, undef>
1555 // ===>
1556 // c = G_CONCAT_VECTORS x, y, z, undef
1557 def combine_shuffle_concat : GICombineRule<
1558   (defs root:$root, concat_matchinfo:$matchinfo),
1559   (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
1560         [{ return Helper.matchCombineShuffleConcat(*${root}, ${matchinfo}); }]),
1561   (apply [{ Helper.applyCombineShuffleConcat(*${root}, ${matchinfo}); }])>;
1562
1563 def insert_vector_element_idx_undef : GICombineRule<
1564    (defs root:$root),
1565    (match (G_IMPLICIT_DEF $idx),
1566           (G_INSERT_VECTOR_ELT $root, $src, $elt, $idx)),
1567    (apply (G_IMPLICIT_DEF $root))>;
1568
1569 def insert_vector_element_elt_undef : GICombineRule<
1570    (defs root:$root),
1571    (match (G_IMPLICIT_DEF $elt),
1572           (G_INSERT_VECTOR_ELT $root, $src, $elt, $idx),
1573           [{ return isGuaranteedNotToBePoison(${src}.getReg(), MRI); }]),
1574    (apply (GIReplaceReg $root, $src))>;
1575
1576 def insert_vector_element_extract_vector_element : GICombineRule<
1577    (defs root:$root),
1578    (match (G_EXTRACT_VECTOR_ELT $elt, $src, $idx),
1579           (G_INSERT_VECTOR_ELT $root, $src, $elt, $idx)),
1580    (apply (GIReplaceReg $root, $src))>;
1581
1582 def insert_vector_elt_oob : GICombineRule<
1583   (defs root:$root, build_fn_matchinfo:$matchinfo),
1584   (match (wip_match_opcode G_INSERT_VECTOR_ELT):$root,
1585          [{ return Helper.matchInsertVectorElementOOB(*${root}, ${matchinfo}); }]),
1586   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1587
1588 // Combine v8i8 (buildvector i8 (trunc(unmerge)), i8 (trunc), i8 (trunc), i8 (trunc), undef, undef, undef, undef)
1589 def combine_use_vector_truncate : GICombineRule<
1590   (defs root:$root, register_matchinfo:$matchinfo),
1591   (match (G_BUILD_VECTOR $dst, GIVariadic<>:$unused):$root,
1592          [{ return Helper.matchUseVectorTruncate(*${root}, ${matchinfo}); }]),
1593   (apply [{ Helper.applyUseVectorTruncate(*${root}, ${matchinfo}); }])>;
1594
1595 def add_of_vscale : GICombineRule<
1596    (defs root:$root, build_fn_matchinfo:$matchinfo),
1597    (match (G_VSCALE $left, $imm1),
1598           (G_VSCALE $right, $imm2),
1599           (G_ADD $root, $left, $right, (MIFlags NoSWrap)),
1600    [{ return Helper.matchAddOfVScale(${root}, ${matchinfo}); }]),
1601    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1602
1603 def mul_of_vscale : GICombineRule<
1604    (defs root:$root, build_fn_matchinfo:$matchinfo),
1605    (match (G_VSCALE $left, $scale),
1606           (G_CONSTANT $x, $imm1),
1607           (G_MUL $root, $left, $x, (MIFlags NoSWrap)),
1608    [{ return Helper.matchMulOfVScale(${root}, ${matchinfo}); }]),
1609    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1610
1611 def shl_of_vscale : GICombineRule<
1612    (defs root:$root, build_fn_matchinfo:$matchinfo),
1613    (match (G_VSCALE $left, $imm),
1614           (G_CONSTANT $x, $imm1),
1615           (G_SHL $root, $left, $x, (MIFlags NoSWrap)),
1616    [{ return Helper.matchShlOfVScale(${root}, ${matchinfo}); }]),
1617    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1618
1619 def sub_of_vscale : GICombineRule<
1620    (defs root:$root, build_fn_matchinfo:$matchinfo),
1621    (match (G_VSCALE $right, $imm),
1622           (G_SUB $root, $x, $right, (MIFlags NoSWrap)),
1623    [{ return Helper.matchSubOfVScale(${root}, ${matchinfo}); }]),
1624    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1625
1626 def expand_const_fpowi : GICombineRule<
1627    (defs root:$root),
1628    (match (G_CONSTANT $int, $imm),
1629           (G_FPOWI $dst, $float, $int):$root,
1630           [{ return Helper.matchFPowIExpansion(*${root}, ${imm}.getCImm()->getSExtValue()); }]),
1631    (apply [{ Helper.applyExpandFPowI(*${root}, ${imm}.getCImm()->getSExtValue()); }])>;
1632
1633 def combine_shuffle_undef_rhs : GICombineRule<
1634   (defs root:$root, build_fn_matchinfo:$matchinfo),
1635   (match (G_IMPLICIT_DEF $undef),
1636          (G_SHUFFLE_VECTOR $root, $src1, $undef, $mask):$root,
1637         [{ return Helper.matchShuffleUndefRHS(*${root}, ${matchinfo}); }]),
1638   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])
1639 >;
1640
1641 def combine_shuffle_disjoint_mask : GICombineRule<
1642   (defs root:$root, build_fn_matchinfo:$matchinfo),
1643   (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
1644         [{ return Helper.matchShuffleDisjointMask(*${root}, ${matchinfo}); }]),
1645   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])
1646 >;
1647
1648 // match_extract_of_element and insert_vector_elt_oob must be the first!
1649 def vector_ops_combines: GICombineGroup<[
1650 match_extract_of_element_undef_vector,
1651 match_extract_of_element_undef_index,
1652 insert_vector_element_idx_undef,
1653 insert_vector_element_elt_undef,
1654 match_extract_of_element,
1655 insert_vector_elt_oob,
1656 extract_vector_element_not_const,
1657 extract_vector_element_different_indices,
1658 extract_vector_element_build_vector,
1659 extract_vector_element_build_vector_trunc2,
1660 extract_vector_element_build_vector_trunc3,
1661 extract_vector_element_build_vector_trunc4,
1662 extract_vector_element_build_vector_trunc5,
1663 extract_vector_element_build_vector_trunc6,
1664 extract_vector_element_build_vector_trunc7,
1665 extract_vector_element_build_vector_trunc8,
1666 extract_vector_element_shuffle_vector,
1667 insert_vector_element_extract_vector_element,
1668 add_of_vscale,
1669 mul_of_vscale,
1670 shl_of_vscale,
1671 sub_of_vscale,
1672 ]>;
1673
1674
1675 // fold ((0-A) + B) -> B-A
1676 def ZeroMinusAPlusB : GICombineRule<
1677    (defs root:$root),
1678    (match (G_SUB $sub, 0, $A),
1679           (G_ADD $root, $sub, $B)),
1680    (apply (G_SUB $root, $B, $A))>;
1681
1682 // fold (A + (0-B)) -> A-B
1683 def APlusZeroMinusB : GICombineRule<
1684    (defs root:$root),
1685    (match (G_SUB $sub, 0, $B),
1686           (G_ADD $root, $A, $sub)),
1687    (apply (G_SUB $root, $A, $B))>;
1688
1689  // fold (A+(B-A)) -> B
1690  def APlusBMinusB : GICombineRule<
1691    (defs root:$root),
1692    (match (G_SUB $sub, $B, $A),
1693           (G_ADD $root, $A, $sub)),
1694    (apply (GIReplaceReg $root, $B))>;
1695
1696 // fold ((B-A)+A) -> B
1697  def BMinusAPlusA : GICombineRule<
1698    (defs root:$root),
1699    (match (G_SUB $sub, $B, $A),
1700           (G_ADD $root, $sub, $A)),
1701    (apply (GIReplaceReg $root, $B))>;
1702
1703 // fold ((A-B)+(C-A)) -> (C-B)
1704 def AMinusBPlusCMinusA : GICombineRule<
1705    (defs root:$root),
1706    (match (G_SUB $sub1, $A, $B),
1707           (G_SUB $sub2, $C, $A),
1708           (G_ADD $root, $sub1, $sub2)),
1709    (apply (G_SUB $root, $C, $B))>;
1710
1711 // fold ((A-B)+(B-C)) -> (A-C)
1712 def AMinusBPlusBMinusC : GICombineRule<
1713    (defs root:$root),
1714    (match (G_SUB $sub1, $A, $B),
1715           (G_SUB $sub2, $B, $C),
1716           (G_ADD $root, $sub1, $sub2)),
1717    (apply (G_SUB $root, $A, $C))>;
1718
1719 // fold (A+(B-(A+C))) to (B-C)
1720 def APlusBMinusAplusC : GICombineRule<
1721    (defs root:$root),
1722    (match (G_ADD $add1, $A, $C),
1723           (G_SUB $sub1, $B, $add1),
1724           (G_ADD $root, $A, $sub1)),
1725    (apply (G_SUB $root, $B, $C))>;
1726
1727 // fold (A+(B-(C+A))) to (B-C)
1728 def APlusBMinusCPlusA : GICombineRule<
1729    (defs root:$root),
1730    (match (G_ADD $add1, $C, $A),
1731           (G_SUB $sub1, $B, $add1),
1732           (G_ADD $root, $A, $sub1)),
1733    (apply (G_SUB $root, $B, $C))>;
1734
1735 // fold (A+C1)-C2 -> A+(C1-C2)
1736 def APlusC1MinusC2: GICombineRule<
1737    (defs root:$root, build_fn_matchinfo:$matchinfo),
1738    (match (G_CONSTANT $c2, $imm2),
1739           (G_CONSTANT $c1, $imm1),
1740           (G_ADD $add, $A, $c1),
1741           (G_SUB $root, $add, $c2):$root,
1742    [{ return Helper.matchFoldAPlusC1MinusC2(*${root}, ${matchinfo}); }]),
1743    (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1744
1745 // fold C2-(A+C1) -> (C2-C1)-A
1746 def C2MinusAPlusC1: GICombineRule<
1747    (defs root:$root, build_fn_matchinfo:$matchinfo),
1748    (match (G_CONSTANT $c2, $imm2),
1749           (G_CONSTANT $c1, $imm1),
1750           (G_ADD $add, $A, $c1),
1751           (G_SUB $root, $c2, $add):$root,
1752    [{ return Helper.matchFoldC2MinusAPlusC1(*${root}, ${matchinfo}); }]),
1753    (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1754
1755 // fold (A-C1)-C2 -> A-(C1+C2)
1756 def AMinusC1MinusC2: GICombineRule<
1757    (defs root:$root, build_fn_matchinfo:$matchinfo),
1758    (match (G_CONSTANT $c2, $imm2),
1759           (G_CONSTANT $c1, $imm1),
1760           (G_SUB $sub1, $A, $c1),
1761           (G_SUB $root, $sub1, $c2):$root,
1762    [{ return Helper.matchFoldAMinusC1MinusC2(*${root}, ${matchinfo}); }]),
1763    (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1764
1765 // fold (C1-A)-C2 -> (C1-C2)-A
1766 def C1Minus2MinusC2: GICombineRule<
1767    (defs root:$root, build_fn_matchinfo:$matchinfo),
1768    (match (G_CONSTANT $c2, $imm2),
1769           (G_CONSTANT $c1, $imm1),
1770           (G_SUB $sub1, $c1, $A),
1771           (G_SUB $root, $sub1, $c2):$root,
1772    [{ return Helper.matchFoldC1Minus2MinusC2(*${root}, ${matchinfo}); }]),
1773    (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1774
1775 // fold ((A-C1)+C2) -> (A+(C2-C1))
1776 def AMinusC1PlusC2: GICombineRule<
1777    (defs root:$root, build_fn_matchinfo:$matchinfo),
1778    (match (G_CONSTANT $c2, $imm2),
1779           (G_CONSTANT $c1, $imm1),
1780           (G_SUB $sub, $A, $c1),
1781           (G_ADD $root, $sub, $c2):$root,
1782    [{ return Helper.matchFoldAMinusC1PlusC2(*${root}, ${matchinfo}); }]),
1783    (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1784
1785 def integer_reassoc_combines: GICombineGroup<[
1786   ZeroMinusAPlusB,
1787   APlusZeroMinusB,
1788   APlusBMinusB,
1789   BMinusAPlusA,
1790   AMinusBPlusCMinusA,
1791   AMinusBPlusBMinusC,
1792   APlusBMinusAplusC,
1793   APlusBMinusCPlusA,
1794   APlusC1MinusC2,
1795   C2MinusAPlusC1,
1796   AMinusC1MinusC2,
1797   C1Minus2MinusC2,
1798   AMinusC1PlusC2
1799 ]>;
1800
1801 def freeze_of_non_undef_non_poison : GICombineRule<
1802    (defs root:$root),
1803    (match (G_FREEZE $root, $src),
1804           [{ return isGuaranteedNotToBeUndefOrPoison(${src}.getReg(), MRI); }]),
1805    (apply (GIReplaceReg $root, $src))>;
1806
1807 def freeze_combines: GICombineGroup<[
1808   freeze_of_non_undef_non_poison,
1809   push_freeze_to_prevent_poison_from_propagating
1810 ]>;
1811
1812 /// Transform trunc ([asz]ext x) to x or ([asz]ext x) or (trunc x).
1813 class truncate_of_opcode<Instruction extOpcode> : GICombineRule <
1814   (defs root:$root, build_fn_matchinfo:$matchinfo),
1815   (match (extOpcode $ext, $src):$ExtMI,
1816          (G_TRUNC $root, $ext):$root,
1817          [{ return Helper.matchTruncateOfExt(*${root}, *${ExtMI}, ${matchinfo}); }]),
1818   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1819
1820 def truncate_of_zext : truncate_of_opcode<G_ZEXT>;
1821 def truncate_of_sext : truncate_of_opcode<G_SEXT>;
1822 def truncate_of_anyext : truncate_of_opcode<G_ANYEXT>;
1823
1824 // Push cast through select.
1825 class select_of_opcode<Instruction castOpcode> : GICombineRule <
1826   (defs root:$root, build_fn_matchinfo:$matchinfo),
1827   (match (G_SELECT $select, $cond, $true, $false):$Select,
1828          (castOpcode $root, $select):$Cast,
1829          [{ return Helper.matchCastOfSelect(*${Cast}, *${Select}, ${matchinfo}); }]),
1830   (apply [{ Helper.applyBuildFn(*${Cast}, ${matchinfo}); }])>;
1831
1832 def select_of_zext : select_of_opcode<G_ZEXT>;
1833 def select_of_anyext : select_of_opcode<G_ANYEXT>;
1834 def select_of_truncate : select_of_opcode<G_TRUNC>;
1835
1836 // Fold ([asz]ext ([asz]ext x)) -> ([asz]ext x).
1837 class ext_of_ext_opcodes<Instruction ext1Opcode, Instruction ext2Opcode> : GICombineRule <
1838   (defs root:$root, build_fn_matchinfo:$matchinfo),
1839   (match (ext2Opcode $second, $src):$Second,
1840          (ext1Opcode $root, $second):$First,
1841          [{ return Helper.matchExtOfExt(*${First}, *${Second}, ${matchinfo}); }]),
1842   (apply [{ Helper.applyBuildFn(*${First}, ${matchinfo}); }])>;
1843
1844 def zext_of_zext : ext_of_ext_opcodes<G_ZEXT, G_ZEXT>;
1845 def zext_of_anyext : ext_of_ext_opcodes<G_ZEXT, G_ANYEXT>;
1846 def sext_of_sext : ext_of_ext_opcodes<G_SEXT, G_SEXT>;
1847 def sext_of_anyext : ext_of_ext_opcodes<G_SEXT, G_ANYEXT>;
1848 def anyext_of_anyext : ext_of_ext_opcodes<G_ANYEXT, G_ANYEXT>;
1849 def anyext_of_zext : ext_of_ext_opcodes<G_ANYEXT, G_ZEXT>;
1850 def anyext_of_sext : ext_of_ext_opcodes<G_ANYEXT, G_SEXT>;
1851
1852 // Push cast through build vector.
1853 class buildvector_of_opcode<Instruction castOpcode> : GICombineRule <
1854   (defs root:$root, build_fn_matchinfo:$matchinfo),
1855   (match (G_BUILD_VECTOR $bv, GIVariadic<>:$unused):$Build,
1856          (castOpcode $root, $bv):$Cast,
1857          [{ return Helper.matchCastOfBuildVector(*${Cast}, *${Build}, ${matchinfo}); }]),
1858   (apply [{ Helper.applyBuildFn(*${Cast}, ${matchinfo}); }])>;
1859
1860 def buildvector_of_truncate : buildvector_of_opcode<G_TRUNC>;
1861
1862 // narrow binop.
1863 // trunc (binop X, C) --> binop (trunc X, trunc C)
1864 class narrow_binop_opcode<Instruction binopOpcode> : GICombineRule <
1865   (defs root:$root, build_fn_matchinfo:$matchinfo),
1866   (match (G_CONSTANT $const, $imm),
1867          (binopOpcode $binop, $x, $const):$Binop,
1868          (G_TRUNC $root, $binop):$Trunc,
1869          [{ return Helper.matchNarrowBinop(*${Trunc}, *${Binop}, ${matchinfo}); }]),
1870   (apply [{ Helper.applyBuildFn(*${Trunc}, ${matchinfo}); }])>;
1871
1872 def narrow_binop_add : narrow_binop_opcode<G_ADD>;
1873 def narrow_binop_sub : narrow_binop_opcode<G_SUB>;
1874 def narrow_binop_mul : narrow_binop_opcode<G_MUL>;
1875 def narrow_binop_and : narrow_binop_opcode<G_AND>;
1876 def narrow_binop_or  : narrow_binop_opcode<G_OR>;
1877 def narrow_binop_xor : narrow_binop_opcode<G_XOR>;
1878
1879 // Cast of integer.
1880 class integer_of_opcode<Instruction castOpcode> : GICombineRule <
1881   (defs root:$root, apint_matchinfo:$matchinfo),
1882   (match (G_CONSTANT $int, $imm),
1883          (castOpcode $root, $int):$Cast,
1884          [{ return Helper.matchCastOfInteger(*${Cast}, ${matchinfo}); }]),
1885   (apply [{ Helper.replaceInstWithConstant(*${Cast}, ${matchinfo}); }])>;
1886
1887 def integer_of_truncate : integer_of_opcode<G_TRUNC>;
1888
1889 def cast_of_cast_combines: GICombineGroup<[
1890   truncate_of_zext,
1891   truncate_of_sext,
1892   truncate_of_anyext,
1893   zext_of_zext,
1894   zext_of_anyext,
1895   sext_of_sext,
1896   sext_of_anyext,
1897   anyext_of_anyext,
1898   anyext_of_zext,
1899   anyext_of_sext
1900 ]>;
1901
1902 def cast_combines: GICombineGroup<[
1903   cast_of_cast_combines,
1904   select_of_zext,
1905   select_of_anyext,
1906   select_of_truncate,
1907   buildvector_of_truncate,
1908   narrow_binop_add,
1909   narrow_binop_sub,
1910   narrow_binop_mul,
1911   narrow_binop_and,
1912   narrow_binop_or,
1913   narrow_binop_xor,
1914   integer_of_truncate
1915 ]>;
1916
1917 def canonicalize_icmp : GICombineRule<
1918   (defs root:$root, build_fn_matchinfo:$matchinfo),
1919   (match (G_ICMP $root, $pred, $lhs, $rhs):$cmp,
1920          [{ return Helper.matchCanonicalizeICmp(*${cmp}, ${matchinfo}); }]),
1921   (apply [{ Helper.applyBuildFn(*${cmp}, ${matchinfo}); }])>;
1922
1923 def canonicalize_fcmp : GICombineRule<
1924   (defs root:$root, build_fn_matchinfo:$matchinfo),
1925   (match (G_FCMP $root, $pred, $lhs, $rhs):$cmp,
1926          [{ return Helper.matchCanonicalizeFCmp(*${cmp}, ${matchinfo}); }]),
1927   (apply [{ Helper.applyBuildFn(*${cmp}, ${matchinfo}); }])>;
1928
1929 def cmp_combines: GICombineGroup<[
1930   canonicalize_icmp,
1931   canonicalize_fcmp,
1932   icmp_to_true_false_known_bits,
1933   icmp_to_lhs_known_bits,
1934   double_icmp_zero_and_combine,
1935   double_icmp_zero_or_combine,
1936   redundant_binop_in_equality
1937 ]>;
1938
1939
1940 def overflow_combines: GICombineGroup<[
1941   match_addos,
1942   match_subo_no_overflow
1943 ]>;
1944
1945 // FIXME: These should use the custom predicate feature once it lands.
1946 def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero,
1947                                      undef_to_negative_one,
1948                                      binop_left_undef_to_zero,
1949                                      binop_right_undef_to_undef,
1950                                      unary_undef_to_zero,
1951                                      unary_undef_to_undef,
1952                                      propagate_undef_any_op,
1953                                      propagate_undef_all_ops,
1954                                      propagate_undef_shuffle_mask,
1955                                      erase_undef_store,
1956                                      insert_extract_vec_elt_out_of_bounds]>;
1957
1958 def identity_combines : GICombineGroup<[select_same_val, right_identity_zero,
1959                                         binop_same_val, binop_left_to_zero,
1960                                         binop_right_to_zero, p2i_to_i2p,
1961                                         i2p_to_p2i, anyext_trunc_fold,
1962                                         fneg_fneg_fold, right_identity_one,
1963                                         add_sub_reg, buildvector_identity_fold,
1964                                         trunc_buildvector_fold,
1965                                         trunc_lshr_buildvector_fold,
1966                                         bitcast_bitcast_fold, fptrunc_fpext_fold,
1967                                         right_identity_neg_zero_fp,
1968                                         right_identity_neg_one_fp]>;
1969
1970 def const_combines : GICombineGroup<[constant_fold_fp_ops, const_ptradd_to_i2p,
1971                                      overlapping_and, mulo_by_2, mulo_by_0,
1972                                      adde_to_addo,
1973                                      combine_minmax_nan, expand_const_fpowi]>;
1974
1975 def known_bits_simplifications : GICombineGroup<[
1976   redundant_and, redundant_sext_inreg, redundant_or, urem_pow2_to_mask,
1977   zext_trunc_fold,
1978   sext_inreg_to_zext_inreg]>;
1979
1980 def width_reduction_combines : GICombineGroup<[reduce_shl_of_extend,
1981                                                narrow_binop_feeding_and]>;
1982
1983 def phi_combines : GICombineGroup<[extend_through_phis]>;
1984
1985 def bitreverse_shift : GICombineGroup<[bitreverse_shl, bitreverse_lshr]>;
1986
1987 def select_combines : GICombineGroup<[select_undef_cmp, select_constant_cmp,
1988                                       select_to_iminmax, match_selects]>;
1989
1990 def trivial_combines : GICombineGroup<[copy_prop, mul_to_shl, sub_to_add,
1991                                        add_p2i_to_ptradd, mul_by_neg_one,
1992                                        idempotent_prop]>;
1993
1994 def fma_combines : GICombineGroup<[combine_fadd_fmul_to_fmad_or_fma,
1995   combine_fadd_fpext_fmul_to_fmad_or_fma, combine_fadd_fma_fmul_to_fmad_or_fma,
1996   combine_fadd_fpext_fma_fmul_to_fmad_or_fma, combine_fsub_fmul_to_fmad_or_fma,
1997   combine_fsub_fneg_fmul_to_fmad_or_fma, combine_fsub_fpext_fmul_to_fmad_or_fma,
1998   combine_fsub_fpext_fneg_fmul_to_fmad_or_fma]>;
1999
2000 def constant_fold_binops : GICombineGroup<[constant_fold_binop,
2001                                            constant_fold_fp_binop]>;
2002
2003 def prefer_sign_combines : GICombineGroup<[nneg_zext]>;
2004
2005 def shuffle_combines : GICombineGroup<[combine_shuffle_concat,
2006                                        combine_shuffle_undef_rhs,
2007                                        combine_shuffle_disjoint_mask]>;
2008
2009 def all_combines : GICombineGroup<[integer_reassoc_combines, trivial_combines,
2010     vector_ops_combines, freeze_combines, cast_combines,
2011     insert_vec_elt_combines, extract_vec_elt_combines, combines_for_extload,
2012     combine_extracted_vector_load,
2013     undef_combines, identity_combines, phi_combines,
2014     simplify_add_to_sub, hoist_logic_op_with_same_opcode_hands, shifts_too_big,
2015     reassocs, ptr_add_immed_chain, cmp_combines,
2016     shl_ashr_to_sext_inreg, sext_inreg_of_load,
2017     width_reduction_combines, select_combines,
2018     known_bits_simplifications, trunc_shift,
2019     not_cmp_fold, opt_brcond_by_inverting_cond,
2020     const_combines, xor_of_and_with_same_reg, ptr_add_with_zero,
2021     shift_immed_chain, shift_of_shifted_logic_chain, load_or_combine,
2022     div_rem_to_divrem, funnel_shift_combines, bitreverse_shift, commute_shift,
2023     form_bitfield_extract, constant_fold_binops, constant_fold_fma,
2024     constant_fold_cast_op, fabs_fneg_fold,
2025     intdiv_combines, mulh_combines, redundant_neg_operands,
2026     and_or_disjoint_mask, fma_combines, fold_binop_into_select,
2027     sub_add_reg, select_to_minmax,
2028     fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors,
2029     simplify_neg_minmax, combine_concat_vector,
2030     sext_trunc, zext_trunc, prefer_sign_combines, shuffle_combines,
2031     combine_use_vector_truncate, merge_combines, overflow_combines]>;
2032
2033 // A combine group used to for prelegalizer combiners at -O0. The combines in
2034 // this group have been selected based on experiments to balance code size and
2035 // compile time performance.
2036 def optnone_combines : GICombineGroup<[trivial_combines,
2037     ptr_add_immed_chain, combines_for_extload,
2038     not_cmp_fold, opt_brcond_by_inverting_cond, combine_concat_vector]>;