22 #ifndef LLVM_CODEGEN_TARGETLOWERING_H
23 #define LLVM_CODEGEN_TARGETLOWERING_H
70 class FunctionLoweringInfo;
76 class LegacyDivergenceAnalysis;
78 class MachineBasicBlock;
79 class MachineFunction;
81 class MachineJumpTableInfo;
83 class MachineRegisterInfo;
87 class ProfileSummaryInfo;
88 class TargetLibraryInfo;
90 class TargetRegisterClass;
91 class TargetRegisterInfo;
92 class TargetTransformInfo;
115 bool DstAlignCanChange;
131 bool MemcpyStrSrc =
false) {
134 Op.DstAlignCanChange = DstAlignCanChange;
135 Op.DstAlign = DstAlign;
138 Op.ZeroMemset =
false;
139 Op.MemcpyStrSrc = MemcpyStrSrc;
140 Op.SrcAlign = SrcAlign;
148 Op.DstAlignCanChange = DstAlignCanChange;
149 Op.DstAlign = DstAlign;
152 Op.ZeroMemset = IsZeroMemset;
153 Op.MemcpyStrSrc =
false;
159 assert(!DstAlignCanChange);
167 return isMemcpy() && !DstAlignCanChange;
394 bool LegalTypes =
true)
const;
450 return HasMultipleConditionRegisters;
486 unsigned DefinedValues)
const {
487 return DefinedValues < 3;
544 return BypassSlowDivWidths;
602 unsigned AddrSpace)
const {
752 unsigned KeptBits)
const {
766 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
808 unsigned &Cost)
const {
853 return BooleanVectorContents;
854 return isFloat ? BooleanFloatContents : BooleanContents;
872 return DAG.
getNode(ExtendCode, dl, BoolVT, Bool);
877 return SchedPreferenceInfo;
892 assert(RC &&
"This value type is not natively supported!");
900 const Value *)
const {
919 return RepRegClassCostForVT[VT.
SimpleTy];
950 return ValueTypeActions[VT.
SimpleTy];
954 ValueTypeActions[VT.
SimpleTy] = Action;
959 return ValueTypeActions;
967 return getTypeConversion(
Context, VT).first;
980 return getTypeConversion(
Context, VT).second;
1011 EVT &IntermediateVT,
1012 unsigned &NumIntermediates,
1013 MVT &RegisterVT)
const;
1020 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
1055 bool ForCodeSize =
false)
const {
1101 unsigned Scale)
const {
1110 unsigned Scale)
const {
1112 if (Action !=
Legal)
1133 return Supported ? Action :
Expand;
1142 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1143 case ISD::STRICT_##DAGN: EqOpc = ISD::DAGN; break;
1144 #define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1145 case ISD::STRICT_##DAGN: EqOpc = ISD::SETCC; break;
1146 #include "llvm/IR/ConstrainedOps.def"
1157 bool LegalOnly =
false)
const {
1171 bool LegalOnly =
false)
const {
1185 bool LegalOnly =
false)
const {
1227 EVT ConditionVT)
const;
1252 return (NumDests == 1 && NumCmps >= 3) || (NumDests == 2 && NumCmps >= 5) ||
1253 (NumDests == 3 && NumCmps >= 6);
1279 unsigned Shift = 4 * ExtType;
1303 "Table isn't big enough!");
1304 return TruncStoreActions[ValI][MemI];
1322 bool LegalOnly)
const {
1333 return getIndexedModeAction(IdxMode, VT, IMAB_Load);
1347 return getIndexedModeAction(IdxMode, VT, IMAB_Store);
1361 return getIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad);
1375 return getIndexedModeAction(IdxMode, VT, IMAB_MaskedStore);
1400 "Table isn't big enough!");
1405 assert(Action !=
Promote &&
"Can't promote condition code!");
1425 "This operation isn't promoted!");
1428 std::map<std::pair<unsigned, MVT::SimpleValueType>,
1430 PromoteToType.find(std::make_pair(
Op, VT.
SimpleTy));
1431 if (PTTI != PromoteToType.end())
return PTTI->second;
1434 "Cannot autopromote this type, add it with AddPromotedToType.");
1440 "Didn't find type to promote to!");
1447 bool AllowUnknown =
false)
const {
1456 bool AllowUnknown =
false)
const {
1458 if (
auto *PTy = dyn_cast<PointerType>(Ty))
1461 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
1462 Type *EltTy = VTy->getElementType();
1464 if (
auto *PTy = dyn_cast<PointerType>(EltTy)) {
1469 VTy->getElementCount());
1476 bool AllowUnknown =
false)
const {
1480 else if (
VectorType *VTy = dyn_cast<VectorType>(Ty)) {
1481 Type *Elm = VTy->getElementType();
1482 if (
PointerType *PT = dyn_cast<PointerType>(Elm)) {
1487 VTy->getElementCount());
1496 bool AllowUnknown =
false)
const {
1508 return RegisterTypeForVT[VT.
SimpleTy];
1521 unsigned NumIntermediates;
1523 NumIntermediates, RegisterVT);
1554 unsigned NumIntermediates;
1560 return (
BitWidth + RegWidth - 1) / RegWidth;
1586 return DL.getABITypeAlign(ArgTy);
1617 return TargetDAGCombineArray[
NT >> 3] & (1 << (
NT&7));
1690 bool * =
nullptr)
const {
1698 bool * =
nullptr)
const {
1708 unsigned AddrSpace = 0,
Align Alignment =
Align(1),
1710 bool *Fast =
nullptr)
const;
1719 bool *Fast =
nullptr)
const;
1727 unsigned AddrSpace = 0,
Align Alignment =
Align(1),
1729 bool *Fast =
nullptr)
const;
1737 bool *Fast =
nullptr)
const;
1742 bool *Fast =
nullptr)
const;
1785 return StackPointerRegisterToSaveRestore;
1808 return MinStackArgumentAlignment;
1890 unsigned & )
const {
1916 return MaxAtomicSizeInBitsSupported;
1962 llvm_unreachable(
"Masked atomicrmw expansion unimplemented on this target");
1970 "Bit test atomicrmw expansion unimplemented on this target");
2162 bool IsSigned)
const {
2174 BooleanContents = Ty;
2175 BooleanFloatContents = Ty;
2181 BooleanContents = IntTy;
2182 BooleanFloatContents = FloatTy;
2188 BooleanVectorContents = Ty;
2193 SchedPreferenceInfo = Pref;
2206 StackPointerRegisterToSaveRestore = R;
2215 HasMultipleConditionRegisters = hasManyRegs;
2223 HasExtractBitsInsn = hasExtractInsn;
2233 BypassSlowDivWidths[SlowBitWidth] = FastBitWidth;
2246 virtual std::pair<const TargetRegisterClass *, uint8_t>
2258 for (
auto Op : Ops) {
2260 OpActions[(unsigned)VT.
SimpleTy][
Op] = Action;
2273 for (
auto ExtType : ExtTypes) {
2275 MemVT.
isValid() &&
"Table isn't big enough!");
2276 assert((
unsigned)Action < 0x10 &&
"too many bits for bitfield array");
2277 unsigned Shift = 4 * ExtType;
2287 for (
auto MemVT : MemVTs)
2305 for (
auto IdxMode : IdxModes)
2306 setIndexedModeAction(IdxMode, VT, IMAB_Load, Action);
2322 for (
auto IdxMode : IdxModes)
2323 setIndexedModeAction(IdxMode, VT, IMAB_Store, Action);
2339 setIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad, Action);
2349 setIndexedModeAction(IdxMode, VT, IMAB_MaskedStore, Action);
2356 for (
auto CC : CCs) {
2358 "Table isn't big enough!");
2359 assert((
unsigned)Action < 0x10 &&
"too many bits for bitfield array");
2393 for (
auto NT : NTs) {
2395 TargetDAGCombineArray[
NT >> 3] |= 1 << (
NT & 7);
2401 MinFunctionAlignment = Alignment;
2407 PrefFunctionAlignment = Alignment;
2415 MaxBytesForAlignment = MaxBytes;
2420 MinStackArgumentAlignment = Alignment;
2428 MaxAtomicSizeInBitsSupported = SizeInBits;
2433 MinCmpXchgSizeInBits = SizeInBits;
2438 SupportsUnalignedAtomics = UnalignedSupported;
2483 Type *Ty,
unsigned AddrSpace,
2495 unsigned AS = 0)
const {
2546 return (
From->isIntegerTy() ||
From->isFloatingPointTy()) &&
2586 default:
return false;
2654 switch (
I->getOpcode()) {
2655 case Instruction::FPExt:
2660 case Instruction::ZExt:
2661 if (
isZExtFree(
I->getOperand(0)->getType(),
I->getType()))
2664 case Instruction::SExt:
2691 if (isa<ZExtInst>(
Ext))
2694 assert(isa<SExtInst>(
Ext) &&
"Unexpected ext type!");
2785 unsigned Factor)
const {
2796 unsigned Factor)
const {
2812 "invalid fpext types");
2820 LLT DestTy,
LLT SrcTy)
const {
2828 EVT DestVT,
EVT SrcVT)
const {
2830 "invalid fpext types");
2888 assert((
MI.getOpcode() == TargetOpcode::G_FADD ||
2889 MI.getOpcode() == TargetOpcode::G_FSUB ||
2890 MI.getOpcode() == TargetOpcode::G_FMUL) &&
2891 "unexpected node in FMAD forming combine");
2912 "unexpected node in FMAD forming combine");
2953 unsigned Index)
const {
2976 bool MathUsed)
const {
3025 for (
auto Call : Calls)
3026 LibcallRoutineNames[Call] =
Name;
3031 return LibcallRoutineNames[Call];
3037 CmpLibcallCCs[Call] = CC;
3043 return CmpLibcallCCs[Call];
3048 LibcallCallingConvs[Call] = CC;
3053 return LibcallCallingConvs[Call];
3077 bool HasMultipleConditionRegisters;
3083 bool HasExtractBitsInsn;
3094 bool JumpIsExpensive;
3113 Align MinStackArgumentAlignment;
3117 Align MinFunctionAlignment;
3121 Align PrefFunctionAlignment;
3124 Align PrefLoopAlignment;
3126 unsigned MaxBytesForAlignment;
3130 unsigned MaxAtomicSizeInBitsSupported;
3134 unsigned MinCmpXchgSizeInBits;
3137 bool SupportsUnalignedAtomics;
3141 Register StackPointerRegisterToSaveRestore;
3202 ValueTypeActionImpl ValueTypeActions;
3223 const char *LibcallRoutineNames[RTLIB::UNKNOWN_LIBCALL + 1];
3233 void InitLibcalls(
const Triple &TT);
3237 enum IndexedModeActionsBits {
3240 IMAB_MaskedStore = 8,
3241 IMAB_MaskedLoad = 12
3244 void setIndexedModeAction(
unsigned IdxMode, MVT VT,
unsigned Shift,
3247 (
unsigned)Action < 0xf &&
"Table isn't big enough!");
3248 unsigned Ty = (unsigned)VT.SimpleTy;
3249 IndexedModeActions[Ty][IdxMode] &= ~(0xf <<
Shift);
3250 IndexedModeActions[Ty][IdxMode] |= ((
uint16_t)Action) <<
Shift;
3254 unsigned Shift)
const {
3256 "Table isn't big enough!");
3257 unsigned Ty = (unsigned)VT.SimpleTy;
3461 bool IsSignaling =
false)
const;
3467 MakeLibCallOptions CallOptions,
3475 const uint32_t *CallerPreservedMask,
3513 const MemOp &
Op,
unsigned DstAS,
unsigned SrcAS,
3521 const APInt &DemandedElts,
3522 TargetLoweringOpt &TLO)
const;
3526 TargetLoweringOpt &TLO)
const;
3533 const APInt &DemandedElts,
3542 TargetLoweringOpt &TLO)
const;
3559 TargetLoweringOpt &TLO,
unsigned Depth = 0,
3560 bool AssumeSingleUse =
false)
const;
3565 KnownBits &Known, TargetLoweringOpt &TLO,
3567 bool AssumeSingleUse =
false)
const;
3572 DAGCombinerInfo &DCI)
const;
3577 const APInt &DemandedElts,
3578 DAGCombinerInfo &DCI)
const;
3584 const APInt &DemandedElts,
3586 unsigned Depth = 0)
const;
3592 unsigned Depth = 0)
const;
3597 const APInt &DemandedElts,
3599 unsigned Depth = 0)
const;
3617 TargetLoweringOpt &TLO,
unsigned Depth = 0,
3618 bool AssumeSingleUse =
false)
const;
3623 DAGCombinerInfo &DCI)
const;
3639 const APInt &DemandedElts,
3641 unsigned Depth = 0)
const;
3649 const APInt &DemandedElts,
3651 unsigned Depth = 0)
const;
3660 unsigned Depth = 0)
const;
3674 const APInt &DemandedElts,
3676 unsigned Depth = 0)
const;
3684 const APInt &DemandedElts,
3686 unsigned Depth = 0)
const;
3695 APInt &KnownZero, TargetLoweringOpt &TLO,
unsigned Depth = 0)
const;
3704 const APInt &DemandedElts,
3706 TargetLoweringOpt &TLO,
3707 unsigned Depth = 0)
const;
3721 bool PoisonOnly,
unsigned Depth)
const;
3741 unsigned Depth = 0)
const;
3747 unsigned Depth = 0)
const;
3790 bool foldBooleans, DAGCombinerInfo &DCI,
3791 const SDLoc &dl)
const;
3886 bool LegalOps,
bool OptForSize,
3888 unsigned Depth = 0)
const;
3893 bool LegalOps,
bool OptForSize,
3894 unsigned Depth = 0)
const {
3909 bool OptForSize,
unsigned Depth = 0)
const {
3923 unsigned NumParts,
MVT PartVT,
3931 const SDValue *Parts,
unsigned NumParts,
4030 IsInReg = Call.hasRetAttr(Attribute::InReg);
4032 Call.doesNotReturn() ||
4033 (!isa<InvokeInst>(Call) && isa<UnreachableInst>(Call.getNextNode()));
4036 RetSExt = Call.hasRetAttr(Attribute::SExt);
4037 RetZExt = Call.hasRetAttr(Attribute::ZExt);
4038 NoMerge = Call.hasFnAttr(Attribute::NoMerge);
4148 bool Value =
true) {
4160 std::pair<SDValue, SDValue>
LowerCallTo(CallLoweringInfo &CLI)
const;
4219 return "__clear_cache";
4239 return VT.
bitsLT(MinVT) ? MinVT : VT;
4258 return DL.isLittleEndian();
4284 assert(
SI.isAtomic() &&
"violated precondition");
4460 virtual std::pair<unsigned, const TargetRegisterClass *>
4465 if (ConstraintCode ==
"m")
4467 if (ConstraintCode ==
"o")
4469 if (ConstraintCode ==
"X")
4471 if (ConstraintCode ==
"p")
4484 std::vector<SDValue> &Ops,
4490 const AsmOperandInfo &OpInfo,
4544 int Enabled,
int &RefinementSteps,
4545 bool &UseOneConstNR,
bool Reciprocal)
const {