22#ifndef LLVM_CODEGEN_TARGETLOWERING_H
23#define LLVM_CODEGEN_TARGETLOWERING_H
70class FunctionLoweringInfo;
78class MachineBasicBlock;
81class MachineJumpTableInfo;
83class MachineRegisterInfo;
87class ProfileSummaryInfo;
88class TargetLibraryInfo;
90class TargetRegisterClass;
91class TargetRegisterInfo;
92class TargetTransformInfo;
115 bool DstAlignCanChange;
130 Align SrcAlign,
bool IsVolatile,
131 bool MemcpyStrSrc =
false) {
134 Op.DstAlignCanChange = DstAlignCanChange;
135 Op.DstAlign = DstAlign;
136 Op.AllowOverlap = !IsVolatile;
138 Op.ZeroMemset =
false;
139 Op.MemcpyStrSrc = MemcpyStrSrc;
140 Op.SrcAlign = SrcAlign;
145 bool IsZeroMemset,
bool IsVolatile) {
148 Op.DstAlignCanChange = DstAlignCanChange;
149 Op.DstAlign = DstAlign;
150 Op.AllowOverlap = !IsVolatile;
152 Op.ZeroMemset = IsZeroMemset;
153 Op.MemcpyStrSrc =
false;
159 assert(!DstAlignCanChange);
167 return isMemcpy() && !DstAlignCanChange;
407 bool LegalTypes =
true)
const;
471 return HasMultipleConditionRegisters;
507 unsigned DefinedValues)
const {
508 return DefinedValues < 3;
565 return BypassSlowDivWidths;
609 unsigned AddrSpace)
const {
751 unsigned KeptBits)
const {
765 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
773 if (OldShiftOpcode ==
ISD::SHL &&
CC->isOne())
777 if (XC && NewShiftOpcode ==
ISD::SHL && XC->isOne())
816 unsigned &Cost)
const {
861 return BooleanVectorContents;
862 return isFloat ? BooleanFloatContents : BooleanContents;
885 return SchedPreferenceInfo;
900 assert(RC &&
"This value type is not natively supported!");
908 const Value *)
const {
927 return RepRegClassCostForVT[VT.
SimpleTy];
939 unsigned ExpansionFactor)
const {
940 if (ExpansionFactor == 1)
961 std::fill(std::begin(ValueTypeActions), std::end(ValueTypeActions),
966 return ValueTypeActions[VT.
SimpleTy];
970 ValueTypeActions[VT.
SimpleTy] = Action;
975 return ValueTypeActions;
1043 EVT &IntermediateVT,
1044 unsigned &NumIntermediates,
1045 MVT &RegisterVT)
const;
1052 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
1091 bool ForCodeSize =
false)
const {
1129 if (Op >= std::size(OpActions[0]))
1138 unsigned Scale)
const {
1147 unsigned Scale)
const {
1149 if (Action !=
Legal)
1170 return Supported ? Action :
Expand;
1179#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1180 case ISD::STRICT_##DAGN: EqOpc = ISD::DAGN; break;
1181#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1182 case ISD::STRICT_##DAGN: EqOpc = ISD::SETCC; break;
1183#include "llvm/IR/ConstrainedOps.def"
1194 bool LegalOnly =
false)
const {
1208 bool LegalOnly =
false)
const {
1222 bool LegalOnly =
false)
const {
1264 EVT ConditionVT)
const;
1289 return (NumDests == 1 && NumCmps >= 3) || (NumDests == 2 && NumCmps >= 5) ||
1290 (NumDests == 3 && NumCmps >= 6);
1316 unsigned Shift = 4 * ExtType;
1317 return (
LegalizeAction)((LoadExtActions[ValI][MemI] >> Shift) & 0xf);
1340 "Table isn't big enough!");
1341 return TruncStoreActions[ValI][MemI];
1359 bool LegalOnly)
const {
1370 return getIndexedModeAction(IdxMode, VT, IMAB_Load);
1384 return getIndexedModeAction(IdxMode, VT, IMAB_Store);
1398 return getIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad);
1412 return getIndexedModeAction(IdxMode, VT, IMAB_MaskedStore);
1439 if (Scale != ElemSize && Scale != 1)
1449 assert((
unsigned)
CC < std::size(CondCodeActions) &&
1450 ((
unsigned)VT.
SimpleTy >> 3) < std::size(CondCodeActions[0]) &&
1451 "Table isn't big enough!");
1456 assert(Action !=
Promote &&
"Can't promote condition code!");
1476 "This operation isn't promoted!");
1479 std::map<std::pair<unsigned, MVT::SimpleValueType>,
1481 PromoteToType.find(std::make_pair(Op, VT.
SimpleTy));
1482 if (PTTI != PromoteToType.end())
return PTTI->second;
1485 "Cannot autopromote this type, add it with AddPromotedToType.");
1491 "Didn't find type to promote to!");
1498 bool AllowUnknown =
false)
const {
1507 bool AllowUnknown =
false)
const {
1509 if (
auto *PTy = dyn_cast<PointerType>(Ty))
1512 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
1513 Type *EltTy = VTy->getElementType();
1515 if (
auto *PTy = dyn_cast<PointerType>(EltTy)) {
1520 VTy->getElementCount());
1527 bool AllowUnknown =
false)
const {
1529 if (
auto *PTy = dyn_cast<PointerType>(Ty))
1532 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
1533 Type *EltTy = VTy->getElementType();
1534 if (
auto *PTy = dyn_cast<PointerType>(EltTy)) {
1539 VTy->getElementCount());
1548 bool AllowUnknown =
false)
const {
1560 return RegisterTypeForVT[VT.
SimpleTy];
1570 unsigned NumIntermediates;
1572 NumIntermediates, RegisterVT);
1594 std::optional<MVT> RegisterVT = std::nullopt)
const {
1597 std::size(NumRegistersForVT));
1603 unsigned NumIntermediates;
1609 return (
BitWidth + RegWidth - 1) / RegWidth;
1635 return DL.getABITypeAlign(ArgTy);
1649 if (NewVT.
isVector() && !Load->hasOneUse())
1669 assert(
unsigned(NT >> 3) < std::size(TargetDAGCombineArray));
1670 return TargetDAGCombineArray[NT >> 3] & (1 << (NT&7));
1744 unsigned * =
nullptr)
const {
1752 unsigned * =
nullptr)
const {
1762 unsigned AddrSpace = 0,
Align Alignment =
Align(1),
1764 unsigned *
Fast =
nullptr)
const;
1773 unsigned *
Fast =
nullptr)
const;
1781 unsigned AddrSpace = 0,
Align Alignment =
Align(1),
1783 unsigned *
Fast =
nullptr)
const;
1791 unsigned *
Fast =
nullptr)
const;
1796 unsigned *
Fast =
nullptr)
const;
1839 return StackPointerRegisterToSaveRestore;
1862 return MinStackArgumentAlignment;
1966 return MaxAtomicSizeInBitsSupported;
1972 return MaxDivRemBitWidthSupported;
1978 return MaxLargeFPConvertBitWidthSupported;
2032 llvm_unreachable(
"Masked atomicrmw expansion unimplemented on this target");
2040 "Generic atomicrmw expansion unimplemented on this target");
2048 "Bit test atomicrmw expansion unimplemented on this target");
2056 "Compare arith atomicrmw expansion unimplemented on this target");
2154 if (
SI->getValueOperand()->getType()->isFloatingPointTy())
2278 bool IsSigned)
const {
2290 return !OptForSize ||
2302 BooleanContents = Ty;
2303 BooleanFloatContents = Ty;
2309 BooleanContents = IntTy;
2310 BooleanFloatContents = FloatTy;
2316 BooleanVectorContents = Ty;
2321 SchedPreferenceInfo = Pref;
2334 StackPointerRegisterToSaveRestore = R;
2343 HasMultipleConditionRegisters = hasManyRegs;
2351 HasExtractBitsInsn = hasExtractInsn;
2361 BypassSlowDivWidths[SlowBitWidth] = FastBitWidth;
2374 virtual std::pair<const TargetRegisterClass *, uint8_t>
2385 assert(Op < std::size(OpActions[0]) &&
"Table isn't big enough!");
2404 MemVT.
isValid() &&
"Table isn't big enough!");
2405 assert((
unsigned)Action < 0x10 &&
"too many bits for bitfield array");
2406 unsigned Shift = 4 * ExtType;
2412 for (
auto ExtType : ExtTypes)
2417 for (
auto MemVT : MemVTs)
2435 for (
auto IdxMode : IdxModes)
2436 setIndexedModeAction(IdxMode, VT, IMAB_Load, Action);
2452 for (
auto IdxMode : IdxModes)
2453 setIndexedModeAction(IdxMode, VT, IMAB_Store, Action);
2469 setIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad, Action);
2479 setIndexedModeAction(IdxMode, VT, IMAB_MaskedStore, Action);
2486 for (
auto CC : CCs) {
2488 "Table isn't big enough!");
2489 assert((
unsigned)Action < 0x10 &&
"too many bits for bitfield array");
2523 for (
auto NT : NTs) {
2524 assert(
unsigned(NT >> 3) < std::size(TargetDAGCombineArray));
2525 TargetDAGCombineArray[NT >> 3] |= 1 << (NT & 7);
2531 MinFunctionAlignment = Alignment;
2537 PrefFunctionAlignment = Alignment;
2545 MaxBytesForAlignment = MaxBytes;
2550 MinStackArgumentAlignment = Alignment;
2558 MaxAtomicSizeInBitsSupported = SizeInBits;
2564 MaxDivRemBitWidthSupported = SizeInBits;
2570 MaxLargeFPConvertBitWidthSupported = SizeInBits;
2575 MinCmpXchgSizeInBits = SizeInBits;
2580 SupportsUnalignedAtomics = UnalignedSupported;
2625 Type *Ty,
unsigned AddrSpace,
2672 return (
From->isIntegerTy() ||
From->isFloatingPointTy()) &&
2714 default:
return false;
2782 switch (
I->getOpcode()) {
2783 case Instruction::FPExt:
2788 case Instruction::ZExt:
2789 if (
isZExtFree(
I->getOperand(0)->getType(),
I->getType()))
2792 case Instruction::SExt:
2819 if (isa<ZExtInst>(Ext))
2822 assert(isa<SExtInst>(Ext) &&
"Unexpected ext type!");
2927 unsigned Factor)
const {
2938 unsigned Factor)
const {
2947 "invalid fpext types");
2955 LLT DestTy,
LLT SrcTy)
const {
2963 EVT DestVT,
EVT SrcVT)
const {
2965 "invalid fpext types");
3023 assert((
MI.getOpcode() == TargetOpcode::G_FADD ||
3024 MI.getOpcode() == TargetOpcode::G_FSUB ||
3025 MI.getOpcode() == TargetOpcode::G_FMUL) &&
3026 "unexpected node in FMAD forming combine");
3047 "unexpected node in FMAD forming combine");
3088 unsigned Index)
const {
3111 bool MathUsed)
const {
3180 LibcallRoutineNames[Call] =
Name;
3183 for (
auto Call : Calls)
3189 return LibcallRoutineNames[Call];
3195 CmpLibcallCCs[Call] =
CC;
3201 return CmpLibcallCCs[Call];
3206 LibcallCallingConvs[Call] =
CC;
3211 return LibcallCallingConvs[Call];
3235 bool HasMultipleConditionRegisters;
3241 bool HasExtractBitsInsn;
3247 DenseMap <unsigned int, unsigned int> BypassSlowDivWidths;
3252 bool JumpIsExpensive;
3271 Align MinStackArgumentAlignment;
3275 Align MinFunctionAlignment;
3279 Align PrefFunctionAlignment;
3282 Align PrefLoopAlignment;
3284 unsigned MaxBytesForAlignment;
3288 unsigned MaxAtomicSizeInBitsSupported;
3292 unsigned MaxDivRemBitWidthSupported;
3296 unsigned MaxLargeFPConvertBitWidthSupported;
3300 unsigned MinCmpXchgSizeInBits;
3303 bool SupportsUnalignedAtomics;
3307 Register StackPointerRegisterToSaveRestore;
3368 ValueTypeActionImpl ValueTypeActions;
3387 const char *LibcallRoutineNames[RTLIB::UNKNOWN_LIBCALL + 1];
3397 void InitLibcalls(
const Triple &TT);
3401 enum IndexedModeActionsBits {
3404 IMAB_MaskedStore = 8,
3405 IMAB_MaskedLoad = 12
3408 void setIndexedModeAction(
unsigned IdxMode, MVT VT,
unsigned Shift,
3411 (
unsigned)Action < 0xf &&
"Table isn't big enough!");
3412 unsigned Ty = (
unsigned)VT.SimpleTy;
3413 IndexedModeActions[Ty][IdxMode] &= ~(0xf << Shift);
3414 IndexedModeActions[Ty][IdxMode] |= ((
uint16_t)Action) << Shift;
3418 unsigned Shift)
const {
3420 "Table isn't big enough!");
3421 unsigned Ty = (
unsigned)VT.SimpleTy;
3422 return (
LegalizeAction)((IndexedModeActions[Ty][IdxMode] >> Shift) & 0xf);
3624 unsigned OpNo)
const {
3640 bool IsSignaling =
false)
const;
3646 MakeLibCallOptions CallOptions,
3654 const uint32_t *CallerPreservedMask,
3693 const MemOp &Op,
unsigned DstAS,
unsigned SrcAS,
3701 const APInt &DemandedElts,
3702 TargetLoweringOpt &TLO)
const;
3706 TargetLoweringOpt &TLO)
const;
3713 const APInt &DemandedElts,
3723 TargetLoweringOpt &TLO)
const;
3740 TargetLoweringOpt &TLO,
unsigned Depth = 0,
3741 bool AssumeSingleUse =
false)
const;
3746 KnownBits &Known, TargetLoweringOpt &TLO,
3748 bool AssumeSingleUse =
false)
const;
3753 DAGCombinerInfo &DCI)
const;
3758 const APInt &DemandedElts,
3759 DAGCombinerInfo &DCI)
const;
3765 const APInt &DemandedElts,
3767 unsigned Depth = 0)
const;
3773 unsigned Depth = 0)
const;
3778 const APInt &DemandedElts,
3780 unsigned Depth = 0)
const;
3798 TargetLoweringOpt &TLO,
unsigned Depth = 0,
3799 bool AssumeSingleUse =
false)
const;
3804 DAGCombinerInfo &DCI)
const;
3820 const APInt &DemandedElts,
3822 unsigned Depth = 0)
const;
3830 const APInt &DemandedElts,
3832 unsigned Depth = 0)
const;
3841 unsigned Depth = 0)
const;
3855 const APInt &DemandedElts,
3857 unsigned Depth = 0)
const;
3865 const APInt &DemandedElts,
3867 unsigned Depth = 0)
const;
3876 APInt &KnownZero, TargetLoweringOpt &TLO,
unsigned Depth = 0)
const;
3885 const APInt &DemandedElts,
3887 TargetLoweringOpt &TLO,
3888 unsigned Depth = 0)
const;
3902 bool PoisonOnly,
unsigned Depth)
const;
3910 bool ConsiderFlags,
unsigned Depth)
const;
3930 unsigned Depth = 0)
const;
3937 unsigned Depth = 0)
const;
3986 bool foldBooleans, DAGCombinerInfo &DCI,
3987 const SDLoc &dl)
const;
4116 bool LegalOps,
bool OptForSize,
4118 unsigned Depth = 0)
const;
4123 unsigned Depth = 0)
const {
4142 bool LegalOps,
bool OptForSize,
4143 unsigned Depth = 0)
const {
4151 bool OptForSize,
unsigned Depth = 0)
const {
4165 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
4182 unsigned &PhysReg,
int &Cost)
const {
4189 const SDValue *Parts,
unsigned NumParts,
4191 std::optional<CallingConv::ID>
CC)
const {
4267 Args = std::move(ArgsList);
4280 Args = std::move(ArgsList);
4289 IsInReg = Call.hasRetAttr(Attribute::InReg);
4291 Call.doesNotReturn() ||
4292 (!isa<InvokeInst>(Call) && isa<UnreachableInst>(Call.getNextNode()));
4295 RetSExt = Call.hasRetAttr(Attribute::SExt);
4296 RetZExt = Call.hasRetAttr(Attribute::ZExt);
4297 NoMerge = Call.hasFnAttr(Attribute::NoMerge);
4303 Args = std::move(ArgsList);