84#include "llvm/IR/IntrinsicsARM.h"
122#define DEBUG_TYPE "arm-isel"
125STATISTIC(NumMovwMovt,
"Number of GAs materialized with movw + movt");
126STATISTIC(NumLoopByVals,
"Number of loops generated for byval arguments");
128 "Number of constants with their storage promoted into constant pools");
132 cl::desc(
"Enable / disable ARM interworking (for debugging only)"),
137 cl::desc(
"Enable / disable promotion of unnamed_addr constants into "
142 cl::desc(
"Maximum size of constant to promote into a constant pool"),
146 cl::desc(
"Maximum size of ALL constants to promote into a constant pool"),
151 cl::desc(
"Maximum interleave factor for MVE VLDn to generate."),
156 ARM::R0, ARM::R1, ARM::R2, ARM::R3
170void ARMTargetLowering::addTypeForNEON(
MVT VT,
MVT PromotedLdStVT) {
171 if (VT != PromotedLdStVT) {
180 if (ElemTy != MVT::f64)
184 if (ElemTy == MVT::i32) {
228void ARMTargetLowering::addDRTypeForNEON(
MVT VT) {
230 addTypeForNEON(VT, MVT::f64);
233void ARMTargetLowering::addQRTypeForNEON(
MVT VT) {
235 addTypeForNEON(VT, MVT::v2f64);
238void ARMTargetLowering::setAllExpand(
MVT VT) {
251void ARMTargetLowering::addAllExtLoads(
const MVT From,
const MVT To,
252 LegalizeAction Action) {
258void ARMTargetLowering::addMVEVectorTypes(
bool HasMVEFP) {
259 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
261 for (
auto VT : IntTypes) {
335 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
336 for (
auto VT : FloatTypes) {
405 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
406 for (
auto VT : LongTypes) {
423 addAllExtLoads(MVT::v8i16, MVT::v8i8,
Legal);
424 addAllExtLoads(MVT::v4i32, MVT::v4i16,
Legal);
425 addAllExtLoads(MVT::v4i32, MVT::v4i8,
Legal);
442 for (
auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
451 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
452 for (
auto VT : pTypes) {
508 for (
int LCID = 0; LCID < RTLIB::UNKNOWN_LIBCALL; ++LCID)
516 if (Subtarget->isThumb() && Subtarget->
hasVFP2Base() &&
517 Subtarget->
hasARMOps() && !Subtarget->useSoftFloat()) {
518 static const struct {
520 const char *
const Name;
542 { RTLIB::UO_F32,
"__unordsf2vfp",
ISD::SETNE },
551 { RTLIB::UO_F64,
"__unorddf2vfp",
ISD::SETNE },
576 for (
const auto &LC : LibraryCalls) {
596 static const struct {
598 const char *
const Name;
683 for (
const auto &LC : LibraryCalls) {
693 static const struct {
695 const char *
const Name;
698 } MemOpsLibraryCalls[] = {
706 for (
const auto &LC : MemOpsLibraryCalls) {
716 static const struct {
718 const char *
const Name;
731 for (
const auto &LC : LibraryCalls) {
763 static const struct {
765 const char *
const Name;
773 for (
const auto &LC : LibraryCalls) {
784 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only() &&
785 Subtarget->hasFPRegs()) {
795 setAllExpand(MVT::f32);
796 if (!Subtarget->hasFP64())
797 setAllExpand(MVT::f64);
800 if (Subtarget->hasFullFP16()) {
809 if (Subtarget->hasBF16()) {
811 setAllExpand(MVT::bf16);
812 if (!Subtarget->hasFullFP16())
819 addAllExtLoads(VT, InnerVT,
Expand);
834 if (Subtarget->hasMVEIntegerOps())
835 addMVEVectorTypes(Subtarget->hasMVEFloatOps());
838 if (Subtarget->hasLOB()) {
842 if (Subtarget->hasNEON()) {
843 addDRTypeForNEON(MVT::v2f32);
844 addDRTypeForNEON(MVT::v8i8);
845 addDRTypeForNEON(MVT::v4i16);
846 addDRTypeForNEON(MVT::v2i32);
847 addDRTypeForNEON(MVT::v1i64);
849 addQRTypeForNEON(MVT::v4f32);
850 addQRTypeForNEON(MVT::v2f64);
851 addQRTypeForNEON(MVT::v16i8);
852 addQRTypeForNEON(MVT::v8i16);
853 addQRTypeForNEON(MVT::v4i32);
854 addQRTypeForNEON(MVT::v2i64);
856 if (Subtarget->hasFullFP16()) {
857 addQRTypeForNEON(MVT::v8f16);
858 addDRTypeForNEON(MVT::v4f16);
861 if (Subtarget->hasBF16()) {
862 addQRTypeForNEON(MVT::v8bf16);
863 addDRTypeForNEON(MVT::v4bf16);
867 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
907 if (Subtarget->hasNEON()) {
1021 for (
MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
1030 for (
auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
1039 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
1047 if (Subtarget->hasMVEIntegerOps()) {
1052 if (Subtarget->hasMVEFloatOps()) {
1056 if (!Subtarget->hasFP64()) {
1103 if (Subtarget->hasFullFP16()) {
1109 if (!Subtarget->hasFP16()) {
1157 if (Subtarget->hasDSP()) {
1179 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6Ops()
1180 || (Subtarget->
isThumb2() && !Subtarget->hasDSP()))
1195 if (Subtarget->hasMVEIntegerOps())
1205 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops())
1216 if (!Subtarget->hasV5TOps() || Subtarget->
isThumb1Only()) {
1225 if (Subtarget->hasPerfMon())
1229 if (!Subtarget->hasV6Ops())
1232 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1233 : Subtarget->hasDivideInARMMode();
1240 if (Subtarget->
isTargetWindows() && !Subtarget->hasDivideInThumbMode()) {
1257 HasStandaloneRem =
false;
1262 const char *
const Name;
1264 } LibraryCalls[] = {
1276 for (
const auto &LC : LibraryCalls) {
1283 const char *
const Name;
1285 } LibraryCalls[] = {
1297 for (
const auto &LC : LibraryCalls) {
1341 InsertFencesForAtomic =
false;
1343 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
1347 if (!Subtarget->isThumb() || !Subtarget->
isMClass())
1352 if (!Subtarget->hasAcquireRelease() ||
1355 InsertFencesForAtomic =
true;
1361 if (Subtarget->hasDataBarrier())
1362 InsertFencesForAtomic =
true;
1382 if (!InsertFencesForAtomic) {
1390 (!Subtarget->
isMClass() && Subtarget->hasV6Ops())) {
1402 }
else if ((Subtarget->
isMClass() && Subtarget->hasV8MBaselineOps()) ||
1403 Subtarget->hasForced32BitAtomics()) {
1417 if (!Subtarget->hasV6Ops()) {
1423 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1455 if (Subtarget->hasFullFP16()) {
1465 if (Subtarget->hasFullFP16())
1480 if (!Subtarget->useSoftFloat() && Subtarget->
hasVFP2Base() &&
1494 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only()) {
1502 if (!Subtarget->hasFP16()) {
1533 if (Subtarget->hasNEON()) {
1540 if (Subtarget->hasFP64()) {
1553 if (Subtarget->hasFullFP16()) {
1572 if (Subtarget->hasNEON()) {
1584 if (Subtarget->hasFullFP16()) {
1616 if (Subtarget->hasMVEIntegerOps())
1619 if (Subtarget->hasV6Ops())
1624 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
1631 if (Subtarget->useSoftFloat() || Subtarget->
isThumb1Only() ||
1659 return Subtarget->useSoftFloat();
1672std::pair<const TargetRegisterClass *, uint8_t>
1683 case MVT::f32:
case MVT::f64:
case MVT::v8i8:
case MVT::v4i16:
1684 case MVT::v2i32:
case MVT::v1i64:
case MVT::v2f32:
1685 RRC = &ARM::DPRRegClass;
1693 case MVT::v16i8:
case MVT::v8i16:
case MVT::v4i32:
case MVT::v2i64:
1694 case MVT::v4f32:
case MVT::v2f64:
1695 RRC = &ARM::DPRRegClass;
1699 RRC = &ARM::DPRRegClass;
1703 RRC = &ARM::DPRRegClass;
1707 return std::make_pair(RRC,
Cost);
1711#define MAKE_CASE(V) \
1930 if ((Subtarget->hasMVEIntegerOps() &&
1931 (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
1932 VT == MVT::v16i8)) ||
1933 (Subtarget->hasMVEFloatOps() &&
1934 (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16)))
1948 if (Subtarget->hasNEON()) {
1949 if (VT == MVT::v4i64)
1950 return &ARM::QQPRRegClass;
1951 if (VT == MVT::v8i64)
1952 return &ARM::QQQQPRRegClass;
1954 if (Subtarget->hasMVEIntegerOps()) {
1955 if (VT == MVT::v4i64)
1956 return &ARM::MQQPRRegClass;
1957 if (VT == MVT::v8i64)
1958 return &ARM::MQQQQPRRegClass;
1967 Align &PrefAlign)
const {
1968 if (!isa<MemIntrinsic>(CI))
1986 unsigned NumVals =
N->getNumValues();
1990 for (
unsigned i = 0; i != NumVals; ++i) {
1991 EVT VT =
N->getValueType(i);
1992 if (VT == MVT::Glue || VT == MVT::Other)
1998 if (!
N->isMachineOpcode())
2022 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2023 return Const->getZExtValue() == 16;
2030 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2031 return Const->getZExtValue() == 16;
2038 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2039 return Const->getZExtValue() == 16;
2108 bool isVarArg)
const {
2129 else if (Subtarget->hasFPRegs() && !Subtarget->
isThumb1Only() &&
2150 bool isVarArg)
const {
2151 return CCAssignFnForNode(
CC,
false, isVarArg);
2155 bool isVarArg)
const {
2156 return CCAssignFnForNode(
CC,
true, isVarArg);
2163 bool isVarArg)
const {
2164 switch (getEffectiveCallingConv(
CC, isVarArg)) {
2190 if (Subtarget->hasFullFP16()) {
2203 if (Subtarget->hasFullFP16()) {
2217SDValue ARMTargetLowering::LowerCallResult(
2221 SDValue ThisVal,
bool isCmseNSCall)
const {
2229 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
2234 if (i == 0 && isThisReturn) {
2236 "unexpected return calling convention register assignment");
2247 Chain =
Lo.getValue(1);
2248 InGlue =
Lo.getValue(2);
2252 Chain =
Hi.getValue(1);
2253 InGlue =
Hi.getValue(2);
2265 Chain =
Lo.getValue(1);
2266 InGlue =
Lo.getValue(2);
2269 Chain =
Hi.getValue(1);
2270 InGlue =
Hi.getValue(2);
2314std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
2316 bool IsTailCall,
int SPDiff)
const {
2338 return std::make_pair(DstAddr, DstInfo);
2343 RegsToPassVector &RegsToPass,
2350 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
2351 unsigned id = Subtarget->
isLittle() ? 0 : 1;
2364 std::tie(DstAddr, DstInfo) =
2365 computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff);
2397 bool isStructRet = (Outs.
empty()) ?
false : Outs[0].
Flags.isSRet();
2398 bool isThisReturn =
false;
2399 bool isCmseNSCall =
false;
2400 bool isSibCall =
false;
2401 bool PreferIndirect =
false;
2402 bool GuardWithBTI =
false;
2412 !Subtarget->noBTIAtReturnTwice())
2417 isCmseNSCall =
true;
2429 if (isa<GlobalAddressSDNode>(Callee)) {
2433 auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
2436 PreferIndirect = Subtarget->isThumb() && Subtarget->
hasMinSize() &&
2438 return isa<Instruction>(U) &&
2439 cast<Instruction>(U)->getParent() == BB;
2446 IsEligibleForTailCallOptimization(CLI, CCInfo, ArgLocs, PreferIndirect);
2460 "site marked musttail");
2463 unsigned NumBytes = CCInfo.getStackSize();
2472 if (isTailCall && !isSibCall) {
2479 NumBytes =
alignTo(NumBytes, StackAlign);
2484 SPDiff = NumReusableBytes - NumBytes;
2488 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (
unsigned)-SPDiff)
2504 RegsToPassVector RegsToPass;
2512 bool AfterFormalArgLoads =
false;
2516 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.
size();
2518 ++i, ++realArgIdx) {
2520 SDValue Arg = OutVals[realArgIdx];
2522 bool isByVal =
Flags.isByVal();
2542 if (isTailCall && VA.
isMemLoc() && !AfterFormalArgLoads) {
2544 AfterFormalArgLoads =
true;
2556 auto ArgVT = Outs[realArgIdx].ArgVT;
2557 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2575 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
2576 StackPtr, MemOpChains, isTailCall, SPDiff);
2580 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
2581 StackPtr, MemOpChains, isTailCall, SPDiff);
2586 std::tie(DstAddr, DstInfo) =
2587 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2591 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
2592 StackPtr, MemOpChains, isTailCall, SPDiff);
2594 if (realArgIdx == 0 &&
Flags.isReturned() && !
Flags.isSwiftSelf() &&
2595 Outs[0].VT == MVT::i32) {
2597 "unexpected calling convention register assignment");
2598 assert(!
Ins.empty() && Ins[0].VT == MVT::i32 &&
2599 "unexpected use of 'returned'");
2600 isThisReturn =
true;
2604 CSInfo.ArgRegPairs.emplace_back(VA.
getLocReg(), i);
2605 RegsToPass.push_back(std::make_pair(VA.
getLocReg(), Arg));
2606 }
else if (isByVal) {
2608 unsigned offset = 0;
2612 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2613 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2615 if (CurByValIdx < ByValArgsCount) {
2617 unsigned RegBegin, RegEnd;
2618 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
2623 for (i = 0, j = RegBegin;
j < RegEnd; i++,
j++) {
2630 RegsToPass.push_back(std::make_pair(j, Load));
2635 offset = RegEnd - RegBegin;
2637 CCInfo.nextInRegsParam();
2640 if (
Flags.getByValSize() > 4*offset) {
2644 std::tie(Dst, DstInfo) =
2645 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2654 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2662 std::tie(DstAddr, DstInfo) =
2663 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2670 if (!MemOpChains.
empty())
2676 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
2677 Chain = DAG.
getCopyToReg(Chain, dl, RegsToPass[i].first,
2678 RegsToPass[i].second, InGlue);
2685 bool isDirect =
false;
2690 GVal =
G->getGlobal();
2691 bool isStub = !
TM.shouldAssumeDSOLocal(GVal) && Subtarget->
isTargetMachO();
2693 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->
isMClass());
2694 bool isLocalARMFunc =
false;
2697 if (Subtarget->genLongCalls()) {
2699 "long-calls codegen is not position independent!");
2703 if (isa<GlobalAddressSDNode>(Callee)) {
2704 if (Subtarget->genExecuteOnly()) {
2723 const char *
Sym = S->getSymbol();
2725 if (Subtarget->genExecuteOnly()) {
2744 }
else if (isa<GlobalAddressSDNode>(Callee)) {
2745 if (!PreferIndirect) {
2750 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !
ARMInterworking);
2752 if (isStub && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2764 "Windows is the only supported COFF target");
2768 else if (!
TM.shouldAssumeDSOLocal(GVal))
2784 const char *
Sym = S->getSymbol();
2785 if (isARMFunc && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2789 ARMPCLabelIndex, 4);
2803 assert(!isARMFunc && !isDirect &&
2804 "Cannot handle call to ARM function or direct call");
2807 "call to non-secure function would "
2808 "require passing arguments on stack",
2815 "call to non-secure function would return value through pointer",
2823 if (Subtarget->isThumb()) {
2826 else if (isCmseNSCall)
2828 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2833 if (!isDirect && !Subtarget->hasV5TOps())
2835 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2848 if (isTailCall && !isSibCall) {
2853 std::vector<SDValue> Ops;
2854 Ops.push_back(Chain);
2855 Ops.push_back(Callee);
2863 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
2864 Ops.push_back(DAG.
getRegister(RegsToPass[i].first,
2865 RegsToPass[i].second.getValueType()));
2877 isThisReturn =
false;
2883 assert(Mask &&
"Missing call preserved mask for calling convention");
2887 Ops.push_back(InGlue);
2899 Chain = DAG.
getNode(CallOpc, dl, NodeTys, Ops);
2911 Chain = DAG.
getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, dl);
2917 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
2918 InVals, isThisReturn,
2919 isThisReturn ? OutVals[0] :
SDValue(), isCmseNSCall);
2926void ARMTargetLowering::HandleByVal(
CCState *State,
unsigned &
Size,
2927 Align Alignment)
const {
2929 Alignment = std::max(Alignment,
Align(4));
2935 unsigned AlignInRegs = Alignment.
value() / 4;
2936 unsigned Waste = (ARM::R4 -
Reg) % AlignInRegs;
2937 for (
unsigned i = 0; i < Waste; ++i)
2943 unsigned Excess = 4 * (ARM::R4 -
Reg);
2950 if (NSAAOffset != 0 &&
Size > Excess) {
2962 unsigned ByValRegBegin =
Reg;
2963 unsigned ByValRegEnd = std::min<unsigned>(Reg +
Size / 4, ARM::R4);
2967 for (
unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2973 Size = std::max<int>(
Size - Excess, 0);
2984 int FI = std::numeric_limits<int>::max();
2992 if (!Flags.isByVal()) {
2998 }
else if (
LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
2999 if (Flags.isByVal())
3014 assert(FI != std::numeric_limits<int>::max());
3025bool ARMTargetLowering::IsEligibleForTailCallOptimization(
3047 if (Outs.
size() >= 4 &&
3048 (!isa<GlobalAddressSDNode>(
Callee.getNode()) || isIndirect)) {
3066 return CalleeCC == CallerCC;
3070 bool isCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
3072 if (isCalleeStructRet || isCallerStructRet)
3086 (!
TT.isOSWindows() ||
TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO()))
3093 getEffectiveCallingConv(CalleeCC, isVarArg),
3094 getEffectiveCallingConv(CallerCC, CallerF.
isVarArg()), MF,
C, Ins,
3100 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
3101 if (CalleeCC != CallerCC) {
3102 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
3103 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
3116 if (!Outs.
empty()) {
3123 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.
size();
3125 ++i, ++realArgIdx) {
3128 SDValue Arg = OutVals[realArgIdx];
3132 if (VA.
needsCustom() && (RegVT == MVT::f64 || RegVT == MVT::v2f64)) {
3139 if (!ArgLocs[++i].isRegLoc())
3141 if (RegVT == MVT::v2f64) {
3142 if (!ArgLocs[++i].isRegLoc())
3144 if (!ArgLocs[++i].isRegLoc())
3169 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
3178 StringRef IntKind =
F.getFnAttribute(
"interrupt").getValueAsString();
3191 if (IntKind ==
"" || IntKind ==
"IRQ" || IntKind ==
"FIQ" ||
3194 else if (IntKind ==
"SWI" || IntKind ==
"UNDEF")
3198 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
3225 bool isLittleEndian = Subtarget->
isLittle();
3237 "secure entry function would return value through pointer",
3243 for (
unsigned i = 0, realRVLocIdx = 0;
3245 ++i, ++realRVLocIdx) {
3249 SDValue Arg = OutVals[realRVLocIdx];
3250 bool ReturnF16 =
false;
3285 auto RetVT = Outs[realRVLocIdx].ArgVT;
3307 DAG.
getVTList(MVT::i32, MVT::i32), Half);
3311 HalfGPRs.
getValue(isLittleEndian ? 0 : 1), Glue);
3317 HalfGPRs.
getValue(isLittleEndian ? 1 : 0), Glue);
3329 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
3331 fmrrd.
getValue(isLittleEndian ? 0 : 1), Glue);
3336 fmrrd.
getValue(isLittleEndian ? 1 : 0), Glue);
3380 return DAG.
getNode(RetNode, dl, MVT::Other, RetOps);
3383bool ARMTargetLowering::isUsedByReturnOnly(
SDNode *
N,
SDValue &Chain)
const {
3384 if (
N->getNumValues() != 1)
3386 if (!
N->hasNUsesOfValue(1, 0))
3394 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3396 TCChain =
Copy->getOperand(0);
3410 SDValue UseChain =
U->getOperand(0);
3418 if (
U->getOperand(
U->getNumOperands() - 1).getValueType() == MVT::Glue)
3426 if (!
Copy->hasOneUse())
3433 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3435 TCChain =
Copy->getOperand(0);
3440 bool HasRet =
false;
3455bool ARMTargetLowering::mayBeEmittedAsTailCall(
const CallInst *CI)
const {
3473 &&
"LowerWRITE_REGISTER called for non-i64 type argument.");
3489 EVT PtrVT =
Op.getValueType();
3499 if (Subtarget->genExecuteOnly()) {
3501 auto T =
const_cast<Type*
>(
CP->getType());
3502 auto C =
const_cast<Constant*
>(
CP->getConstVal());
3513 return LowerGlobalAddress(GA, DAG);
3518 Align CPAlign =
CP->getAlign();
3520 CPAlign = std::max(CPAlign,
Align(4));
3521 if (
CP->isMachineConstantPoolEntry())
3533 if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps())
3542 unsigned ARMPCLabelIndex = 0;
3545 const BlockAddress *BA = cast<BlockAddressSDNode>(
Op)->getBlockAddress();
3548 if (!IsPositionIndependent) {
3551 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3562 if (!IsPositionIndependent)
3593ARMTargetLowering::LowerGlobalTLSAddressDarwin(
SDValue Op,
3596 "This function expects a Darwin target");
3601 SDValue DescAddr = LowerGlobalAddressDarwin(
Op, DAG);
3607 MVT::i32,
DL, Chain, DescAddr,
3631 Chain, FuncTLVGet, DAG.
getRegister(ARM::R0, MVT::i32),
3637ARMTargetLowering::LowerGlobalTLSAddressWindows(
SDValue Op,
3654 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
3681 const auto *GA = cast<GlobalAddressSDNode>(
Op);
3698 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3720 Args.push_back(Entry);
3728 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
3729 return CallResult.first;
3751 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3759 PtrVT, dl, Chain,
Offset,
3761 Chain =
Offset.getValue(1);
3767 PtrVT, dl, Chain,
Offset,
3777 PtrVT, dl, Chain,
Offset,
3793 return LowerGlobalTLSAddressDarwin(
Op, DAG);
3796 return LowerGlobalTLSAddressWindows(
Op, DAG);
3805 return LowerToTLSGeneralDynamicModel(GA, DAG);
3808 return LowerToTLSExecModels(GA, DAG, model);
3817 while (!Worklist.
empty()) {
3819 if (isa<ConstantExpr>(U)) {
3824 auto *
I = dyn_cast<Instruction>(U);
3825 if (!
I ||
I->getParent()->getParent() !=
F)
3853 auto *GVar = dyn_cast<GlobalVariable>(GV);
3854 if (!GVar || !GVar->hasInitializer() ||
3855 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3856 !GVar->hasLocalLinkage())
3861 auto *
Init = GVar->getInitializer();
3863 Init->needsDynamicRelocation())
3872 auto *CDAInit = dyn_cast<ConstantDataArray>(
Init);
3875 unsigned RequiredPadding = 4 - (
Size % 4);
3876 bool PaddingPossible =
3877 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3882 unsigned PaddedSize =
Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3906 if (RequiredPadding != 4) {
3911 while (RequiredPadding--)
3923 ++NumConstpoolPromoted;
3928 if (
const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
3929 if (!(GV = GA->getAliaseeObject()))
3931 if (
const auto *V = dyn_cast<GlobalVariable>(GV))
3932 return V->isConstant();
3933 return isa<Function>(GV);
3941 return LowerGlobalAddressWindows(
Op, DAG);
3943 return LowerGlobalAddressELF(
Op, DAG);
3945 return LowerGlobalAddressDarwin(
Op, DAG);
3953 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
3957 if (GV->
isDSOLocal() && !Subtarget->genExecuteOnly())
3970 }
else if (Subtarget->
isROPI() && IsRO) {
3975 }
else if (Subtarget->
isRWPI() && !IsRO) {
4000 if (Subtarget->
useMovt() || Subtarget->genExecuteOnly()) {
4019 "ROPI/RWPI not currently supported for Darwin");
4022 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4045 "Windows on ARM expects to use movw/movt");
4047 "ROPI/RWPI not currently supported for Windows");
4050 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4054 else if (!
TM.shouldAssumeDSOLocal(GV))
4078 DAG.
getVTList(MVT::i32, MVT::Other),
Op.getOperand(0),
4079 Op.getOperand(1), Val);
4096SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
4099 Op.getConstantOperandVal(
Op.getOperand(0).getValueType() == MVT::Other);
4103 case Intrinsic::arm_gnu_eabi_mcount: {
4112 assert(Mask &&
"Missing call preserved mask for calling convention");
4117 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
4121 if (Subtarget->isThumb())
4124 ARM::tBL_PUSHLR, dl, ResultTys,
4125 {ReturnAddress, DAG.getTargetConstant(ARMCC::AL, dl, PtrVT),
4126 DAG.getRegister(0, PtrVT), Callee, RegisterMask, Chain}),
4130 {ReturnAddress, Callee, RegisterMask, Chain}),
4139 unsigned IntNo =
Op.getConstantOperandVal(0);
4143 case Intrinsic::thread_pointer: {
4147 case Intrinsic::arm_cls: {
4148 const SDValue &Operand =
Op.getOperand(1);
4149 const EVT VTy =
Op.getValueType();
4160 case Intrinsic::arm_cls64: {
4163 const SDValue &Operand =
Op.getOperand(1);
4164 const EVT VTy =
Op.getValueType();
4187 case Intrinsic::eh_sjlj_lsda: {
4194 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
4204 if (IsPositionIndependent) {
4210 case Intrinsic::arm_neon_vabs:
4213 case Intrinsic::arm_neon_vabds:
4214 if (
Op.getValueType().isInteger())
4216 Op.getOperand(1),
Op.getOperand(2));
4218 case Intrinsic::arm_neon_vabdu:
4220 Op.getOperand(1),
Op.getOperand(2));
4221 case Intrinsic::arm_neon_vmulls:
4222 case Intrinsic::arm_neon_vmullu: {
4223 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
4226 Op.getOperand(1),
Op.getOperand(2));
4228 case Intrinsic::arm_neon_vminnm:
4229 case Intrinsic::arm_neon_vmaxnm: {
4230 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
4233 Op.getOperand(1),
Op.getOperand(2));
4235 case Intrinsic::arm_neon_vminu:
4236 case Intrinsic::arm_neon_vmaxu: {
4237 if (
Op.getValueType().isFloatingPoint())
4239 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
4242 Op.getOperand(1),
Op.getOperand(2));
4244 case Intrinsic::arm_neon_vmins:
4245 case Intrinsic::arm_neon_vmaxs: {
4247 if (!
Op.getValueType().isFloatingPoint()) {
4248 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4251 Op.getOperand(1),
Op.getOperand(2));
4253 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4256 Op.getOperand(1),
Op.getOperand(2));
4258 case Intrinsic::arm_neon_vtbl1:
4260 Op.getOperand(1),
Op.getOperand(2));
4261 case Intrinsic::arm_neon_vtbl2:
4263 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4264 case Intrinsic::arm_mve_pred_i2v:
4265 case Intrinsic::arm_mve_pred_v2i:
4268 case Intrinsic::arm_mve_vreinterpretq:
4271 case Intrinsic::arm_mve_lsll:
4273 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4274 case Intrinsic::arm_mve_asrl:
4276 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4287 if (!Subtarget->hasDataBarrier()) {
4291 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
4292 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
4303 }
else if (Subtarget->preferISHSTBarriers() &&
4312 DAG.
getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
4320 (!Subtarget->
isThumb1Only() && Subtarget->hasV5TEOps())))
4322 return Op.getOperand(0);
4325 unsigned isRead =
~Op.getConstantOperandVal(2) & 1;
4327 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
4329 return Op.getOperand(0);
4331 unsigned isData =
Op.getConstantOperandVal(4);
4332 if (Subtarget->isThumb()) {
4334 isRead = ~isRead & 1;
4335 isData = ~isData & 1;
4352 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
4353 return DAG.
getStore(
Op.getOperand(0), dl, FR,
Op.getOperand(1),
4361 const SDLoc &dl)
const {
4367 RC = &ARM::tGPRRegClass;
4369 RC = &ARM::GPRRegClass;
4383 MVT::i32, dl, Root, FIN,
4404 const Value *OrigArg,
4405 unsigned InRegsParamRecordIdx,
4406 int ArgOffset,
unsigned ArgSize)
const {
4421 unsigned RBegin, REnd;
4431 ArgOffset = -4 * (ARM::R4 - RBegin);
4441 for (
unsigned Reg = RBegin, i = 0;
Reg < REnd; ++
Reg, ++i) {
4450 if (!MemOps.
empty())
4459 unsigned TotalArgRegsSaveSize,
4460 bool ForceMutable)
const {
4471 CCInfo.
getStackSize(), std::max(4U, TotalArgRegsSaveSize));
4475bool ARMTargetLowering::splitValueIntoRegisterParts(
4477 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
4479 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4491SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4493 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
4494 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4507SDValue ARMTargetLowering::LowerFormalArguments(
4523 unsigned CurArgIdx = 0;
4535 unsigned ArgRegBegin = ARM::R4;
4536 for (
unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4543 if (!
Flags.isByVal())
4547 unsigned RBegin, REnd;
4549 ArgRegBegin = std::min(ArgRegBegin, RBegin);
4555 int lastInsIndex = -1;
4559 ArgRegBegin = std::min(ArgRegBegin, (
unsigned)
GPRArgRegs[RegIdx]);
4562 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4566 for (
unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4568 if (Ins[VA.
getValNo()].isOrigArg()) {
4569 std::advance(CurOrigArg,
4570 Ins[VA.
getValNo()].getOrigArgIndex() - CurArgIdx);
4582 GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4589 MVT::f64, dl, Chain, FIN,
4592 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4600 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4604 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4605 RC = &ARM::HPRRegClass;
4606 else if (RegVT == MVT::f32)
4607 RC = &ARM::SPRRegClass;
4608 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4609 RegVT == MVT::v4bf16)
4610 RC = &ARM::DPRRegClass;
4611 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4612 RegVT == MVT::v8bf16)
4613 RC = &ARM::QPRRegClass;
4614 else if (RegVT == MVT::i32)
4616 : &ARM::GPRRegClass;
4662 assert(VA.
getValVT() != MVT::i64 &&
"i64 should already be lowered");
4668 if (index != lastInsIndex)
4676 if (
Flags.isByVal()) {
4677 assert(Ins[index].isOrigArg() &&
4678 "Byval arguments cannot be implicit");
4682 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
4697 lastInsIndex = index;
4704 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.
getStackSize(),
4705 TotalArgRegsSaveSize);
4709 "secure entry function must not be variadic", dl.
getDebugLoc());
4720 StackArgSize =
alignTo(StackArgSize,
DL.getStackAlignment());
4729 "secure entry function requires arguments on stack", dl.
getDebugLoc());
4739 return CFP->getValueAPF().isPosZero();
4743 SDValue WrapperOp =
Op.getOperand(1).getOperand(0);
4745 if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
4746 return CFP->getValueAPF().isPosZero();
4749 Op->getValueType(0) == MVT::f64) {
4764 const SDLoc &dl)
const {
4766 unsigned C = RHSC->getZExtValue();
4832 LHS.getValueType() == MVT::i32 && isa<ConstantSDNode>(RHS) &&
4834 unsigned Mask =
LHS.getConstantOperandVal(1);
4835 auto *RHSC = cast<ConstantSDNode>(
RHS.getNode());
4836 uint64_t RHSV = RHSC->getZExtValue();
4837 if (
isMask_32(Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4839 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4854 isa<ConstantSDNode>(RHS) &&
RHS->getAsZExtVal() == 0x80000000U &&
4856 LHS.getConstantOperandVal(1) < 31) {
4857 unsigned ShiftAmt =
LHS.getConstantOperandVal(1) + 1;
4897 return DAG.
getNode(CompareType, dl, MVT::Glue, LHS, RHS);
4903 bool Signaling)
const {
4904 assert(Subtarget->hasFP64() ||
RHS.getValueType() != MVT::f64);
4908 dl, MVT::Glue, LHS, RHS);
4911 dl, MVT::Glue, LHS);
4919 unsigned Opc =
Cmp.getOpcode();
4922 return DAG.
getNode(Opc,
DL, MVT::Glue,
Cmp.getOperand(0),
Cmp.getOperand(1));
4926 Opc =
Cmp.getOpcode();
4940std::pair<SDValue, SDValue>
4943 assert(
Op.getValueType() == MVT::i32 &&
"Unsupported value type");
4955 switch (
Op.getOpcode()) {
4968 DAG.
getVTList(
Op.getValueType(), MVT::i32), LHS, RHS)
5007 return std::make_pair(
Value, OverflowCmp);
5018 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Op, DAG, ARMcc);
5024 EVT VT =
Op.getValueType();
5027 ARMcc, CCR, OverflowCmp);
5067 EVT VT =
Op.getValueType();
5071 switch (
Op.getOpcode()) {
5096 EVT VT =
Op.getValueType();
5097 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP() || Subtarget->
isThumb1Only())
5107 switch (
Op->getOpcode()) {
5123 switch (
Op->getOpcode()) {
5142 DAG.
getNode(NewOpcode, dl, MVT::i32,
5153 unsigned Opc =
Cond.getOpcode();
5155 if (
Cond.getResNo() == 1 &&
5163 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5165 EVT VT =
Op.getValueType();
5167 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR,
5178 dyn_cast<ConstantSDNode>(
Cond.getOperand(0));
5180 dyn_cast<ConstantSDNode>(
Cond.getOperand(1));
5182 if (CMOVTrue && CMOVFalse) {
5188 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
5190 False = SelectFalse;
5191 }
else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
5197 EVT VT =
Op.getValueType();
5202 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG);
5218 bool &swpCmpOps,
bool &swpVselOps) {
5246 swpCmpOps = !swpCmpOps;
5247 swpVselOps = !swpVselOps;
5270 if (!Subtarget->hasFP64() && VT == MVT::f64) {
5272 DAG.
getVTList(MVT::i32, MVT::i32), FalseVal);
5274 DAG.
getVTList(MVT::i32, MVT::i32), TrueVal);
5284 ARMcc, CCR, duplicateCmp(Cmp, DAG));
5311 ((K ==
LHS && K == TrueVal) || (K ==
RHS && K == FalseVal))) ||
5313 ((K ==
RHS && K == TrueVal) || (K ==
LHS && K == FalseVal)));
5334 EVT VT =
Op.getValueType();
5341 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
5356 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5364 if (!isa<ConstantSDNode>(K1) || !isa<ConstantSDNode>(K2))
5367 int64_t Val1 = cast<ConstantSDNode>(K1)->getSExtValue();
5368 int64_t Val2 = cast<ConstantSDNode>(K2)->getSExtValue();
5369 int64_t PosVal = std::max(Val1, Val2);
5370 int64_t NegVal = std::min(Val1, Val2);
5416 SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal;
5417 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5422 if (*K != KTmp || V != VTmp)
5433bool ARMTargetLowering::isUnsupportedFloatingType(
EVT VT)
const {
5437 return !Subtarget->hasFP64();
5439 return !Subtarget->hasFullFP16();
5444 EVT VT =
Op.getValueType();
5448 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->
isThumb2())
5460 if (VT == MVT::i32 &&
5480 if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal &&
5481 LHS.getValueType() == MVT::i32 &&
RHS.getValueType() == MVT::i32) {
5484 unsigned Opcode = 0;
5486 if (TVal == ~FVal) {
5488 }
else if (TVal == ~FVal + 1) {
5490 }
else if (TVal + 1 == FVal) {
5492 }
else if (TVal == FVal + 1) {
5524 return DAG.
getNode(Opcode, dl, VT, TrueVal, FalseVal, ARMcc, Cmp);
5528 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5530 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5534 if (!
RHS.getNode()) {
5540 if (
LHS.getValueType() == MVT::i32) {
5552 TrueVal.getValueType() == MVT::f32 ||
5553 TrueVal.getValueType() == MVT::f64)) {
5568 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5580 (
TrueVal.getValueType() == MVT::f16 ||
5581 TrueVal.getValueType() == MVT::f32 ||
5582 TrueVal.getValueType() == MVT::f64)) {
5583 bool swpCmpOps =
false;
5584 bool swpVselOps =
false;
5599 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5603 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
5604 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG);
5614 if (!
N->hasOneUse())
5617 if (!
N->getNumValues())
5619 EVT VT =
Op.getValueType();
5620 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
5637 return DAG.
getLoad(MVT::i32,
SDLoc(
Op), Ld->getChain(), Ld->getBasePtr(),
5638 Ld->getPointerInfo(), Ld->getAlign(),
5639 Ld->getMemOperand()->getFlags());
5657 DAG.
getLoad(MVT::i32, dl, Ld->getChain(),
Ptr, Ld->getPointerInfo(),
5658 Ld->getAlign(), Ld->getMemOperand()->getFlags());
5660 EVT PtrType =
Ptr.getValueType();
5663 RetVal2 = DAG.
getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
5664 Ld->getPointerInfo().getWithOffset(4),
5666 Ld->getMemOperand()->getFlags());
5684 bool LHSSeenZero =
false;
5686 bool RHSSeenZero =
false;
5688 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5699 if (
LHS.getValueType() == MVT::f32) {
5707 Chain, Dest, ARMcc, CCR, Cmp);
5719 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5734 unsigned Opc =
Cond.getOpcode();
5737 if (
Cond.getResNo() == 1 &&
5747 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5771 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5773 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5777 if (!
RHS.getNode()) {
5785 unsigned Opc =
LHS.getOpcode();
5799 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
LHS.getValue(0), DAG, ARMcc);
5814 if (
LHS.getValueType() == MVT::i32) {
5819 Chain, Dest, ARMcc, CCR, Cmp);
5825 if (
SDValue Result = OptimizeVFPBrcond(
Op, DAG))
5836 SDValue Ops[] = { Chain, Dest, ARMcc, CCR,
Cmp };
5858 if (Subtarget->
isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5864 Addr,
Op.getOperand(2), JTI);
5870 Chain =
Addr.getValue(1);
5877 Chain =
Addr.getValue(1);
5883 EVT VT =
Op.getValueType();
5886 if (
Op.getValueType().getVectorElementType() == MVT::i32) {
5887 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
5895 const EVT OpTy =
Op.getOperand(0).getValueType();
5896 if (OpTy == MVT::v4f32)
5898 else if (OpTy == MVT::v4f16 && HasFullFP16)
5900 else if (OpTy == MVT::v8f16 && HasFullFP16)
5905 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5908 Op = DAG.
getNode(
Op.getOpcode(), dl, NewTy,
Op.getOperand(0));
5913 EVT VT =
Op.getValueType();
5917 bool IsStrict =
Op->isStrictFPOpcode();
5918 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
5920 if (isUnsupportedFloatingType(SrcVal.
getValueType())) {
5930 MakeLibCallOptions CallOptions;
5933 std::tie(Result, Chain) =
makeLibCall(DAG, LC,
Op.getValueType(), SrcVal,
5934 CallOptions, Loc, Chain);
5944 Loc,
Op.getValueType(), SrcVal);
5953 EVT VT =
Op.getValueType();
5954 EVT ToVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
5955 EVT FromVT =
Op.getOperand(0).getValueType();
5957 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
5959 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
5960 Subtarget->hasFP64())
5962 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
5963 Subtarget->hasFullFP16())
5965 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
5966 Subtarget->hasMVEFloatOps())
5968 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
5969 Subtarget->hasMVEFloatOps())
5972 if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
5989 EVT VT =
Op.getValueType();
5992 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
5998 assert((
Op.getOperand(0).getValueType() == MVT::v4i16 ||
5999 Op.getOperand(0).getValueType() == MVT::v8i16) &&
6000 "Invalid type for custom lowering!");
6005 if (VT == MVT::v4f32)
6006 DestVecType = MVT::v4i32;
6007 else if (VT == MVT::v4f16 && HasFullFP16)
6008 DestVecType = MVT::v4i16;
6009 else if (VT == MVT::v8f16 && HasFullFP16)
6010 DestVecType = MVT::v8i16;
6016 switch (
Op.getOpcode()) {
6028 Op = DAG.
getNode(CastOpc, dl, DestVecType,
Op.getOperand(0));
6033 EVT VT =
Op.getValueType();
6036 if (isUnsupportedFloatingType(VT)) {
6044 MakeLibCallOptions CallOptions;
6046 CallOptions,
SDLoc(
Op)).first;
6057 EVT VT =
Op.getValueType();
6061 bool UseNEON = !InGPR && Subtarget->hasNEON();
6068 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
6075 if (SrcVT == MVT::f32) {
6081 }
else if (VT == MVT::f32)
6097 if (VT == MVT::f32) {
6109 if (SrcVT == MVT::f64)
6118 if (VT == MVT::f32) {
6142 EVT VT =
Op.getValueType();
6144 unsigned Depth =
Op.getConstantOperandVal(0);
6146 SDValue FrameAddr = LowerFRAMEADDR(
Op, DAG);
6165 EVT VT =
Op.getValueType();
6167 unsigned Depth =
Op.getConstantOperandVal(0);
6181 .
Case(
"sp", ARM::SP)
6196 assert(
N->getValueType(0) == MVT::i64
6197 &&
"ExpandREAD_REGISTER called for non-i64 type result.");
6200 DAG.
getVTList(MVT::i32, MVT::i32, MVT::Other),
6206 Results.push_back(Read.getOperand(0));
6240 const APInt &APIntIndex =
Index->getAPIntValue();
6242 NewIndex *= APIntIndex;
6272 EVT SrcVT =
Op.getValueType();
6273 EVT DstVT =
N->getValueType(0);
6275 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
6276 (DstVT == MVT::f16 || DstVT == MVT::bf16))
6280 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
6281 (SrcVT == MVT::f16 || SrcVT == MVT::bf16))
6286 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
6290 if (SrcVT == MVT::i64 && TLI.
isTypeLegal(DstVT)) {
6302 if (DstVT == MVT::i64 && TLI.
isTypeLegal(SrcVT)) {
6339 EVT VT =
Op.getValueType();
6358 SDValue LoBigShift = DAG.
getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
6366 ? DAG.
getNode(Opc, dl, VT, ShOpHi,
6383 EVT VT =
Op.getValueType();
6426 DAG.
getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32)};
6478 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6506 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6536 EVT VT =
N->getValueType(0);
6537 if (VT.
isVector() && ST->hasNEON()) {
6546 if (ElemTy == MVT::i8) {
6554 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
6569 if (ElemTy == MVT::i64) {
6582 if (!ST->hasV6T2Ops())
6591 EVT VT =
N->getValueType(0);
6594 assert(ST->hasNEON() &&
"Custom ctpop lowering requires NEON.");
6595 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6596 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6597 "Unexpected type for custom ctpop lowering");
6605 unsigned EltSize = 8;
6628 Op =
Op.getOperand(0);
6630 APInt SplatBits, SplatUndef;
6631 unsigned SplatBitSize;
6634 !BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6636 SplatBitSize > ElementBits)
6647 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6651 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6662 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6667 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6668 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6677 EVT VT =
N->getValueType(0);
6699 "unexpected vector shift opcode");
6701 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
6702 unsigned VShiftOpc =
6704 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
6710 EVT ShiftVT =
N->getOperand(1).getValueType();
6713 unsigned VShiftOpc =
6715 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0), NegatedCount);
6720 EVT VT =
N->getValueType(0);
6729 "Unknown shift to lower!");
6731 unsigned ShOpc =
N->getOpcode();
6732 if (ST->hasMVEIntegerOps()) {
6762 DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6776 if (ST->isThumb1Only())
6781 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6797 bool Invert =
false;
6804 EVT VT =
Op.getValueType();
6812 assert(ST->hasMVEIntegerOps() &&
6813 "No hardware support for integer vector comparison!");
6815 if (
Op.getValueType().getVectorElementType() != MVT::i1)
6840 Merged = DAG.
getNOT(dl, Merged, CmpVT);
6850 switch (SetCCOpcode) {
6854 if (ST->hasMVEFloatOps()) {
6857 Invert =
true; [[fallthrough]];
6862 case ISD::SETLT: Swap =
true; [[fallthrough]];
6866 case ISD::SETLE: Swap =
true; [[fallthrough]];
6882 Result = DAG.
getNOT(dl, Result, VT);
6885 case ISD::SETUO: Invert =
true; [[fallthrough]];
6894 Result = DAG.
getNOT(dl, Result, VT);
6900 switch (SetCCOpcode) {
6903 if (ST->hasMVEIntegerOps()) {
6906 Invert =
true; [[fallthrough]];
6909 case ISD::SETLT: Swap =
true; [[fallthrough]];
6911 case ISD::SETLE: Swap =
true; [[fallthrough]];
6920 if (ST->hasNEON() && Opc ==
ARMCC::EQ) {
6936 Result = DAG.
getNOT(dl, Result, VT);
6970 Result = DAG.
getNOT(dl, Result, VT);
6982 assert(
LHS.getSimpleValueType().isInteger() &&
"SETCCCARRY is integer only.");
7012 unsigned OpCmode, Imm;
7023 switch (SplatBitSize) {
7028 assert((SplatBits & ~0xff) == 0 &&
"one byte splat value is too big");
7031 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
7036 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
7037 if ((SplatBits & ~0xff) == 0) {
7043 if ((SplatBits & ~0xff00) == 0) {
7046 Imm = SplatBits >> 8;
7056 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
7057 if ((SplatBits & ~0xff) == 0) {
7063 if ((SplatBits & ~0xff00) == 0) {
7066 Imm = SplatBits >> 8;
7069 if ((SplatBits & ~0xff0000) == 0) {
7072 Imm = SplatBits >> 16;
7075 if ((SplatBits & ~0xff000000) == 0) {
7078 Imm = SplatBits >> 24;
7085 if ((SplatBits & ~0xffff) == 0 &&
7086 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
7089 Imm = SplatBits >> 8;
7097 if ((SplatBits & ~0xffffff) == 0 &&
7098 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
7101 Imm = SplatBits >> 16;
7117 unsigned ImmMask = 1;
7119 for (
int ByteNum = 0; ByteNum < 8; ++ByteNum) {
7120 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
7122 }
else if ((SplatBits & BitMask) != 0) {
7132 unsigned Mask = (1 << BytesPerElem) - 1;
7133 unsigned NumElems = 8 / BytesPerElem;
7134 unsigned NewImm = 0;
7135 for (
unsigned ElemNum = 0; ElemNum < NumElems; ++ElemNum) {
7136 unsigned Elem = ((Imm >> ElemNum * BytesPerElem) & Mask);
7137 NewImm |= Elem << (NumElems - ElemNum - 1) * BytesPerElem;
7144 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
7158 EVT VT =
Op.getValueType();
7159 bool IsDouble = (VT == MVT::f64);
7165 if (
ST->genExecuteOnly()) {
7167 assert((!
ST->isThumb1Only() ||
ST->hasV8MBaselineOps()) &&
7168 "Unexpected architecture");
7191 if (!
ST->hasVFP3Base())
7196 if (IsDouble && !Subtarget->hasFP64())
7203 if (IsDouble || !
ST->useNEONForSinglePrecisionFP()) {
7221 if (!
ST->hasNEON() || (!IsDouble && !
ST->useNEONForSinglePrecisionFP()))
7230 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
7284 unsigned ExpectedElt = Imm;
7285 for (
unsigned i = 1; i < NumElts; ++i) {
7289 if (ExpectedElt == NumElts)
7292 if (M[i] < 0)
continue;
7293 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7301 bool &ReverseVEXT,
unsigned &Imm) {
7303 ReverseVEXT =
false;
7314 unsigned ExpectedElt = Imm;
7315 for (
unsigned i = 1; i < NumElts; ++i) {
7319 if (ExpectedElt == NumElts * 2) {
7324 if (M[i] < 0)
continue;
7325 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7340 return VT == MVT::v8i8 && M.size() == 8;
7345 if (Mask.size() == Elements * 2)
7346 return Index / Elements;
7347 return Mask[
Index] == 0 ? 0 : 1;
7377 if (M.size() != NumElts && M.size() != NumElts*2)
7385 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7387 for (
unsigned j = 0; j < NumElts; j += 2) {
7388 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7389 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + NumElts + WhichResult))
7394 if (M.size() == NumElts*2)
7409 if (M.size() != NumElts && M.size() != NumElts*2)
7412 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7414 for (
unsigned j = 0; j < NumElts; j += 2) {
7415 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7416 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + WhichResult))
7421 if (M.size() == NumElts*2)
7441 if (M.size() != NumElts && M.size() != NumElts*2)
7444 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7446 for (
unsigned j = 0; j < NumElts; ++j) {
7447 if (M[i+j] >= 0 && (
unsigned) M[i+j] != 2 * j + WhichResult)
7452 if (M.size() == NumElts*2)
7471 if (M.size() != NumElts && M.size() != NumElts*2)
7474 unsigned Half = NumElts / 2;
7475 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7477 for (
unsigned j = 0; j < NumElts; j += Half) {
7478 unsigned Idx = WhichResult;
7479 for (
unsigned k = 0; k < Half; ++k) {
7480 int MIdx = M[i + j + k];
7481 if (MIdx >= 0 && (
unsigned) MIdx !=
Idx)
7488 if (M.size() == NumElts*2)
7512 if (M.size() != NumElts && M.size() != NumElts*2)
7515 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7517 unsigned Idx = WhichResult * NumElts / 2;
7518 for (
unsigned j = 0; j < NumElts; j += 2) {
7519 if ((M[i+j] >= 0 && (
unsigned) M[i+j] !=
Idx) ||
7520 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] !=
Idx + NumElts))
7526 if (M.size() == NumElts*2)
7545 if (M.size() != NumElts && M.size() != NumElts*2)
7548 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7550 unsigned Idx = WhichResult * NumElts / 2;
7551 for (
unsigned j = 0; j < NumElts; j += 2) {
7552 if ((M[i+j] >= 0 && (
unsigned) M[i+j] !=
Idx) ||
7553 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] !=
Idx))
7559 if (M.size() == NumElts*2)
7572 unsigned &WhichResult,
7575 if (
isVTRNMask(ShuffleMask, VT, WhichResult))
7577 if (
isVUZPMask(ShuffleMask, VT, WhichResult))
7579 if (
isVZIPMask(ShuffleMask, VT, WhichResult))
7597 if (NumElts != M.size())
7601 for (
unsigned i = 0; i != NumElts; ++i)
7602 if (M[i] >= 0 && M[i] != (
int) (NumElts - 1 - i))
7611 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7619 int Ofs = Top ? 1 : 0;
7620 int Upper = SingleSource ? 0 : NumElts;
7621 for (
int i = 0, e = NumElts / 2; i != e; ++i) {
7622 if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
7624 if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs +
Upper)
7633 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7642 unsigned Offset = Top ? 0 : 1;
7643 unsigned N = SingleSource ? 0 : NumElts;
7644 for (
unsigned i = 0; i < NumElts; i += 2) {
7645 if (M[i] >= 0 && M[i] != (
int)i)
7647 if (M[i + 1] >= 0 && M[i + 1] != (
int)(
N + i +
Offset))
7656 if (NumElts != M.size())
7664 unsigned Off0 = rev ? NumElts / 2 : 0;
7665 unsigned Off1 = rev ? 0 : NumElts / 2;
7666 for (
unsigned i = 0; i < NumElts; i += 2) {
7667 if (M[i] >= 0 && M[i] != (
int)(Off0 + i / 2))
7669 if (M[i + 1] >= 0 && M[i + 1] != (
int)(Off1 + i / 2))
7685 if (!ST->hasMVEFloatOps())
7690 if (VT != MVT::v8f16)
7711 for (
unsigned i = 1; i < 4; i++) {
7738 if (!ST->hasMVEFloatOps())
7743 if (VT != MVT::v4f32)
7759 for (
unsigned i = 1; i < 4; i++) {
7780 if (!isa<ConstantSDNode>(
N))
7782 Val =
N->getAsZExtVal();
7784 if (ST->isThumb1Only()) {
7785 if (Val <= 255 || ~Val <= 255)
7797 EVT VT =
Op.getValueType();
7799 assert(ST->hasMVEIntegerOps() &&
"LowerBUILD_VECTOR_i1 called without MVE!");
7803 unsigned BitsPerBool;
7807 }
else if (NumElts == 4) {
7810 }
else if (NumElts == 8) {
7813 }
else if (NumElts == 16) {
7822 if (!isa<ConstantSDNode>(FirstOp) &&
7824 return U.get().isUndef() || U.get() == FirstOp;
7832 unsigned Bits32 = 0;
7833 for (
unsigned i = 0; i < NumElts; ++i) {
7835 if (!isa<ConstantSDNode>(V) && !V.isUndef())
7837 bool BitSet = V.isUndef() ?
false : V->getAsZExtVal();
7839 Bits32 |= BoolMask << (i * BitsPerBool);
7845 for (
unsigned i = 0; i < NumElts; ++i) {
7847 if (isa<ConstantSDNode>(V) || V.isUndef())
7858 if (!ST->hasMVEIntegerOps())
7862 EVT VT =
Op.getValueType();
7872 if (
N != 1 &&
N != 2 &&
N != 4 &&
N != 8)
7876 for (
unsigned I = 2;
I < NumElts;
I++) {
7892 switch (
N->getOpcode()) {
7901 return N->getOperand(1).getNode() ==
Op;
7903 switch (
N->getConstantOperandVal(0)) {
7904 case Intrinsic::arm_mve_add_predicated:
7905 case Intrinsic::arm_mve_mul_predicated:
7906 case Intrinsic::arm_mve_qadd_predicated:
7907 case Intrinsic::arm_mve_vhadd:
7908 case Intrinsic::arm_mve_hadd_predicated:
7909 case Intrinsic::arm_mve_vqdmulh:
7910 case Intrinsic::arm_mve_qdmulh_predicated:
7911 case Intrinsic::arm_mve_vqrdmulh:
7912 case Intrinsic::arm_mve_qrdmulh_predicated:
7913 case Intrinsic::arm_mve_vqdmull:
7914 case Intrinsic::arm_mve_vqdmull_predicated:
7916 case Intrinsic::arm_mve_sub_predicated:
7917 case Intrinsic::arm_mve_qsub_predicated:
7918 case Intrinsic::arm_mve_vhsub:
7919 case Intrinsic::arm_mve_hsub_predicated:
7920 return N->getOperand(2).getNode() ==
Op;
7935 EVT VT =
Op.getValueType();
7943 APInt SplatBits, SplatUndef;
7944 unsigned SplatBitSize;
7946 if (BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7953 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
7955 [BVN](
const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) {
7956 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7957 : SplatBitSize == 16 ? MVT::v8i16
7964 if ((
ST->hasNEON() && SplatBitSize <= 64) ||
7965 (
ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7970 SplatBitSize, DAG, dl, VmovVT, VT,
VMOVModImm);
7978 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7980 NegatedImm, SplatUndef.
getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
7988 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
7998 if (
ST->hasMVEIntegerOps() &&
7999 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
8000 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
8001 : SplatBitSize == 16 ? MVT::v8i16
8018 bool isOnlyLowElement =
true;
8019 bool usesOnlyOneValue =
true;
8020 bool hasDominantValue =
false;
8027 for (
unsigned i = 0; i < NumElts; ++i) {
8032 isOnlyLowElement =
false;
8033 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
8036 ValueCounts.
insert(std::make_pair(V, 0));
8037 unsigned &Count = ValueCounts[
V];
8040 if (++Count > (NumElts / 2)) {
8041 hasDominantValue =
true;
8045 if (ValueCounts.
size() != 1)
8046 usesOnlyOneValue =
false;
8047 if (!
Value.getNode() && !ValueCounts.
empty())
8050 if (ValueCounts.
empty())
8062 if (hasDominantValue && EltSize <= 32) {
8073 (constIndex = dyn_cast<ConstantSDNode>(
Value->getOperand(1)))) {
8078 if (VT !=
Value->getOperand(0).getValueType()) {
8091 if (!usesOnlyOneValue) {
8094 for (
unsigned I = 0;
I < NumElts; ++
I) {
8109 assert(FVT == MVT::f32 || FVT == MVT::f16);
8110 MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
8111 for (
unsigned i = 0; i < NumElts; ++i)
8116 Val = LowerBUILD_VECTOR(Val, DAG, ST);
8120 if (usesOnlyOneValue) {
8147 if (
ST->hasNEON() && VT.
is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
8167 if (EltSize >= 32) {
8173 for (
unsigned i = 0; i < NumElts; ++i)
8187 for (
unsigned i = 0 ; i < NumElts; ++i) {
8206 EVT VT =
Op.getValueType();
8209 struct ShuffleSourceInfo {
8211 unsigned MinElt = std::numeric_limits<unsigned>::max();
8212 unsigned MaxElt = 0;
8222 int WindowScale = 1;
8224 ShuffleSourceInfo(
SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
8226 bool operator ==(
SDValue OtherVec) {
return Vec == OtherVec; }
8232 for (
unsigned i = 0; i < NumElts; ++i) {
8240 }
else if (!isa<ConstantSDNode>(
V.getOperand(1))) {
8247 SDValue SourceVec =
V.getOperand(0);
8249 if (Source == Sources.
end())
8253 unsigned EltNo =
V.getConstantOperandVal(1);
8260 if (Sources.
size() > 2)
8266 for (
auto &Source : Sources) {
8267 EVT SrcEltTy =
Source.Vec.getValueType().getVectorElementType();
8268 if (SrcEltTy.
bitsLT(SmallestEltTy))
8269 SmallestEltTy = SrcEltTy;
8271 unsigned ResMultiplier =
8279 for (
auto &Src : Sources) {
8280 EVT SrcVT = Src.ShuffleVec.getValueType();
8284 if (SrcVTSize == VTSize)
8293 if (SrcVTSize < VTSize) {
8294 if (2 * SrcVTSize != VTSize)
8300 DAG.
getUNDEF(Src.ShuffleVec.getValueType()));
8304 if (SrcVTSize != 2 * VTSize)
8307 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
8312 if (Src.MinElt >= NumSrcElts) {
8317 Src.WindowBase = -NumSrcElts;
8318 }
else if (Src.MaxElt < NumSrcElts) {
8335 Src.WindowBase = -Src.MinElt;
8342 for (
auto &Src : Sources) {
8343 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8344 if (SrcEltTy == SmallestEltTy)
8349 Src.WindowBase *= Src.WindowScale;
8355 assert(Src.ShuffleVec.getValueType() == ShuffleVT););
8362 if (
Entry.isUndef())
8366 int EltNo = cast<ConstantSDNode>(
Entry.getOperand(1))->getSExtValue();
8371 EVT OrigEltTy =
Entry.getOperand(0).getValueType().getVectorElementType();
8374 int LanesDefined = BitsDefined / BitsPerShuffleLane;
8378 int *LaneMask = &
Mask[i * ResMultiplier];
8380 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8381 ExtractBase += NumElts * (Src - Sources.begin());
8382 for (
int j = 0;
j < LanesDefined; ++
j)
8383 LaneMask[j] = ExtractBase + j;
8389 assert(Sources.size() <= 2 &&
"Too many sources!");
8392 for (
unsigned i = 0; i < Sources.size(); ++i)
8421 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8441 unsigned PFIndexes[4];
8442 for (
unsigned i = 0; i != 4; ++i) {
8446 PFIndexes[i] = M[i];
8450 unsigned PFTableIndex =
8451 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8453 unsigned Cost = (PFEntry >> 30);
8459 bool ReverseVEXT, isV_UNDEF;
8460 unsigned Imm, WhichResult;
8463 if (EltSize >= 32 ||
8470 else if (Subtarget->hasNEON() &&
8475 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8478 else if (Subtarget->hasMVEIntegerOps() &&
8482 else if (Subtarget->hasMVEIntegerOps() &&
8496 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8497 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8498 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8501 if (LHSID == (1*9+2)*9+3)
return LHS;
8502 assert(LHSID == ((4*9+5)*9+6)*9+7 &&
"Illegal OP_COPY!");
8559 for (
int I : ShuffleMask)
8562 if (V2.getNode()->isUndef())
8572 EVT VT =
Op.getValueType();
8574 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8575 "Expect an v8i16/v16i8 type");
8581 std::vector<int> NewMask;
8585 NewMask.push_back(i);
8625 if (VT != MVT::v16i1)
8641 EVT VT =
Op.getValueType();
8645 assert(ST->hasMVEIntegerOps() &&
8646 "No support for vector shuffle of boolean predicates");
8672 "Expected identical vector type in expanded i1 shuffle!");
8676 PredAsVector2, ShuffleMask);
8681 if (VT == MVT::v2i1) {
8698 EVT VT =
Op.getValueType();
8702 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8703 "Unexpected vector type");
8705 int QuarterSize = NumElts / 4;
8714 for (
int i = 0; i <
Length; i++) {
8715 if (ShuffleMask[Start + i] >= 0) {
8716 if (ShuffleMask[Start + i] %
Length != i)
8718 MovIdx = ShuffleMask[Start + i] /
Length;
8726 for (
int i = 1; i <
Length; i++) {
8727 if (ShuffleMask[Start + i] >= 0 &&
8728 (ShuffleMask[Start + i] /
Length != MovIdx ||
8729 ShuffleMask[Start + i] %
Length != i))
8735 for (
int Part = 0; Part < 4; ++Part) {
8737 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8741 Input =
Op->getOperand(1);
8751 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8756 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8758 for (
int Part = 0; Part < 4; ++Part)
8759 for (
int i = 0; i < QuarterSize; i++)
8761 Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8763 VT, dl,
Op->getOperand(0),
Op->getOperand(1), NewShuffleMask);
8766 for (
int Part = 0; Part < 4; ++Part)
8782 EVT VT =
Op.getValueType();
8794 for (
int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8798 if (Mask[i] != i + BaseOffset) {
8799 if (OffElement == -1)
8805 return NonUndef > 2 && OffElement != -1;
8809 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8811 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8822 ShuffleMask[OffElement] < (
int)NumElts ? V1 : V2,
8833 EVT VT =
Op.getValueType();
8837 if (ST->hasMVEIntegerOps() && EltSize == 1)
8848 if (EltSize <= 32) {
8852 if (Lane == -1) Lane = 0;
8863 bool IsScalarToVector =
true;
8866 IsScalarToVector =
false;
8869 if (IsScalarToVector)
8876 bool ReverseVEXT =
false;
8878 if (ST->hasNEON() &&
isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) {
8902 unsigned WhichResult = 0;
8903 bool isV_UNDEF =
false;
8904 if (ST->hasNEON()) {
8906 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8913 if (ST->hasMVEIntegerOps()) {
8948 }) &&
"Unexpected shuffle index into UNDEF operand!");
8951 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
8954 assert((WhichResult == 0) &&
8955 "In-place shuffle of concat can only have one result!");
8964 if (ST->hasMVEIntegerOps() && EltSize <= 32) {
8968 for (
bool Top : {
false,
true}) {
8969 for (
bool SingleSource : {
false,
true}) {
8970 if (
isTruncMask(ShuffleMask, VT, Top, SingleSource)) {
8975 SingleSource ? V1 : V2);
8991 unsigned PFIndexes[4];
8992 for (
unsigned i = 0; i != 4; ++i) {
8993 if (ShuffleMask[i] < 0)
8996 PFIndexes[i] = ShuffleMask[i];
9000 unsigned PFTableIndex =
9001 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
9003 unsigned Cost = (PFEntry >> 30);
9009 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
9010 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
9020 if (EltSize >= 32) {
9028 for (
unsigned i = 0; i < NumElts; ++i) {
9029 if (ShuffleMask[i] < 0)
9033 ShuffleMask[i] < (
int)NumElts ? V1 : V2,
9041 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
9045 if (ST->hasNEON() && VT == MVT::v8i8)
9049 if (ST->hasMVEIntegerOps())
9058 EVT VecVT =
Op.getOperand(0).getValueType();
9061 assert(ST->hasMVEIntegerOps() &&
9062 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9066 unsigned Lane =
Op.getConstantOperandVal(2);
9067 unsigned LaneWidth =
9069 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
9081 if (!isa<ConstantSDNode>(Lane))
9087 if (Subtarget->hasMVEIntegerOps() &&
9088 Op.getValueType().getScalarSizeInBits() == 1)
9112 IVecIn, IElt, Lane);
9121 EVT VecVT =
Op.getOperand(0).getValueType();
9124 assert(ST->hasMVEIntegerOps() &&
9125 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9129 unsigned Lane =
Op.getConstantOperandVal(1);
9130 unsigned LaneWidth =
9141 if (!isa<ConstantSDNode>(Lane))
9161 assert(
Op.getValueType().getScalarSizeInBits() == 1 &&
9162 "Unexpected custom CONCAT_VECTORS lowering");
9164 "Unexpected custom CONCAT_VECTORS lowering");
9165 assert(ST->hasMVEIntegerOps() &&
9166 "CONCAT_VECTORS lowering only supported for MVE");
9170 EVT Op2VT = V2.getValueType();
9171 assert(Op1VT == Op2VT &&
"Operand types don't match!");
9172 assert((Op1VT == MVT::v2i1 || Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) &&
9173 "Unexpected i1 concat operations!");
9186 if (Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) {
9200 auto ExtractInto = [&DAG, &dl](
SDValue NewV,
SDValue ConVec,
unsigned &j) {
9201 EVT NewVT = NewV.getValueType();
9202 EVT ConcatVT = ConVec.getValueType();
9203 unsigned ExtScale = 1;
9204 if (NewVT == MVT::v2f64) {
9218 ConVec = ExtractInto(NewV1, ConVec, j);
9219 ConVec = ExtractInto(NewV2, ConVec, j);
9229 while (ConcatOps.
size() > 1) {
9230 for (
unsigned I = 0, E = ConcatOps.
size();
I != E;
I += 2) {
9233 ConcatOps[
I / 2] = ConcatPair(V1, V2);
9237 return ConcatOps[0];
9242 EVT VT =
Op->getValueType(0);
9249 "unexpected CONCAT_VECTORS");
9270 EVT VT =
Op.getValueType();
9273 unsigned Index = V2->getAsZExtVal();
9276 "Unexpected custom EXTRACT_SUBVECTOR lowering");
9277 assert(ST->hasMVEIntegerOps() &&
9278 "EXTRACT_SUBVECTOR lowering only supported for MVE");
9288 EVT SubVT = MVT::v4i32;
9290 for (
unsigned i =
Index, j = 0; i < (
Index + NumElts); i++, j += 2) {
9305 for (
unsigned i =
Index, j = 0; i < (
Index + NumElts); i++, j++) {
9321 assert(ST->hasMVEIntegerOps() &&
"Expected MVE!");
9322 EVT VT =
N->getValueType(0);
9323 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&
9324 "Expected a vector i1 type!");
9326 EVT FromVT =
Op.getValueType();
9337 if (!Subtarget->hasMVEIntegerOps())
9340 EVT ToVT =
N->getValueType(0);
9383 if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8)
9385 EVT FromVT =
N->getOperand(0).getValueType();
9386 if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16)
9397 if (!Subtarget->hasMVEIntegerOps())
9402 EVT ToVT =
N->getValueType(0);
9403 if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16)
9406 EVT FromVT =
Op.getValueType();
9407 if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8)
9418 SDValue Ext1 = Ext.getValue(1);
9421 Ext = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext);
9422 Ext1 = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext1);
9434 EVT VT =
N->getValueType(0);
9436 SDNode *BVN =
N->getOperand(0).getNode();
9441 unsigned HiElt = 1 - LoElt;
9446 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
9462 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
9463 SDNode *Elt =
N->getOperand(i).getNode();
9466 unsigned HalfSize = EltSize / 2;
9468 if (!
isIntN(HalfSize,
C->getSExtValue()))
9471 if (!
isUIntN(HalfSize,
C->getZExtValue()))
9510 switch (OrigSimpleTy) {
9526 unsigned ExtOpcode) {
9549 if (ExtendedTy == LD->getMemoryVT())
9550 return DAG.
getLoad(LD->getMemoryVT(),
SDLoc(LD), LD->getChain(),
9551 LD->getBasePtr(), LD->getPointerInfo(), LD->getAlign(),
9552 LD->getMemOperand()->getFlags());
9558 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
9559 LD->getMemoryVT(), LD->getAlign(),
9560 LD->getMemOperand()->getFlags());
9573 N->getOperand(0)->getValueType(0),
9579 "Expected extending load");
9585 DAG.
getNode(Opcode,
SDLoc(newLoad), LD->getValueType(0), newLoad);
9594 SDNode *BVN =
N->getOperand(0).getNode();
9596 BVN->
getValueType(0) == MVT::v4i32 &&
"expected v4i32 BUILD_VECTOR");
9604 EVT VT =
N->getValueType(0);
9610 for (
unsigned i = 0; i != NumElts; ++i) {
9611 const APInt &CInt =
N->getConstantOperandAPInt(i);
9620 unsigned Opcode =
N->getOpcode();
9622 SDNode *N0 =
N->getOperand(0).getNode();
9623 SDNode *N1 =
N->getOperand(1).getNode();
9631 unsigned Opcode =
N->getOpcode();
9633 SDNode *N0 =
N->getOperand(0).getNode();
9634 SDNode *N1 =
N->getOperand(1).getNode();
9644 EVT VT =
Op.getValueType();
9646 "unexpected type for custom-lowering ISD::MUL");
9647 SDNode *N0 =
Op.getOperand(0).getNode();
9648 SDNode *N1 =
Op.getOperand(1).getNode();
9649 unsigned NewOpc = 0;
9653 if (isN0SExt && isN1SExt)
9658 if (isN0ZExt && isN1ZExt)
9660 else if (isN1SExt || isN1ZExt) {
9677 if (VT == MVT::v2i64)
9694 "unexpected types for extended operands to VMULL");
9695 return DAG.
getNode(NewOpc,
DL, VT, Op0, Op1);
9730 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9764 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9767 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9788 EVT VT =
Op.getValueType();
9789 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9790 "unexpected type for custom-lowering ISD::SDIV");
9797 if (VT == MVT::v8i8) {
9825 EVT VT =
Op.getValueType();
9826 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9827 "unexpected type for custom-lowering ISD::UDIV");
9834 if (VT == MVT::v8i8) {
9873 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9876 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9880 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9901 EVT VT =
N->getValueType(0);
9915 Op.getOperand(1), Carry);
9929 Op.getOperand(1), Carry);
9962 bool ShouldUseSRet = Subtarget->
isAPCS_ABI();
9964 if (ShouldUseSRet) {
9974 Entry.IsSExt =
false;
9975 Entry.IsZExt =
false;
9976 Entry.IsSRet =
true;
9977 Args.push_back(Entry);
9984 Entry.IsSExt =
false;
9985 Entry.IsZExt =
false;
9986 Args.push_back(Entry);
9989 (ArgVT == MVT::f64) ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
9999 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
10001 if (!ShouldUseSRet)
10002 return CallResult.first;
10021 EVT VT =
Op.getValueType();
10022 assert((VT == MVT::i32 || VT == MVT::i64) &&
10023 "unexpected type for custom lowering DIV");
10029 const char *
Name =
nullptr;
10031 Name = (VT == MVT::i32) ?
"__rt_sdiv" :
"__rt_sdiv64";
10033 Name = (VT == MVT::i32) ?
"__rt_udiv" :
"__rt_udiv64";
10039 for (
auto AI : {1, 0}) {
10041 Arg.Node =
Op.getOperand(AI);
10042 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.
getContext());
10043 Args.push_back(Arg);
10046 CallLoweringInfo CLI(DAG);
10050 ES, std::move(Args));
10060ARMTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
10068 const bool MinSize =
ST.hasMinSize();
10069 const bool HasDivide =
ST.isThumb() ?
ST.hasDivideInThumbMode()
10070 :
ST.hasDivideInARMMode();
10074 if (
N->getOperand(0).getValueType().isVector())
10079 if (!(MinSize && HasDivide))
10092 if (Divisor.
sgt(128))
10100 assert(
Op.getValueType() == MVT::i32 &&
10101 "unexpected type for custom lowering DIV");
10107 return LowerWindowsDIVLibCall(
Op, DAG,
Signed, DBZCHK);
10113 if (
N->getValueType(0) == MVT::i32)
10121void ARMTargetLowering::ExpandDIV_Windows(
10127 assert(
Op.getValueType() == MVT::i64 &&
10128 "unexpected type for custom lowering DIV");
10145 EVT MemVT = LD->getMemoryVT();
10146 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10147 MemVT == MVT::v16i1) &&
10148 "Expected a predicate type!");
10149 assert(MemVT ==
Op.getValueType());
10151 "Expected a non-extending load");
10152 assert(LD->isUnindexed() &&
"Expected a unindexed load");
10166 ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(),
10168 LD->getMemOperand());
10175 if (MemVT != MVT::v16i1)
10184 EVT MemVT =
LD->getMemoryVT();
10185 assert(
LD->isUnindexed() &&
"Loads should be unindexed at this point.");
10187 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10193 {LD->getChain(), LD->getBasePtr()}, MemVT,
LD->getMemOperand());
10203 EVT MemVT = ST->getMemoryVT();
10204 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10205 MemVT == MVT::v16i1) &&
10206 "Expected a predicate type!");
10207 assert(MemVT == ST->getValue().getValueType());
10208 assert(!ST->isTruncatingStore() &&
"Expected a non-extending store");
10209 assert(ST->isUnindexed() &&
"Expected a unindexed store");
10214 SDValue Build = ST->getValue();
10215 if (MemVT != MVT::v16i1) {
10234 ST->getChain(), dl, GRP, ST->getBasePtr(),
10236 ST->getMemOperand());
10242 EVT MemVT = ST->getMemoryVT();
10243 assert(ST->isUnindexed() &&
"Stores should be unindexed at this point.");
10245 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10261 {ST->getChain(), Lo, Hi, ST->getBasePtr()},
10262 MemVT, ST->getMemOperand());
10263 }
else if (Subtarget->hasMVEIntegerOps() &&
10264 ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10265 MemVT == MVT::v16i1))) {
10280 MVT VT =
Op.getSimpleValueType();
10282 SDValue PassThru =
N->getPassThru();
10293 VT, dl,
N->getChain(),
N->getBasePtr(),
N->getOffset(), Mask, ZeroVec,
10294 N->getMemoryVT(),
N->getMemOperand(),
N->getAddressingMode(),
10295 N->getExtensionType(),
N->isExpandingLoad());
10300 if (!PassThru.
isUndef() && !PassThruIsCastZero)
10307 if (!ST->hasMVEIntegerOps())
10311 unsigned BaseOpcode = 0;
10312 switch (
Op->getOpcode()) {
10328 unsigned NumActiveLanes = NumElts;
10330 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10331 NumActiveLanes == 2) &&
10332 "Only expected a power 2 vector size");
10336 while (NumActiveLanes > 4) {
10339 Op0 = DAG.
getNode(BaseOpcode, dl, VT, Op0, Rev);
10340 NumActiveLanes /= 2;
10344 if (NumActiveLanes == 4) {
10354 SDValue Res0 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10355 SDValue Res1 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext2, Ext3,
Op->getFlags());
10356 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Res0, Res1,
Op->getFlags());
10362 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10366 if (EltVT !=
Op->getValueType(0))
10373 if (!ST->hasMVEFloatOps())
10380 if (!ST->hasNEON())
10388 unsigned PairwiseIntrinsic = 0;
10389 switch (
Op->getOpcode()) {
10393 PairwiseIntrinsic = Intrinsic::arm_neon_vpminu;
10396 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxu;
10399 PairwiseIntrinsic = Intrinsic::arm_neon_vpmins;
10402 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxs;
10408 unsigned NumActiveLanes = NumElts;
10410 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10411 NumActiveLanes == 2) &&
10412 "Only expected a power 2 vector size");
10418 VT =
Lo.getValueType();
10420 NumActiveLanes /= 2;
10424 while (NumActiveLanes > 1) {
10426 NumActiveLanes /= 2;
10433 if (EltVT !=
Op.getValueType()) {
10434 unsigned Extend = 0;
10435 switch (
Op->getOpcode()) {
10447 Res = DAG.
getNode(Extend, dl,
Op.getValueType(), Res);
10469 SDValue Ops[] = {
N->getOperand(0),
10479 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
10486 SDLoc dl(V.getNode());
10487 auto [VLo, VHi] = DAG.
SplitScalar(V, dl, MVT::i32, MVT::i32);
10495 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
10497 DAG.
getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
10503 assert(
N->getValueType(0) == MVT::i64 &&
10504 "AtomicCmpSwap on types less than 64 should be legal");
10505 SDValue Ops[] = {
N->getOperand(1),
10510 ARM::CMP_SWAP_64,
SDLoc(
N),
10511 DAG.
getVTList(MVT::Untyped, MVT::i32, MVT::Other), Ops);
10530 EVT VT =
Op.getValueType();
10539 if (isUnsupportedFloatingType(
LHS.getValueType())) {
10541 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS, Chain, IsSignaling);
10542 if (!
RHS.getNode()) {
10563 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10564 SDValue Result = getCMOV(dl, VT, False, True, ARMcc, CCR, Cmp, DAG);
10566 ARMcc = DAG.
getConstant(CondCode2, dl, MVT::i32);
10567 Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10568 Result = getCMOV(dl, VT, Result, True, ARMcc, CCR, Cmp, DAG);
10584 switch (
Op.getOpcode()) {
10616 case ISD::BITCAST:
return ExpandBITCAST(
Op.getNode(), DAG, Subtarget);
10620 case ISD::SREM:
return LowerREM(
Op.getNode(), DAG);
10621 case ISD::UREM:
return LowerREM(
Op.getNode(), DAG);
10643 return LowerSET_FPMODE(
Op, DAG);
10645 return LowerRESET_FPMODE(
Op, DAG);
10649 return LowerDIV_Windows(
Op, DAG,
true);
10653 return LowerDIV_Windows(
Op, DAG,
false);
10660 return LowerSignedALUO(
Op, DAG);
10663 return LowerUnsignedALUO(
Op, DAG);
10697 return LowerDYNAMIC_STACKALLOC(
Op, DAG);
10706 return LowerSPONENTRY(
Op, DAG);
10713 unsigned IntNo =
N->getConstantOperandVal(0);
10715 if (IntNo == Intrinsic::arm_smlald)
10717 else if (IntNo == Intrinsic::arm_smlaldx)
10719 else if (IntNo == Intrinsic::arm_smlsld)
10721 else if (IntNo == Intrinsic::arm_smlsldx)
10728 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(3), dl, MVT::i32, MVT::i32);
10732 N->getOperand(1),
N->getOperand(2),
10744 switch (
N->getOpcode()) {
10751 Res = ExpandBITCAST(
N, DAG, Subtarget);
10760 Res = LowerREM(
N, DAG);
10764 Res = LowerDivRem(
SDValue(
N, 0), DAG);
10818 "ROPI/RWPI not currently supported with SjLj");
10827 bool isThumb = Subtarget->isThumb();
10828 bool isThumb2 = Subtarget->
isThumb2();
10831 unsigned PCAdj = (
isThumb || isThumb2) ? 4 : 8;
10837 : &ARM::GPRRegClass;
10855 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10861 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10867 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10885 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10890 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10895 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10900 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
10906 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
10921 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10927 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10951 : &ARM::GPRnopcRegClass;
10956 unsigned MaxCSNum = 0;
10964 if (!
II.isEHLabel())
10968 if (!MF->hasCallSiteLandingPad(
Sym))
continue;
10971 for (
unsigned Idx : CallSiteIdxs) {
10972 CallSiteNumToLPad[
Idx].push_back(&BB);
10973 MaxCSNum = std::max(MaxCSNum,
Idx);
10980 std::vector<MachineBasicBlock*> LPadList;
10982 LPadList.reserve(CallSiteNumToLPad.
size());
10983 for (
unsigned I = 1;
I <= MaxCSNum; ++
I) {
10986 LPadList.push_back(
MBB);
10991 assert(!LPadList.empty() &&
10992 "No landing pad destinations for the dispatch jump table!");
11006 unsigned trap_opcode;
11007 if (Subtarget->isThumb())
11008 trap_opcode = ARM::tTRAP;
11010 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP;
11019 MF->insert(MF->end(), DispatchBB);
11020 MF->insert(MF->end(), DispContBB);
11021 MF->insert(MF->end(), TrapBB);
11025 SetupEntryBlockForSjLj(
MI,
MBB, DispatchBB, FI);
11032 MIB =
BuildMI(DispatchBB, dl,
TII->get(ARM::Int_eh_sjlj_dispatchsetup));
11044 unsigned NumLPads = LPadList.size();
11046 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11047 BuildMI(DispatchBB, dl,
TII->get(ARM::t2LDRi12), NewVReg1)
11053 if (NumLPads < 256) {
11054 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPri))
11056 .
addImm(LPadList.size())
11059 Register VReg1 =
MRI->createVirtualRegister(TRC);
11060 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVi16), VReg1)
11061 .
addImm(NumLPads & 0xFFFF)
11064 unsigned VReg2 = VReg1;
11065 if ((NumLPads & 0xFFFF0000) != 0) {
11066 VReg2 =
MRI->createVirtualRegister(TRC);
11067 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVTi16), VReg2)
11073 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPrr))
11079 BuildMI(DispatchBB, dl,
TII->get(ARM::t2Bcc))
11084 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11085 BuildMI(DispContBB, dl,
TII->get(ARM::t2LEApcrelJT), NewVReg3)
11089 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11090 BuildMI(DispContBB, dl,
TII->get(ARM::t2ADDrs), NewVReg4)
11097 BuildMI(DispContBB, dl,
TII->get(ARM::t2BR_JT))
11101 }
else if (Subtarget->isThumb()) {
11102 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11103 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRspi), NewVReg1)
11109 if (NumLPads < 256) {
11110 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPi8))
11117 const Constant *
C = ConstantInt::get(Int32Ty, NumLPads);
11120 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
11123 Register VReg1 =
MRI->createVirtualRegister(TRC);
11124 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRpci))
11128 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPr))
11134 BuildMI(DispatchBB, dl,
TII->get(ARM::tBcc))
11139 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
11140 BuildMI(DispContBB, dl,
TII->get(ARM::tLSLri), NewVReg2)
11146 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11147 BuildMI(DispContBB, dl,
TII->get(ARM::tLEApcrelJT), NewVReg3)
11151 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11152 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg4)
11162 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11163 BuildMI(DispContBB, dl,
TII->get(ARM::tLDRi), NewVReg5)
11169 unsigned NewVReg6 = NewVReg5;
11170 if (IsPositionIndependent) {
11171 NewVReg6 =
MRI->createVirtualRegister(TRC);
11172 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg6)
11179 BuildMI(DispContBB, dl,
TII->get(ARM::tBR_JTr))
11183 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11184 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRi12), NewVReg1)
11190 if (NumLPads < 256) {
11191 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPri))
11195 }
else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) {
11196 Register VReg1 =
MRI->createVirtualRegister(TRC);
11197 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVi16), VReg1)
11198 .
addImm(NumLPads & 0xFFFF)
11201 unsigned VReg2 = VReg1;
11202 if ((NumLPads & 0xFFFF0000) != 0) {
11203 VReg2 =
MRI->createVirtualRegister(TRC);
11204 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVTi16), VReg2)
11210 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11217 const Constant *
C = ConstantInt::get(Int32Ty, NumLPads);
11220 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
11223 Register VReg1 =
MRI->createVirtualRegister(TRC);
11224 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRcp))
11229 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11240 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11241 BuildMI(DispContBB, dl,
TII->get(ARM::MOVsi), NewVReg3)
11246 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11247 BuildMI(DispContBB, dl,
TII->get(ARM::LEApcrelJT), NewVReg4)
11254 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11255 BuildMI(DispContBB, dl,
TII->get(ARM::LDRrs), NewVReg5)
11262 if (IsPositionIndependent) {
11263 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTadd))
11268 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTr))
11277 if (SeenMBBs.
insert(CurMBB).second)
11289 while (!Successors.empty()) {
11298 BB->normalizeSuccProbs();
11305 II = BB->rbegin(), IE = BB->rend();
II != IE; ++
II) {
11306 if (!
II->isCall())
continue;
11310 OI =
II->operands_begin(), OE =
II->operands_end();
11312 if (!OI->isReg())
continue;
11313 DefRegs[OI->getReg()] =
true;
11318 for (
unsigned i = 0; SavedRegs[i] != 0; ++i) {
11319 unsigned Reg = SavedRegs[i];
11321 !ARM::tGPRRegClass.contains(Reg) &&
11322 !ARM::hGPRRegClass.contains(Reg))
11324 if (Subtarget->
isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
11326 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
11339 MBBLPad->setIsEHPad(
false);
11342 MI.eraseFromParent();
11355static unsigned getLdOpcode(
unsigned LdSize,
bool IsThumb1,
bool IsThumb2) {
11357 return LdSize == 16 ? ARM::VLD1q32wb_fixed
11358 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
11360 return LdSize == 4 ? ARM::tLDRi
11361 : LdSize == 2 ? ARM::tLDRHi
11362 : LdSize == 1 ? ARM::tLDRBi : 0;
11364 return LdSize == 4 ? ARM::t2LDR_POST
11365 : LdSize == 2 ? ARM::t2LDRH_POST
11366 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
11367 return LdSize == 4 ? ARM::LDR_POST_IMM
11368 : LdSize == 2 ? ARM::LDRH_POST
11369 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
11374static unsigned getStOpcode(
unsigned StSize,
bool IsThumb1,
bool IsThumb2) {
11376 return StSize == 16 ? ARM::VST1q32wb_fixed
11377 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
11379 return StSize == 4 ? ARM::tSTRi
11380 : StSize == 2 ? ARM::tSTRHi
11381 : StSize == 1 ? ARM::tSTRBi : 0;
11383 return StSize == 4 ? ARM::t2STR_POST
11384 : StSize == 2 ? ARM::t2STRH_POST
11385 : StSize == 1 ? ARM::t2STRB_POST : 0;
11386 return StSize == 4 ? ARM::STR_POST_IMM
11387 : StSize == 2 ? ARM::STRH_POST
11388 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
11395 unsigned LdSize,
unsigned Data,
unsigned AddrIn,
11396 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11397 unsigned LdOpc =
getLdOpcode(LdSize, IsThumb1, IsThumb2);
11398 assert(LdOpc != 0 &&
"Should have a load opcode");
11405 }
else if (IsThumb1) {
11411 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11416 }
else if (IsThumb2) {
11436 unsigned StSize,
unsigned Data,
unsigned AddrIn,
11437 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11438 unsigned StOpc =
getStOpcode(StSize, IsThumb1, IsThumb2);
11439 assert(StOpc != 0 &&
"Should have a store opcode");
11441 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11446 }
else if (IsThumb1) {
11453 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11458 }
else if (IsThumb2) {
11459 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11465 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11486 unsigned SizeVal =
MI.getOperand(2).getImm();
11487 unsigned Alignment =
MI.getOperand(3).getImm();
11492 unsigned UnitSize = 0;
11497 bool IsThumb2 = Subtarget->
isThumb2();
11498 bool IsThumb = Subtarget->isThumb();
11500 if (Alignment & 1) {
11502 }
else if (Alignment & 2) {
11507 Subtarget->hasNEON()) {
11508 if ((Alignment % 16 == 0) && SizeVal >= 16)
11510 else if ((Alignment % 8 == 0) && SizeVal >= 8)
11519 bool IsNeon = UnitSize >= 8;
11520 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
11522 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
11523 : UnitSize == 8 ? &ARM::DPRRegClass
11526 unsigned BytesLeft = SizeVal % UnitSize;
11527 unsigned LoopSize = SizeVal - BytesLeft;
11529 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
11533 unsigned srcIn = src;
11534 unsigned destIn = dest;
11535 for (
unsigned i = 0; i < LoopSize; i+=UnitSize) {
11536 Register srcOut =
MRI.createVirtualRegister(TRC);
11537 Register destOut =
MRI.createVirtualRegister(TRC);
11538 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11540 IsThumb1, IsThumb2);
11542 IsThumb1, IsThumb2);
11550 for (
unsigned i = 0; i < BytesLeft; i++) {
11551 Register srcOut =
MRI.createVirtualRegister(TRC);
11552 Register destOut =
MRI.createVirtualRegister(TRC);
11553 Register scratch =
MRI.createVirtualRegister(TRC);
11555 IsThumb1, IsThumb2);
11557 IsThumb1, IsThumb2);
11561 MI.eraseFromParent();
11587 MF->
insert(It, loopMBB);
11588 MF->
insert(It, exitMBB);
11591 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
11601 Register varEnd =
MRI.createVirtualRegister(TRC);
11603 BuildMI(BB, dl,
TII->get(IsThumb ? ARM::t2MOVi32imm : ARM::MOVi32imm),
11606 }
else if (Subtarget->genExecuteOnly()) {
11607 assert(IsThumb &&
"Non-thumb expected to have used movt");
11612 const Constant *
C = ConstantInt::get(Int32Ty, LoopSize);
11643 Register varLoop =
MRI.createVirtualRegister(TRC);
11644 Register varPhi =
MRI.createVirtualRegister(TRC);
11645 Register srcLoop =
MRI.createVirtualRegister(TRC);
11646 Register srcPhi =
MRI.createVirtualRegister(TRC);
11647 Register destLoop =
MRI.createVirtualRegister(TRC);
11648 Register destPhi =
MRI.createVirtualRegister(TRC);
11656 BuildMI(BB, dl,
TII->get(ARM::PHI), destPhi)
11662 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11664 IsThumb1, IsThumb2);
11666 IsThumb1, IsThumb2);
11670 BuildMI(*BB, BB->
end(), dl,
TII->get(ARM::tSUBi8), varLoop)
11678 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop);
11687 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
11696 auto StartOfExit = exitMBB->
begin();
11700 unsigned srcIn = srcLoop;
11701 unsigned destIn = destLoop;
11702 for (
unsigned i = 0; i < BytesLeft; i++) {
11703 Register srcOut =
MRI.createVirtualRegister(TRC);
11704 Register destOut =
MRI.createVirtualRegister(TRC);
11705 Register scratch =
MRI.createVirtualRegister(TRC);
11706 emitPostLd(BB, StartOfExit,
TII, dl, 1, scratch, srcIn, srcOut,
11707 IsThumb1, IsThumb2);
11708 emitPostSt(BB, StartOfExit,
TII, dl, 1, scratch, destIn, destOut,
11709 IsThumb1, IsThumb2);
11714 MI.eraseFromParent();
11726 "__chkstk is only supported on Windows");
11727 assert(Subtarget->
isThumb2() &&
"Windows on ARM requires Thumb-2 mode");
11747 switch (
TM.getCodeModel()) {
11789 MI.eraseFromParent();
11813 .
addReg(
MI.getOperand(0).getReg())
11821 MI.eraseFromParent();
11845 if (miI == BB->
end()) {
11847 if (Succ->isLiveIn(ARM::CPSR))
11853 SelectItr->addRegisterKilled(ARM::CPSR,
TRI);
11865 Register AddDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11866 BuildMI(TpEntry, Dl,
TII->get(ARM::t2ADDri), AddDestReg)
11872 Register LsrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11873 BuildMI(TpEntry, Dl,
TII->get(ARM::t2LSRri), LsrDestReg)
11879 Register TotalIterationsReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11880 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopSetup), TotalIterationsReg)
11883 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopStart))
11884 .
addUse(TotalIterationsReg)
11891 return TotalIterationsReg;
11902 Register TotalIterationsReg,
bool IsMemcpy) {
11909 SrcPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11910 CurrSrcReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11911 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), SrcPhiReg)
11919 Register DestPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11920 Register CurrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11921 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), DestPhiReg)
11928 Register LoopCounterPhiReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11929 Register RemainingLoopIterationsReg =
11930 MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11931 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), LoopCounterPhiReg)
11932 .
addUse(TotalIterationsReg)
11934 .
addUse(RemainingLoopIterationsReg)
11938 Register PredCounterPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11939 Register RemainingElementsReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11940 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), PredCounterPhiReg)
11941 .
addUse(ElementCountReg)
11943 .
addUse(RemainingElementsReg)
11947 Register VccrReg =
MRI.createVirtualRegister(&ARM::VCCRRegClass);
11948 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VCTP8), VccrReg)
11949 .
addUse(PredCounterPhiReg)
11954 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2SUBri), RemainingElementsReg)
11955 .
addUse(PredCounterPhiReg)
11963 SrcValueReg =
MRI.createVirtualRegister(&ARM::MQPRRegClass);
11964 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VLDRBU8_post))
11973 SrcValueReg = OpSrcReg;
11975 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VSTRBU8_post))
11986 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopDec), RemainingLoopIterationsReg)
11987 .
addUse(LoopCounterPhiReg)
11990 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopEnd))
11991 .
addUse(RemainingLoopIterationsReg)
12004 bool isThumb2 = Subtarget->
isThumb2();
12005 switch (
MI.getOpcode()) {
12012 case ARM::tLDR_postidx: {
12016 .
add(
MI.getOperand(2))
12017 .
add(
MI.getOperand(3))
12018 .
add(
MI.getOperand(4))
12019 .
add(
MI.getOperand(0))
12021 MI.eraseFromParent();
12025 case ARM::MVE_MEMCPYLOOPINST:
12026 case ARM::MVE_MEMSETLOOPINST: {
12056 Register OpDestReg =
MI.getOperand(0).getReg();
12057 Register OpSrcReg =
MI.getOperand(1).getReg();
12058 Register OpSizeReg =
MI.getOperand(2).getReg();
12078 if (TpExit == BB) {
12080 "block containing memcpy/memset Pseudo");
12093 bool IsMemcpy =
MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
12095 OpDestReg, OpSizeReg, TotalIterationsReg, IsMemcpy);
12110 MI.eraseFromParent();
12120 case ARM::t2STR_preidx:
12121 MI.setDesc(
TII->get(ARM::t2STR_PRE));
12123 case ARM::t2STRB_preidx:
12124 MI.setDesc(
TII->get(ARM::t2STRB_PRE));
12126 case ARM::t2STRH_preidx:
12127 MI.setDesc(
TII->get(ARM::t2STRH_PRE));
12130 case ARM::STRi_preidx:
12131 case ARM::STRBi_preidx: {
12132 unsigned NewOpc =
MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
12133 : ARM::STRB_PRE_IMM;
12135 unsigned Offset =
MI.getOperand(4).getImm();
12143 .
add(
MI.getOperand(0))
12144 .
add(
MI.getOperand(1))
12145 .
add(
MI.getOperand(2))
12147 .
add(
MI.getOperand(5))
12148 .
add(
MI.getOperand(6))
12150 MI.eraseFromParent();
12153 case ARM::STRr_preidx:
12154 case ARM::STRBr_preidx:
12155 case ARM::STRH_preidx: {
12157 switch (
MI.getOpcode()) {
12159 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG;
break;
12160 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG;
break;
12161 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE;
break;
12166 MI.eraseFromParent();
12170 case ARM::tMOVCCr_pseudo: {
12188 F->insert(It, copy0MBB);
12189 F->insert(It, sinkMBB);
12192 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
12198 if (!
MI.killsRegister(ARM::CPSR,
nullptr) &&
12214 .
addImm(
MI.getOperand(3).getImm())
12215 .
addReg(
MI.getOperand(4).getReg());
12230 .
addReg(
MI.getOperand(1).getReg())
12232 .
addReg(
MI.getOperand(2).getReg())
12235 MI.eraseFromParent();
12240 case ARM::BCCZi64: {
12246 bool RHSisZero =
MI.getOpcode() == ARM::BCCZi64;
12251 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12255 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12261 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12265 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12275 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
12284 MI.eraseFromParent();
12288 case ARM::Int_eh_sjlj_setjmp:
12289 case ARM::Int_eh_sjlj_setjmp_nofp:
12290 case ARM::tInt_eh_sjlj_setjmp:
12291 case ARM::t2Int_eh_sjlj_setjmp:
12292 case ARM::t2Int_eh_sjlj_setjmp_nofp:
12295 case ARM::Int_eh_sjlj_setup_dispatch:
12296 EmitSjLjDispatchBlock(
MI, BB);
12319 Fn->
insert(BBI, SinkBB);
12321 Register ABSSrcReg =
MI.getOperand(1).getReg();
12322 Register ABSDstReg =
MI.getOperand(0).getReg();
12323 bool ABSSrcKIll =
MI.getOperand(1).isKill();
12324 bool isThumb2 = Subtarget->
isThumb2();
12328 Register NewRsbDstReg =
MRI.createVirtualRegister(
12329 isThumb2 ? &ARM::rGPRRegClass : &ARM::GPRRegClass);
12343 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12350 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).
addMBB(SinkBB)
12357 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg)
12366 TII->get(ARM::PHI), ABSDstReg)
12371 MI.eraseFromParent();
12376 case ARM::COPY_STRUCT_BYVAL_I32:
12378 return EmitStructByval(
MI, BB);
12379 case ARM::WIN__CHKSTK:
12380 return EmitLowered__chkstk(
MI, BB);
12381 case ARM::WIN__DBZCHK:
12382 return EmitLowered__dbzchk(
MI, BB);
12399 if (!
Node->hasAnyUseOfValue(0)) {
12400 MI.getOperand(0).setIsDead(
true);
12402 if (!
Node->hasAnyUseOfValue(1)) {
12403 MI.getOperand(1).setIsDead(
true);
12407 for (
unsigned I = 0;
I !=
MI.getOperand(4).getImm(); ++
I) {
12408 Register TmpReg =
MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass
12409 : &ARM::GPRRegClass);
12416 if (
MI.getOpcode() == ARM::MEMCPY) {
12434 MCID = &
TII->get(NewOpc);
12437 MI.getDesc().getNumOperands() + 5 -
MI.getDesc().getSize()
12438 &&
"converted opcode should be the same except for cc_out"
12439 " (and, on Thumb1, pred)");
12449 MI.addOperand(
MI.getOperand(1));
12450 MI.removeOperand(1);
12454 for (
unsigned i =
MI.getNumOperands(); i--;) {
12456 if (
op.isReg() &&
op.isUse()) {
12459 MI.tieOperands(DefIdx, i);
12473 if (!
MI.hasOptionalDef() || !MCID->
operands()[ccOutIdx].isOptionalDef()) {
12474 assert(!NewOpc &&
"Optional cc_out operand required");
12479 bool definesCPSR =
false;
12480 bool deadCPSR =
false;
12481 for (
unsigned i = MCID->
getNumOperands(), e =
MI.getNumOperands(); i != e;
12485 definesCPSR =
true;
12488 MI.removeOperand(i);
12492 if (!definesCPSR) {
12493 assert(!NewOpc &&
"Optional cc_out operand required");
12496 assert(deadCPSR == !Node->hasAnyUseOfValue(1) &&
"inconsistent dead flag");
12498 assert(!
MI.getOperand(ccOutIdx).getReg() &&
12499 "expect uninitialized optional cc_out operand");
12537 switch (
N->getOpcode()) {
12538 default:
return false;
12540 CC =
N->getOperand(0);
12562 EVT VT =
N->getValueType(0);
12563 CC =
N->getOperand(0);
12564 if (
CC.getValueType() != MVT::i1 ||
CC.getOpcode() !=
ISD::SETCC)
12610 EVT VT =
N->getValueType(0);
12613 bool SwapSelectOps;
12615 NonConstantVal, DAG))
12621 OtherOp, NonConstantVal);
12627 CCOp, TrueVal, FalseVal);
12651 if (
N->getOpcode() ==
ARMISD::VTRN &&
N->getValueType(0) == MVT::v2i32)
12666 if (!
N->getValueType(0).is64BitVector())
12674 EVT VT =
N->getValueType(0);
12713 EVT VT =
N->getValueType(0);
12719 Opcode = Intrinsic::arm_neon_vpaddls;
12721 Opcode = Intrinsic::arm_neon_vpaddlu;
12749 EVT VT =
N->getValueType(0);
12764 unsigned nextIndex = 0;
12832 return DAG.
getNode(ExtOp, dl, VT, tmp);
12863 if (SRA.getOpcode() !=
ISD::SRA) {
12869 if (
auto Const = dyn_cast<ConstantSDNode>(SRA.getOperand(1))) {
12870 if (Const->getZExtValue() != 31)
12875 if (SRA.getOperand(0) !=
Mul)
12879 SDLoc dl(AddcNode);
12880 unsigned Opcode = 0;
12886 Op0 =
Mul.getOperand(0);
12887 Op1 =
Mul.getOperand(1);
12890 Op0 =
Mul.getOperand(0);
12891 Op1 =
Mul.getOperand(1).getOperand(0);
12894 Op0 =
Mul.getOperand(0).getOperand(0);
12895 Op1 =
Mul.getOperand(1);
12898 Op0 =
Mul->getOperand(0).getOperand(0);
12899 Op1 =
Mul->getOperand(1).getOperand(0);
12908 SDValue HiMLALResult(SMLAL.getNode(), 1);
12909 SDValue LoMLALResult(SMLAL.getNode(), 0);
12915 SDValue resNode(AddcNode, 0);
12944 "Expect an ADDE or SUBE");
12948 "ADDE node has the wrong inputs");
12967 "Expect ADDC with two result values. First: i32");
12987 bool IsLeftOperandMUL =
false;
12992 IsLeftOperandMUL =
true;
13003 SDValue *LowAddSub =
nullptr;
13006 if ((AddeSubeOp0 != MULOp.
getValue(1)) && (AddeSubeOp1 != MULOp.
getValue(1)))
13009 if (IsLeftOperandMUL)
13010 HiAddSub = &AddeSubeOp1;
13012 HiAddSub = &AddeSubeOp0;
13017 if (AddcSubcOp0 == MULOp.
getValue(0)) {
13018 LoMul = &AddcSubcOp0;
13019 LowAddSub = &AddcSubcOp1;
13021 if (AddcSubcOp1 == MULOp.
getValue(0)) {
13022 LoMul = &AddcSubcOp1;
13023 LowAddSub = &AddcSubcOp0;
13031 if (AddcSubcNode == HiAddSub->getNode() ||
13047 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->
useMulOps() &&
13061 return SDValue(AddeSubeNode, 0);
13072 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13082 return SDValue(AddeSubeNode, 0);
13094 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13103 SDNode *UmlalNode =
nullptr;
13128 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13142 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13147 SDNode* AddcNode =
N->getOperand(2).getNode();
13148 SDNode* AddeNode =
N->getOperand(3).getNode();
13156 {N->getOperand(0), N->getOperand(1),
13157 AddcNode->getOperand(0), AddcNode->getOperand(1)});
13181 int32_t imm =
C->getSExtValue();
13182 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
13187 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
N->getOperand(0),
RHS);
13202 int64_t imm =
C->getSExtValue();
13213 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
13214 N->getOperand(0),
RHS,
N->getOperand(2));
13226 if (!Subtarget->hasMVEIntegerOps())
13239 SetCC =
N->getOperand(0);
13242 CC = cast<CondCodeSDNode>(SetCC->
getOperand(2))->get();
13243 TrueVal =
N->getOperand(1);
13244 FalseVal =
N->getOperand(2);
13246 LHS =
N->getOperand(0);
13247 RHS =
N->getOperand(1);
13248 CC = cast<CondCodeSDNode>(
N->getOperand(4))->get();
13249 TrueVal =
N->getOperand(2);
13250 FalseVal =
N->getOperand(3);
13255 unsigned int Opcode = 0;
13284 switch (TrueVal->getOpcode()) {
13303 if (TrueVal !=
LHS || FalseVal !=
RHS)
13306 EVT LeftType =
LHS->getValueType(0);
13307 EVT RightType =
RHS->getValueType(0);
13310 if (LeftType != VectorScalarType || RightType != VectorScalarType)
13314 if (VectorScalarType != MVT::i32)
13322 if (VectorScalarType != MVT::i32)
13335 EVT VT =
N->getValueType(0);
13343 Shft =
N->getOperand(0);
13349 cast<CondCodeSDNode>(Cmp.getOperand(2))->get() !=
ISD::SETLT ||
13350 Cmp.getOperand(0) !=
N->getOperand(1) ||
13351 Cmp.getOperand(1) !=
N->getOperand(2))
13353 Shft =
N->getOperand(1);
13365 ScalarType = MVT::i8;
13368 case (1 << 15) - 1:
13369 ScalarType = MVT::i16;
13372 case (1ULL << 31) - 1:
13373 ScalarType = MVT::i32;
13404 unsigned LegalLanes = 128 / (ShftAmt + 1);
13428 for (
unsigned I = 0;
I < NumParts; ++
I) {
13445 if (!Subtarget->hasMVEIntegerOps())
13460 if (
N->getOperand(0).getOpcode() !=
ISD::XOR)
13470 if (!Const || !Const->isOne())
13488 EVT VT =
N->getValueType(0);
13490 if (!Subtarget->hasMVEIntegerOps() ||
13519 Opc = Intrinsic::arm_mve_vctp64;
13522 Opc = Intrinsic::arm_mve_vctp32;
13525 Opc = Intrinsic::arm_mve_vctp16;
13528 Opc = Intrinsic::arm_mve_vctp8;
13582 EVT VT =
N->getValueType(0);
13588 switch (
Op.getOpcode()) {
13605 !isa<ConstantSDNode>(N0) && N1->
hasOneUse()) {
13613 unsigned N0RedOp = 0;
13620 unsigned N1RedOp = 0;
13634 if (
SDValue R = DistrubuteAddAddVecReduce(N0, N1))
13636 if (
SDValue R = DistrubuteAddAddVecReduce(N1, N0))
13643 auto DistrubuteVecReduceLoad = [&](
SDValue N0,
SDValue N1,
bool IsForward) {
13657 LoadSDNode *Load0 = dyn_cast<LoadSDNode>(N0);
13658 LoadSDNode *Load1 = dyn_cast<LoadSDNode>(N1);
13667 if (!BaseLocDecomp0.getBase() ||
13668 BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
13669 !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
13671 if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
13673 if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
13683 if (IsBefore < 0) {
13686 }
else if (IsBefore > 0) {
13699 }
else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
13709 if (!IsVecReduce(N0) || !IsVecReduce(N1))
13719 if (
SDValue R = DistrubuteVecReduceLoad(N0, N1,
true))
13721 if (
SDValue R = DistrubuteVecReduceLoad(N1, N0,
false))
13728 if (!Subtarget->hasMVEIntegerOps())
13734 EVT VT =
N->getValueType(0);
13739 if (VT != MVT::i64)
13750 auto MakeVecReduce = [&](
unsigned Opcode,
unsigned OpcodeA,
SDValue NA,
13768 std::tie(Ops[0], Ops[1]) = DAG.
SplitScalar(NA, dl, MVT::i32, MVT::i32);
13770 unsigned S = VecRed->
getOpcode() == OpcodeA ? 2 : 0;
13819 "Expected shift op");
13837 if (
auto *Const = dyn_cast<ConstantSDNode>(N1->
getOperand(1))) {
13838 if (Const->getAPIntValue().ult(256))
13841 Const->getAPIntValue().sgt(-256))
13857 (
N->getOperand(0).getOpcode() ==
ISD::SHL ||
13858 N->getOperand(0).getOpcode() ==
ISD::SRL) &&
13859 "Expected XOR(SHIFT) pattern");
13862 auto *XorC = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13863 auto *ShiftC = dyn_cast<ConstantSDNode>(
N->getOperand(0).getOperand(1));
13864 if (XorC && ShiftC) {
13865 unsigned MaskIdx, MaskLen;
13866 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) {
13867 unsigned ShiftAmt = ShiftC->getZExtValue();
13868 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
13869 if (
N->getOperand(0).getOpcode() ==
ISD::SHL)
13870 return MaskIdx == ShiftAmt && MaskLen == (
BitWidth - ShiftAmt);
13871 return MaskIdx == 0 && MaskLen == (
BitWidth - ShiftAmt);
13881 N->getOperand(0).getOpcode() ==
ISD::SRL) ||
13883 N->getOperand(0).getOpcode() ==
ISD::SHL)) &&
13884 "Expected shift-shift mask");
13897 return Subtarget->hasMVEIntegerOps() &&
isTypeLegal(VT);
13901 if (!Subtarget->hasNEON()) {
13920 return Subtarget->hasFP64();
13923 return Subtarget->hasMVEFloatOps();
13952 if (ST->isThumb() && ST->isThumb1Only())
13956 for (
auto *U :
N->uses()) {
13957 switch(U->getOpcode()) {
13970 if (isa<ConstantSDNode>(U->getOperand(0)) ||
13971 isa<ConstantSDNode>(U->getOperand(1)))
13975 if (U->getOperand(0).getOpcode() ==
ISD::SHL ||
13976 U->getOperand(1).getOpcode() ==
ISD::SHL)
13986 if (
N->getOperand(0).getOpcode() !=
ISD::SHL)
13991 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13992 auto *C2 = dyn_cast<ConstantSDNode>(SHL.getOperand(1));
13993 if (!C1ShlC2 || !C2)
13996 APInt C2Int = C2->getAPIntValue();
13997 APInt C1Int = C1ShlC2->getAPIntValue();
13999 if (C2Int.
uge(C2Width))
14005 if ((C1Int & Mask) != C1Int)
14012 auto LargeImm = [](
const APInt &Imm) {
14013 unsigned Zeros = Imm.countl_zero() + Imm.countr_zero();
14014 return Imm.getBitWidth() - Zeros > 8;
14017 if (LargeImm(C1Int) || LargeImm(C2Int))
14029 SHL.dump();
N->dump());
14064 if (CSINC.getOpcode() !=
ARMISD::CSINC || !CSINC.hasOneUse())
14073 CSINC.getOperand(0)),
14074 CSINC.getOperand(1), CSINC.getOperand(2),
14075 CSINC.getOperand(3));
14094 if (!Subtarget->hasMVEIntegerOps() || !
N->getValueType(0).isVector())
14136 if (!Subtarget->hasVMLxForwarding())
14155 EVT VT =
N->getValueType(0);
14166 EVT VT =
N->getValueType(0);
14167 if (VT != MVT::v2i64)
14176 EVT VT = cast<VTSDNode>(
Op->getOperand(1))->getVT();
14178 return Op->getOperand(0);
14192 And =
And->getOperand(0);
14197 Mask = Mask->getOperand(0);
14200 Mask.getValueType() != MVT::v4i32)
14206 return And->getOperand(0);
14211 if (
SDValue Op0 = IsSignExt(N0)) {
14212 if (
SDValue Op1 = IsSignExt(N1)) {
14218 if (
SDValue Op0 = IsZeroExt(N0)) {
14219 if (
SDValue Op1 = IsZeroExt(N1)) {
14234 EVT VT =
N->getValueType(0);
14235 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
14246 if (VT != MVT::i32)
14253 int64_t MulAmt =
C->getSExtValue();
14254 unsigned ShiftAmt = llvm::countr_zero<uint64_t>(MulAmt);
14256 ShiftAmt = ShiftAmt & (32 - 1);
14261 MulAmt >>= ShiftAmt;
14264 if (llvm::has_single_bit<uint32_t>(MulAmt - 1)) {
14272 }
else if (llvm::has_single_bit<uint32_t>(MulAmt + 1)) {
14284 if (llvm::has_single_bit<uint32_t>(MulAmtAbs + 1)) {
14292 }
else if (llvm::has_single_bit<uint32_t>(MulAmtAbs - 1)) {
14322 if (
N->getValueType(0) != MVT::i32)
14331 if (C1 == 255 || C1 == 65535)
14334 SDNode *N0 =
N->getOperand(0).getNode();
14348 if (!C2 || C2 >= 32)
14392 if (Trailing == C2 && C2 + C3 < 32) {
14405 if (Leading == C2 && C2 + C3 < 32) {
14433 EVT VT =
N->getValueType(0);
14437 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
14440 APInt SplatBits, SplatUndef;
14441 unsigned SplatBitSize;
14443 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14444 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14445 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14446 SplatBitSize == 64) {
14480 if (!Subtarget->hasV6Ops() ||
14481 (Subtarget->isThumb() &&
14482 (!Subtarget->hasThumb2() || !Subtarget->hasDSP())))
14485 SDValue SRL = OR->getOperand(0);
14486 SDValue SHL = OR->getOperand(1);
14489 SRL = OR->getOperand(1);
14490 SHL = OR->getOperand(0);
14497 if ((SRL.getOperand(0).getNode() != SHL.getOperand(0).getNode()) ||
14501 SDNode *SMULLOHI = SRL.getOperand(0).getNode();
14502 if (SRL.getOperand(0) !=
SDValue(SMULLOHI, 0) ||
14503 SHL.getOperand(0) !=
SDValue(SMULLOHI, 1))
14522 unsigned Opcode = 0;
14523 if (
isS16(OpS16, DAG))
14541 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6T2Ops())
14544 EVT VT =
N->getValueType(0);
14559 if (VT != MVT::i32)
14572 if (Mask == 0xffff)
14579 if ((Val & ~Mask) != Val)
14604 (Mask == ~Mask2)) {
14607 if (Subtarget->hasDSP() &&
14608 (Mask == 0xffff || Mask == 0xffff0000))
14621 (~Mask == Mask2)) {
14624 if (Subtarget->hasDSP() &&
14625 (Mask2 == 0xffff || Mask2 == 0xffff0000))
14698 EVT VT =
N->getValueType(0);
14703 auto IsFreelyInvertable = [&](
SDValue V) {
14710 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
14726 EVT VT =
N->getValueType(0);
14732 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
14733 VT == MVT::v8i1 || VT == MVT::v16i1))
14736 APInt SplatBits, SplatUndef;
14737 unsigned SplatBitSize;
14739 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14740 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14741 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14742 SplatBitSize == 64) {
14778 unsigned SplatBitSize;
14781 APInt SplatBits0, SplatBits1;
14785 if (BVN0 && BVN0->
isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
14786 HasAnyUndefs) && !HasAnyUndefs) {
14787 if (BVN1 && BVN1->
isConstantSplat(SplatBits1, SplatUndef, SplatBitSize,
14788 HasAnyUndefs) && !HasAnyUndefs) {
14793 SplatBits0 == ~SplatBits1) {
14823 EVT VT =
N->getValueType(0);
14838 if (Subtarget->hasMVEIntegerOps()) {
14869 ToMask = ~N->getConstantOperandAPInt(2);
14875 isa<ConstantSDNode>(
From->getOperand(1))) {
14876 APInt Shift =
From->getConstantOperandAPInt(1);
14889 unsigned LastActiveBitInA =
A.countr_zero();
14890 unsigned FirstActiveBitInB =
B.getBitWidth() -
B.countl_zero() - 1;
14891 return LastActiveBitInA - 1 == FirstActiveBitInB;
14896 APInt ToMask, FromMask;
14904 APInt NewToMask, NewFromMask;
14906 if (NewFrom !=
From)
14910 if ((NewToMask & ToMask).getBoolValue())
14935 unsigned InvMask =
N->getConstantOperandVal(2);
14937 unsigned Width = llvm::bit_width<unsigned>(~InvMask) - LSB;
14939 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
14940 "undefined behavior");
14941 unsigned Mask = (1u << Width) - 1;
14943 if ((Mask & (~Mask2)) == 0)
14945 N->getOperand(0), N1.
getOperand(0),
N->getOperand(2));
14952 APInt ToMask1, FromMask1;
14955 APInt ToMask2, FromMask2;
14961 APInt NewFromMask = FromMask1 | FromMask2;
14962 APInt NewToMask = ToMask1 | ToMask2;
14964 EVT VT =
N->getValueType(0);
14967 if (NewFromMask[0] == 0)
14979 APInt ToMask1 = ~N->getConstantOperandAPInt(2);
14980 APInt ToMask2 = ~N0.getConstantOperandAPInt(2);
14982 if (!N0.
hasOneUse() || (ToMask1 & ToMask2) != 0 ||
14986 EVT VT =
N->getValueType(0);
14989 N->getOperand(1),
N->getOperand(2));
15003 SDValue CSInc = Cmp->getOperand(0);
15053 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ)
15054 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15057 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
15059 N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15072 SDValue InDouble =
N->getOperand(0);
15081 !cast<LoadSDNode>(InNode)->isVolatile()) {
15087 SDValue BasePtr = LD->getBasePtr();
15089 DAG.
getLoad(MVT::i32,
DL, LD->getChain(), BasePtr, LD->getPointerInfo(),
15090 LD->getAlign(), LD->getMemOperand()->getFlags());
15096 LD->getPointerInfo().getWithOffset(4),
15098 LD->getMemOperand()->getFlags());
15110 isa<ConstantSDNode>(InDouble.
getOperand(1))) {
15131 if (!Subtarget->
isLittle() && BVSwap)
15141 if (isa<ConstantSDNode>(BV.
getOperand(2))) {
15149 if (!Subtarget->
isLittle() && BVSwap)
15194 if (Copy.getValueType() == MVT::f32 &&
15196 bool HasGlue = Copy->getNumOperands() == 3;
15197 SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1),
15198 HasGlue ? Copy->getOperand(2) :
SDValue()};
15199 EVT OutTys[] = {
N->getValueType(0), MVT::Other, MVT::Glue};
15217 if (
LoadSDNode *LN0 = dyn_cast<LoadSDNode>(Op0)) {
15218 if (LN0->hasOneUse() && LN0->isUnindexed() &&
15219 LN0->getMemoryVT() == MVT::i16) {
15222 LN0->getBasePtr(), LN0->getMemOperand());
15240 EVT VT =
N->getValueType(0);
15274 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
15275 for (
unsigned i = 0; i < NumElts; ++i) {
15276 SDNode *Elt =
N->getOperand(i).getNode();
15293 if (
N->getNumOperands() == 2)
15299 EVT VT =
N->getValueType(0);
15305 for (
unsigned i = 0; i < NumElts; ++i) {
15331 EVT VT =
N->getValueType(0);
15339 assert(EltVT == MVT::f32 &&
"Unexpected type!");
15344 Use->getValueType(0).isFloatingPoint())
15352 unsigned NumOfBitCastedElts = 0;
15354 unsigned NumOfRelevantElts = NumElts;
15355 for (
unsigned Idx = 0;
Idx < NumElts; ++
Idx) {
15360 ++NumOfBitCastedElts;
15361 }
else if (Elt.
isUndef() || isa<ConstantSDNode>(Elt))
15364 --NumOfRelevantElts;
15368 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
15386 for (
unsigned Idx = 0 ;
Idx < NumElts; ++
Idx) {
15391 V->getOperand(0).getValueType() == MVT::i32)
15393 V = V.getOperand(0);
15410 EVT VT =
N->getValueType(0);
15417 if (
Op->getOperand(0).getValueType() == VT)
15418 return Op->getOperand(0);
15433 if (
Op.getValueType() == MVT::i32) {
15444 EVT VT =
N->getValueType(0);
15449 if (ST->isLittle())
15459 if (
Op->getOperand(0).getValueType() == VT)
15460 return Op->getOperand(0);
15469 if (!Subtarget->hasMVEIntegerOps())
15472 EVT VT =
N->getValueType(0);
15503 EVT VT =
N->getValueType(0);
15504 SDNode *Elt =
N->getOperand(1).getNode();
15519 Vec, V,
N->getOperand(2));
15529 EVT VT =
N->getValueType(0);
15538 Ext.getOperand(0).getValueType() == MVT::f32)
15539 Ext = Ext.getOperand(0);
15541 !isa<ConstantSDNode>(Ext.getOperand(1)) ||
15542 Ext.getConstantOperandVal(1) % 2 != 0)
15544 if (Ext->use_size() == 1 &&
15549 SDValue Op0 = Ext.getOperand(0);
15552 unsigned Lane = Ext.getConstantOperandVal(1);
15558 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15559 isa<ConstantSDNode>(V->getOperand(1)) &&
15560 V->getConstantOperandVal(1) == Lane + 1 &&
15561 V->getOperand(0).getResNo() == ResNo;
15563 if (OtherIt == Op0->
uses().end())
15568 SDValue OtherExt(*OtherIt, 0);
15581 DCI.
DAG.
getConstant(Ext.getConstantOperandVal(1) / 2, dl, MVT::i32));
15593 EVT VT =
N->getValueType(0);
15599 if (VT == MVT::f16 &&
X.getValueType() == MVT::i32)
15601 if (VT == MVT::i32 &&
X.getValueType() == MVT::f16)
15603 if (VT == MVT::f32 &&
X.getValueType() == MVT::i32)
15606 while (
X.getValueType() != VT &&
X->getOpcode() ==
ISD::BITCAST)
15607 X =
X->getOperand(0);
15608 if (
X.getValueType() == VT)
15614 isa<ConstantSDNode>(
N->getOperand(1)) &&
15616 return Op0.
getOperand(
N->getConstantOperandVal(1));
15621 isa<ConstantSDNode>(
N->getOperand(1)) &&
15626 unsigned Offset =
N->getConstantOperandVal(1);
15638 unsigned Idx =
N->getConstantOperandVal(1);
15652 EVT VT =
N->getValueType(0);
15656 cast<VTSDNode>(
N->getOperand(1))->getVT() ==
15657 Op.getOperand(0).getValueType().getScalarType())
15667 SDValue SubVec =
N->getOperand(1);
15668 uint64_t IdxVal =
N->getConstantOperandVal(2);
15679 if (IdxVal == 0 && Vec.
isUndef())
15685 (IdxVal != 0 && IdxVal != NumSubElts))
15758 EVT VT =
N->getValueType(0);
15769 unsigned HalfElts = NumElts/2;
15771 for (
unsigned n = 0; n < NumElts; ++n) {
15774 if (MaskElt < (
int)HalfElts)
15776 else if (MaskElt >= (
int)NumElts && MaskElt < (
int)(NumElts + HalfElts))
15777 NewElt = HalfElts + MaskElt - NumElts;
15804 bool SimpleConstIncOnly,
15812 bool isLoadOp =
true;
15813 bool isLaneOp =
false;
15816 bool hasAlignment =
true;
15817 unsigned NewOpc = 0;
15818 unsigned NumVecs = 0;
15819 if (
Target.isIntrinsic) {
15820 unsigned IntNo =
N->getConstantOperandVal(1);
15824 case Intrinsic::arm_neon_vld1:
15828 case Intrinsic::arm_neon_vld2:
15832 case Intrinsic::arm_neon_vld3:
15836 case Intrinsic::arm_neon_vld4:
15840 case Intrinsic::arm_neon_vld1x2:
15843 hasAlignment =
false;
15845 case Intrinsic::arm_neon_vld1x3:
15848 hasAlignment =
false;
15850 case Intrinsic::arm_neon_vld1x4:
15853 hasAlignment =
false;
15855 case Intrinsic::arm_neon_vld2dup:
15859 case Intrinsic::arm_neon_vld3dup:
15863 case Intrinsic::arm_neon_vld4dup:
15867 case Intrinsic::arm_neon_vld2lane:
15872 case Intrinsic::arm_neon_vld3lane:
15877 case Intrinsic::arm_neon_vld4lane:
15882 case Intrinsic::arm_neon_vst1:
15887 case Intrinsic::arm_neon_vst2:
15892 case Intrinsic::arm_neon_vst3:
15897 case Intrinsic::arm_neon_vst4:
15902 case Intrinsic::arm_neon_vst2lane:
15908 case Intrinsic::arm_neon_vst3lane:
15914 case Intrinsic::arm_neon_vst4lane:
15920 case Intrinsic::arm_neon_vst1x2:
15924 hasAlignment =
false;
15926 case Intrinsic::arm_neon_vst1x3:
15930 hasAlignment =
false;
15932 case Intrinsic::arm_neon_vst1x4:
15936 hasAlignment =
false;
15941 switch (
N->getOpcode()) {
15977 VecTy =
N->getValueType(0);
15978 }
else if (
Target.isIntrinsic) {
15979 VecTy =
N->getOperand(
Target.AddrOpIdx + 1).getValueType();
15982 "Node has to be a load, a store, or an intrinsic!");
15983 VecTy =
N->getOperand(1).getValueType();
15991 if (isLaneOp || isVLDDUPOp)
15994 if (NumBytes >= 3 * 16 &&
User.ConstInc != NumBytes) {
16000 if (SimpleConstIncOnly &&
User.ConstInc != NumBytes)
16006 EVT AlignedVecTy = VecTy;
16023 if (isa<LSBaseSDNode>(
N)) {
16026 assert(NumVecs == 1 &&
"Unexpected multi-element generic load/store.");
16027 assert(!isLaneOp &&
"Unexpected generic load/store lane.");
16038 Alignment =
Align(1);
16044 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16046 for (n = 0; n < NumResultVecs; ++n)
16047 Tys[n] = AlignedVecTy;
16048 Tys[n++] = MVT::i32;
16049 Tys[n] = MVT::Other;
16064 unsigned LastOperand =
16065 hasAlignment ?
N->getNumOperands() - 1 :
N->getNumOperands();
16066 for (
unsigned i =
Target.AddrOpIdx + 1; i < LastOperand; ++i)
16075 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::STORE) {
16086 for (
unsigned i = 0; i < NumResultVecs; ++i)
16091 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::LOAD) {
16092 SDValue &LdVal = NewResults[0];
16128 switch (
N->getOpcode()) {
16131 if (isa<ConstantSDNode>(
N->getOperand(1))) {
16132 *
Ptr =
N->getOperand(0);
16133 *CInc =
N->getOperand(1);
16139 if (isa<ConstantSDNode>(
N->getOperand(2))) {
16140 *
Ptr =
N->getOperand(1);
16141 *CInc =
N->getOperand(2);
16176 const unsigned AddrOpIdx = ((isIntrinsic ||
isStore) ? 2 : 1);
16185 UE =
Addr.getNode()->use_end(); UI != UE; ++UI) {
16187 if (UI.getUse().getResNo() !=
Addr.getResNo() ||
16192 unsigned ConstInc =
16210 if (UI.getUse().getResNo() !=
Base.getResNo() ||
User ==
Addr.getNode() ||
16215 unsigned UserOffset =
16218 if (!UserOffset || UserOffset <=
Offset)
16221 unsigned NewConstInc = UserOffset -
Offset;
16231 unsigned NumValidUpd = BaseUpdates.
size();
16232 for (
unsigned I = 0;
I < NumValidUpd;) {
16236 std::swap(BaseUpdates[
I], BaseUpdates[NumValidUpd]);
16244 BaseUpdates.
resize(NumValidUpd);
16249 std::stable_sort(BaseUpdates.
begin(), BaseUpdates.
end(),
16251 return LHS.ConstInc < RHS.ConstInc;
16280 unsigned IntNo =
N->getConstantOperandVal(1);
16281 if (IntNo == Intrinsic::arm_mve_vst2q &&
N->getConstantOperandVal(5) != 1)
16283 if (IntNo == Intrinsic::arm_mve_vst4q &&
N->getConstantOperandVal(7) != 3)
16288 UE =
Addr.getNode()->use_end();
16292 UI.getUse().getResNo() !=
Addr.getResNo())
16308 bool isLoadOp =
true;
16309 unsigned NewOpc = 0;
16310 unsigned NumVecs = 0;
16314 case Intrinsic::arm_mve_vld2q:
16318 case Intrinsic::arm_mve_vld4q:
16322 case Intrinsic::arm_mve_vst2q:
16327 case Intrinsic::arm_mve_vst4q:
16337 VecTy =
N->getValueType(0);
16339 VecTy =
N->getOperand(3).getValueType();
16353 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16355 for (n = 0; n < NumResultVecs; ++n)
16357 Tys[n++] = MVT::i32;
16358 Tys[n] = MVT::Other;
16367 for (
unsigned i = 3; i <
N->getNumOperands(); ++i)
16375 for (
unsigned i = 0; i < NumResultVecs; ++i)
16394 EVT VT =
N->getValueType(0);
16400 SDNode *VLD =
N->getOperand(0).getNode();
16403 unsigned NumVecs = 0;
16404 unsigned NewOpc = 0;
16406 if (IntNo == Intrinsic::arm_neon_vld2lane) {
16409 }
else if (IntNo == Intrinsic::arm_neon_vld3lane) {
16412 }
else if (IntNo == Intrinsic::arm_neon_vld4lane) {
16425 if (UI.getUse().getResNo() == NumVecs)
16429 VLDLaneNo !=
User->getConstantOperandVal(1))
16436 for (n = 0; n < NumVecs; ++n)
16438 Tys[n] = MVT::Other;
16449 unsigned ResNo = UI.getUse().
getResNo();
16451 if (ResNo == NumVecs)
16459 std::vector<SDValue> VLDDupResults;
16460 for (
unsigned n = 0; n < NumVecs; ++n)
16474 EVT VT =
N->getValueType(0);
16477 if (Subtarget->hasMVEIntegerOps()) {
16481 ExtractVT = MVT::i32;
16483 N->getOperand(0),
N->getOperand(1));
16495 Op =
Op.getOperand(0);
16500 unsigned EltSize =
Op.getScalarValueSizeInBits();
16502 unsigned Imm =
Op.getConstantOperandVal(0);
16518 if (Subtarget->hasMVEIntegerOps()) {
16521 if (
Op.getValueType() == MVT::f32)
16524 else if (
Op.getValueType() == MVT::f16)
16529 if (!Subtarget->hasNEON())
16536 if (LD &&
Op.hasOneUse() && LD->isUnindexed() &&
16537 LD->getMemoryVT() ==
N->getValueType(0).getVectorElementType()) {
16538 SDValue Ops[] = {LD->getOperand(0), LD->getOperand(1),
16543 LD->getMemoryVT(), LD->getMemOperand());
16554 EVT VT =
N->getValueType(0);
16576 assert(StVT != VT &&
"Cannot truncate to the same type");
16586 if (0 != (NumElems * FromEltSz) % ToEltSz)
16589 unsigned SizeRatio = FromEltSz / ToEltSz;
16594 NumElems * SizeRatio);
16600 for (
unsigned i = 0; i < NumElems; ++i)
16614 MVT StoreType = MVT::i8;
16616 if (TLI.
isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
16635 unsigned E = (ToEltSz * NumElems) / StoreType.
getSizeInBits();
16636 for (
unsigned I = 0;
I < E;
I++) {
16667 if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16)
16670 unsigned NumElements = 4;
16687 unsigned Off0 = Rev ? NumElts : 0;
16688 unsigned Off1 = Rev ? 0 : NumElts;
16690 for (
unsigned I = 0;
I < NumElts;
I += 2) {
16691 if (M[
I] >= 0 && M[
I] != (
int)(Off0 +
I / 2))
16693 if (M[
I + 1] >= 0 && M[
I + 1] != (
int)(Off1 +
I / 2))
16700 if (
auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Trunc.
getOperand(0)))
16701 if (isVMOVNShuffle(Shuffle,
false) || isVMOVNShuffle(Shuffle,
true))
16721 unsigned NewOffset = i * NumElements * ToEltVT.
getSizeInBits() / 8;
16736 NewToVT, Alignment, MMOFlags, AAInfo);
16769 unsigned NewOffset =
16777 NewToVT, Alignment, MMOFlags, AAInfo);
16799 {Extract.getOperand(0), Extract.getOperand(1)});
16830 if (Subtarget->hasNEON())
16834 if (Subtarget->hasMVEFloatOps())
16838 if (Subtarget->hasMVEIntegerOps()) {
16914 if (!Subtarget->hasNEON())
16918 if (!
Op.getValueType().isVector() || !
Op.getValueType().isSimple() ||
16923 if (!isa<BuildVectorSDNode>(ConstVec))
16926 MVT FloatTy =
Op.getSimpleValueType().getVectorElementType();
16928 MVT IntTy =
N->getSimpleValueType(0).getVectorElementType();
16930 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
16931 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
16942 if (
C == -1 ||
C == 0 ||
C > 32)
16947 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
16948 Intrinsic::arm_neon_vcvtfp2fxu;
16951 DAG.
getConstant(IntrinsicOpcode, dl, MVT::i32),
Op->getOperand(0),
16954 if (IntBits < FloatBits)
16962 if (!Subtarget->hasMVEFloatOps())
16970 EVT VT =
N->getValueType(0);
16975 auto isIdentitySplat = [&](
SDValue Op,
bool NSZ) {
16979 uint64_t ImmVal =
Op.getOperand(0).getConstantOperandVal(0);
16980 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ)))
16982 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ)))
16995 if (!isIdentitySplat(Op1.
getOperand(2), NSZ))
17006 EVT VT =
N->getValueType(0);
17009 if (!
N->getFlags().hasAllowReassociation())
17016 unsigned Opc =
A.getConstantOperandVal(0);
17017 if (Opc != Intrinsic::arm_mve_vcmlaq)
17022 A.getOperand(3),
A.getOperand(4));
17054 if (!Subtarget->hasNEON())
17058 unsigned OpOpcode =
Op.getNode()->getOpcode();
17059 if (!
N->getValueType(0).isVector() || !
N->getValueType(0).isSimple() ||
17063 SDValue ConstVec =
N->getOperand(1);
17064 if (!isa<BuildVectorSDNode>(ConstVec))
17067 MVT FloatTy =
N->getSimpleValueType(0).getVectorElementType();
17069 MVT IntTy =
Op.getOperand(0).getSimpleValueType().getVectorElementType();
17071 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
17072 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17092 int32_t
C = IntVal.exactLogBase2();
17093 if (
C == -1 ||
C == 0 ||
C > 32)
17099 if (IntBits < FloatBits)
17101 NumLanes == 2 ? MVT::v2i32 : MVT::v4i32, ConvInput);
17103 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfxs2fp
17104 : Intrinsic::arm_neon_vcvtfxu2fp;
17112 if (!ST->hasMVEIntegerOps())
17116 EVT ResVT =
N->getValueType(0);
17144 EVT AVT =
A.getValueType();
17150 auto ExtendIfNeeded = [&](
SDValue A,
unsigned ExtendCode) {
17151 EVT AVT =
A.getValueType();
17163 if (ExtTypeMatches(
A, ExtTypes))
17164 return ExtendIfNeeded(
A, ExtendCode);
17167 auto IsPredVADDV = [&](
MVT RetTy,
unsigned ExtendCode,
17174 if (Ext->getOpcode() != ExtendCode)
17177 if (ExtTypeMatches(
A, ExtTypes))
17178 return ExtendIfNeeded(
A, ExtendCode);
17191 if (ResVT !=
RetTy)
17194 if (
Mul->getOpcode() == ExtendCode &&
17195 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17197 Mul =
Mul->getOperand(0);
17206 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17207 A = ExtendIfNeeded(
A, ExtendCode);
17208 B = ExtendIfNeeded(
B, ExtendCode);
17226 if (
Mul->getOpcode() == ExtendCode &&
17227 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17229 Mul =
Mul->getOperand(0);
17238 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17239 A = ExtendIfNeeded(
A, ExtendCode);
17240 B = ExtendIfNeeded(
B, ExtendCode);
17251 EVT VT = Ops[0].getValueType();
17252 if (VT == MVT::v16i8) {
17254 "Unexpected illegal long reduction opcode");
17259 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[0]);
17262 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[1]);
17283 if (IsVMLAV(MVT::i64,
ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17286 if (IsVMLAV(MVT::i64,
ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17350 Op =
Op->getOperand(1);
17352 Op->getOperand(0)->getOpcode() ==
ISD::MUL) {
17354 if (
Mul->getOperand(0) ==
Mul->getOperand(1) &&
17371 unsigned VecOp =
N->getOperand(0).getValueType().isVector() ? 0 : 2;
17372 auto *Shuf = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(VecOp));
17373 if (!Shuf || !Shuf->getOperand(1).isUndef())
17378 APInt SetElts(Mask.size(), 0);
17379 for (
int E : Mask) {
17380 if (E < 0 || E >= (
int)Mask.size())
17387 if (
N->getNumOperands() != VecOp + 1) {
17388 auto *Shuf2 = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(VecOp + 1));
17389 if (!Shuf2 || !Shuf2->getOperand(1).isUndef() || Shuf2->getMask() != Mask)
17395 if (
Op.getValueType().isVector())
17407 unsigned IsTop =
N->getConstantOperandVal(2);
17414 if (Op0->
isUndef() && !IsTop)
17428 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17430 APInt Op0DemandedElts =
17431 IsTop ? Op1DemandedElts
17446 unsigned IsTop =
N->getConstantOperandVal(2);
17448 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17449 APInt Op0DemandedElts =
17461 EVT VT =
N->getValueType(0);
17465 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(
LHS);
17466 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(
RHS);
17468 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
17469 LHS.getOperand(1).isUndef() &&
RHS.getOperand(1).isUndef() &&
17473 LHS.getOperand(0),
RHS.getOperand(0));
17487 if (
auto C = dyn_cast<ConstantSDNode>(
N->getOperand(2))) {
17488 int ShiftAmt =
C->getSExtValue();
17489 if (ShiftAmt == 0) {
17495 if (ShiftAmt >= -32 && ShiftAmt < 0) {
17496 unsigned NewOpcode =
17512 unsigned IntNo =
N->getConstantOperandVal(0);
17523 case Intrinsic::arm_neon_vshifts:
17524 case Intrinsic::arm_neon_vshiftu:
17525 case Intrinsic::arm_neon_vrshifts:
17526 case Intrinsic::arm_neon_vrshiftu:
17527 case Intrinsic::arm_neon_vrshiftn:
17528 case Intrinsic::arm_neon_vqshifts:
17529 case Intrinsic::arm_neon_vqshiftu:
17530 case Intrinsic::arm_neon_vqshiftsu:
17531 case Intrinsic::arm_neon_vqshiftns:
17532 case Intrinsic::arm_neon_vqshiftnu:
17533 case Intrinsic::arm_neon_vqshiftnsu:
17534 case Intrinsic::arm_neon_vqrshiftns:
17535 case Intrinsic::arm_neon_vqrshiftnu:
17536 case Intrinsic::arm_neon_vqrshiftnsu: {
17537 EVT VT =
N->getOperand(1).getValueType();
17539 unsigned VShiftOpc = 0;
17542 case Intrinsic::arm_neon_vshifts:
17543 case Intrinsic::arm_neon_vshiftu:
17548 if (
isVShiftRImm(
N->getOperand(2), VT,
false,
true, Cnt)) {
17555 case Intrinsic::arm_neon_vrshifts:
17556 case Intrinsic::arm_neon_vrshiftu:
17561 case Intrinsic::arm_neon_vqshifts:
17562 case Intrinsic::arm_neon_vqshiftu:
17567 case Intrinsic::arm_neon_vqshiftsu:
17572 case Intrinsic::arm_neon_vrshiftn:
17573 case Intrinsic::arm_neon_vqshiftns:
17574 case Intrinsic::arm_neon_vqshiftnu:
17575 case Intrinsic::arm_neon_vqshiftnsu:
17576 case Intrinsic::arm_neon_vqrshiftns:
17577 case Intrinsic::arm_neon_vqrshiftnu:
17578 case Intrinsic::arm_neon_vqrshiftnsu:
17590 case Intrinsic::arm_neon_vshifts:
17591 case Intrinsic::arm_neon_vshiftu:
17594 case Intrinsic::arm_neon_vrshifts:
17597 case Intrinsic::arm_neon_vrshiftu:
17600 case Intrinsic::arm_neon_vrshiftn:
17603 case Intrinsic::arm_neon_vqshifts:
17606 case Intrinsic::arm_neon_vqshiftu:
17609 case Intrinsic::arm_neon_vqshiftsu:
17612 case Intrinsic::arm_neon_vqshiftns:
17615 case Intrinsic::arm_neon_vqshiftnu:
17618 case Intrinsic::arm_neon_vqshiftnsu:
17621 case Intrinsic::arm_neon_vqrshiftns:
17624 case Intrinsic::arm_neon_vqrshiftnu:
17627 case Intrinsic::arm_neon_vqrshiftnsu:
17633 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17634 N->getOperand(1), DAG.
getConstant(Cnt, dl, MVT::i32));
17637 case Intrinsic::arm_neon_vshiftins: {
17638 EVT VT =
N->getOperand(1).getValueType();
17640 unsigned VShiftOpc = 0;
17644 else if (
isVShiftRImm(
N->getOperand(3), VT,
false,
true, Cnt))
17651 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17652 N->getOperand(1),
N->getOperand(2),
17656 case Intrinsic::arm_neon_vqrshifts:
17657 case Intrinsic::arm_neon_vqrshiftu:
17661 case Intrinsic::arm_mve_vqdmlah:
17662 case Intrinsic::arm_mve_vqdmlash:
17663 case Intrinsic::arm_mve_vqrdmlah:
17664 case Intrinsic::arm_mve_vqrdmlash:
17665 case Intrinsic::arm_mve_vmla_n_predicated:
17666 case Intrinsic::arm_mve_vmlas_n_predicated:
17667 case Intrinsic::arm_mve_vqdmlah_predicated:
17668 case Intrinsic::arm_mve_vqdmlash_predicated:
17669 case Intrinsic::arm_mve_vqrdmlah_predicated:
17670 case Intrinsic::arm_mve_vqrdmlash_predicated: {
17675 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
17682 case Intrinsic::arm_mve_minv:
17683 case Intrinsic::arm_mve_maxv:
17684 case Intrinsic::arm_mve_minav:
17685 case Intrinsic::arm_mve_maxav:
17686 case Intrinsic::arm_mve_minv_predicated:
17687 case Intrinsic::arm_mve_maxv_predicated:
17688 case Intrinsic::arm_mve_minav_predicated:
17689 case Intrinsic::arm_mve_maxav_predicated: {
17692 unsigned BitWidth =
N->getOperand(2)->getValueType(0).getScalarSizeInBits();
17699 case Intrinsic::arm_mve_addv: {
17702 bool Unsigned =
N->getConstantOperandVal(2);
17707 case Intrinsic::arm_mve_addlv:
17708 case Intrinsic::arm_mve_addlv_predicated: {
17711 bool Unsigned =
N->getConstantOperandVal(2);
17712 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
17717 for (
unsigned i = 1, e =
N->getNumOperands(); i < e; i++)
17740 EVT VT =
N->getValueType(0);
17742 if (ST->isThumb1Only() &&
N->getOpcode() ==
ISD::SHL && VT == MVT::i32 &&
17743 N->getOperand(0)->getOpcode() ==
ISD::AND &&
17744 N->getOperand(0)->hasOneUse()) {
17752 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(
N->getOperand(1));
17761 if (AndMask == 255 || AndMask == 65535)
17765 if (MaskedBits > ShiftAmt) {
17780 if (ST->hasMVEIntegerOps())
17785 switch (
N->getOpcode()) {
17798 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
17799 unsigned VShiftOpc =
17802 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
17818 if (!LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed() ||
17821 EVT FromVT = LD->getValueType(0);
17822 EVT ToVT =
N->getValueType(0);
17829 unsigned NumElements = 0;
17830 if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8)
17832 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
17834 if (NumElements == 0 ||
17844 SDValue BasePtr = LD->getBasePtr();
17845 Align Alignment = LD->getOriginalAlign();
17866 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
17867 Alignment, MMOFlags, AAInfo);
17873 if (FromEltVT == MVT::f16) {
17876 for (
unsigned i = 0; i < Loads.
size(); i++) {
17902 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
17906 EVT VT =
N->getValueType(0);
17910 if (VT == MVT::i32 &&
17911 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
17913 isa<ConstantSDNode>(Lane)) {
17916 switch (
N->getOpcode()) {
17930 if (ST->hasMVEIntegerOps())
17939 if (ST->hasMVEFloatOps())
17950 if ((Subtarget->isThumb() || !Subtarget->hasV6Ops()) &&
17954 EVT VT =
Op.getValueType();
17957 if (VT != MVT::i32 ||
17959 !isa<ConstantSDNode>(
Op.getOperand(1)) ||
17970 APInt MaxC = Max.getConstantOperandAPInt(1);
17973 !(MinC + 1).isPowerOf2())
17991 EVT VT =
N->getValueType(0);
17994 if (VT == MVT::i32)
17997 if (!ST->hasMVEIntegerOps())
18003 if (VT != MVT::v4i32 && VT != MVT::v8i16)
18006 auto IsSignedSaturate = [&](
SDNode *Min,
SDNode *Max) {
18014 if (VT == MVT::v4i32)
18015 SaturateC =
APInt(32, (1 << 15) - 1,
true);
18017 SaturateC =
APInt(16, (1 << 7) - 1,
true);
18024 MaxC != ~SaturateC)
18029 if (IsSignedSaturate(
N, N0.
getNode())) {
18032 if (VT == MVT::v4i32) {
18033 HalfVT = MVT::v8i16;
18034 ExtVT = MVT::v4i16;
18036 HalfVT = MVT::v16i8;
18051 auto IsUnsignedSaturate = [&](
SDNode *Min) {
18057 if (VT == MVT::v4i32)
18058 SaturateC =
APInt(32, (1 << 16) - 1,
true);
18060 SaturateC =
APInt(16, (1 << 8) - 1,
true);
18069 if (IsUnsignedSaturate(
N)) {
18073 if (VT == MVT::v4i32) {
18074 HalfVT = MVT::v8i16;
18075 ExtConst = 0x0000FFFF;
18077 HalfVT = MVT::v16i8;
18099 const APInt *CV = &
C->getAPIntValue();
18117 SDValue Op0 = CMOV->getOperand(0);
18118 SDValue Op1 = CMOV->getOperand(1);
18119 auto CC = CMOV->getConstantOperandAPInt(2).getLimitedValue();
18120 SDValue CmpZ = CMOV->getOperand(4);
18156 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
18163 if ((OrCI & Known.
Zero) != OrCI)
18169 EVT VT =
X.getValueType();
18170 unsigned BitInX = AndC->
logBase2();
18178 for (
unsigned BitInY = 0, NumActiveBits = OrCI.
getActiveBits();
18179 BitInY < NumActiveBits; ++BitInY) {
18180 if (OrCI[BitInY] == 0)
18183 Mask.setBit(BitInY);
18200 switch (
N->getOpcode()) {
18204 if (!isa<ConstantSDNode>(
N.getOperand(1)))
18206 if (!cast<ConstantSDNode>(
N.getOperand(1))->isOne())
18212 auto *Const = dyn_cast<ConstantSDNode>(
N.getOperand(1));
18215 if (Const->isZero())
18217 else if (Const->isOne())
18221 CC = cast<CondCodeSDNode>(
N.getOperand(2))->get();
18225 unsigned IntOp =
N.getConstantOperandVal(1);
18226 if (IntOp != Intrinsic::test_start_loop_iterations &&
18227 IntOp != Intrinsic::loop_decrement_reg)
18253 bool Negate =
false;
18259 Cond =
N->getOperand(1);
18260 Dest =
N->getOperand(2);
18263 CC = cast<CondCodeSDNode>(
N->getOperand(1))->get();
18264 Cond =
N->getOperand(2);
18265 Dest =
N->getOperand(4);
18266 if (
auto *Const = dyn_cast<ConstantSDNode>(
N->getOperand(3))) {
18267 if (!Const->isOne() && !Const->isZero())
18269 Imm = Const->getZExtValue();
18297 assert((IsTrueIfZero(
CC, Imm) || IsFalseIfZero(
CC, Imm)) &&
18298 "unsupported condition");
18303 unsigned IntOp =
Int->getConstantOperandVal(1);
18305 &&
"expected single br user");
18306 SDNode *Br = *
N->use_begin();
18316 if (IntOp == Intrinsic::test_start_loop_iterations) {
18320 if (IsTrueIfZero(
CC, Imm)) {
18321 SDValue Ops[] = {Chain, Setup, Dest};
18326 UpdateUncondBr(Br, Dest, DAG);
18328 SDValue Ops[] = {Chain, Setup, OtherTarget};
18341 DAG.
getVTList(MVT::i32, MVT::Other), Args);
18349 if (
Target == OtherTarget)
18350 UpdateUncondBr(Br, Dest, DAG);
18369 EVT VT =
N->getValueType(0);
18388 LHS->getOperand(0)->getOperand(3),
LHS->getOperand(0)->getOperand(4));
18402 EVT VT =
N->getValueType(0);
18406 SDValue FalseVal =
N->getOperand(0);
18407 SDValue TrueVal =
N->getOperand(1);
18412 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops()) {
18438 N->getOperand(3), Cmp);
18443 N->getOperand(3), NewCmp);
18452 LHS->getOperand(2),
LHS->getOperand(3),
18453 LHS->getOperand(4));
18463 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ ||
18467 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
18469 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
18472 N->getOperand(3),
C);
18479 if (!Subtarget->
isThumb1Only() && Subtarget->hasV5TOps()) {
18515 N->getOperand(3), CPSRGlue.
getValue(1));
18530 N->getOperand(3), CPSRGlue.
getValue(1));
18548 const APInt *TrueConst;
18550 ((FalseVal.getOpcode() ==
ARMISD::SUBC && FalseVal.getOperand(0) ==
LHS &&
18551 FalseVal.getOperand(1) ==
RHS) ||
18555 unsigned ShiftAmount = TrueConst->
logBase2();
18570 if (Known.
Zero == 0xfffffffe)
18573 else if (Known.
Zero == 0xffffff00)
18576 else if (Known.
Zero == 0xffff0000)
18589 EVT DstVT =
N->getValueType(0);
18592 if (ST->hasMVEIntegerOps() && Src.getOpcode() ==
ARMISD::VDUP) {
18593 EVT SrcVT = Src.getValueType();
18601 Src = Src.getOperand(0);
18605 EVT SrcVT = Src.getValueType();
18625 EVT VT =
N->getValueType(0);
18633 if (
N->getNumOperands() == 2 &&
18637 N->getOperand(0).getOperand(1),
18638 N->getOperand(1).getOperand(0),
18639 N->getOperand(1).getOperand(1));
18642 if (
N->getNumOperands() == 2 &&
18645 auto *S0 = cast<ShuffleVectorSDNode>(
N->getOperand(0).getNode());
18646 auto *
S1 = cast<ShuffleVectorSDNode>(
N->getOperand(1).getNode());
18648 if (S0->getOperand(0) ==
S1->getOperand(0) &&
18649 S0->getOperand(1) ==
S1->getOperand(1)) {
18652 Mask.append(
S1->getMask().begin(),
S1->getMask().end());
18672 return Op.getOpcode() == ISD::BUILD_VECTOR ||
18673 Op.getOpcode() == ISD::VECTOR_SHUFFLE ||
18674 (Op.getOpcode() == ISD::BITCAST &&
18675 Op.getOperand(0).getOpcode() == ISD::BUILD_VECTOR);
18678 for (
unsigned Op = 0;
Op <
N->getNumOperands();
Op++) {
18680 for (
unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) {
18697 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18698 int NumIns =
N->getNumOperands();
18699 assert((NumIns == 2 || NumIns == 4) &&
18700 "Expected 2 or 4 inputs to an MVETrunc");
18702 if (
N->getNumOperands() == 4)
18706 for (
int I = 0;
I < NumIns;
I++) {
18708 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18728 if (!LD || !LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed())
18731 EVT FromVT = LD->getMemoryVT();
18732 EVT ToVT =
N->getValueType(0);
18739 unsigned NumElements = 0;
18740 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
18742 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
18744 assert(NumElements != 0);
18750 LD->getExtensionType() != NewExtType)
18757 SDValue BasePtr = LD->getBasePtr();
18758 Align Alignment = LD->getOriginalAlign();
18777 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
18778 Alignment, MMOFlags, AAInfo);
18794 EVT VT =
N->getValueType(0);
18796 assert(
N->getNumValues() == 2 &&
"Expected MVEEXT with 2 elements");
18797 assert((VT == MVT::v4i32 || VT == MVT::v8i16) &&
"Unexpected MVEEXT type");
18799 EVT ExtVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18801 auto Extend = [&](
SDValue V) {
18811 SDValue Ext = Extend(
N->getOperand(0));
18816 if (
auto *SVN = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(0))) {
18819 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements());
18824 auto CheckInregMask = [&](
int Start,
int Offset) {
18832 if (CheckInregMask(0, 0))
18834 else if (CheckInregMask(0, 1))
18835 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18836 else if (CheckInregMask(0, Mask.size()))
18838 else if (CheckInregMask(0, Mask.size() + 1))
18839 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18844 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18848 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18855 if (
N->getOperand(0)->getOpcode() ==
ISD::LOAD)
18865 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18866 int NumOuts =
N->getNumValues();
18867 assert((NumOuts == 2 || NumOuts == 4) &&
18868 "Expected 2 or 4 outputs to an MVEEXT");
18869 EVT LoadVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18871 if (
N->getNumOperands() == 4)
18877 StackPtr, MPI,
Align(4));
18880 for (
int I = 0;
I < NumOuts;
I++) {
18882 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18883 DAG.
getConstant(
I * 16 / NumOuts,
DL, StackPtr.getValueType()));
18888 VT, Chain,
Ptr, MPI, LoadVT,
Align(4));
18897 switch (
N->getOpcode()) {
19014 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19021 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19032 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19040 unsigned LowWidth =
N->getOperand(0).getValueType().getSizeInBits();
19042 unsigned HighWidth =
N->getOperand(1).getValueType().getSizeInBits();
19050 unsigned HighWidth =
N->getOperand(0).getValueType().getSizeInBits();
19052 unsigned LowWidth =
N->getOperand(1).getValueType().getSizeInBits();
19060 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19071 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19080 switch (
N->getConstantOperandVal(1)) {
19081 case Intrinsic::arm_neon_vld1:
19082 case Intrinsic::arm_neon_vld1x2:
19083 case Intrinsic::arm_neon_vld1x3:
19084 case Intrinsic::arm_neon_vld1x4:
19085 case Intrinsic::arm_neon_vld2:
19086 case Intrinsic::arm_neon_vld3:
19087 case Intrinsic::arm_neon_vld4:
19088 case Intrinsic::arm_neon_vld2lane:
19089 case Intrinsic::arm_neon_vld3lane:
19090 case Intrinsic::arm_neon_vld4lane:
19091 case Intrinsic::arm_neon_vld2dup:
19092 case Intrinsic::arm_neon_vld3dup:
19093 case Intrinsic::arm_neon_vld4dup:
19094 case Intrinsic::arm_neon_vst1:
19095 case Intrinsic::arm_neon_vst1x2:
19096 case Intrinsic::arm_neon_vst1x3:
19097 case Intrinsic::arm_neon_vst1x4:
19098 case Intrinsic::arm_neon_vst2:
19099 case Intrinsic::arm_neon_vst3:
19100 case Intrinsic::arm_neon_vst4:
19101 case Intrinsic::arm_neon_vst2lane:
19102 case Intrinsic::arm_neon_vst3lane:
19103 case Intrinsic::arm_neon_vst4lane:
19105 case Intrinsic::arm_mve_vld2q:
19106 case Intrinsic::arm_mve_vld4q:
19107 case Intrinsic::arm_mve_vst2q:
19108 case Intrinsic::arm_mve_vst4q:
19125 unsigned *
Fast)
const {
19134 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
19136 if (AllowsUnaligned) {
19138 *
Fast = Subtarget->hasV7Ops();
19143 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
19147 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->
isLittle())) {
19154 if (!Subtarget->hasMVEIntegerOps())
19158 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
19159 Ty == MVT::v2i1)) {
19167 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
19183 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
19184 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
19185 Ty == MVT::v2f64) {
19198 if ((
Op.isMemcpy() ||
Op.isZeroMemset()) && Subtarget->hasNEON() &&
19199 !FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat)) {
19201 if (
Op.size() >= 16 &&
19207 }
else if (
Op.size() >= 8 &&
19228 return (SrcBits == 64 && DestBits == 32);
19237 return (SrcBits == 64 && DestBits == 32);
19273 return Subtarget->hasFullFP16();
19283 return Ext->getType()->getScalarSizeInBits() ==
19284 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
19289 !areExtDoubled(cast<Instruction>(Ext1)) ||
19290 !areExtDoubled(cast<Instruction>(Ext2)))
19301 if (!
I->getType()->isVectorTy())
19304 if (Subtarget->hasNEON()) {
19305 switch (
I->getOpcode()) {
19306 case Instruction::Sub:
19307 case Instruction::Add: {
19319 if (!Subtarget->hasMVEIntegerOps())
19323 if (!
I->hasOneUse())
19325 auto *Sub = cast<Instruction>(*
I->users().begin());
19326 return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(1) ==
I;
19336 switch (
I->getOpcode()) {
19337 case Instruction::Add:
19338 case Instruction::Mul:
19339 case Instruction::FAdd:
19340 case Instruction::ICmp:
19341 case Instruction::FCmp:
19343 case Instruction::FMul:
19344 return !IsFMSMul(
I);
19345 case Instruction::Sub:
19346 case Instruction::FSub:
19347 case Instruction::Shl:
19348 case Instruction::LShr:
19349 case Instruction::AShr:
19350 return Operand == 1;
19351 case Instruction::Call:
19352 if (
auto *
II = dyn_cast<IntrinsicInst>(
I)) {
19353 switch (
II->getIntrinsicID()) {
19354 case Intrinsic::fma:
19356 case Intrinsic::sadd_sat:
19357 case Intrinsic::uadd_sat:
19358 case Intrinsic::arm_mve_add_predicated:
19359 case Intrinsic::arm_mve_mul_predicated:
19360 case Intrinsic::arm_mve_qadd_predicated:
19361 case Intrinsic::arm_mve_vhadd:
19362 case Intrinsic::arm_mve_hadd_predicated:
19363 case Intrinsic::arm_mve_vqdmull:
19364 case Intrinsic::arm_mve_vqdmull_predicated:
19365 case Intrinsic::arm_mve_vqdmulh:
19366 case Intrinsic::arm_mve_qdmulh_predicated:
19367 case Intrinsic::arm_mve_vqrdmulh:
19368 case Intrinsic::arm_mve_qrdmulh_predicated:
19369 case Intrinsic::arm_mve_fma_predicated:
19371 case Intrinsic::ssub_sat:
19372 case Intrinsic::usub_sat:
19373 case Intrinsic::arm_mve_sub_predicated:
19374 case Intrinsic::arm_mve_qsub_predicated:
19375 case Intrinsic::arm_mve_hsub_predicated:
19376 case Intrinsic::arm_mve_vhsub:
19377 return Operand == 1;
19388 for (
auto OpIdx :
enumerate(
I->operands())) {
19389 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
19391 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
19395 if (Shuffle->
getOpcode() == Instruction::BitCast)
19396 Shuffle = dyn_cast<Instruction>(Shuffle->
getOperand(0));
19403 if (!IsSinker(
I, OpIdx.index()))
19408 for (
Use &U :
Op->uses()) {
19410 if (!IsSinker(
Insn, U.getOperandNo()))
19423 if (!Subtarget->hasMVEIntegerOps())
19441 if (
auto *Ld = dyn_cast<MaskedLoadSDNode>(ExtVal.
getOperand(0))) {
19442 if (Ld->isExpandingLoad())
19446 if (Subtarget->hasMVEIntegerOps())
19491bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(
const MachineFunction &MF,
19499 return Subtarget->hasMVEFloatOps();
19517 unsigned Scale = 1;
19534 if ((V & (Scale - 1)) != 0)
19536 return isUInt<5>(V / Scale);
19543 if (VT.
isVector() && Subtarget->hasNEON())
19546 !Subtarget->hasMVEFloatOps())
19549 bool IsNeg =
false;
19555 unsigned NumBytes = std::max((
unsigned)VT.
getSizeInBits() / 8, 1U);
19558 if (VT.
isVector() && Subtarget->hasMVEIntegerOps()) {
19562 return isShiftedUInt<7,2>(V);
19565 return isShiftedUInt<7,1>(V);
19567 return isUInt<7>(V);
19574 if (VT.
isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16())
19575 return isShiftedUInt<8, 1>(V);
19578 return isShiftedUInt<8, 2>(V);
19580 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
19583 return isUInt<8>(V);
19584 return isUInt<12>(V);
19610 default:
return false;
19615 return isUInt<12>(V);
19618 return isUInt<8>(V);
19623 return isShiftedUInt<8, 2>(V);
19629 int Scale = AM.
Scale;
19634 default:
return false;
19642 Scale = Scale & ~1;
19643 return Scale == 2 || Scale == 4 || Scale == 8;
19660 if (Scale & 1)
return false;
19667 const int Scale = AM.
Scale;
19677 return (Scale == 1) || (!AM.
HasBaseReg && Scale == 2);
19693 switch (AM.
Scale) {
19710 int Scale = AM.
Scale;
19712 default:
return false;
19716 if (Scale < 0) Scale = -Scale;
19724 if (Scale == 1 || (AM.
HasBaseReg && Scale == -1))
19737 if (Scale & 1)
return false;
19750 if (!Subtarget->isThumb())
19757 return Imm >= 0 && Imm <= 255;
19766 int64_t AbsImm = std::abs(Imm);
19767 if (!Subtarget->isThumb())
19772 return AbsImm >= 0 && AbsImm <= 255;
19807 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
19811 int RHSC = (int)
RHS->getZExtValue();
19812 if (RHSC < 0 && RHSC > -256) {
19822 }
else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
19825 int RHSC = (int)
RHS->getZExtValue();
19826 if (RHSC < 0 && RHSC > -0x1000) {
19868 int RHSC = (int)
RHS->getZExtValue();
19869 if (RHSC < 0 && RHSC > -0x100) {
19874 }
else if (RHSC > 0 && RHSC < 0x100) {
19885 bool isSEXTLoad,
bool IsMasked,
bool isLE,
19890 if (!isa<ConstantSDNode>(
Ptr->getOperand(1)))
19896 bool CanChangeType = isLE && !IsMasked;
19899 int RHSC = (int)
RHS->getZExtValue();
19901 auto IsInRange = [&](
int RHSC,
int Limit,
int Scale) {
19902 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
19907 }
else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
19918 if (VT == MVT::v4i16) {
19919 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
19921 }
else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
19922 if (IsInRange(RHSC, 0x80, 1))
19924 }
else if (Alignment >= 4 &&
19925 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
19926 IsInRange(RHSC, 0x80, 4))
19928 else if (Alignment >= 2 &&
19929 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
19930 IsInRange(RHSC, 0x80, 2))
19932 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
19951 bool isSEXTLoad =
false;
19952 bool IsMasked =
false;
19954 Ptr = LD->getBasePtr();
19955 VT = LD->getMemoryVT();
19956 Alignment = LD->getAlign();
19958 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
19959 Ptr = ST->getBasePtr();
19960 VT = ST->getMemoryVT();
19961 Alignment = ST->getAlign();
19963 Ptr = LD->getBasePtr();
19964 VT = LD->getMemoryVT();
19965 Alignment = LD->getAlign();
19969 Ptr = ST->getBasePtr();
19970 VT = ST->getMemoryVT();
19971 Alignment = ST->getAlign();
19977 bool isLegal =
false;
19979 isLegal = Subtarget->hasMVEIntegerOps() &&
19981 Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
20009 bool isSEXTLoad =
false, isNonExt;
20010 bool IsMasked =
false;
20012 VT = LD->getMemoryVT();
20013 Ptr = LD->getBasePtr();
20014 Alignment = LD->getAlign();
20017 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20018 VT = ST->getMemoryVT();
20019 Ptr = ST->getBasePtr();
20020 Alignment = ST->getAlign();
20021 isNonExt = !ST->isTruncatingStore();
20023 VT = LD->getMemoryVT();
20024 Ptr = LD->getBasePtr();
20025 Alignment = LD->getAlign();
20030 VT = ST->getMemoryVT();
20031 Ptr = ST->getBasePtr();
20032 Alignment = ST->getAlign();
20033 isNonExt = !ST->isTruncatingStore();
20041 assert(
Op->getValueType(0) == MVT::i32 &&
"Non-i32 post-inc op?!");
20042 if (
Op->getOpcode() !=
ISD::ADD || !isNonExt)
20044 auto *
RHS = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
20045 if (!
RHS ||
RHS->getZExtValue() != 4)
20047 if (Alignment <
Align(4))
20051 Base =
Op->getOperand(0);
20057 bool isLegal =
false;
20059 isLegal = Subtarget->hasMVEIntegerOps() &&
20092 const APInt &DemandedElts,
20094 unsigned Depth)
const {
20097 switch (
Op.getOpcode()) {
20104 if (
Op.getResNo() == 0) {
20130 case Intrinsic::arm_ldaex:
20131 case Intrinsic::arm_ldrex: {
20132 EVT VT = cast<MemIntrinsicSDNode>(
Op)->getMemoryVT();
20146 const APInt &Mask =
Op.getConstantOperandAPInt(2);
20147 Known.
Zero &= Mask;
20153 const SDValue &SrcSV =
Op.getOperand(0);
20159 "VGETLANE index out of bounds");
20164 EVT VT =
Op.getValueType();
20171 Known = Known.
sext(DstSz);
20173 Known = Known.
zext(DstSz);
20181 Known = KnownOp.
zext(32);
20196 true,
false,
false, KnownOp1,
20222 EVT VT =
Op.getValueType();
20228 assert(VT == MVT::i32 &&
"Unexpected integer type");
20235 unsigned Mask =
C->getZExtValue();
20238 unsigned ShrunkMask = Mask & Demanded;
20239 unsigned ExpandedMask = Mask | ~Demanded;
20243 if (ShrunkMask == 0)
20249 if (ExpandedMask == ~0U)
20252 auto IsLegalMask = [ShrunkMask, ExpandedMask](
unsigned Mask) ->
bool {
20253 return (ShrunkMask & Mask) == ShrunkMask && (~ExpandedMask & Mask) == 0;
20255 auto UseMask = [Mask,
Op, VT, &TLO](
unsigned NewMask) ->
bool {
20256 if (NewMask == Mask)
20265 if (IsLegalMask(0xFF))
20266 return UseMask(0xFF);
20269 if (IsLegalMask(0xFFFF))
20270 return UseMask(0xFFFF);
20274 if (ShrunkMask < 256)
20275 return UseMask(ShrunkMask);
20279 if ((
int)ExpandedMask <= -2 && (
int)ExpandedMask >= -256)
20280 return UseMask(ExpandedMask);
20295 unsigned Depth)
const {
20296 unsigned Opc =
Op.getOpcode();
20304 if (
Op.getResNo() == 0 && !
Op->hasAnyUseOfValue(1) &&
20305 isa<ConstantSDNode>(
Op->getOperand(2))) {
20306 unsigned ShAmt =
Op->getConstantOperandVal(2);
20318 unsigned ModImm =
Op.getConstantOperandVal(1);
20319 unsigned EltBits = 0;
20321 if ((OriginalDemandedBits & Mask) == 0)
20327 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO,
Depth);
20336 if (!Subtarget->hasV6Ops())
20342 SplitString(AsmStr, AsmPieces,
";\n");
20344 switch (AsmPieces.
size()) {
20345 default:
return false;
20347 AsmStr = AsmPieces[0];
20349 SplitString(AsmStr, AsmPieces,
" \t,");
20352 if (AsmPieces.
size() == 3 &&
20353 AsmPieces[0] ==
"rev" && AsmPieces[1] ==
"$0" && AsmPieces[2] ==
"$1" &&
20354 IA->getConstraintString().compare(0, 4,
"=l,l") == 0) {
20377 if (ConstraintVT.
isVector() && Subtarget->hasNEON() &&
20389 unsigned S = Constraint.
size();
20391 switch (Constraint[0]) {
20403 }
else if (S == 2) {
20404 switch (Constraint[0]) {
20421 Value *CallOperandVal =
info.CallOperandVal;
20424 if (!CallOperandVal)
20428 switch (*constraint) {
20434 if (Subtarget->isThumb())
20448using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
20452 switch (Constraint.
size()) {
20455 switch (Constraint[0]) {
20457 if (Subtarget->isThumb())
20458 return RCPair(0U, &ARM::tGPRRegClass);
20459 return RCPair(0U, &ARM::GPRRegClass);
20461 if (Subtarget->isThumb())
20462 return RCPair(0U, &ARM::hGPRRegClass);
20466 return RCPair(0U, &ARM::tGPRRegClass);
20467 return RCPair(0U, &ARM::GPRRegClass);
20469 if (VT == MVT::Other)
20471 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20472 return RCPair(0U, &ARM::SPRRegClass);
20474 return RCPair(0U, &ARM::DPRRegClass);
20476 return RCPair(0U, &ARM::QPRRegClass);
20479 if (VT == MVT::Other)
20481 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20482 return RCPair(0U, &ARM::SPR_8RegClass);
20484 return RCPair(0U, &ARM::DPR_8RegClass);
20486 return RCPair(0U, &ARM::QPR_8RegClass);
20489 if (VT == MVT::Other)
20491 if (VT == MVT::f32 || VT == MVT::i32 || VT == MVT::f16 || VT == MVT::bf16)
20492 return RCPair(0U, &ARM::SPRRegClass);
20494 return RCPair(0U, &ARM::DPR_VFP2RegClass);
20496 return RCPair(0U, &ARM::QPR_VFP2RegClass);
20502 if (Constraint[0] ==
'T') {
20503 switch (Constraint[1]) {
20507 return RCPair(0U, &ARM::tGPREvenRegClass);
20509 return RCPair(0U, &ARM::tGPROddRegClass);
20518 if (
StringRef(
"{cc}").equals_insensitive(Constraint))
20519 return std::make_pair(
unsigned(ARM::CPSR), &ARM::CCRRegClass);
20528 std::vector<SDValue> &Ops,
20533 if (Constraint.
size() != 1)
20536 char ConstraintLetter = Constraint[0];
20537 switch (ConstraintLetter) {
20540 case 'I':
case 'J':
case 'K':
case 'L':
20541 case 'M':
case 'N':
case 'O':
20546 int64_t CVal64 =
C->getSExtValue();
20547 int CVal = (int) CVal64;
20550 if (CVal != CVal64)
20553 switch (ConstraintLetter) {
20557 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
20558 if (CVal >= 0 && CVal <= 65535)
20565 if (CVal >= 0 && CVal <= 255)
20567 }
else if (Subtarget->
isThumb2()) {
20586 if (CVal >= -255 && CVal <= -1)
20592 if (CVal >= -4095 && CVal <= 4095)
20605 }
else if (Subtarget->
isThumb2()) {
20628 if (CVal >= -7 && CVal < 7)
20630 }
else if (Subtarget->
isThumb2()) {
20653 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
20659 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
20667 if (CVal >= 0 && CVal <= 31)
20676 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
20685 if (Result.getNode()) {
20686 Ops.push_back(Result);
20696 "Unhandled Opcode in getDivRemLibcall");
20702 case MVT::i8: LC =
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
20703 case MVT::i16: LC =
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
20704 case MVT::i32: LC =
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
20705 case MVT::i64: LC =
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
20714 "Unhandled Opcode in getDivRemArgList");
20719 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
20720 EVT ArgVT =
N->getOperand(i).getValueType();
20722 Entry.Node =
N->getOperand(i);
20726 Args.push_back(Entry);
20737 "Register-based DivRem lowering only");
20738 unsigned Opcode =
Op->getOpcode();
20740 "Invalid opcode for Div/Rem lowering");
20742 EVT VT =
Op->getValueType(0);
20745 if (VT == MVT::i64 && isa<ConstantSDNode>(
Op.getOperand(1))) {
20764 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
20765 : Subtarget->hasDivideInARMMode();
20766 if (hasDivide &&
Op->getValueType(0).isSimple() &&
20767 Op->getSimpleValueType(0) == MVT::i32) {
20769 const SDValue Dividend =
Op->getOperand(0);
20770 const SDValue Divisor =
Op->getOperand(1);
20771 SDValue Div = DAG.
getNode(DivOpcode, dl, VT, Dividend, Divisor);
20775 SDValue Values[2] = {Div, Rem};
20807 EVT VT =
N->getValueType(0);
20809 if (VT == MVT::i64 && isa<ConstantSDNode>(
N->getOperand(1))) {
20813 Result[0], Result[1]);
20817 std::vector<Type*> RetTyParams;
20818 Type *RetTyElement;
20828 RetTyParams.push_back(RetTyElement);
20829 RetTyParams.push_back(RetTyElement);
20846 CallLoweringInfo CLI(DAG);
20850 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
20853 SDNode *ResNode = CallResult.first.getNode();
20868 "no-stack-arg-probe")) {
20870 cast<ConstantSDNode>(
Op.getOperand(2))->getMaybeAlignValue();
20879 SDValue Ops[2] = { SP, Chain };
20896 SDValue Ops[2] = { NewSP, Chain };
20901 bool IsStrict =
Op->isStrictFPOpcode();
20902 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20903 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20905 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&
20906 "Unexpected type for custom-lowering FP_EXTEND");
20909 "With both FP DP and 16, any FP conversion is legal!");
20911 assert(!(DstSz == 32 && Subtarget->hasFP16()) &&
20912 "With FP16, 16 to 32 conversion is legal!");
20915 if (SrcSz == 32 && DstSz == 64 && Subtarget->hasFP64()) {
20920 Loc,
Op.getValueType(), SrcVal);
20933 MakeLibCallOptions CallOptions;
20935 for (
unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
20936 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
20937 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
20938 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
20942 {DstVT, MVT::Other}, {Chain, SrcVal});
20949 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20950 "Unexpected type for custom-lowering FP_EXTEND");
20951 std::tie(SrcVal, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20956 return IsStrict ? DAG.
getMergeValues({SrcVal, Chain}, Loc) : SrcVal;
20960 bool IsStrict =
Op->isStrictFPOpcode();
20962 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20964 EVT DstVT =
Op.getValueType();
20965 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20968 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&
20969 "Unexpected type for custom-lowering FP_ROUND");
20972 "With both FP DP and 16, any FP conversion is legal!");
20977 if (SrcSz == 32 && Subtarget->hasFP16())
20982 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20983 "Unexpected type for custom-lowering FP_ROUND");
20984 MakeLibCallOptions CallOptions;
20987 std::tie(Result, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20999 if (v == 0xffffffff)
21011 bool ForCodeSize)
const {
21014 if (VT == MVT::f16 && Subtarget->hasFullFP16())
21016 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
21019 if (VT == MVT::f32)
21021 if (VT == MVT::f64 && Subtarget->hasFP64())
21032 unsigned Intrinsic)
const {
21033 switch (Intrinsic) {
21034 case Intrinsic::arm_neon_vld1:
21035 case Intrinsic::arm_neon_vld2:
21036 case Intrinsic::arm_neon_vld3:
21037 case Intrinsic::arm_neon_vld4:
21038 case Intrinsic::arm_neon_vld2lane:
21039 case Intrinsic::arm_neon_vld3lane:
21040 case Intrinsic::arm_neon_vld4lane:
21041 case Intrinsic::arm_neon_vld2dup:
21042 case Intrinsic::arm_neon_vld3dup:
21043 case Intrinsic::arm_neon_vld4dup: {
21046 auto &
DL =
I.getDataLayout();
21047 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21049 Info.ptrVal =
I.getArgOperand(0);
21051 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21052 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
21057 case Intrinsic::arm_neon_vld1x2:
21058 case Intrinsic::arm_neon_vld1x3:
21059 case Intrinsic::arm_neon_vld1x4: {
21062 auto &
DL =
I.getDataLayout();
21063 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21065 Info.ptrVal =
I.getArgOperand(
I.arg_size() - 1);
21067 Info.align.reset();
21072 case Intrinsic::arm_neon_vst1:
21073 case Intrinsic::arm_neon_vst2:
21074 case Intrinsic::arm_neon_vst3:
21075 case Intrinsic::arm_neon_vst4:
21076 case Intrinsic::arm_neon_vst2lane:
21077 case Intrinsic::arm_neon_vst3lane:
21078 case Intrinsic::arm_neon_vst4lane: {
21081 auto &
DL =
I.getDataLayout();
21082 unsigned NumElts = 0;
21083 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21084 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21087 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21090 Info.ptrVal =
I.getArgOperand(0);
21092 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21093 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
21098 case Intrinsic::arm_neon_vst1x2:
21099 case Intrinsic::arm_neon_vst1x3:
21100 case Intrinsic::arm_neon_vst1x4: {
21103 auto &
DL =
I.getDataLayout();
21104 unsigned NumElts = 0;
21105 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21106 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21109 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21112 Info.ptrVal =
I.getArgOperand(0);
21114 Info.align.reset();
21119 case Intrinsic::arm_mve_vld2q:
21120 case Intrinsic::arm_mve_vld4q: {
21123 Type *VecTy = cast<StructType>(
I.getType())->getElementType(1);
21124 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
21126 Info.ptrVal =
I.getArgOperand(0);
21133 case Intrinsic::arm_mve_vst2q:
21134 case Intrinsic::arm_mve_vst4q: {
21137 Type *VecTy =
I.getArgOperand(1)->getType();
21138 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
21140 Info.ptrVal =
I.getArgOperand(0);
21147 case Intrinsic::arm_mve_vldr_gather_base:
21148 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
21150 Info.ptrVal =
nullptr;
21156 case Intrinsic::arm_mve_vldr_gather_base_wb:
21157 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
21159 Info.ptrVal =
nullptr;
21165 case Intrinsic::arm_mve_vldr_gather_offset:
21166 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
21168 Info.ptrVal =
nullptr;
21170 unsigned MemSize = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
21177 case Intrinsic::arm_mve_vstr_scatter_base:
21178 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
21180 Info.ptrVal =
nullptr;
21186 case Intrinsic::arm_mve_vstr_scatter_base_wb:
21187 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
21189 Info.ptrVal =
nullptr;
21195 case Intrinsic::arm_mve_vstr_scatter_offset:
21196 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
21198 Info.ptrVal =
nullptr;
21200 unsigned MemSize = cast<ConstantInt>(
I.getArgOperand(3))->getZExtValue();
21207 case Intrinsic::arm_ldaex:
21208 case Intrinsic::arm_ldrex: {
21209 auto &
DL =
I.getDataLayout();
21210 Type *ValTy =
I.getParamElementType(0);
21213 Info.ptrVal =
I.getArgOperand(0);
21215 Info.align =
DL.getABITypeAlign(ValTy);
21219 case Intrinsic::arm_stlex:
21220 case Intrinsic::arm_strex: {
21221 auto &
DL =
I.getDataLayout();
21222 Type *ValTy =
I.getParamElementType(1);
21225 Info.ptrVal =
I.getArgOperand(1);
21227 Info.align =
DL.getABITypeAlign(ValTy);
21231 case Intrinsic::arm_stlexd:
21232 case Intrinsic::arm_strexd:
21234 Info.memVT = MVT::i64;
21235 Info.ptrVal =
I.getArgOperand(2);
21241 case Intrinsic::arm_ldaexd:
21242 case Intrinsic::arm_ldrexd:
21244 Info.memVT = MVT::i64;
21245 Info.ptrVal =
I.getArgOperand(0);
21265 if (Bits == 0 || Bits > 32)
21271 unsigned Index)
const {
21283 if (!Subtarget->hasDataBarrier()) {
21287 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
21324 if (Subtarget->preferISHSTBarriers())
21357 bool has64BitAtomicStore;
21359 has64BitAtomicStore =
false;
21360 else if (Subtarget->isThumb())
21361 has64BitAtomicStore = Subtarget->hasV7Ops();
21363 has64BitAtomicStore = Subtarget->hasV6Ops();
21365 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
21379 bool has64BitAtomicLoad;
21381 has64BitAtomicLoad =
false;
21382 else if (Subtarget->isThumb())
21383 has64BitAtomicLoad = Subtarget->hasV7Ops();
21385 has64BitAtomicLoad = Subtarget->hasV6Ops();
21402 hasAtomicRMW = Subtarget->hasV8MBaselineOps();
21403 else if (Subtarget->isThumb())
21404 hasAtomicRMW = Subtarget->hasV7Ops();
21406 hasAtomicRMW = Subtarget->hasV6Ops();
21407 if (
Size <= (Subtarget->
isMClass() ? 32U : 64U) && hasAtomicRMW) {
21430 bool HasAtomicCmpXchg;
21432 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps();
21433 else if (Subtarget->isThumb())
21434 HasAtomicCmpXchg = Subtarget->hasV7Ops();
21436 HasAtomicCmpXchg = Subtarget->hasV6Ops();
21438 HasAtomicCmpXchg &&
Size <= (Subtarget->
isMClass() ? 32U : 64U))
21445 return InsertFencesForAtomic;
21458 M.getOrInsertGlobal(
"__security_cookie",
21466 F->addParamAttr(0, Attribute::AttrKind::InReg);
21472 return M.getGlobalVariable(
"__security_cookie");
21479 return M.getFunction(
"__security_check_cookie");
21484 unsigned &
Cost)
const {
21486 if (!Subtarget->hasNEON())
21498 if (!isa<ConstantInt>(
Idx))
21513 return Subtarget->hasV6T2Ops();
21517 return Subtarget->hasV6T2Ops();
21522 if (!Subtarget->hasV7Ops())
21528 if (!Mask || Mask->getValue().getBitWidth() > 32u)
21530 auto MaskVal =
unsigned(Mask->getValue().getZExtValue());
21555 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
21567 Lo, Builder.
CreateShl(
Hi, ConstantInt::get(ValueTy, 32)),
"val64");
21570 Type *Tys[] = {
Addr->getType() };
21571 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
21576 0,
Attribute::get(M->getContext(), Attribute::ElementType, ValueTy));
21582 if (!Subtarget->hasV7Ops())
21599 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
21610 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
21611 Type *Tys[] = {
Addr->getType() };
21633 return (
DL.getTypeSizeInBits(VecTy) + 127) / 128;
21640 unsigned VecSize =
DL.getTypeSizeInBits(VecTy);
21643 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
21651 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
21659 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
21662 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
21667 if (Subtarget->hasNEON() && VecSize == 64)
21669 return VecSize % 128 == 0;
21673 if (Subtarget->hasNEON())
21675 if (Subtarget->hasMVEIntegerOps())
21695 "Invalid interleave factor");
21696 assert(!Shuffles.
empty() &&
"Empty shufflevector input");
21698 "Unmatched number of shufflevectors and indices");
21700 auto *VecTy = cast<FixedVectorType>(Shuffles[0]->
getType());
21701 Type *EltTy = VecTy->getElementType();
21724 if (NumLoads > 1) {
21728 VecTy->getNumElements() / NumLoads);
21733 auto createLoadIntrinsic = [&](
Value *BaseAddr) {
21734 if (Subtarget->hasNEON()) {
21736 Type *Tys[] = {VecTy, PtrTy};
21737 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
21738 Intrinsic::arm_neon_vld3,
21739 Intrinsic::arm_neon_vld4};
21747 return Builder.
CreateCall(VldnFunc, Ops,
"vldN");
21749 assert((Factor == 2 || Factor == 4) &&
21750 "expected interleave factor of 2 or 4 for MVE");
21752 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
21754 Type *Tys[] = {VecTy, PtrTy};
21760 return Builder.
CreateCall(VldnFunc, Ops,
"vldN");
21769 for (
unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
21774 VecTy->getNumElements() * Factor);
21776 CallInst *VldN = createLoadIntrinsic(BaseAddr);
21780 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21782 unsigned Index = Indices[i];
21792 SubVecs[SV].push_back(SubVec);
21801 auto &SubVec = SubVecs[SVI];
21804 SVI->replaceAllUsesWith(WideVec);
21838 unsigned Factor)
const {
21840 "Invalid interleave factor");
21842 auto *VecTy = cast<FixedVectorType>(SVI->
getType());
21843 assert(VecTy->getNumElements() % Factor == 0 &&
"Invalid interleaved store");
21845 unsigned LaneLen = VecTy->getNumElements() / Factor;
21846 Type *EltTy = VecTy->getElementType();
21850 Align Alignment = SI->getAlign();
21867 Type *IntTy =
DL.getIntPtrType(EltTy);
21879 Value *BaseAddr = SI->getPointerOperand();
21881 if (NumStores > 1) {
21884 LaneLen /= NumStores;
21892 auto createStoreIntrinsic = [&](
Value *BaseAddr,
21894 if (Subtarget->hasNEON()) {
21895 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
21896 Intrinsic::arm_neon_vst3,
21897 Intrinsic::arm_neon_vst4};
21898 Type *PtrTy = Builder.
getPtrTy(SI->getPointerAddressSpace());
21899 Type *Tys[] = {PtrTy, SubVecTy};
21902 SI->getModule(), StoreInts[Factor - 2], Tys);
21910 assert((Factor == 2 || Factor == 4) &&
21911 "expected interleave factor of 2 or 4 for MVE");
21913 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
21914 Type *PtrTy = Builder.
getPtrTy(SI->getPointerAddressSpace());
21915 Type *Tys[] = {PtrTy, SubVecTy};
21922 for (
unsigned F = 0;
F < Factor;
F++) {
21930 for (
unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
21933 if (StoreCount > 0)
21935 BaseAddr, LaneLen * Factor);
21940 for (
unsigned i = 0; i < Factor; i++) {
21941 unsigned IdxI = StoreCount * LaneLen * Factor + i;
21942 if (Mask[IdxI] >= 0) {
21946 unsigned StartMask = 0;
21947 for (
unsigned j = 1; j < LaneLen; j++) {
21948 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
21949 if (Mask[IdxJ * Factor + IdxI] >= 0) {
21950 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
21965 createStoreIntrinsic(BaseAddr, Shuffles);
21980 if (
auto *ST = dyn_cast<StructType>(Ty)) {
21981 for (
unsigned i = 0; i < ST->getNumElements(); ++i) {
21985 Members += SubMembers;
21987 }
else if (
auto *AT = dyn_cast<ArrayType>(Ty)) {
21991 Members += SubMembers * AT->getNumElements();
22002 }
else if (
auto *VT = dyn_cast<VectorType>(Ty)) {
22009 return VT->getPrimitiveSizeInBits().getFixedValue() == 64;
22011 return VT->getPrimitiveSizeInBits().getFixedValue() == 128;
22013 switch (VT->getPrimitiveSizeInBits().getFixedValue()) {
22026 return (Members > 0 && Members <= 4);
22032 const Align ABITypeAlign =
DL.getABITypeAlign(ArgTy);
22034 return ABITypeAlign;
22038 return std::min(ABITypeAlign,
DL.getStackAlignment());
22047 if (getEffectiveCallingConv(CallConv, isVarArg) !=
22057 return IsHA || IsIntArray;
22061 const Constant *PersonalityFn)
const {
22068 const Constant *PersonalityFn)
const {
22080void ARMTargetLowering::insertCopiesSplitCSR(
22084 const MCPhysReg *IStart =
TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
22094 RC = &ARM::GPRRegClass;
22095 else if (ARM::DPRRegClass.
contains(*
I))
22096 RC = &ARM::DPRRegClass;
22106 assert(Entry->getParent()->getFunction().hasFnAttribute(
22107 Attribute::NoUnwind) &&
22108 "Function should be nounwind in insertCopiesSplitCSR!");
22109 Entry->addLiveIn(*
I);
22114 for (
auto *Exit : Exits)
22116 TII->get(TargetOpcode::COPY), *
I)
22127 return Subtarget->hasMVEIntegerOps();
22132 auto *VTy = dyn_cast<FixedVectorType>(Ty);
22137 unsigned NumElements = VTy->getNumElements();
22144 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy())
22145 return Subtarget->hasMVEFloatOps();
22150 return Subtarget->hasMVEIntegerOps() &&
22151 (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
22152 ScalarTy->isIntegerTy(32));
22164 assert(TyWidth >= 128 &&
"Width of vector type must be at least 128 bits");
22166 if (TyWidth > 128) {
22171 ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
22173 auto *LowerSplitA =
B.CreateShuffleVector(InputA, LowerSplitMask);
22174 auto *LowerSplitB =
B.CreateShuffleVector(InputB, LowerSplitMask);
22175 auto *UpperSplitA =
B.CreateShuffleVector(InputA, UpperSplitMask);
22176 auto *UpperSplitB =
B.CreateShuffleVector(InputB, UpperSplitMask);
22177 Value *LowerSplitAcc =
nullptr;
22178 Value *UpperSplitAcc =
nullptr;
22181 LowerSplitAcc =
B.CreateShuffleVector(
Accumulator, LowerSplitMask);
22182 UpperSplitAcc =
B.CreateShuffleVector(
Accumulator, UpperSplitMask);
22186 B, OperationType, Rotation, LowerSplitA, LowerSplitB, LowerSplitAcc);
22188 B, OperationType, Rotation, UpperSplitA, UpperSplitB, UpperSplitAcc);
22191 return B.CreateShuffleVector(LowerSplitInt, UpperSplitInt, JoinMask);
22198 ConstRotation = ConstantInt::get(IntTy, (
int)Rotation);
22201 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmlaq, Ty,
22203 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmulq, Ty,
22204 {ConstRotation, InputB, InputA});
22209 auto *ConstHalving = ConstantInt::get(IntTy, 1);
22212 ConstRotation = ConstantInt::get(IntTy, 0);
22214 ConstRotation = ConstantInt::get(IntTy, 1);
22216 if (!ConstRotation)
22219 return B.CreateIntrinsic(Intrinsic::arm_mve_vcaddq, Ty,
22220 {ConstHalving, ConstRotation, InputA, InputB});
unsigned const MachineRegisterInfo * MRI
static bool isAddSubSExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
static bool isExtendedBUILD_VECTOR(SDValue N, SelectionDAG &DAG, bool isSigned)
static bool isZeroExtended(SDValue N, SelectionDAG &DAG)
static bool areExtractExts(Value *Ext1, Value *Ext2)
Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
static EVT getExtensionTo64Bits(const EVT &OrigVT)
static const MCPhysReg GPRArgRegs[]
static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1, SDValue V2, unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations t...
static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt)
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift oper...
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG)
static bool isSignExtended(SDValue N, SelectionDAG &DAG)
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
static bool isAddSubZExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left...
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG)
static bool isStore(int Opcode)
static bool isThumb(const MCSubtargetInfo &STI)
static SDValue LowerUADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo &MFI, const MachineRegisterInfo *MRI, const TargetInstrInfo *TII)
MatchingStackOffset - Return true if the given stack call argument is already available in the same p...
static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG)
static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total value size to 64 bits.
static cl::opt< unsigned > ConstpoolPromotionMaxSize("arm-promote-constant-max-size", cl::Hidden, cl::desc("Maximum size of constant to promote into a constant pool"), cl::init(64))
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isVTBLMask(ArrayRef< int > M, EVT VT)
static SDValue PerformSUBCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
static cl::opt< bool > EnableConstpoolPromotion("arm-promote-constant", cl::Hidden, cl::desc("Enable / disable promotion of unnamed_addr constants into " "constant pools"), cl::init(false))
static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG)
static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
static SDValue PerformExtractEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static const APInt * isPowerOf2Constant(SDValue V)
static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD) can replace combinations of ...
static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG)
static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static bool isValidMVECond(unsigned CC, bool IsFloat)
static SDValue PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC)
IntCCToARMCC - Convert a DAG integer condition code to an ARM CC.
static SDValue PerformSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSTORECombine - Target-specific dag combine xforms for ISD::STORE.
static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry, SelectionDAG &DAG)
static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isGTorGE(ISD::CondCode CC)
static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a vldN-lane (N > 1) intrinsic,...
static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask)
static bool isReverseMask(ArrayRef< int > M, EVT VT)
static bool isVZIP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of "vector_shuffle v,...
static SDValue PerformSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVMulVCTPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVMulVCTPCombine - VCVT (fixed-point to floating-point, Advanced SIMD) can replace combinations...
static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG)
static bool findPointerConstIncrement(SDNode *N, SDValue *Ptr, SDValue *CInc)
static bool isVTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool CanInvertMVEVCMP(SDValue N)
static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG)
static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformShiftCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
PerformShiftCombine - Checks for immediate versions of vector shifts and lowers them.
static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode, ARMCC::CondCodes &CondCode2)
FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static EVT getVectorTyFromPredicateVector(EVT VT)
static SDValue PerformFADDVCMLACombine(SDNode *N, SelectionDAG &DAG)
static SDValue handleCMSEValue(const SDValue &Value, const ISD::InputArg &Arg, SelectionDAG &DAG, const SDLoc &DL)
static SDValue PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
static bool isSRL16(const SDValue &Op)
static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformLOADCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC)
static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr, SDValue Inc, const SelectionDAG &DAG)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static Register genTPEntry(MachineBasicBlock *TpEntry, MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpExit, Register OpSizeReg, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI)
Adds logic in loop entry MBB to calculate loop iteration count and adds t2WhileLoopSetup and t2WhileL...
static bool isLTorLE(ISD::CondCode CC)
static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl, SelectionDAG &DAG)
static SDValue PerformBITCASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG)
static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG)
static bool hasNormalLoadOperand(SDNode *N)
hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node are normal,...
static SDValue PerformInsertEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
PerformInsertEltCombine - Target-specific dag combine xforms for ISD::INSERT_VECTOR_ELT.
static SDValue PerformVDUPLANECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVDUPLANECombine - Target-specific dag combine xforms for ARMISD::VDUPLANE.
static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static cl::opt< unsigned > ConstpoolPromotionMaxTotal("arm-promote-constant-max-total", cl::Hidden, cl::desc("Maximum size of ALL constants to promote into a constant pool"), cl::init(128))
static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static RTLIB::Libcall getDivRemLibcall(const SDNode *N, MVT::SimpleValueType SVT)
static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG &DAG)
SkipLoadExtensionForVMULL - return a load of the original vector size that does not do any sign/zero ...
static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static const MCPhysReg GPRArgRegs[]
static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombineWithOperands - Try DAG combinations for an ADD with operands N0 and N1.
static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT, SelectionDAG &DAG)
static bool isVZIPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformORCombineToSMULWBT(SDNode *OR, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isVTRN_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of "vector_shuffle v,...
static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue FindBFIToCombineWith(SDNode *N)
static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT, SelectionDAG &DAG)
static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode, bool &swpCmpOps, bool &swpVselOps)
static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isS16(const SDValue &Op, SelectionDAG &DAG)
static bool isSRA16(const SDValue &Op)
static SDValue AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue LowerInterruptReturn(SmallVectorImpl< SDValue > &RetOps, const SDLoc &DL, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl, SelectionDAG &DAG)
static void expandf64Toi32(SDValue Op, SelectionDAG &DAG, SDValue &RetVal1, SDValue &RetVal2)
static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isSHL16(const SDValue &Op)
static bool isVEXTMask(ArrayRef< int > M, EVT VT, bool &ReverseVEXT, unsigned &Imm)
static SDValue PerformMVEVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isTruncMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue PerformADDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2)
Return the load opcode for a given load size.
static bool isLegalT2AddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
static bool isLegalMVEShuffleOp(unsigned PFEntry)
static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N, SelectionDAG &DAG)
static bool isVUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG)
PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for ISD::VECTOR_SHUFFLE.
static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG)
SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND, ANY_EXTEND,...
static bool isVMOVNTruncMask(ArrayRef< int > M, EVT ToVT, bool rev)
static SDValue PerformVQMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static MachineBasicBlock * OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ)
static SDValue LowerVecReduceMinMax(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformAddcSubcCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static TargetLowering::ArgListTy getDivRemArgList(const SDNode *N, LLVMContext *Context, const ARMSubtarget *Subtarget)
static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl)
getZeroVector - Returns a vector of specified type with all zero elements.
static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static ARMCC::CondCodes getVCMPCondCode(SDValue N)
static cl::opt< bool > ARMInterworking("arm-interworking", cl::Hidden, cl::desc("Enable / disable ARM interworking (for debugging only)"), cl::init(true))
static void ReplaceREADCYCLECOUNTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformORCombineToBFI(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes, SDValue &CC, bool &Invert, SDValue &OtherOp, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVSetCCToVCTPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isZeroVector(SDValue N)
static SDValue PerformAddeSubeCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void ReplaceCMP_SWAP_64Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isLowerSaturate(const SDValue LHS, const SDValue RHS, const SDValue TrueVal, const SDValue FalseVal, const ISD::CondCode CC, const SDValue K)
static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG)
static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned StSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment store operation with given size.
static bool isVMOVNMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue CombineBaseUpdate(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineBaseUpdate - Target-specific DAG combine function for VLDDUP, NEON load/store intrinsics,...
static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVRRDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMOVRRDCombine - Target-specific dag combine xforms for ARMISD::VMOVRRD.
static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue PerformInsertSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain)
static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue PerformVMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMULCombine Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the special multi...
static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG)
static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformORCombine - Target-specific dag combine xforms for ISD::OR.
static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG)
static SDValue PerformTruncatingStoreCombine(StoreSDNode *St, SelectionDAG &DAG)
static unsigned SelectPairHalf(unsigned Elements, ArrayRef< int > Mask, unsigned Index)
static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned LdSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment load operation with given size.
static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG)
static bool isValidBaseUpdate(SDNode *N, SDNode *User)
static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG, const ARMSubtarget *ST, const SDLoc &dl)
static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op)
static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformXORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment, bool isSEXTLoad, bool IsMasked, bool isLE, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
std::pair< unsigned, const TargetRegisterClass * > RCPair
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,...
static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
cl::opt< unsigned > MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden, cl::desc("Maximum interleave factor for MVE VLDn to generate."), cl::init(2))
static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef, unsigned SplatBitSize, SelectionDAG &DAG, const SDLoc &dl, EVT &VT, EVT VectorVT, VMOVModImmType type)
isVMOVModifiedImm - Check if the specified splat value corresponds to a valid vector constant for a N...
static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC, SelectionDAG &DAG)
BC is a bitcast that is about to be turned into a VMOVDRR.
static SDValue promoteToConstantPool(const ARMTargetLowering *TLI, const GlobalValue *GV, SelectionDAG &DAG, EVT PtrVT, const SDLoc &dl)
static unsigned isNEONTwoResultShuffleMask(ArrayRef< int > ShuffleMask, EVT VT, unsigned &WhichResult, bool &isV_UNDEF)
Check if ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN), and return the corresponding AR...
static bool BitsProperlyConcatenate(const APInt &A, const APInt &B)
static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG)
static bool TryCombineBaseUpdate(struct BaseUpdateTarget &Target, struct BaseUpdateUser &User, bool SimpleConstIncOnly, TargetLowering::DAGCombinerInfo &DCI)
static bool allUsersAreInFunction(const Value *V, const Function *F)
Return true if all users of V are within function F, looking through ConstantExprs.
static bool isSingletonVEXTMask(ArrayRef< int > M, EVT VT, unsigned &Imm)
static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG)
PerformVMOVDRRCombine - Target-specific dag combine xforms for ARMISD::VMOVDRR.
static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V, SDValue &SatK)
static bool isLegalAddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
isLegalAddressImmediate - Return true if the integer value can be used as the offset of the target ad...
static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isLegalT1AddressImmediate(int64_t V, EVT VT)
static SDValue CombineANDShift(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSHLSimplify(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformADDECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDECombine - Target-specific dag combine transform from ARMISD::ADDC, ARMISD::ADDE,...
static SDValue PerformReduceShuffleCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformHWLoopCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool isVUZP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of "vector_shuffle v,...
static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base, uint64_t &Members)
static SDValue PerformMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
static SDValue PerformANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG)
static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm, bool &Negate)
static bool canChangeToInt(SDValue Op, bool &SeenZero, const ARMSubtarget *Subtarget)
canChangeToInt - Given the fp compare operand, return true if it is suitable to morph to an integer c...
static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2)
Return the store opcode for a given store size.
static bool IsVUZPShuffleNode(SDNode *N)
static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget, MachineInstr &MI, const SDNode *Node)
Attaches vregs to MEMCPY that it will use as scratch registers when it is expanded into LDM/STM.
static bool isFloatingPointZero(SDValue Op)
isFloatingPointZero - Return true if this is +0.0.
static SDValue findMUL_LOHI(SDValue V)
static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N, SelectionDAG &DAG)
static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG)
static void genTPLoopBody(MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI, Register OpSrcReg, Register OpDestReg, Register ElementCountReg, Register TotalIterationsReg, bool IsMemcpy)
Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and t2DoLoopEnd.
static SDValue PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformBUILD_VECTORCombine - Target-specific dag combine xforms for ISD::BUILD_VECTOR.
static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformMinMaxCombine - Target-specific DAG combining for creating truncating saturates.
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static Function * getFunction(Constant *C)
static bool isSigned(unsigned int Opcode)
const HexagonInstrInfo * TII
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
static Value * LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP)
Emit the code to lower ctpop of V before the specified instruction IP.
loop Loop Strength Reduction
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
This file describes how to lower LLVM code to machine code.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static X86::CondCode getSwappedCondition(X86::CondCode CC)
Assuming the flags are set by MI(a,b), return the condition code if we modify the instructions such t...
static constexpr int Concat[]
bool getExactInverse(APFloat *inv) const
APInt bitcastToAPInt() const
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned logBase2() const
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
An arbitrary precision integer that knows its signedness.
virtual const ARMBaseRegisterInfo & getRegisterInfo() const =0
const uint32_t * getSjLjDispatchPreservedMask(const MachineFunction &MF) const
const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const override
Code Generation virtual methods...
Register getFrameRegister(const MachineFunction &MF) const override
const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override
const uint32_t * getTLSCallPreservedMask(const MachineFunction &MF) const
const uint32_t * getThisReturnPreservedMask(const MachineFunction &MF, CallingConv::ID) const
getThisReturnPreservedMask - Returns a call preserved mask specific to the case that 'returned' is on...
static ARMConstantPoolConstant * Create(const Constant *C, unsigned ID)
static ARMConstantPoolMBB * Create(LLVMContext &C, const MachineBasicBlock *mbb, unsigned ID, unsigned char PCAdj)
static ARMConstantPoolSymbol * Create(LLVMContext &C, StringRef s, unsigned ID, unsigned char PCAdj)
ARMConstantPoolValue - ARM specific constantpool value.
ARMFunctionInfo - This class is derived from MachineFunctionInfo and contains private ARM-specific in...
int getVarArgsFrameIndex() const
int getPromotedConstpoolIncrease() const
SmallPtrSet< const GlobalVariable *, 2 > & getGlobalsPromotedToConstantPool()
void setArgumentStackToRestore(unsigned v)
bool branchTargetEnforcement() const
unsigned createPICLabelUId()
void setPromotedConstpoolIncrease(int Sz)
bool isThumb1OnlyFunction() const
void setArgRegsSaveSize(unsigned s)
bool shouldSignReturnAddress() const
bool isCmseNSEntryFunction() const
void setReturnRegsCount(unsigned s)
void setVarArgsFrameIndex(int Index)
unsigned getArgRegsSaveSize() const
void markGlobalAsPromotedToConstantPool(const GlobalVariable *GV)
Indicate to the backend that GV has had its storage changed to inside a constant pool.
void setIsSplitCSR(bool s)
void setArgumentStackSize(unsigned size)
unsigned getArgumentStackSize() const
bool isTargetMachO() const
bool isTargetAEABI() const
bool supportsTailCall() const
const Triple & getTargetTriple() const
const ARMBaseInstrInfo * getInstrInfo() const override
bool isThumb1Only() const
bool hasFPARMv8Base() const
bool isTargetWindows() const
bool isGVIndirectSymbol(const GlobalValue *GV) const
True if the GV will be accessed via an indirect symbol.
const ARMTargetLowering * getTargetLowering() const override
bool isTargetDarwin() const
const ARMBaseRegisterInfo * getRegisterInfo() const override
bool isTargetAndroid() const
bool isTargetCOFF() const
bool isTargetGNUAEABI() const
bool isTargetWatchOS() const
bool useNEONForSinglePrecisionFP() const
const InstrItineraryData * getInstrItineraryData() const override
getInstrItins - Return the instruction itineraries based on subtarget selection.
bool isTargetWatchABI() const
bool hasAnyDataBarrier() const
bool isTargetDriverKit() const
bool allowsUnalignedMem() const
bool isTargetMuslAEABI() const
bool isTargetLinux() const
unsigned getPrefLoopLogAlignment() const
bool isTargetHardFloat() const
Align getDualLoadStoreAlignment() const
bool isReadOnly(const GlobalValue *GV) const
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL) const
Returns the number of interleaved accesses that will be generated when lowering accesses of the given...
bool shouldInsertFencesForAtomic(const Instruction *I) const override
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const override
Return the correct alignment for the current calling convention.
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const override
Examine constraint string and operand type and determine a weight value.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
isLegalAddressingMode - Return true if the addressing mode represented by AM is legal for this target...
const ARMSubtarget * getSubtarget() const
bool isLegalT2ScaledAddressingMode(const AddrMode &AM, EVT VT) const
bool isLegalT1ScaledAddressingMode(const AddrMode &AM, EVT VT) const
Returns true if the addressing mode representing by AM is legal for the Thumb1 target,...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPreIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mod...
bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, Align &PrefAlign) const override
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
ReplaceNodeResults - Replace the results of node with an illegal result type with new values built ou...
void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
bool isLegalAddImmediate(int64_t Imm) const override
isLegalAddImmediate - Return true if the specified immediate is legal add immediate,...
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
SDValue PerformMVETruncCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const override
isFPImmLegal - Returns true if the target can instruction select the specified FP immediate natively.
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
bool preferIncOfAddToSubOfNot(EVT VT) const override
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
SDValue PerformIntrinsicCombine(SDNode *N, DAGCombinerInfo &DCI) const
PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
bool isDesirableToCommuteXorWithShift(const SDNode *N) const override
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
bool ExpandInlineAsm(CallInst *CI) const override
This hook allows the target to expand an inline asm call to be explicit llvm code if it wants to.
SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const
PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
Value * createComplexDeinterleavingIR(IRBuilderBase &B, ComplexDeinterleavingOperation OperationType, ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB, Value *Accumulator=nullptr) const override
Create the IR node for the given complex deinterleaving operation.
bool isComplexDeinterleavingSupported() const override
Does this target support complex deinterleaving.
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
SDValue PerformMVEExtCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to fold a pair of shifts into a mask.
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &OriginalDemandedBits, const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
getSetCCResultType - Return the value type to use for ISD::SETCC.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override
Perform a store-conditional operation to Addr.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool isVarArg) const
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
createFastISel - This method returns a target specific FastISel object, or null if the target does no...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
getTgtMemIntrinsic - Represent NEON load and store intrinsics as MemIntrinsicNodes.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
isShuffleMaskLegal - Targets can use this to indicate that they only support some VECTOR_SHUFFLE oper...
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const override
getRegClassFor - Return the register class that should be used for the specified value type.
std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const override
Return the largest legal super-reg register class of the register class for the specified type and it...
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vstN intrinsic.
ARMTargetLowering(const TargetMachine &TM, const ARMSubtarget &STI)
bool isComplexDeinterleavingOperationSupported(ComplexDeinterleavingOperation Operation, Type *Ty) const override
Does this target support complex deinterleaving with the given operation and type.
SDValue PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const
PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
Type * shouldConvertSplatType(ShuffleVectorInst *SVI) const override
Given a shuffle vector SVI representing a vector splat, return a new scalar type of size equal to SVI...
Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const override
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
Instruction * makeDMB(IRBuilderBase &Builder, ARM_MB::MemBOpt Domain) const
bool isLegalICmpImmediate(int64_t Imm) const override
isLegalICmpImmediate - Return true if the specified immediate is legal icmp immediate,...
const char * LowerXConstraint(EVT ConstraintVT) const override
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool isDesirableToTransformToIntegerOp(unsigned Opc, EVT VT) const override
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool isVarArg) const
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
allowsMisalignedMemoryAccesses - Returns true if the target allows unaligned memory accesses of the s...
bool isLegalInterleavedAccessType(unsigned Factor, FixedVectorType *VecTy, Align Alignment, const DataLayout &DL) const
Returns true if VecTy is a legal interleaved access type.
bool isVectorLoadExtDesirable(SDValue ExtVal) const override
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const override
Return true if the target can combine store(extractelement VectorTy, Idx).
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vldN intrinsic.
bool useSoftFloat() const override
bool alignLoopsWithOptSize() const override
Should loops be aligned even when the function is marked OptSize (but not MinSize).
SDValue PerformCMOVToBFICombine(SDNode *N, SelectionDAG &DAG) const
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
LowerAsmOperandForConstraint - Lower the specified operand into the Ops vector.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const override
Returns true if an argument of type Ty needs to be passed in a contiguous block of registers in calli...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPostIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mo...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
bool isFloatingPointOperation() const
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
The address of a basic block.
static BranchProbability getZero()
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
CCState - This class holds information needed while lowering arguments and return values.
void getInRegsParamInfo(unsigned InRegsParamRecordIndex, unsigned &BeginReg, unsigned &EndReg) const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
static bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
void rewindByValRegsInfo()
unsigned getInRegsParamsProcessed() const
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
void addInRegsParamInfo(unsigned RegBegin, unsigned RegEnd)
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
unsigned getInRegsParamsCount() const
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
unsigned getValNo() const
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
Value * getCalledOperand() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
const APFloat & getValueAPF() const
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
This is an important base class in LLVM.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
Align getStackAlignment() const
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPreferredAlign(const GlobalVariable *GV) const
Returns the preferred alignment of the specified global.
StringRef getPrivateGlobalPrefix() const
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for unsupported feature in backend.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
const GlobalValue * getGlobal() const
bool hasExternalWeakLinkage() const
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
bool isStrongDefinitionForLinker() const
Returns true if this global's definition will be the one chosen by the linker.
@ InternalLinkage
Rename collisions when linking (static functions).
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
TargetInstrInfo overrides.
Common base class shared among various IRBuilders.
Value * CreateZExtOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
BasicBlock * GetInsertBlock() const
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateTruncOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::optional< unsigned > getOperandCycle(unsigned ItinClassIndx, unsigned OperandIdx) const
Return the cycle for the given class and operand.
bool isEmpty() const
Returns true if there are no itineraries.
bool hasAtomicStore() const LLVM_READONLY
Return true if this atomic instruction stores to memory.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
static bool LowerToByteSwap(CallInst *CI)
Try to replace a call instruction with a call to a bswap intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Describe properties that are true of each instruction in the target description file.
unsigned getSchedClass() const
Return the scheduling class for this instruction.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
ArrayRef< MCOperandInfo > operands() const
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specified operand constraint if it is present.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
bool isEHPad() const
Returns true if the block is a landing pad.
MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool canFallThrough()
Return true if the block can implicitly transfer control to the block after it by falling off the end...
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
pred_iterator pred_begin()
MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void moveAfter(MachineBasicBlock *NewBefore)
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void computeMaxCallFrameSize(MachineFunction &MF, std::vector< MachineBasicBlock::iterator > *FrameSDOps=nullptr)
Computes the maximum size of a callframe.
void setAdjustsStack(bool V)
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool hasVAStart() const
Returns true if the function calls the llvm.va_start intrinsic.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
int getFunctionContextIndex() const
Return the index for the function context object.
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & reset(Property P)
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
const MachineFunctionProperties & getProperties() const
Get the function properties.
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i....
Representation of each machine instruction.
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr reads the specified register.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
const MachineOperand & getOperand(unsigned i) const
unsigned createJumpTableIndex(const std::vector< MachineBasicBlock * > &DestBBs)
createJumpTableIndex - Create a new jump table.
@ EK_Inline
EK_Inline - Jump table entries are emitted inline at their point of use.
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
void setIsDef(bool Val=true)
Change a def to a use, or a use to a def.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class is used to represent an MLOAD node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
virtual void print(raw_ostream &OS, const Module *M) const
print - Print out the internal state of the pass.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< use_iterator > uses()
size_t use_size() const
Return the number of uses of this node.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
void setFlags(SDNodeFlags NewFlags)
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
void addCallSiteInfo(const SDNode *Node, CallSiteInfo &&CallInfo)
Set CallSiteInfo to be associated with Node.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
This instruction constructs a fixed permutation of two input vectors.
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
ArrayRef< int > getMask() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
StringRef - Represent a constant reference to a string, i.e.
const unsigned char * bytes_end() const
constexpr size_t size() const
size - Get the string size.
const unsigned char * bytes_begin() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
void setCmpLibcallCC(RTLIB::Libcall Call, ISD::CondCode CC)
Override the default CondCode to be used to test the result of the comparison libcall against zero.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
void setMinStackArgumentAlignment(Align Alignment)
Set the minimum stack alignment of an argument.
const TargetMachine & getTargetMachine() const
void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC)
Set the CallingConv that should be used for the specified libcall.
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
virtual unsigned getMaxSupportedInterleaveFactor() const
Get the maximum supported factor for interleaved memory accesses.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
bool isPositionIndependent() const
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
const Triple & getTargetTriple() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
bool isOSMSVCRT() const
Is this a "Windows" OS targeting a "MSVCRT.dll" environment.
bool isOSVersionLT(unsigned Major, unsigned Minor=0, unsigned Micro=0) const
Helper function for doing comparisons against version numbers included in the target triple.
bool isWindowsMSVCEnvironment() const
Checks if the environment could be MSVC.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isArrayTy() const
True if this is an instance of ArrayType.
bool isPointerTy() const
True if this is an instance of PointerType.
Type * getArrayElementType() const
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
const Use & getOperandUse(unsigned i) const
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
const ParentTy * getParent() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static CondCodes getOppositeCondition(CondCodes CC)
@ SECREL
Thread Pointer Offset.
@ SBREL
Section Relative (Windows TLS)
@ GOTTPOFF
Global Offset Table, PC Relative.
@ TPOFF
Global Offset Table, Thread Pointer Offset.
TOF
Target Operand Flag enum.
@ MO_NONLAZY
MO_NONLAZY - This is an independent flag, on a symbol operand "FOO" it represents a symbol which,...
@ MO_SBREL
MO_SBREL - On a symbol operand, this represents a static base relative relocation.
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_GOT
MO_GOT - On a symbol operand, this represents a GOT relative relocation.
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
uint64_t decodeVMOVModImm(unsigned ModImm, unsigned &EltBits)
decodeVMOVModImm - Decode a NEON/MVE modified immediate value into the element value and the element ...
unsigned getAM2Offset(unsigned AM2Opc)
bool isThumbImmShiftedVal(unsigned V)
isThumbImmShiftedVal - Return true if the specified value can be obtained by left shifting a 8-bit im...
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
unsigned createVMOVModImm(unsigned OpCmode, unsigned Val)
int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
int getFP32FP16Imm(const APInt &Imm)
If this is a FP16Imm encoded as a fp32 value, return the 8-bit encoding for it.
AddrOpc getAM2Op(unsigned AM2Opc)
bool isBitFieldInvertedMask(unsigned v)
const unsigned FPStatusBits
const unsigned FPReservedBits
const unsigned RoundingBitsPos
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Swift
Calling convention for Swift.
@ ARM_APCS
ARM Procedure Calling Standard (obsolete, but still used on some targets).
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ ARM_AAPCS
ARM Architecture Procedure Calling Standard calling convention (aka EABI).
@ CXX_FAST_TLS
Used for access functions.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ ARM_AAPCS_VFP
Same as ARM_AAPCS, but uses hard floating point ABI.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SET_FPENV
Sets the current floating-point environment.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
static const int LAST_INDEXED_MODE
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool CC_ARM_APCS_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool RetCC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns true if Val1 has a lower Constant Materialization Cost than Val2.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool FastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool RetCC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
void shuffle(Iterator first, Iterator last, RNG &&g)
static std::array< MachineOperand, 2 > predOps(ARMCC::CondCodes Pred, unsigned PredReg=0)
Get the operands corresponding to the given Pred value.
bool CC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
bool CC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
bool RetCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_Win32_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
ComplexDeinterleavingOperation
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
ComplexDeinterleavingRotation
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
unsigned ConstantMaterializationCost(unsigned Val, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns the number of instructions required to materialize the given constant in a register,...
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool RetFastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
DWARFExpression::Operation Op
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
static MachineOperand t1CondCodeOp(bool isDead=false)
Get the operand corresponding to the conditional code result for Thumb1.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
static MachineOperand condCodeOp(unsigned CCReg=0)
Get the operand corresponding to the conditional code result.
bool isVREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isVREVMask - Check if a vector shuffle corresponds to a VREV instruction with the specified blocksize...
unsigned gettBLXrOpcode(const MachineFunction &MF)
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
unsigned convertAddSubFlagsOpcode(unsigned OldOpc)
Map pseudo instructions that imply an 'S' bit onto real opcodes.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
static const unsigned PerfectShuffleTable[6561+1]
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Load/store instruction that can be merged with a base address update.
SDNode * N
Instruction that updates a pointer.
unsigned ConstInc
Pointer increment value if it is a constant, or 0 otherwise.
SDValue Inc
Pointer increment operand.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
static constexpr roundingMode rmTowardZero
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
bool isUnknown() const
Returns true if we don't know any bits.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoSignedZeros() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This contains information for each constraint that we are lowering.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setInRegister(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)