25#include "llvm/IR/IntrinsicsS390.h"
37#define DEBUG_TYPE "systemz-lower"
43 cl::desc(
"Verify that narrow int args are properly extended per the "
50 : Op0(Op0In), Op1(Op1In), Chain(ChainIn),
51 Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
101 if (Subtarget.hasHighWord())
107 if (Subtarget.hasVector()) {
116 if (Subtarget.hasVectorEnhancements1())
121 if (Subtarget.hasVector()) {
131 if (Subtarget.hasVector())
136 if (!Subtarget.hasVector())
162 for (
unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
163 I <= MVT::LAST_FP_VALUETYPE;
189 for (
unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
190 I <= MVT::LAST_INTEGER_VALUETYPE;
221 if (Subtarget.hasPopulationCount())
247 (!Subtarget.hasFPExtension() && VT == MVT::i32) ?
Promote :
Custom;
268 if (!Subtarget.hasVectorEnhancements3()) {
295 if (Subtarget.hasVectorEnhancements3()) {
320 {MVT::i8, MVT::i16},
Custom);
332 {MVT::i8, MVT::i16, MVT::i32},
Legal);
334 {MVT::i8, MVT::i16},
Legal);
355 if (Subtarget.hasMiscellaneousExtensions4()) {
362 if (Subtarget.hasMiscellaneousExtensions3()) {
447 if (!Subtarget.hasVector()) {
465 if (VT != MVT::v2i64 || Subtarget.hasVectorEnhancements3()) {
470 if (Subtarget.hasVectorEnhancements3() &&
471 VT != MVT::v16i8 && VT != MVT::v8i16) {
481 if (Subtarget.hasVectorEnhancements1())
515 if (Subtarget.hasVector()) {
537 if (Subtarget.hasVectorEnhancements2()) {
563 for (
MVT VT : {MVT::f32, MVT::f64, MVT::f128}) {
577 for (
unsigned I = MVT::FIRST_FP_VALUETYPE;
578 I <= MVT::LAST_FP_VALUETYPE;
586 if (Subtarget.hasFPExtension()) {
614 if (Subtarget.hasFPExtension()) {
630 if (Subtarget.hasVector()) {
681 if (Subtarget.hasVectorEnhancements1()) {
688 if (Subtarget.hasVectorEnhancements1()) {
705 for (
MVT Type : {MVT::f64, MVT::v2f64, MVT::f32, MVT::v4f32, MVT::f128}) {
730 for (
auto VT : { MVT::f32, MVT::f64, MVT::f128,
731 MVT::v4f32, MVT::v2f64 }) {
742 if (!Subtarget.hasVectorEnhancements1()) {
748 if (Subtarget.hasVectorEnhancements1())
758 if (Subtarget.hasVectorEnhancements1()) {
770 if (!Subtarget.hasVector()) {
781 if (Subtarget.isTargetzOS()) {
846 return Subtarget.hasSoftFloat();
867 if (!Subtarget.hasVector() || VT.
getScalarType() == MVT::f16)
877 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
879 if (Subtarget.hasVector() && VT.
isVectorOf(MVT::f16)) {
880 IntermediateVT = RegisterVT = MVT::v8f16;
881 return NumIntermediates =
885 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
888 if (RegisterVT == MVT::v2i64 && !Subtarget.hasVector()) {
889 IntermediateVT = RegisterVT = MVT::i64;
890 NumIntermediates *= 2;
905 if (Subtarget.hasVector() && VT.
isVectorOf(MVT::f16))
909 if (RegisterVT == MVT::v2i64 && !Subtarget.hasVector())
917 if (Subtarget.hasVector() && VT.
isVectorOf(MVT::f16))
923 if (RegisterVT == MVT::v2i64 && !Subtarget.hasVector())
950 return Subtarget.hasVectorEnhancements1();
964 (isFP128 && !
Subtarget.hasVectorEnhancements1()))
973 uint64_t Byte = IntBits.lshr(
I * 8).trunc(8).getZExtValue();
980 Opcode = SystemZISD::BYTE_MASK;
986 if (SplatBitSize > 64)
989 auto TryValue = [&](uint64_t
Value) ->
bool {
993 OpVals.push_back(((
unsigned) SignedValue));
994 Opcode = SystemZISD::REPLICATE;
1000 unsigned Start,
End;
1001 if (
TII->isRxSBGMask(
Value, SplatBitSize, Start,
End)) {
1005 OpVals.push_back(Start - (64 - SplatBitSize));
1006 OpVals.push_back(
End - (64 - SplatBitSize));
1007 Opcode = SystemZISD::ROTATE_MASK;
1019 uint64_t SplatBitsZ = SplatBits.getZExtValue();
1020 uint64_t SplatUndefZ = SplatUndef.getZExtValue();
1032 return TryValue(SplatBitsZ | Middle);
1041 assert(IntBits.getBitWidth() == 128 &&
"Unsupported APInt.");
1047 unsigned HalfSize = Width / 2;
1052 if (HighValue != LowValue || 8 > HalfSize)
1055 SplatBits = HighValue;
1059 SplatBitSize = Width;
1067 BVN->
isConstantSplat(IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, 128,
1071 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, 8,
1077 bool ForCodeSize)
const {
1079 if (
Imm.isZero() ||
Imm.isNegZero())
1102 assert(
TRI->isTypeLegalForClass(*RC, MVT::i32) &&
"Invalid destination!");
1104 Register MainDstReg =
MRI.createVirtualRegister(RC);
1105 Register RestoreDstReg =
MRI.createVirtualRegister(RC);
1108 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
"Invalid Pointer Size!");
1161 const int64_t FPOffset = 0;
1170 Register LabelReg =
MRI.createVirtualRegister(PtrRC);
1182 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1183 bool HasFP = Subtarget.getFrameLowering()->hasFP(*MF);
1186 .
addReg(SpecialRegs->getFramePointerRegister())
1194 .
addReg(SpecialRegs->getStackPointerRegister())
1202 Register BCReg =
MRI.createVirtualRegister(PtrRC);
1205 .
addReg(SpecialRegs->getStackPointerRegister())
1206 .
addImm(TFL->getBackchainOffset(*MF))
1217 MIB =
BuildMI(*ThisMBB,
MI,
DL,
TII->get(SystemZ::EH_SjLj_Setup))
1221 MIB.
addRegMask(RegInfo->getNoPreservedMask());
1242 MI.eraseFromParent();
1258 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
"Invalid Pointer Size!");
1261 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1268 const int64_t FPOffset = 0;
1280 SpecialRegs->getFramePointerRegister())
1302 SpecialRegs->getStackPointerRegister())
1311 .
addReg(SpecialRegs->getStackPointerRegister())
1312 .
addImm(TFL->getBackchainOffset(*MF))
1318 MI.eraseFromParent();
1325 return MemVT != MVT::v2i64 || Subtarget.hasVector();
1331 return !Subtarget.hasVector();
1361 if (Subtarget.hasInterlockedAccess1() &&
1394 EVT VT =
Y.getValueType();
1397 if (VT == MVT::i32 || VT == MVT::i64)
1398 return Subtarget.hasMiscellaneousExtensions3();
1401 if (VT.
isVector() || VT == MVT::i128)
1402 return Subtarget.hasVector();
1430 bool MVC = Ty->isIntegerTy(8);
1436static AddressingMode
1439 switch (
II->getIntrinsicID()) {
1441 case Intrinsic::memset:
1442 case Intrinsic::memmove:
1443 case Intrinsic::memcpy:
1450 if (SingleUser->getParent() ==
I->getParent()) {
1453 if (
C->getBitWidth() <= 64 &&
1463 if (LoadI->hasOneUse() && LoadI->getParent() ==
I->getParent())
1477 I->getOperand(0)->getType());
1479 bool IsVectorAccess = MemAccessTy->isVectorTy();
1484 Value *DataOp =
I->getOperand(0);
1486 IsVectorAccess =
true;
1492 User *LoadUser = *
I->user_begin();
1494 IsVectorAccess =
true;
1497 if (IsFPAccess || IsVectorAccess)
1516 Subtarget.hasVector() && (Ty->isVectorTy() || Ty->isIntegerTy(128));
1526 return AM.
Scale == 0;
1533 LLVMContext &Context, std::vector<EVT> &MemOps,
unsigned Limit,
1534 const MemOp &
Op,
unsigned DstAS,
unsigned SrcAS,
1538 "Expected EmitTargetCodeForMemXXX() to handle AlwaysInline cases.");
1540 if (
Op.isZeroMemset())
1543 const int MVCFastLen = 16;
1546 if (
Op.isMemcpy() &&
Op.size() <= MVCFastLen)
1548 if (
Op.isMemset() &&
Op.size() - 1 <= MVCFastLen)
1552 if ((
Op.size() >= 16 && !
Op.isAligned(
Align(8))) ||
1553 (
Op.size() >= 25 &&
Op.size() <= 31))
1557 Context, MemOps, Limit,
Op, DstAS, SrcAS, FuncAttributes, LargestVT);
1563 return Subtarget.hasVector() ? MVT::v2i64 : MVT::Other;
1567 if (!FromType->isIntegerTy() || !ToType->
isIntegerTy())
1569 unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedValue();
1571 return FromBits > ToBits;
1579 return FromBits > ToBits;
1583 unsigned Opc =
N->getOpcode();
1596 if (Constraint.
size() == 1) {
1597 switch (Constraint[0]) {
1623 }
else if (Constraint.
size() == 2 && Constraint[0] ==
'Z') {
1624 switch (Constraint[1]) {
1635 if (
StringRef(
"{@cc}").compare(Constraint) == 0)
1645 Value *CallOperandVal = Info.CallOperandVal;
1648 if (!CallOperandVal)
1652 switch (*Constraint) {
1671 if (Subtarget.hasVector())
1702 if (
C->getZExtValue() == 0x7fffffff)
1712static std::pair<unsigned, const TargetRegisterClass *>
1714 const unsigned *Map,
unsigned Size) {
1715 assert(*(Constraint.
end()-1) ==
'}' &&
"Missing '}'");
1716 if (isdigit(Constraint[2])) {
1721 return std::make_pair(Map[Index], RC);
1723 return std::make_pair(0U,
nullptr);
1726std::pair<unsigned, const TargetRegisterClass *>
1729 if (Constraint.
size() == 1) {
1731 switch (Constraint[0]) {
1737 if (VT == MVT::Other)
1740 return std::make_pair(0U, &SystemZ::GR64BitRegClass);
1742 return std::make_pair(0U, &SystemZ::GR128BitRegClass);
1743 return std::make_pair(0U, &SystemZ::GR32BitRegClass);
1747 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass);
1748 else if (VT == MVT::i128)
1749 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass);
1750 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass);
1753 return std::make_pair(0U, &SystemZ::GRH32BitRegClass);
1757 if (VT == MVT::Other)
1760 return std::make_pair(0U, &SystemZ::FP16BitRegClass);
1762 return std::make_pair(0U, &SystemZ::FP64BitRegClass);
1764 return std::make_pair(0U, &SystemZ::FP128BitRegClass);
1765 return std::make_pair(0U, &SystemZ::FP32BitRegClass);
1770 if (Subtarget.hasVector()) {
1771 if (VT == MVT::Other)
1774 return std::make_pair(0U, &SystemZ::VR16BitRegClass);
1776 return std::make_pair(0U, &SystemZ::VR32BitRegClass);
1778 return std::make_pair(0U, &SystemZ::VR64BitRegClass);
1779 return std::make_pair(0U, &SystemZ::VR128BitRegClass);
1788 auto getVTSizeInBits = [&VT]() {
1796 if (Constraint[1] ==
'r') {
1797 if (getVTSizeInBits() == 32)
1800 if (getVTSizeInBits() == 128)
1806 if (Constraint[1] ==
'f') {
1808 return std::make_pair(
1810 if (getVTSizeInBits() == 16)
1813 if (getVTSizeInBits() == 32)
1816 if (getVTSizeInBits() == 128)
1822 if (Constraint[1] ==
'v') {
1823 if (!Subtarget.hasVector())
1824 return std::make_pair(
1826 if (getVTSizeInBits() == 16)
1829 if (getVTSizeInBits() == 32)
1832 if (getVTSizeInBits() == 64)
1838 if (Constraint[1] ==
'@') {
1839 if (
StringRef(
"{@cc}").compare(Constraint) == 0)
1840 return std::make_pair(SystemZ::CC, &SystemZ::CCRRegClass);
1853 .
Case(
"r4", Subtarget.isTargetXPLINK64() ? SystemZ::R4D
1854 : SystemZ::NoRegister)
1856 Subtarget.isTargetELF() ? SystemZ::R15D : SystemZ::NoRegister)
1864 return Subtarget.isTargetXPLINK64() ? SystemZ::R1D : SystemZ::R6D;
1869 return Subtarget.isTargetXPLINK64() ? SystemZ::R2D : SystemZ::R7D;
1884 if (
StringRef(
"{@cc}").compare(OpInfo.ConstraintCode) != 0)
1888 if (OpInfo.ConstraintVT.isVector() || !OpInfo.ConstraintVT.isInteger() ||
1889 OpInfo.ConstraintVT.getSizeInBits() < 8)
1904 if (Constraint.
size() == 1) {
1905 switch (Constraint[0]) {
1910 Op.getValueType()));
1917 Op.getValueType()));
1924 C->getSExtValue(),
SDLoc(
Op),
Op.getValueType()));
1931 C->getSExtValue(),
SDLoc(
Op),
Op.getValueType()));
1936 if (
C->getZExtValue() == 0x7fffffff)
1938 Op.getValueType()));
1949#define GET_CALLING_CONV_IMPL
1950#include "SystemZGenCallingConv.inc"
1954 static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D,
1960 Type *ToType)
const {
2023 if (BitCastToType == MVT::v2i64)
2047 return DAG.
getNode(SystemZISD::PAIR128,
DL, MVT::v2i64,
Hi,
Lo);
2070 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
2083 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
2095template <
class ArgTy>
2099 if (!Args[
I].Flags.isSplit())
2103 PartVT = ArgLocs[
I].getValVT();
2105 for (
unsigned PartIdx =
I + 1;; ++PartIdx) {
2106 assert(PartIdx != ArgLocs.
size() &&
"SplitEnd not found.");
2107 assert(ArgLocs[PartIdx].getValVT() == PartVT &&
"Unsupported split.");
2109 if (Args[PartIdx].Flags.isSplitEnd())
2127 ADAvReg =
MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
2130 MRI.addLiveIn(Regs->getADARegister(), ADAvReg);
2153 unsigned NumFixedGPRs = 0;
2154 unsigned NumFixedFPRs = 0;
2155 for (
unsigned I = 0, E = ArgLocs.
size();
I != E; ++
I) {
2168 RC = &SystemZ::GR32BitRegClass;
2172 RC = &SystemZ::GR64BitRegClass;
2176 RC = &SystemZ::FP16BitRegClass;
2180 RC = &SystemZ::FP32BitRegClass;
2184 RC = &SystemZ::FP64BitRegClass;
2188 RC = &SystemZ::FP128BitRegClass;
2197 RC = &SystemZ::VR128BitRegClass;
2211 if (Subtarget.isTargetXPLINK64()) {
2214 ArgSPOffset += XPRegs.getCallFrameSize();
2225 unsigned SlotOffs = VA.
getLocVT() == MVT::f16 ? 6 : 4;
2229 ArgValue = DAG.
getLoad(LocVT,
DL, Chain, FIN,
2243 for (
unsigned PartIdx = 1; PartIdx <
NumParts; ++PartIdx) {
2246 unsigned PartOffset = Ins[
I].PartOffset;
2251 assert(PartOffset &&
"Offset should be non-zero.");
2254 }
else if (Subtarget.isTargetXPLINK64() &&
2257 Ins[
I].ArgVT.isSimple()) {
2262 MVT OrigVT = Ins[
I].ArgVT.getSimpleVT();
2268 if (IsVarArg && Subtarget.isTargetXPLINK64()) {
2274 Subtarget.getSpecialRegisters());
2280 int64_t VarArgOffset = CCInfo.
getStackSize() + Regs->getCallFrameSize();
2285 if (IsVarArg && Subtarget.isTargetELF()) {
2298 int64_t RegSaveOffset =
2313 &SystemZ::FP64BitRegClass);
2334 for (
unsigned I = 0,
E = ArgLocs.
size();
I !=
E; ++
I) {
2341 if (
Reg == SystemZ::R6H ||
Reg == SystemZ::R6L ||
Reg == SystemZ::R6D)
2343 if (Outs[
I].Flags.isSwiftSelf() || Outs[
I].Flags.isSwiftError())
2350 unsigned Offset,
bool LoadAdr =
false) {
2372 bool LoadAddr =
false;
2394 unsigned ADADelta = 0;
2395 unsigned EPADelta = 8;
2401 bool IsInternal = (
G->getGlobal()->hasInternalLinkage() ||
2402 G->getGlobal()->hasPrivateLinkage());
2407 Callee = DAG.
getNode(SystemZISD::PCREL_WRAPPER,
DL, PtrVT, Callee);
2453 if (Subtarget.isTargetXPLINK64()) {
2460 verifyNarrowIntegerArgs_Call(CLI, Outs, &MF.
getFunction(), Callee);
2464 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx);
2474 "site marked musttail");
2487 for (
unsigned I = 0, E = ArgLocs.
size();
I != E; ++
I) {
2499 SlotVT = Outs[
I].VT;
2506 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot, StackPtrInfo));
2509 assert(Outs[
I].PartOffset == 0);
2510 for (
unsigned PartIdx = 1; PartIdx <
NumParts; ++PartIdx) {
2513 unsigned PartOffset = Outs[
I].PartOffset;
2519 assert(PartOffset &&
"Offset should be non-zero.");
2521 SlotVT.
getStoreSize()) &&
"Not enough space for argument part!");
2523 ArgValue = SpillSlot;
2540 if (!StackPtr.getNode())
2547 else if (VA.
getLocVT() == MVT::f16)
2560 if (Subtarget.isTargetXPLINK64() && VA.
needsCustom()) {
2564 RegsToPass.
push_back(std::make_pair(SystemZ::R3D, ShadowArgValue));
2570 if (!MemOpChains.
empty())
2578 if (Subtarget.isTargetXPLINK64()) {
2583 ->getAddressOfCalleeRegister();
2586 Callee = DAG.
getRegister(CalleeReg, Callee.getValueType());
2593 Callee = DAG.
getNode(SystemZISD::PCREL_WRAPPER,
DL, PtrVT, Callee);
2596 Callee = DAG.
getNode(SystemZISD::PCREL_WRAPPER,
DL, PtrVT, Callee);
2597 }
else if (IsTailCall) {
2600 Callee = DAG.
getRegister(SystemZ::R1D, Callee.getValueType());
2605 for (
const auto &[Reg,
N] : RegsToPass) {
2612 Ops.push_back(Chain);
2613 Ops.push_back(Callee);
2617 for (
const auto &[Reg,
N] : RegsToPass)
2622 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
2623 assert(Mask &&
"Missing call preserved mask for calling convention");
2628 Ops.push_back(Glue);
2637 Chain = DAG.
getNode(SystemZISD::CALL,
DL, NodeTys,
Ops);
2647 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx);
2654 VA.getLocVT(), Glue);
2671 bool DoesNotReturn,
bool IsReturnValueUsed)
const {
2673 Args.reserve(
Ops.size());
2679 Entry.IsZExt = !Entry.IsSExt;
2680 Args.push_back(Entry);
2691 .
setCallee(CallConv, RetTy, Callee, std::move(Args))
2702 const Type *RetTy)
const {
2705 for (
auto &Out : Outs)
2706 if (Out.ArgVT.isScalarInteger() && Out.ArgVT.getSizeInBits() > 64)
2710 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Context);
2711 return RetCCInfo.
CheckReturn(Outs, RetCC_SystemZ);
2723 verifyNarrowIntegerArgs_Ret(CallConv, Outs, &MF.
getFunction());
2731 if (RetLocs.
empty())
2732 return DAG.
getNode(SystemZISD::RET_GLUE,
DL, MVT::Other, Chain);
2741 for (
unsigned I = 0, E = RetLocs.
size();
I != E; ++
I) {
2763 return DAG.
getNode(SystemZISD::RET_GLUE,
DL, MVT::Other, RetOps);
2770 unsigned &CCValid) {
2771 unsigned Id =
Op.getConstantOperandVal(1);
2773 case Intrinsic::s390_tbegin:
2774 Opcode = SystemZISD::TBEGIN;
2778 case Intrinsic::s390_tbegin_nofloat:
2779 Opcode = SystemZISD::TBEGIN_NOFLOAT;
2783 case Intrinsic::s390_tend:
2784 Opcode = SystemZISD::TEND;
2797 unsigned Id =
Op.getConstantOperandVal(0);
2799 case Intrinsic::s390_vpkshs:
2800 case Intrinsic::s390_vpksfs:
2801 case Intrinsic::s390_vpksgs:
2802 Opcode = SystemZISD::PACKS_CC;
2806 case Intrinsic::s390_vpklshs:
2807 case Intrinsic::s390_vpklsfs:
2808 case Intrinsic::s390_vpklsgs:
2809 Opcode = SystemZISD::PACKLS_CC;
2813 case Intrinsic::s390_vceqbs:
2814 case Intrinsic::s390_vceqhs:
2815 case Intrinsic::s390_vceqfs:
2816 case Intrinsic::s390_vceqgs:
2817 case Intrinsic::s390_vceqqs:
2818 Opcode = SystemZISD::VICMPES;
2822 case Intrinsic::s390_vchbs:
2823 case Intrinsic::s390_vchhs:
2824 case Intrinsic::s390_vchfs:
2825 case Intrinsic::s390_vchgs:
2826 case Intrinsic::s390_vchqs:
2827 Opcode = SystemZISD::VICMPHS;
2831 case Intrinsic::s390_vchlbs:
2832 case Intrinsic::s390_vchlhs:
2833 case Intrinsic::s390_vchlfs:
2834 case Intrinsic::s390_vchlgs:
2835 case Intrinsic::s390_vchlqs:
2836 Opcode = SystemZISD::VICMPHLS;
2840 case Intrinsic::s390_vtm:
2841 Opcode = SystemZISD::VTM;
2845 case Intrinsic::s390_vfaebs:
2846 case Intrinsic::s390_vfaehs:
2847 case Intrinsic::s390_vfaefs:
2848 Opcode = SystemZISD::VFAE_CC;
2852 case Intrinsic::s390_vfaezbs:
2853 case Intrinsic::s390_vfaezhs:
2854 case Intrinsic::s390_vfaezfs:
2855 Opcode = SystemZISD::VFAEZ_CC;
2859 case Intrinsic::s390_vfeebs:
2860 case Intrinsic::s390_vfeehs:
2861 case Intrinsic::s390_vfeefs:
2862 Opcode = SystemZISD::VFEE_CC;
2866 case Intrinsic::s390_vfeezbs:
2867 case Intrinsic::s390_vfeezhs:
2868 case Intrinsic::s390_vfeezfs:
2869 Opcode = SystemZISD::VFEEZ_CC;
2873 case Intrinsic::s390_vfenebs:
2874 case Intrinsic::s390_vfenehs:
2875 case Intrinsic::s390_vfenefs:
2876 Opcode = SystemZISD::VFENE_CC;
2880 case Intrinsic::s390_vfenezbs:
2881 case Intrinsic::s390_vfenezhs:
2882 case Intrinsic::s390_vfenezfs:
2883 Opcode = SystemZISD::VFENEZ_CC;
2887 case Intrinsic::s390_vistrbs:
2888 case Intrinsic::s390_vistrhs:
2889 case Intrinsic::s390_vistrfs:
2890 Opcode = SystemZISD::VISTR_CC;
2894 case Intrinsic::s390_vstrcbs:
2895 case Intrinsic::s390_vstrchs:
2896 case Intrinsic::s390_vstrcfs:
2897 Opcode = SystemZISD::VSTRC_CC;
2901 case Intrinsic::s390_vstrczbs:
2902 case Intrinsic::s390_vstrczhs:
2903 case Intrinsic::s390_vstrczfs:
2904 Opcode = SystemZISD::VSTRCZ_CC;
2908 case Intrinsic::s390_vstrsb:
2909 case Intrinsic::s390_vstrsh:
2910 case Intrinsic::s390_vstrsf:
2911 Opcode = SystemZISD::VSTRS_CC;
2915 case Intrinsic::s390_vstrszb:
2916 case Intrinsic::s390_vstrszh:
2917 case Intrinsic::s390_vstrszf:
2918 Opcode = SystemZISD::VSTRSZ_CC;
2922 case Intrinsic::s390_vfcedbs:
2923 case Intrinsic::s390_vfcesbs:
2924 Opcode = SystemZISD::VFCMPES;
2928 case Intrinsic::s390_vfchdbs:
2929 case Intrinsic::s390_vfchsbs:
2930 Opcode = SystemZISD::VFCMPHS;
2934 case Intrinsic::s390_vfchedbs:
2935 case Intrinsic::s390_vfchesbs:
2936 Opcode = SystemZISD::VFCMPHES;
2940 case Intrinsic::s390_vftcidb:
2941 case Intrinsic::s390_vftcisb:
2942 Opcode = SystemZISD::VFTCI;
2946 case Intrinsic::s390_tdc:
2947 Opcode = SystemZISD::TDC;
2960 unsigned NumOps =
Op.getNumOperands();
2963 Ops.push_back(
Op.getOperand(0));
2965 Ops.push_back(
Op.getOperand(
I));
2967 assert(
Op->getNumValues() == 2 &&
"Expected only CC result and chain");
2981 unsigned NumOps =
Op.getNumOperands();
2987 assert((
Op.getConstantOperandVal(0) == Intrinsic::s390_tdc &&
I == 1) &&
2988 "Unhandled intrinsic with f16 operand.");
2991 Ops.push_back(CurrOper);
3005 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
3006 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
3007 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
3033 if (!ConstOp1 || ConstOp1->getValueSizeInBits(0) > 64)
3036 int64_t
Value = ConstOp1->getSExtValue();
3052 if (!
C.Op0.hasOneUse() ||
3059 unsigned NumBits =
Load->getMemoryVT().getSizeInBits();
3060 if ((NumBits != 8 && NumBits != 16) ||
3061 NumBits !=
Load->getMemoryVT().getStoreSizeInBits())
3067 if (!ConstOp1 || ConstOp1->getValueSizeInBits(0) > 64)
3070 uint64_t Mask = (1 << NumBits) - 1;
3073 int64_t SignedValue = ConstOp1->getSExtValue();
3080 }
else if (NumBits == 8) {
3106 if (
C.Op0.getValueType() != MVT::i32 ||
3107 Load->getExtensionType() != ExtType) {
3109 Load->getBasePtr(),
Load->getPointerInfo(),
3110 Load->getMemoryVT(),
Load->getAlign(),
3111 Load->getMemOperand()->getFlags());
3117 if (
C.Op1.getValueType() != MVT::i32 ||
3118 Value != ConstOp1->getZExtValue())
3128 if (
Load->getMemoryVT() == MVT::i8)
3131 switch (
Load->getExtensionType()) {
3149 if (
C.Op0.isMachineOpcode() &&
3150 (
C.Op0.getMachineOpcode() == SystemZ::LOAD_STACK_GUARD))
3154 if (
C.Op0.getValueType() == MVT::i128)
3156 if (
C.Op0.getValueType() == MVT::f128)
3168 if (ConstOp1 && ConstOp1->getZExtValue() == 0)
3197 unsigned Opcode0 =
C.Op0.getOpcode();
3204 C.Op0.getConstantOperandVal(1) == 0xffffffff)
3219 ((
N->getOperand(0) ==
C.Op0 &&
N->getOperand(1) ==
C.Op1) ||
3220 (
N->getOperand(0) ==
C.Op1 &&
N->getOperand(1) ==
C.Op0))) {
3242 if (
C1 &&
C1->isZero()) {
3261 if (
C.Op0.getOpcode() ==
ISD::SHL &&
C.Op0.getValueType() == MVT::i64 &&
3264 if (
C1 &&
C1->getZExtValue() == 32) {
3265 SDValue ShlOp0 =
C.Op0.getOperand(0);
3284 C.Op0.getOperand(0).getOpcode() ==
ISD::LOAD &&
3287 C.Op1->getAsZExtVal() == 0) {
3289 if (L->getMemoryVT().getStoreSizeInBits().getFixedValue() <=
3290 C.Op0.getValueSizeInBits().getFixedValue()) {
3291 unsigned Type = L->getExtensionType();
3294 C.Op0 =
C.Op0.getOperand(0);
3310 if (
C.Opcode != SystemZISD::ICMP)
3320 if (!
C.Op1.isMachineOpcode() ||
3321 C.Op1.getMachineOpcode() != SystemZ::LOAD_STACK_GUARD)
3326 C.Opcode = SystemZISD::CMP_STACKGUARD;
3337 uint64_t Amount = Shift->getZExtValue();
3338 if (Amount >=
N.getValueSizeInBits())
3353 unsigned ICmpType) {
3354 assert(Mask != 0 &&
"ANDs with zero should have been removed by now");
3376 if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <=
Low) {
3382 if (EffectivelyUnsigned && CmpVal <
Low) {
3390 if (CmpVal == Mask) {
3396 if (EffectivelyUnsigned && CmpVal >= Mask -
Low && CmpVal < Mask) {
3402 if (EffectivelyUnsigned && CmpVal > Mask -
Low && CmpVal <= Mask) {
3410 if (EffectivelyUnsigned && CmpVal >= Mask -
High && CmpVal <
High) {
3416 if (EffectivelyUnsigned && CmpVal > Mask -
High && CmpVal <=
High) {
3445 if (
C.Op0.getValueType() == MVT::i128) {
3451 if (Mask && Mask->getAPIntValue() == 0) {
3452 C.Opcode = SystemZISD::VTM;
3469 uint64_t CmpVal = ConstOp1->getZExtValue();
3476 NewC.Op0 =
C.Op0.getOperand(0);
3477 NewC.Op1 =
C.Op0.getOperand(1);
3481 MaskVal = Mask->getZExtValue();
3501 MaskVal = -(CmpVal & -CmpVal);
3510 unsigned NewCCMask, ShiftVal;
3514 (MaskVal >> ShiftVal != 0) &&
3515 ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
3517 MaskVal >> ShiftVal,
3521 MaskVal >>= ShiftVal;
3525 (MaskVal << ShiftVal != 0) &&
3526 ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
3528 MaskVal << ShiftVal,
3532 MaskVal <<= ShiftVal;
3541 C.Opcode = SystemZISD::TM;
3543 if (Mask && Mask->getZExtValue() == MaskVal)
3548 C.CCMask = NewCCMask;
3554 if (
C.Opcode != SystemZISD::ICMP)
3556 if (
C.Op0.getValueType() != MVT::i128)
3567 Src = Src.getOperand(0);
3570 unsigned Opcode = 0;
3571 if (Src.hasOneUse()) {
3572 switch (Src.getOpcode()) {
3573 case SystemZISD::VICMPE: Opcode = SystemZISD::VICMPES;
break;
3574 case SystemZISD::VICMPH: Opcode = SystemZISD::VICMPHS;
break;
3575 case SystemZISD::VICMPHL: Opcode = SystemZISD::VICMPHLS;
break;
3576 case SystemZISD::VFCMPE: Opcode = SystemZISD::VFCMPES;
break;
3577 case SystemZISD::VFCMPH: Opcode = SystemZISD::VFCMPHS;
break;
3578 case SystemZISD::VFCMPHE: Opcode = SystemZISD::VFCMPHES;
break;
3584 C.Op0 = Src->getOperand(0);
3585 C.Op1 = Src->getOperand(1);
3589 C.CCMask ^=
C.CCValid;
3601 C.Opcode = SystemZISD::VICMPES;
3613 bool Swap =
false, Invert =
false;
3625 C.Opcode = SystemZISD::UCMP128HI;
3627 C.Opcode = SystemZISD::SCMP128HI;
3632 C.CCMask ^=
C.CCValid;
3643 if (!Mask || Mask->getValueSizeInBits(0) > 64)
3646 if ((~
Known.Zero).getZExtValue() & ~Mask->getZExtValue())
3649 C.Op0 =
C.Op0.getOperand(0);
3661 C.CCValid = CCValid;
3664 C.CCMask =
CC < 4 ? 1 << (3 -
CC) : 0;
3667 C.CCMask =
CC < 4 ? ~(1 << (3 -
CC)) : -1;
3671 C.CCMask =
CC < 4 ? ~0U << (4 -
CC) : -1;
3674 C.CCMask =
CC < 4 ? ~(~0U << (4 -
CC)) : 0;
3678 C.CCMask =
CC < 4 ? ~0U << (3 -
CC) : -1;
3681 C.CCMask =
CC < 4 ? ~(~0U << (3 -
CC)) : 0;
3684 C.CCMask &= CCValid;
3692 bool IsSignaling =
false) {
3695 unsigned Opcode, CCValid;
3707 Comparison
C(CmpOp0, CmpOp1, Chain);
3709 if (
C.Op0.getValueType().isFloatingPoint()) {
3712 C.Opcode = SystemZISD::FCMP;
3713 else if (!IsSignaling)
3714 C.Opcode = SystemZISD::STRICT_FCMP;
3716 C.Opcode = SystemZISD::STRICT_FCMPS;
3721 C.Opcode = SystemZISD::ICMP;
3757 if (!
C.Op1.getNode()) {
3758 if (
C.Opcode == SystemZISD::CMP_STACKGUARD)
3759 return DAG.
getNode(SystemZISD::CMP_STACKGUARD,
DL, MVT::i32,
C.Op0);
3761 switch (
C.Op0.getOpcode()) {
3772 if (
C.Opcode == SystemZISD::ICMP)
3773 return DAG.
getNode(SystemZISD::ICMP,
DL, MVT::i32,
C.Op0,
C.Op1,
3775 if (
C.Opcode == SystemZISD::TM) {
3778 return DAG.
getNode(SystemZISD::TM,
DL, MVT::i32,
C.Op0,
C.Op1,
3781 if (
C.Opcode == SystemZISD::VICMPES ||
3782 C.Opcode == SystemZISD::VICMPHS ||
3783 C.Opcode == SystemZISD::VICMPHLS ||
3784 C.Opcode == SystemZISD::VFCMPES ||
3785 C.Opcode == SystemZISD::VFCMPHS ||
3786 C.Opcode == SystemZISD::VFCMPHES) {
3787 EVT IntVT =
C.Op0.getValueType().changeVectorElementTypeToInteger();
3794 return DAG.
getNode(
C.Opcode,
DL, VTs,
C.Chain,
C.Op0,
C.Op1);
3796 return DAG.
getNode(
C.Opcode,
DL, MVT::i32,
C.Op0,
C.Op1);
3805 Op0 = DAG.
getNode(Extend,
DL, MVT::i64, Op0);
3806 Op1 = DAG.
getNode(Extend,
DL, MVT::i64, Op1);
3831 unsigned CCValid,
unsigned CCMask) {
3836 return DAG.
getNode(SystemZISD::SELECT_CCMASK,
DL, MVT::i32,
Ops);
3914 int Mask[] = { Start, -1, Start + 1, -1 };
3918 return DAG.
getNode(SystemZISD::STRICT_VEXTEND,
DL, VTs, Chain,
Op);
3920 return DAG.
getNode(SystemZISD::VEXTEND,
DL, MVT::v2f64,
Op);
3934 !Subtarget.hasVectorEnhancements1()) {
3940 SDVTList VTs = DAG.
getVTList(MVT::v2i64, MVT::Other);
3953 return DAG.
getNode(SystemZISD::PACK,
DL, VT, HRes, LRes);
3956 SDVTList VTs = DAG.
getVTList(VT, MVT::Other);
3957 return DAG.
getNode(Opcode,
DL, VTs, Chain, CmpOp0, CmpOp1);
3959 return DAG.
getNode(Opcode,
DL, VT, CmpOp0, CmpOp1);
3972 bool IsSignaling)
const {
3975 assert (!IsSignaling || Chain);
3978 bool Invert =
false;
3986 assert(IsFP &&
"Unexpected integer comparison");
3988 DL, VT, CmpOp1, CmpOp0, Chain);
3990 DL, VT, CmpOp0, CmpOp1, Chain);
3994 LT.getValue(1),
GE.getValue(1));
4003 assert(IsFP &&
"Unexpected integer comparison");
4005 DL, VT, CmpOp1, CmpOp0, Chain);
4007 DL, VT, CmpOp0, CmpOp1, Chain);
4011 LT.getValue(1),
GT.getValue(1));
4032 Cmp = getVectorCmp(DAG, Opcode,
DL, VT, CmpOp0, CmpOp1, Chain);
4036 Cmp = getVectorCmp(DAG, Opcode,
DL, VT, CmpOp1, CmpOp0, Chain);
4041 Chain =
Cmp.getValue(1);
4049 if (Chain && Chain.
getNode() !=
Cmp.getNode()) {
4062 EVT VT =
Op.getValueType();
4064 return lowerVectorSETCC(DAG,
DL, VT,
CC, CmpOp0, CmpOp1);
4073 bool IsSignaling)
const {
4079 EVT VT =
Op.getNode()->getValueType(0);
4081 SDValue Res = lowerVectorSETCC(DAG,
DL, VT,
CC, CmpOp0, CmpOp1,
4082 Chain, IsSignaling);
4104 SystemZISD::BR_CCMASK,
DL,
Op.getValueType(),
Op.getOperand(0),
4138 C.CCMask ^=
C.CCValid;
4146 Op = SystemZISD::VICMPE;
4150 Op = SystemZISD::VICMPHL;
4152 Op = SystemZISD::VICMPH;
4191 C.Op1->getAsZExtVal() == 0) {
4198 if (Subtarget.hasVectorEnhancements3() &&
4199 C.Opcode == SystemZISD::ICMP &&
4200 C.Op0.getValueType() == MVT::i128 &&
4210 return DAG.
getNode(SystemZISD::SELECT_CCMASK,
DL,
Op.getValueType(),
Ops);
4216 const GlobalValue *GV =
Node->getGlobal();
4222 if (Subtarget.isPC32DBLSymbol(GV, CM)) {
4244 }
else if (Subtarget.isTargetELF()) {
4249 }
else if (Subtarget.isTargetzOS()) {
4280 Chain = DAG.
getCopyToReg(Chain,
DL, SystemZ::R2D, GOTOffset, Glue);
4285 Ops.push_back(Chain);
4287 Node->getValueType(0),
4296 const TargetRegisterInfo *
TRI = Subtarget.getRegisterInfo();
4297 const uint32_t *
Mask =
4299 assert(Mask &&
"Missing call preserved mask for calling convention");
4303 Ops.push_back(Glue);
4306 SDVTList NodeTys = DAG.
getVTList(MVT::Other, MVT::Glue);
4314SDValue SystemZTargetLowering::lowerThreadPointer(
const SDLoc &
DL,
4338 const GlobalValue *GV =
Node->getGlobal();
4346 SDValue TP = lowerThreadPointer(
DL, DAG);
4353 SystemZConstantPoolValue *CPV =
4362 Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_GDCALL,
Offset);
4368 SystemZConstantPoolValue *CPV =
4377 Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_LDCALL,
Offset);
4382 SystemZMachineFunctionInfo* MFI =
4411 SystemZConstantPoolValue *CPV =
4445 return DAG.
getNode(SystemZISD::PCREL_WRAPPER,
DL, PtrVT, Result);
4462 return DAG.
getNode(SystemZISD::PCREL_WRAPPER,
DL, PtrVT, Result);
4467 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4469 MachineFrameInfo &MFI = MF.getFrameInfo();
4473 unsigned Depth =
Op.getConstantOperandVal(0);
4480 int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
4485 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4491 MachinePointerInfo());
4506 unsigned Depth =
Op.getConstantOperandVal(0);
4511 if (!MF.
getSubtarget<SystemZSubtarget>().hasBackChain())
4514 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
4515 const auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4516 int Offset = TFL->getReturnAddressOffset(MF);
4520 MachinePointerInfo());
4525 SystemZCallingConventionRegisters *CCR = Subtarget.getSpecialRegisters();
4527 &SystemZ::GR64BitRegClass);
4535 EVT InVT =
In.getValueType();
4536 EVT ResVT =
Op.getValueType();
4544 LoadN->getBasePtr(), LoadN->getMemOperand());
4550 if (InVT == MVT::i32 && ResVT == MVT::f32) {
4552 if (Subtarget.hasHighWord()) {
4556 MVT::i64,
SDValue(U64, 0), In);
4564 DL, MVT::f32, Out64);
4566 if (InVT == MVT::f32 && ResVT == MVT::i32) {
4569 MVT::f64,
SDValue(U64, 0), In);
4571 if (Subtarget.hasHighWord())
4584 if (Subtarget.isTargetXPLINK64())
4585 return lowerVASTART_XPLINK(
Op, DAG);
4587 return lowerVASTART_ELF(
Op, DAG);
4593 SystemZMachineFunctionInfo *FuncInfo =
4594 MF.
getInfo<SystemZMachineFunctionInfo>();
4604 MachinePointerInfo(SV));
4610 SystemZMachineFunctionInfo *FuncInfo =
4611 MF.
getInfo<SystemZMachineFunctionInfo>();
4620 const unsigned NumFields = 4;
4631 for (
unsigned I = 0;
I < NumFields; ++
I) {
4636 MemOps[
I] = DAG.
getStore(Chain,
DL, Fields[
I], FieldAddr,
4637 MachinePointerInfo(SV,
Offset));
4657 nullptr, std::nullopt, MachinePointerInfo(DstSV),
4658 MachinePointerInfo(SrcSV));
4662SystemZTargetLowering::lowerDYNAMIC_STACKALLOC(
SDValue Op,
4664 if (Subtarget.isTargetXPLINK64())
4665 return lowerDYNAMIC_STACKALLOC_XPLINK(
Op, DAG);
4667 return lowerDYNAMIC_STACKALLOC_ELF(
Op, DAG);
4671SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_XPLINK(
SDValue Op,
4673 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4683 uint64_t AlignVal = (RealignOpt ?
Align->getAsZExtVal() : 0);
4686 uint64_t RequiredAlign = std::max(AlignVal, StackAlign);
4687 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4693 if (ExtraAlignSpace)
4697 bool IsSigned =
false;
4698 bool DoesNotReturn =
false;
4699 bool IsReturnValueUsed =
false;
4700 EVT VT =
Op.getValueType();
4710 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
4722 if (ExtraAlignSpace) {
4734SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_ELF(
SDValue Op,
4736 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4739 bool StoreBackchain = MF.
getSubtarget<SystemZSubtarget>().hasBackChain();
4748 uint64_t AlignVal = (RealignOpt ?
Align->getAsZExtVal() : 0);
4751 uint64_t RequiredAlign = std::max(AlignVal, StackAlign);
4752 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4763 Backchain = DAG.
getLoad(MVT::i64,
DL, Chain, getBackchainAddress(OldSP, DAG),
4764 MachinePointerInfo());
4767 if (ExtraAlignSpace)
4774 NewSP = DAG.
getNode(SystemZISD::PROBED_ALLOCA,
DL,
4775 DAG.
getVTList(MVT::i64, MVT::Other), Chain, OldSP, NeededSpace);
4791 if (RequiredAlign > StackAlign) {
4801 Chain = DAG.
getStore(Chain,
DL, Backchain, getBackchainAddress(NewSP, DAG),
4802 MachinePointerInfo());
4808SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
4812 return DAG.
getNode(SystemZISD::ADJDYNALLOC,
DL, MVT::i64);
4817 unsigned Opcode)
const {
4818 EVT VT =
Op.getValueType();
4824 assert(Subtarget.hasMiscellaneousExtensions2());
4829 Op.getOperand(0),
Op.getOperand(1), Even, Odd);
4835 EVT VT =
Op.getValueType();
4843 else if (Subtarget.hasMiscellaneousExtensions2())
4848 Op.getOperand(0),
Op.getOperand(1),
Ops[1],
Ops[0]);
4883 EVT VT =
Op.getValueType();
4896 Op.getOperand(0),
Op.getOperand(1),
Ops[1],
Ops[0]);
4904 EVT VT =
Op.getValueType();
4924 EVT VT =
Op.getValueType();
4931 Op.getOperand(0),
Op.getOperand(1),
Ops[1],
Ops[0]);
4936 assert(
Op.getValueType() == MVT::i64 &&
"Should be 64-bit operation");
4946 Known[1].Zero.getZExtValue() };
4948 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff)
4950 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff)
4987 MVT::i64, HighOp, Low32);
4993 SDNode *
N =
Op.getNode();
4998 if (
N->getValueType(0) == MVT::i128) {
4999 unsigned BaseOp = 0;
5000 unsigned FlagOp = 0;
5001 bool IsBorrow =
false;
5002 switch (
Op.getOpcode()) {
5006 FlagOp = SystemZISD::VACC;
5010 FlagOp = SystemZISD::VSCBI;
5025 unsigned BaseOp = 0;
5026 unsigned CCValid = 0;
5027 unsigned CCMask = 0;
5029 switch (
Op.getOpcode()) {
5032 BaseOp = SystemZISD::SADDO;
5037 BaseOp = SystemZISD::SSUBO;
5042 BaseOp = SystemZISD::UADDO;
5047 BaseOp = SystemZISD::USUBO;
5053 SDVTList VTs = DAG.
getVTList(
N->getValueType(0), MVT::i32);
5057 if (
N->getValueType(1) == MVT::i1)
5083 SDNode *
N =
Op.getNode();
5084 MVT VT =
N->getSimpleValueType(0);
5095 if (VT == MVT::i128) {
5096 unsigned BaseOp = 0;
5097 unsigned FlagOp = 0;
5098 bool IsBorrow =
false;
5099 switch (
Op.getOpcode()) {
5102 BaseOp = SystemZISD::VAC;
5103 FlagOp = SystemZISD::VACCC;
5106 BaseOp = SystemZISD::VSBI;
5107 FlagOp = SystemZISD::VSBCBI;
5126 unsigned BaseOp = 0;
5127 unsigned CCValid = 0;
5128 unsigned CCMask = 0;
5130 switch (
Op.getOpcode()) {
5136 BaseOp = SystemZISD::ADDCARRY;
5144 BaseOp = SystemZISD::SUBCARRY;
5155 SDVTList VTs = DAG.
getVTList(VT, MVT::i32);
5159 if (
N->getValueType(1) == MVT::i1)
5167 EVT VT =
Op.getValueType();
5169 Op =
Op.getOperand(0);
5192 Op = DAG.
getNode(SystemZISD::VSRL_BY_SCALAR,
DL, VT,
Op, Shift);
5204 Op = DAG.
getNode(SystemZISD::VSUM,
DL, MVT::v4i32,
Op, Tmp);
5218 if (NumSignificantBits == 0)
5224 BitSize = std::min(BitSize, OrigBitSize);
5233 for (int64_t
I = BitSize / 2;
I >= 8;
I =
I / 2) {
5235 if (BitSize != OrigBitSize)
5272 EVT RegVT =
Op.getValueType();
5274 return lowerATOMIC_LDST_I128(
Op, DAG);
5275 return lowerLoadF16(
Op, DAG);
5281 if (
Node->getMemoryVT().getSizeInBits() == 128)
5282 return lowerATOMIC_LDST_I128(
Op, DAG);
5283 return lowerStoreF16(
Op, DAG);
5290 (
Node->getMemoryVT() == MVT::i128 ||
Node->getMemoryVT() == MVT::f128) &&
5291 "Only custom lowering i128 or f128.");
5304 EVT WideVT = MVT::i32;
5325void SystemZTargetLowering::lowerATOMIC_LOAD_OP(
5327 unsigned Opcode)
const {
5330 EVT NarrowVT =
Node->getMemoryVT();
5331 EVT WideVT = MVT::i32;
5336 MachineMemOperand *MMO =
Node->getMemOperand();
5340 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
5342 Opcode = SystemZISD::ATOMIC_LOADW_ADD;
5346 SDValue AlignedAddr, BitShift, NegBitShift;
5355 if (Opcode != SystemZISD::ATOMIC_SWAPW)
5358 if (Opcode == SystemZISD::ATOMIC_LOADW_AND ||
5359 Opcode == SystemZISD::ATOMIC_LOADW_NAND)
5364 SDVTList VTList = DAG.
getVTList(WideVT, MVT::Other);
5365 SDValue Ops[] = { ChainIn, AlignedAddr,
Src2, BitShift, NegBitShift,
5384 EVT MemVT =
Node->getMemoryVT();
5385 assert(
Op.getValueType() == MemVT &&
"Mismatched VTs");
5386 assert(Subtarget.hasInterlockedAccess1() &&
5387 "Should have been expanded by AtomicExpand pass.");
5393 Node->getBasePtr(), NegSrc2,
Node->getMemOperand());
5404 MachineMemOperand *MMO =
Node->getMemOperand();
5407 if (
Node->getMemoryVT() == MVT::i128) {
5416 EVT NarrowVT =
Node->getMemoryVT();
5417 EVT WideVT = NarrowVT == MVT::i64 ? MVT::i64 : MVT::i32;
5418 if (NarrowVT == WideVT) {
5419 SDVTList Tys = DAG.
getVTList(WideVT, MVT::i32, MVT::Other);
5420 SDValue Ops[] = { ChainIn, Addr, CmpVal, SwapVal };
5422 DL, Tys,
Ops, NarrowVT, MMO);
5436 SDValue AlignedAddr, BitShift, NegBitShift;
5440 SDVTList VTList = DAG.
getVTList(WideVT, MVT::i32, MVT::Other);
5441 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
5444 VTList,
Ops, NarrowVT, MMO);
5458SystemZTargetLowering::getTargetMMOFlags(
const Instruction &
I)
const {
5481 auto *Regs = Subtarget.getSpecialRegisters();
5484 "in GHC calling convention");
5486 Regs->getStackPointerRegister(),
Op.getValueType());
5492 auto *Regs = Subtarget.getSpecialRegisters();
5496 "in GHC calling convention");
5511 if (Subtarget.isTargetXPLINK64()) {
5515 return DAG.
getNode(SystemZISD::XPLINK_STACKRESTORE,
DL, MVT::Other, Chain,
5519 bool StoreBackchain = MF.
getSubtarget<SystemZSubtarget>().hasBackChain();
5521 if (StoreBackchain) {
5523 Chain,
DL, Regs->getStackPointerRegister(), MVT::i64);
5525 DAG.
getLoad(MVT::i64,
DL, Chain, getBackchainAddress(OldSP, DAG),
5526 MachinePointerInfo());
5529 Chain = DAG.
getCopyToReg(Chain,
DL, Regs->getStackPointerRegister(), NewSP);
5532 Chain = DAG.
getStore(Chain,
DL, Backchain, getBackchainAddress(NewSP, DAG),
5533 MachinePointerInfo());
5540 bool IsData =
Op.getConstantOperandVal(4);
5543 return Op.getOperand(0);
5546 bool IsWrite =
Op.getConstantOperandVal(2);
5553 Node->getMemoryVT(),
Node->getMemOperand());
5557SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(
SDValue Op,
5559 unsigned Opcode, CCValid;
5561 assert(
Op->getNumValues() == 2 &&
"Expected only CC result and chain");
5572SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(
SDValue Op,
5574 unsigned Opcode, CCValid;
5577 if (
Op->getNumValues() == 1)
5579 assert(
Op->getNumValues() == 2 &&
"Expected a CC and non-CC result");
5584 unsigned Id =
Op.getConstantOperandVal(0);
5586 case Intrinsic::thread_pointer:
5587 return lowerThreadPointer(SDLoc(
Op), DAG);
5589 case Intrinsic::s390_vpdi:
5590 return DAG.
getNode(SystemZISD::PERMUTE_DWORDS, SDLoc(
Op),
Op.getValueType(),
5591 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
5593 case Intrinsic::s390_vperm:
5594 return DAG.
getNode(SystemZISD::PERMUTE, SDLoc(
Op),
Op.getValueType(),
5595 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
5597 case Intrinsic::s390_vuphb:
5598 case Intrinsic::s390_vuphh:
5599 case Intrinsic::s390_vuphf:
5600 case Intrinsic::s390_vuphg:
5601 return DAG.
getNode(SystemZISD::UNPACK_HIGH, SDLoc(
Op),
Op.getValueType(),
5604 case Intrinsic::s390_vuplhb:
5605 case Intrinsic::s390_vuplhh:
5606 case Intrinsic::s390_vuplhf:
5607 case Intrinsic::s390_vuplhg:
5608 return DAG.
getNode(SystemZISD::UNPACKL_HIGH, SDLoc(
Op),
Op.getValueType(),
5611 case Intrinsic::s390_vuplb:
5612 case Intrinsic::s390_vuplhw:
5613 case Intrinsic::s390_vuplf:
5614 case Intrinsic::s390_vuplg:
5615 return DAG.
getNode(SystemZISD::UNPACK_LOW, SDLoc(
Op),
Op.getValueType(),
5618 case Intrinsic::s390_vupllb:
5619 case Intrinsic::s390_vupllh:
5620 case Intrinsic::s390_vupllf:
5621 case Intrinsic::s390_vupllg:
5622 return DAG.
getNode(SystemZISD::UNPACKL_LOW, SDLoc(
Op),
Op.getValueType(),
5625 case Intrinsic::s390_vsumb:
5626 case Intrinsic::s390_vsumh:
5627 case Intrinsic::s390_vsumgh:
5628 case Intrinsic::s390_vsumgf:
5629 case Intrinsic::s390_vsumqf:
5630 case Intrinsic::s390_vsumqg:
5631 return DAG.
getNode(SystemZISD::VSUM, SDLoc(
Op),
Op.getValueType(),
5632 Op.getOperand(1),
Op.getOperand(2));
5634 case Intrinsic::s390_vaq:
5636 Op.getOperand(1),
Op.getOperand(2));
5637 case Intrinsic::s390_vaccb:
5638 case Intrinsic::s390_vacch:
5639 case Intrinsic::s390_vaccf:
5640 case Intrinsic::s390_vaccg:
5641 case Intrinsic::s390_vaccq:
5642 return DAG.
getNode(SystemZISD::VACC, SDLoc(
Op),
Op.getValueType(),
5643 Op.getOperand(1),
Op.getOperand(2));
5644 case Intrinsic::s390_vacq:
5645 return DAG.
getNode(SystemZISD::VAC, SDLoc(
Op),
Op.getValueType(),
5646 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
5647 case Intrinsic::s390_vacccq:
5648 return DAG.
getNode(SystemZISD::VACCC, SDLoc(
Op),
Op.getValueType(),
5649 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
5651 case Intrinsic::s390_vsq:
5653 Op.getOperand(1),
Op.getOperand(2));
5654 case Intrinsic::s390_vscbib:
5655 case Intrinsic::s390_vscbih:
5656 case Intrinsic::s390_vscbif:
5657 case Intrinsic::s390_vscbig:
5658 case Intrinsic::s390_vscbiq:
5659 return DAG.
getNode(SystemZISD::VSCBI, SDLoc(
Op),
Op.getValueType(),
5660 Op.getOperand(1),
Op.getOperand(2));
5661 case Intrinsic::s390_vsbiq:
5662 return DAG.
getNode(SystemZISD::VSBI, SDLoc(
Op),
Op.getValueType(),
5663 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
5664 case Intrinsic::s390_vsbcbiq:
5665 return DAG.
getNode(SystemZISD::VSBCBI, SDLoc(
Op),
Op.getValueType(),
5666 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
5668 case Intrinsic::s390_vmhb:
5669 case Intrinsic::s390_vmhh:
5670 case Intrinsic::s390_vmhf:
5671 case Intrinsic::s390_vmhg:
5672 case Intrinsic::s390_vmhq:
5674 Op.getOperand(1),
Op.getOperand(2));
5675 case Intrinsic::s390_vmlhb:
5676 case Intrinsic::s390_vmlhh:
5677 case Intrinsic::s390_vmlhf:
5678 case Intrinsic::s390_vmlhg:
5679 case Intrinsic::s390_vmlhq:
5681 Op.getOperand(1),
Op.getOperand(2));
5683 case Intrinsic::s390_vmahb:
5684 case Intrinsic::s390_vmahh:
5685 case Intrinsic::s390_vmahf:
5686 case Intrinsic::s390_vmahg:
5687 case Intrinsic::s390_vmahq:
5688 return DAG.
getNode(SystemZISD::VMAH, SDLoc(
Op),
Op.getValueType(),
5689 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
5690 case Intrinsic::s390_vmalhb:
5691 case Intrinsic::s390_vmalhh:
5692 case Intrinsic::s390_vmalhf:
5693 case Intrinsic::s390_vmalhg:
5694 case Intrinsic::s390_vmalhq:
5695 return DAG.
getNode(SystemZISD::VMALH, SDLoc(
Op),
Op.getValueType(),
5696 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
5698 case Intrinsic::s390_vmeb:
5699 case Intrinsic::s390_vmeh:
5700 case Intrinsic::s390_vmef:
5701 case Intrinsic::s390_vmeg:
5702 return DAG.
getNode(SystemZISD::VME, SDLoc(
Op),
Op.getValueType(),
5703 Op.getOperand(1),
Op.getOperand(2));
5704 case Intrinsic::s390_vmleb:
5705 case Intrinsic::s390_vmleh:
5706 case Intrinsic::s390_vmlef:
5707 case Intrinsic::s390_vmleg:
5708 return DAG.
getNode(SystemZISD::VMLE, SDLoc(
Op),
Op.getValueType(),
5709 Op.getOperand(1),
Op.getOperand(2));
5710 case Intrinsic::s390_vmob:
5711 case Intrinsic::s390_vmoh:
5712 case Intrinsic::s390_vmof:
5713 case Intrinsic::s390_vmog:
5714 return DAG.
getNode(SystemZISD::VMO, SDLoc(
Op),
Op.getValueType(),
5715 Op.getOperand(1),
Op.getOperand(2));
5716 case Intrinsic::s390_vmlob:
5717 case Intrinsic::s390_vmloh:
5718 case Intrinsic::s390_vmlof:
5719 case Intrinsic::s390_vmlog:
5720 return DAG.
getNode(SystemZISD::VMLO, SDLoc(
Op),
Op.getValueType(),
5721 Op.getOperand(1),
Op.getOperand(2));
5723 case Intrinsic::s390_vmaeb:
5724 case Intrinsic::s390_vmaeh:
5725 case Intrinsic::s390_vmaef:
5726 case Intrinsic::s390_vmaeg:
5728 DAG.
getNode(SystemZISD::VME, SDLoc(
Op),
Op.getValueType(),
5729 Op.getOperand(1),
Op.getOperand(2)),
5731 case Intrinsic::s390_vmaleb:
5732 case Intrinsic::s390_vmaleh:
5733 case Intrinsic::s390_vmalef:
5734 case Intrinsic::s390_vmaleg:
5736 DAG.
getNode(SystemZISD::VMLE, SDLoc(
Op),
Op.getValueType(),
5737 Op.getOperand(1),
Op.getOperand(2)),
5739 case Intrinsic::s390_vmaob:
5740 case Intrinsic::s390_vmaoh:
5741 case Intrinsic::s390_vmaof:
5742 case Intrinsic::s390_vmaog:
5744 DAG.
getNode(SystemZISD::VMO, SDLoc(
Op),
Op.getValueType(),
5745 Op.getOperand(1),
Op.getOperand(2)),
5747 case Intrinsic::s390_vmalob:
5748 case Intrinsic::s390_vmaloh:
5749 case Intrinsic::s390_vmalof:
5750 case Intrinsic::s390_vmalog:
5752 DAG.
getNode(SystemZISD::VMLO, SDLoc(
Op),
Op.getValueType(),
5753 Op.getOperand(1),
Op.getOperand(2)),
5774 { SystemZISD::MERGE_HIGH, 8,
5775 { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } },
5777 { SystemZISD::MERGE_HIGH, 4,
5778 { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } },
5780 { SystemZISD::MERGE_HIGH, 2,
5781 { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } },
5783 { SystemZISD::MERGE_HIGH, 1,
5784 { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } },
5786 { SystemZISD::MERGE_LOW, 8,
5787 { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } },
5789 { SystemZISD::MERGE_LOW, 4,
5790 { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } },
5792 { SystemZISD::MERGE_LOW, 2,
5793 { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } },
5795 { SystemZISD::MERGE_LOW, 1,
5796 { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } },
5798 { SystemZISD::PACK, 4,
5799 { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } },
5801 { SystemZISD::PACK, 2,
5802 { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } },
5804 { SystemZISD::PACK, 1,
5805 { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } },
5807 { SystemZISD::PERMUTE_DWORDS, 4,
5808 { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } },
5810 { SystemZISD::PERMUTE_DWORDS, 1,
5811 { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } }
5825 OpNo0 = OpNo1 = OpNos[1];
5826 }
else if (OpNos[1] < 0) {
5827 OpNo0 = OpNo1 = OpNos[0];
5845 unsigned &OpNo0,
unsigned &OpNo1) {
5846 int OpNos[] = { -1, -1 };
5859 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5861 OpNos[ModelOpNo] = RealOpNo;
5869 unsigned &OpNo0,
unsigned &OpNo1) {
5886 int Elt = Bytes[From];
5889 Transform[From] = -1;
5891 while (
P.Bytes[To] != Elt) {
5896 Transform[From] = To;
5920 Bytes.
resize(NumElements * BytesPerElement, -1);
5921 for (
unsigned I = 0;
I < NumElements; ++
I) {
5922 int Index = VSN->getMaskElt(
I);
5924 for (
unsigned J = 0; J < BytesPerElement; ++J)
5925 Bytes[
I * BytesPerElement + J] = Index * BytesPerElement + J;
5929 if (SystemZISD::SPLAT == ShuffleOp.
getOpcode() &&
5932 Bytes.
resize(NumElements * BytesPerElement, -1);
5933 for (
unsigned I = 0;
I < NumElements; ++
I)
5934 for (
unsigned J = 0; J < BytesPerElement; ++J)
5935 Bytes[
I * BytesPerElement + J] = Index * BytesPerElement + J;
5946 unsigned BytesPerElement,
int &
Base) {
5948 for (
unsigned I = 0;
I < BytesPerElement; ++
I) {
5949 if (Bytes[Start +
I] >= 0) {
5950 unsigned Elem = Bytes[Start +
I];
5954 if (
unsigned(
Base) % Bytes.
size() + BytesPerElement > Bytes.
size())
5956 }
else if (
unsigned(
Base) != Elem -
I)
5969 unsigned &StartIndex,
unsigned &OpNo0,
5971 int OpNos[] = { -1, -1 };
5973 for (
unsigned I = 0;
I < 16; ++
I) {
5974 int Index = Bytes[
I];
5980 Shift = ExpectedShift;
5981 else if (Shift != ExpectedShift)
5985 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5987 OpNos[ModelOpNo] = RealOpNo;
6000 unsigned InBytes = (
P.Opcode == SystemZISD::PERMUTE_DWORDS ? 8 :
6001 P.Opcode == SystemZISD::PACK ?
P.Operand * 2 :
6009 if (
P.Opcode == SystemZISD::PERMUTE_DWORDS) {
6011 Op = DAG.
getNode(SystemZISD::PERMUTE_DWORDS,
DL, InVT, Op0, Op1, Op2);
6012 }
else if (
P.Opcode == SystemZISD::PACK) {
6015 Op = DAG.
getNode(SystemZISD::PACK,
DL, OutVT, Op0, Op1);
6024 N =
N->getOperand(0);
6027 return Op->getZExtValue() == 0;
6033 for (
unsigned I = 0;
I < Num ;
I++)
6045 for (
unsigned I = 0;
I < 2; ++
I)
6049 unsigned StartIndex, OpNo0, OpNo1;
6051 return DAG.
getNode(SystemZISD::SHL_DOUBLE,
DL, MVT::v16i8,
Ops[OpNo0],
6058 if (ZeroVecIdx != UINT32_MAX) {
6059 bool MaskFirst =
true;
6064 if (OpNo == ZeroVecIdx &&
I == 0) {
6069 if (OpNo != ZeroVecIdx && Byte == 0) {
6076 if (ZeroIdx != -1) {
6079 if (Bytes[
I] >= 0) {
6082 if (OpNo == ZeroVecIdx)
6094 return DAG.
getNode(SystemZISD::PERMUTE,
DL, MVT::v16i8, Mask, Src,
6097 return DAG.
getNode(SystemZISD::PERMUTE,
DL, MVT::v16i8, Src, Mask,
6109 return DAG.
getNode(SystemZISD::PERMUTE,
DL, MVT::v16i8,
Ops[0],
6115struct GeneralShuffle {
6116 GeneralShuffle(EVT vt)
6117 : VT(vt), UnpackFromEltSize(UINT_MAX), UnpackLow(
false) {}
6121 void tryPrepareForUnpack();
6122 bool unpackWasPrepared() {
return UnpackFromEltSize <= 4; }
6137 unsigned UnpackFromEltSize;
6144void GeneralShuffle::addUndef() {
6146 for (
unsigned I = 0;
I < BytesPerElement; ++
I)
6147 Bytes.push_back(-1);
6156bool GeneralShuffle::add(
SDValue Op,
unsigned Elem) {
6162 EVT FromVT =
Op.getNode() ?
Op.getValueType() : VT;
6167 if (FromBytesPerElement < BytesPerElement)
6171 (FromBytesPerElement - BytesPerElement));
6174 while (
Op.getNode()) {
6176 Op =
Op.getOperand(0);
6192 }
else if (
Op.isUndef()) {
6201 for (; OpNo <
Ops.size(); ++OpNo)
6202 if (
Ops[OpNo] ==
Op)
6204 if (OpNo ==
Ops.size())
6209 for (
unsigned I = 0;
I < BytesPerElement; ++
I)
6210 Bytes.push_back(
Base +
I);
6219 if (
Ops.size() == 0)
6223 tryPrepareForUnpack();
6226 if (
Ops.size() == 1)
6238 unsigned Stride = 1;
6239 for (; Stride * 2 <
Ops.size(); Stride *= 2) {
6240 for (
unsigned I = 0;
I <
Ops.size() - Stride;
I += Stride * 2) {
6250 else if (OpNo ==
I + Stride)
6261 if (NewBytes[J] >= 0) {
6263 "Invalid double permute");
6266 assert(NewBytesMap[J] < 0 &&
"Invalid double permute");
6272 if (NewBytes[J] >= 0)
6288 unsigned OpNo0, OpNo1;
6292 else if (
const Permute *
P =
matchPermute(Bytes, OpNo0, OpNo1))
6297 Op = insertUnpackIfPrepared(DAG,
DL,
Op);
6304 dbgs() <<
Msg.c_str() <<
" { ";
6305 for (
unsigned I = 0;
I < Bytes.
size();
I++)
6306 dbgs() << Bytes[
I] <<
" ";
6314void GeneralShuffle::tryPrepareForUnpack() {
6316 if (ZeroVecOpNo == UINT32_MAX ||
Ops.size() == 1)
6321 if (
Ops.size() > 2 &&
6326 UnpackFromEltSize = 1;
6327 for (; UnpackFromEltSize <= 4; UnpackFromEltSize *= 2) {
6328 bool MatchUnpack =
true;
6331 unsigned ToEltSize = UnpackFromEltSize * 2;
6332 bool IsZextByte = (Elt % ToEltSize) < UnpackFromEltSize;
6335 if (Bytes[Elt] != -1) {
6337 if (IsZextByte != (OpNo == ZeroVecOpNo)) {
6338 MatchUnpack =
false;
6344 if (
Ops.size() == 2) {
6346 bool CanUseUnpackLow =
true, CanUseUnpackHigh =
true;
6348 if (SrcBytes[i] == -1)
6350 if (SrcBytes[i] % 16 !=
int(i))
6351 CanUseUnpackHigh =
false;
6353 CanUseUnpackLow =
false;
6354 if (!CanUseUnpackLow && !CanUseUnpackHigh) {
6355 UnpackFromEltSize = UINT_MAX;
6359 if (!CanUseUnpackHigh)
6365 if (UnpackFromEltSize > 4)
6368 LLVM_DEBUG(
dbgs() <<
"Preparing for final unpack of element size "
6369 << UnpackFromEltSize <<
". Zero vector is Op#" << ZeroVecOpNo
6371 dumpBytes(Bytes,
"Original Bytes vector:"););
6380 Elt += UnpackFromEltSize;
6381 for (
unsigned i = 0; i < UnpackFromEltSize; i++, Elt++,
B++)
6382 Bytes[
B] = Bytes[Elt];
6390 Ops.erase(&
Ops[ZeroVecOpNo]);
6392 if (Bytes[
I] >= 0) {
6394 if (OpNo > ZeroVecOpNo)
6405 if (!unpackWasPrepared())
6407 unsigned InBits = UnpackFromEltSize * 8;
6411 unsigned OutBits = InBits * 2;
6414 return DAG.
getNode(UnpackLow ? SystemZISD::UNPACKL_LOW
6415 : SystemZISD::UNPACKL_HIGH,
6416 DL, OutVT, PackedOp);
6421 for (
unsigned I = 1,
E =
Op.getNumOperands();
I !=
E; ++
I)
6422 if (!
Op.getOperand(
I).isUndef())
6438 if (
Value.isUndef())
6450 return DAG.
getNode(SystemZISD::REPLICATE,
DL, VT, Op1);
6453 return DAG.
getNode(SystemZISD::REPLICATE,
DL, VT, Op0);
6454 return DAG.
getNode(SystemZISD::MERGE_HIGH,
DL, VT,
6475 return DAG.
getNode(SystemZISD::JOIN_DWORDS,
DL, MVT::v2i64, Op0, Op1);
6491 GeneralShuffle GS(VT);
6493 bool FoundOne =
false;
6494 for (
unsigned I = 0;
I < NumElements; ++
I) {
6497 Op =
Op.getOperand(0);
6500 unsigned Elem =
Op.getConstantOperandVal(1);
6501 if (!GS.add(
Op.getOperand(0), Elem))
6504 }
else if (
Op.isUndef()) {
6518 if (!ResidueOps.
empty()) {
6519 while (ResidueOps.
size() < NumElements)
6521 for (
auto &
Op : GS.Ops) {
6522 if (!
Op.getNode()) {
6528 return GS.getNode(DAG,
SDLoc(BVN));
6531bool SystemZTargetLowering::isVectorElementLoad(
SDValue Op)
const {
6537 if (Subtarget.hasVectorEnhancements2() &&
Op.getOpcode() == SystemZISD::LRV)
6548 "Handling full vectors only.");
6568 if (Op01.
getOpcode() == SystemZISD::REPLICATE && Op01 == Op23)
6580 unsigned int NumElements = Elems.
size();
6581 unsigned int Count = 0;
6582 for (
auto Elem : Elems) {
6583 if (!Elem.isUndef()) {
6586 else if (Elem !=
Single) {
6610 bool AllLoads =
true;
6611 for (
auto Elem : Elems)
6612 if (!isVectorElementLoad(Elem)) {
6618 if (VT == MVT::v2i64 && !AllLoads)
6622 if (VT == MVT::v2f64 && !AllLoads)
6632 if (VT == MVT::v4f32 && !AllLoads)
6636 if (VT == MVT::v8f16 && !AllLoads) {
6645 if (Op0123.
getOpcode() == SystemZISD::REPLICATE && Op0123 == Op4567)
6654 unsigned NumConstants = 0;
6655 for (
unsigned I = 0;
I < NumElements; ++
I) {
6669 if (NumConstants > 0) {
6670 for (
unsigned I = 0;
I < NumElements; ++
I)
6681 std::map<const SDNode*, unsigned> UseCounts;
6682 SDNode *LoadMaxUses =
nullptr;
6683 for (
unsigned I = 0;
I < NumElements; ++
I)
6684 if (isVectorElementLoad(Elems[
I])) {
6685 SDNode *Ld = Elems[
I].getNode();
6686 unsigned Count = ++UseCounts[Ld];
6687 if (LoadMaxUses ==
nullptr || UseCounts[LoadMaxUses] <
Count)
6690 if (LoadMaxUses !=
nullptr) {
6691 ReplicatedVal =
SDValue(LoadMaxUses, 0);
6695 unsigned I1 = NumElements / 2 - 1;
6696 unsigned I2 = NumElements - 1;
6697 bool Def1 = !Elems[
I1].isUndef();
6698 bool Def2 = !Elems[I2].isUndef();
6712 for (
unsigned I = 0;
I < NumElements; ++
I)
6713 if (!
Done[
I] && !Elems[
I].
isUndef() && Elems[
I] != ReplicatedVal)
6723 EVT VT =
Op.getValueType();
6725 if (BVN->isConstant()) {
6726 if (SystemZVectorConstantInfo(BVN).isVectorConstantLegal(Subtarget))
6744 for (
unsigned I = 0;
I < NumElements; ++
I)
6746 return buildVector(DAG,
DL, VT,
Ops);
6753 EVT VT =
Op.getValueType();
6756 if (VSN->isSplat()) {
6758 unsigned Index = VSN->getSplatIndex();
6760 "Splat index should be defined and in first operand");
6766 return DAG.
getNode(SystemZISD::SPLAT,
DL, VT,
Op.getOperand(0),
6770 GeneralShuffle
GS(VT);
6771 for (
unsigned I = 0;
I < NumElements; ++
I) {
6772 int Elt = VSN->getMaskElt(
I);
6775 else if (!
GS.add(
Op.getOperand(
unsigned(Elt) / NumElements),
6776 unsigned(Elt) % NumElements))
6779 return GS.getNode(DAG, SDLoc(VSN));
6794 assert(
Op.getSimpleValueType() == MVT::i64 &&
6795 "Expexted to convert i64 to f16.");
6807 assert(
Op.getSimpleValueType() == MVT::f16 &&
6808 "Expected to convert f16 to i64.");
6830 EVT VT =
Op.getValueType();
6833 if (!Subtarget.hasVector())
6858SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(
SDValue Op,
6864 EVT VT =
Op.getValueType();
6871 if (!Subtarget.hasVector())
6877 MVT ExtrVT = IntVT == MVT::i16 ? MVT::i32 : IntVT;
6885SDValue SystemZTargetLowering::
6888 EVT OutVT =
Op.getValueType();
6892 unsigned StartOffset = 0;
6899 ArrayRef<int> ShuffleMask = SVN->
getMask();
6904 if (ToBits == 64 && OutNumElts == 2) {
6905 int NumElem = ToBits / FromBits;
6906 if (ShuffleMask[0] == NumElem - 1 && ShuffleMask[1] == 2 * NumElem - 1)
6912 int StartOffsetCandidate = -1;
6913 for (
int Elt = 0; Elt < OutNumElts; Elt++) {
6914 if (ShuffleMask[Elt] == -1)
6916 if (ShuffleMask[Elt] % OutNumElts == Elt) {
6917 if (StartOffsetCandidate == -1)
6918 StartOffsetCandidate = ShuffleMask[Elt] - Elt;
6919 if (StartOffsetCandidate == ShuffleMask[Elt] - Elt)
6922 StartOffsetCandidate = -1;
6925 if (StartOffsetCandidate != -1) {
6929 unsigned Offset = StartOffsetCandidate;
6930 unsigned OpNo =
Offset >= InNumElts ? 1 : 0;
6931 StartOffset =
Offset - OpNo * InNumElts;
6940 unsigned Opcode = SystemZISD::UNPACK_HIGH;
6941 if (StartOffset >= OutNumElts) {
6942 Opcode = SystemZISD::UNPACK_LOW;
6943 StartOffset -= OutNumElts;
6945 PackedOp = DAG.
getNode(Opcode, SDLoc(PackedOp), OutVT, PackedOp);
6946 }
while (FromBits != ToBits);
6951SDValue SystemZTargetLowering::
6955 EVT OutVT =
Op.getValueType();
6959 unsigned NumInPerOut = InNumElts / OutNumElts;
6964 SmallVector<int, 16>
Mask(InNumElts);
6965 unsigned ZeroVecElt = InNumElts;
6966 for (
unsigned PackedElt = 0; PackedElt < OutNumElts; PackedElt++) {
6967 unsigned MaskElt = PackedElt * NumInPerOut;
6968 unsigned End = MaskElt + NumInPerOut - 1;
6969 for (; MaskElt <
End; MaskElt++)
6970 Mask[MaskElt] = ZeroVecElt++;
6971 Mask[MaskElt] = PackedElt;
6978 unsigned ByScalar)
const {
6983 EVT VT =
Op.getValueType();
6988 APInt SplatBits, SplatUndef;
6989 unsigned SplatBitSize;
6993 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6994 ElemBitSize,
true) &&
6995 SplatBitSize == ElemBitSize) {
6998 return DAG.
getNode(ByScalar,
DL, VT, Op0, Shift);
7001 BitVector UndefElements;
7007 return DAG.
getNode(ByScalar,
DL, VT, Op0, Shift);
7014 if (VSN->isSplat()) {
7015 SDValue VSNOp0 = VSN->getOperand(0);
7016 unsigned Index = VSN->getSplatIndex();
7018 "Splat index should be defined and in first operand");
7025 return DAG.
getNode(ByScalar,
DL, VT, Op0, Shift);
7043 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
7044 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
7047 if (ShiftAmt > 120) {
7051 DAG.
getNode(SystemZISD::SHR_DOUBLE_BIT,
DL, MVT::v16i8, Op0, Op1,
7055 SmallVector<int, 16>
Mask(16);
7056 for (
unsigned Elt = 0; Elt < 16; Elt++)
7057 Mask[Elt] = (ShiftAmt >> 3) + Elt;
7059 if ((ShiftAmt & 7) == 0)
7063 DAG.
getNode(SystemZISD::SHL_DOUBLE_BIT,
DL, MVT::v16i8, Shuf1, Shuf2,
7081 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
7082 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
7085 if (ShiftAmt > 120) {
7089 DAG.
getNode(SystemZISD::SHL_DOUBLE_BIT,
DL, MVT::v16i8, Op0, Op1,
7093 SmallVector<int, 16>
Mask(16);
7094 for (
unsigned Elt = 0; Elt < 16; Elt++)
7095 Mask[Elt] = 16 - (ShiftAmt >> 3) + Elt;
7097 if ((ShiftAmt & 7) == 0)
7101 DAG.
getNode(SystemZISD::SHR_DOUBLE_BIT,
DL, MVT::v16i8, Shuf2, Shuf1,
7113 MVT DstVT =
Op.getSimpleValueType();
7116 unsigned SrcAS =
N->getSrcAddressSpace();
7118 assert(SrcAS !=
N->getDestAddressSpace() &&
7119 "addrspacecast must be between different address spaces");
7127 }
else if (DstVT == MVT::i32) {
7141 if (
In.getSimpleValueType() != MVT::f16)
7148 SDValue Chain,
bool IsStrict)
const {
7149 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected request for libcall!");
7152 std::tie(Result, Chain) =
7161 bool IsStrict =
Op->isStrictFPOpcode();
7163 MVT VT =
Op.getSimpleValueType();
7164 SDValue InOp =
Op.getOperand(IsStrict ? 1 : 0);
7172 if (!
Subtarget.hasFPExtension() && !IsSigned)
7183 if (VT == MVT::i128) {
7186 return useLibCall(DAG, LC, VT, InOp,
DL, Chain, IsStrict);
7196 bool IsStrict =
Op->isStrictFPOpcode();
7198 MVT VT =
Op.getSimpleValueType();
7199 SDValue InOp =
Op.getOperand(IsStrict ? 1 : 0);
7204 if (VT == MVT::f16) {
7211 if (!Subtarget.hasFPExtension() && !IsSigned)
7214 if (InVT == MVT::i128) {
7217 return useLibCall(DAG, LC, VT, InOp,
DL, Chain, IsStrict);
7226 EVT RegVT =
Op.getValueType();
7227 assert(RegVT == MVT::f16 &&
"Expected to lower an f16 load.");
7234 assert(EVT(RegVT) == AtomicLd->getMemoryVT() &&
"Unhandled f16 load");
7236 AtomicLd->getChain(), AtomicLd->getBasePtr(),
7237 AtomicLd->getMemOperand());
7240 assert(EVT(RegVT) == Ld->getMemoryVT() &&
"Unhandled f16 load");
7242 Ld->getBasePtr(), Ld->getPointerInfo(), MVT::i16,
7243 Ld->getBaseAlign(), Ld->getMemOperand()->getFlags());
7257 Shft, AtomicSt->getBasePtr(),
7258 AtomicSt->getMemOperand());
7261 return DAG.
getTruncStore(St->getChain(),
DL, Shft, St->getBasePtr(), MVT::i16,
7262 St->getMemOperand());
7268 MVT ResultVT =
Op.getSimpleValueType();
7270 unsigned Check =
Op.getConstantOperandVal(1);
7272 unsigned TDCMask = 0;
7307 MachinePointerInfo MPI =
7313 SystemZISD::STCKF,
DL, DAG.
getVTList(MVT::Other), StoreOps, MVT::i64,
7317 return DAG.
getLoad(MVT::i64,
DL, Chain, StackPtr, MPI);
7322 switch (
Op.getOpcode()) {
7324 return lowerFRAMEADDR(
Op, DAG);
7326 return lowerRETURNADDR(
Op, DAG);
7328 return lowerBR_CC(
Op, DAG);
7330 return lowerSELECT_CC(
Op, DAG);
7332 return lowerSETCC(
Op, DAG);
7334 return lowerSTRICT_FSETCC(
Op, DAG,
false);
7336 return lowerSTRICT_FSETCC(
Op, DAG,
true);
7348 return lowerBITCAST(
Op, DAG);
7350 return lowerVASTART(
Op, DAG);
7352 return lowerVACOPY(
Op, DAG);
7354 return lowerDYNAMIC_STACKALLOC(
Op, DAG);
7356 return lowerGET_DYNAMIC_AREA_OFFSET(
Op, DAG);
7358 return lowerMULH(
Op, DAG, SystemZISD::SMUL_LOHI);
7360 return lowerMULH(
Op, DAG, SystemZISD::UMUL_LOHI);
7362 return lowerSMUL_LOHI(
Op, DAG);
7364 return lowerUMUL_LOHI(
Op, DAG);
7366 return lowerSDIVREM(
Op, DAG);
7368 return lowerUDIVREM(
Op, DAG);
7373 return lowerXALUO(
Op, DAG);
7376 return lowerUADDSUBO_CARRY(
Op, DAG);
7378 return lowerOR(
Op, DAG);
7380 return lowerCTPOP(
Op, DAG);
7382 return lowerVECREDUCE_ADD(
Op, DAG);
7384 return lowerATOMIC_FENCE(
Op, DAG);
7386 return lowerATOMIC_STORE(
Op, DAG);
7388 return lowerATOMIC_LOAD(
Op, DAG);
7390 return lowerATOMIC_LOAD_SUB(
Op, DAG);
7392 return lowerATOMIC_CMP_SWAP(
Op, DAG);
7394 return lowerSTACKSAVE(
Op, DAG);
7396 return lowerSTACKRESTORE(
Op, DAG);
7398 return lowerPREFETCH(
Op, DAG);
7400 return lowerINTRINSIC_W_CHAIN(
Op, DAG);
7402 return lowerINTRINSIC_WO_CHAIN(
Op, DAG);
7404 return lowerBUILD_VECTOR(
Op, DAG);
7406 return lowerVECTOR_SHUFFLE(
Op, DAG);
7408 return lowerSCALAR_TO_VECTOR(
Op, DAG);
7410 return lowerINSERT_VECTOR_ELT(
Op, DAG);
7412 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
7414 return lowerSIGN_EXTEND_VECTOR_INREG(
Op, DAG);
7416 return lowerZERO_EXTEND_VECTOR_INREG(
Op, DAG);
7418 return lowerShift(
Op, DAG, SystemZISD::VSHL_BY_SCALAR);
7420 return lowerShift(
Op, DAG, SystemZISD::VSRL_BY_SCALAR);
7422 return lowerShift(
Op, DAG, SystemZISD::VSRA_BY_SCALAR);
7426 return lowerShift(
Op, DAG, SystemZISD::VROTL_BY_SCALAR);
7428 return lowerFSHL(
Op, DAG);
7430 return lowerFSHR(
Op, DAG);
7433 return lowerFP_EXTEND(
Op, DAG);
7438 return lower_FP_TO_INT(
Op, DAG);
7443 return lower_INT_TO_FP(
Op, DAG);
7445 return lowerLoadF16(
Op, DAG);
7447 return lowerStoreF16(
Op, DAG);
7449 return lowerIS_FPCLASS(
Op, DAG);
7451 return lowerGET_ROUNDING(
Op, DAG);
7453 return lowerREADCYCLECOUNTER(
Op, DAG);
7475 &SystemZ::FP128BitRegClass);
7484 SystemZ::REG_SEQUENCE, SL, MVT::f128,
7499 &SystemZ::FP128BitRegClass);
7516 switch (
N->getOpcode()) {
7520 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1) };
7523 DL, Tys,
Ops, MVT::i128, MMO);
7526 if (
N->getValueType(0) == MVT::f128)
7540 SDValue Ops[] = {
N->getOperand(0), Val,
N->getOperand(2)};
7543 DL, Tys,
Ops, MVT::i128, MMO);
7549 MVT::Other, Res), 0);
7561 DL, Tys,
Ops, MVT::i128, MMO);
7575 EVT SrcVT = Src.getValueType();
7576 EVT ResVT =
N->getValueType(0);
7577 if (ResVT == MVT::i128 && SrcVT == MVT::f128)
7579 else if (SrcVT == MVT::i16 && ResVT == MVT::f16) {
7580 if (Subtarget.hasVector()) {
7588 }
else if (SrcVT == MVT::f16 && ResVT == MVT::i16) {
7590 Subtarget.hasVector()
7604 bool IsStrict =
N->isStrictFPOpcode();
7606 SDValue InOp =
N->getOperand(IsStrict ? 1 : 0);
7607 EVT ResVT =
N->getValueType(0);
7609 if (ResVT == MVT::f16) {
7632 bool IsStrict =
N->isStrictFPOpcode();
7634 EVT ResVT =
N->getValueType(0);
7635 SDValue InOp =
N->getOperand(IsStrict ? 1 : 0);
7638 if (InVT == MVT::f16) {
7644 std::tie(InF32, Chain) =
7656 lowerATOMIC_LOAD_OP(
N,
Results, DAG, SystemZISD::ATOMIC_SWAPW);
7659 lowerATOMIC_LOAD_OP(
N,
Results, DAG, SystemZISD::ATOMIC_LOADW_ADD);
7662 lowerATOMIC_LOAD_OP(
N,
Results, DAG, SystemZISD::ATOMIC_LOADW_SUB);
7665 lowerATOMIC_LOAD_OP(
N,
Results, DAG, SystemZISD::ATOMIC_LOADW_AND);
7668 lowerATOMIC_LOAD_OP(
N,
Results, DAG, SystemZISD::ATOMIC_LOADW_OR);
7671 lowerATOMIC_LOAD_OP(
N,
Results, DAG, SystemZISD::ATOMIC_LOADW_XOR);
7674 lowerATOMIC_LOAD_OP(
N,
Results, DAG, SystemZISD::ATOMIC_LOADW_NAND);
7677 lowerATOMIC_LOAD_OP(
N,
Results, DAG, SystemZISD::ATOMIC_LOADW_MIN);
7680 lowerATOMIC_LOAD_OP(
N,
Results, DAG, SystemZISD::ATOMIC_LOADW_MAX);
7683 lowerATOMIC_LOAD_OP(
N,
Results, DAG, SystemZISD::ATOMIC_LOADW_UMIN);
7686 lowerATOMIC_LOAD_OP(
N,
Results, DAG, SystemZISD::ATOMIC_LOADW_UMAX);
7702bool SystemZTargetLowering::canTreatAsByteVector(
EVT VT)
const {
7717 DAGCombinerInfo &DCI,
7725 unsigned Opcode =
Op.getOpcode();
7728 Op =
Op.getOperand(0);
7730 canTreatAsByteVector(
Op.getValueType())) {
7739 BytesPerElement,
First))
7746 if (Byte % BytesPerElement != 0)
7749 Index = Byte / BytesPerElement;
7753 canTreatAsByteVector(
Op.getValueType())) {
7756 EVT OpVT =
Op.getValueType();
7758 if (OpBytesPerElement < BytesPerElement)
7762 unsigned End = (
Index + 1) * BytesPerElement;
7763 if (
End % OpBytesPerElement != 0)
7766 Op =
Op.getOperand(
End / OpBytesPerElement - 1);
7770 if (!
Op.getValueType().isInteger()) {
7775 DCI.AddToWorklist(
Op.getNode());
7778 if (ResIntVT != ResVT) {
7779 DCI.AddToWorklist(
Op.getNode());
7786 canTreatAsByteVector(
Op.getValueType()) &&
7787 canTreatAsByteVector(
Op.getOperand(0).getValueType())) {
7789 EVT ExtVT =
Op.getValueType();
7790 EVT OpVT =
Op.getOperand(0).getValueType();
7793 unsigned Byte =
Index * BytesPerElement;
7794 unsigned SubByte =
Byte % ExtBytesPerElement;
7795 unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement;
7796 if (SubByte < MinSubByte ||
7797 SubByte + BytesPerElement > ExtBytesPerElement)
7800 Byte =
Byte / ExtBytesPerElement * OpBytesPerElement;
7802 Byte += SubByte - MinSubByte;
7803 if (Byte % BytesPerElement != 0)
7805 Op =
Op.getOperand(0);
7812 if (
Op.getValueType() != VecVT) {
7814 DCI.AddToWorklist(
Op.getNode());
7824SDValue SystemZTargetLowering::combineTruncateExtract(
7833 if (canTreatAsByteVector(VecVT)) {
7837 if (BytesPerElement % TruncBytes == 0) {
7843 unsigned Scale = BytesPerElement / TruncBytes;
7844 unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1;
7851 EVT ResVT = (TruncBytes < 4 ? MVT::i32 : TruncVT);
7852 return combineExtract(
DL, ResVT, VecVT, Vec, NewIndex, DCI,
true);
7860SDValue SystemZTargetLowering::combineZERO_EXTEND(
7861 SDNode *
N, DAGCombinerInfo &DCI)
const {
7863 SelectionDAG &DAG = DCI.DAG;
7865 EVT VT =
N->getValueType(0);
7866 if (N0.
getOpcode() == SystemZISD::SELECT_CCMASK) {
7869 if (TrueOp && FalseOp) {
7879 DCI.CombineTo(N0.
getNode(), TruncSelect);
7922 return DAG.
getNode(SystemZISD::VSCBI, SDLoc(N0), VT, Op0, Op1);
7940SDValue SystemZTargetLowering::combineSIGN_EXTEND_INREG(
7941 SDNode *
N, DAGCombinerInfo &DCI)
const {
7945 SelectionDAG &DAG = DCI.DAG;
7947 EVT VT =
N->getValueType(0);
7961SDValue SystemZTargetLowering::combineSIGN_EXTEND(
7962 SDNode *
N, DAGCombinerInfo &DCI)
const {
7966 SelectionDAG &DAG = DCI.DAG;
7968 EVT VT =
N->getValueType(0);
7975 unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra;
7976 unsigned NewSraAmt = SraAmt->getZExtValue() + Extra;
7992SDValue SystemZTargetLowering::combineMERGE(
7993 SDNode *
N, DAGCombinerInfo &DCI)
const {
7994 SelectionDAG &DAG = DCI.DAG;
7995 unsigned Opcode =
N->getOpcode();
8003 if (Op1 ==
N->getOperand(0))
8008 if (ElemBytes <= 4) {
8009 Opcode = (Opcode == SystemZISD::MERGE_HIGH ?
8010 SystemZISD::UNPACKL_HIGH : SystemZISD::UNPACKL_LOW);
8016 DCI.AddToWorklist(Op1.
getNode());
8019 DCI.AddToWorklist(
Op.getNode());
8028 LoPart = HiPart =
nullptr;
8033 if (
Use.getResNo() != 0)
8038 bool IsLoPart =
true;
8063 LoPart = HiPart =
nullptr;
8068 if (
Use.getResNo() != 0)
8074 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
8077 switch (
User->getConstantOperandVal(1)) {
8078 case SystemZ::subreg_l64:
8083 case SystemZ::subreg_h64:
8095SDValue SystemZTargetLowering::combineLOAD(
8096 SDNode *
N, DAGCombinerInfo &DCI)
const {
8097 SelectionDAG &DAG = DCI.DAG;
8098 EVT LdVT =
N->getValueType(0);
8102 MVT LoadNodeVT = LN->getBasePtr().getSimpleValueType();
8103 if (PtrVT != LoadNodeVT) {
8107 return DAG.
getExtLoad(LN->getExtensionType(),
DL, LN->getValueType(0),
8108 LN->getChain(), AddrSpaceCast, LN->getMemoryVT(),
8109 LN->getMemOperand());
8119 SDNode *LoPart, *HiPart;
8127 LD->getPointerInfo(),
LD->getBaseAlign(),
8128 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
8130 DCI.CombineTo(HiPart, EltLoad,
true);
8137 LD->getPointerInfo().getWithOffset(8),
LD->getBaseAlign(),
8138 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
8140 DCI.CombineTo(LoPart, EltLoad,
true);
8147 DCI.AddToWorklist(Chain.
getNode());
8162 for (SDUse &Use :
N->uses()) {
8163 if (
Use.getUser()->getOpcode() == SystemZISD::REPLICATE) {
8167 }
else if (
Use.getResNo() == 0)
8170 if (!Replicate || OtherUses.
empty())
8176 for (SDNode *U : OtherUses) {
8179 Ops.push_back((
Op.getNode() ==
N &&
Op.getResNo() == 0) ? Extract0 :
Op);
8185bool SystemZTargetLowering::canLoadStoreByteSwapped(
EVT VT)
const {
8186 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64)
8188 if (Subtarget.hasVectorEnhancements2())
8189 if (VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v2i64 || VT == MVT::i128)
8201 for (
unsigned i = 0; i < NumElts; ++i) {
8202 if (M[i] < 0)
continue;
8203 if ((
unsigned) M[i] != NumElts - 1 - i)
8211 for (
auto *U : StoredVal->
users()) {
8213 EVT CurrMemVT = ST->getMemoryVT().getScalarType();
8272SDValue SystemZTargetLowering::combineSTORE(
8273 SDNode *
N, DAGCombinerInfo &DCI)
const {
8274 SelectionDAG &DAG = DCI.DAG;
8277 EVT MemVT = SN->getMemoryVT();
8281 MVT StoreNodeVT = SN->getBasePtr().getSimpleValueType();
8282 if (PtrVT != StoreNodeVT) {
8286 return DAG.
getStore(SN->getChain(),
DL, SN->getValue(), AddrSpaceCast,
8287 SN->getPointerInfo(), SN->getBaseAlign(),
8288 SN->getMemOperand()->getFlags(), SN->getAAInfo());
8296 if (MemVT.
isInteger() && SN->isTruncatingStore()) {
8298 combineTruncateExtract(SDLoc(
N), MemVT, SN->getValue(), DCI)) {
8299 DCI.AddToWorklist(
Value.getNode());
8303 SN->getBasePtr(), SN->getMemoryVT(),
8304 SN->getMemOperand());
8315 return DAG.
getNode(SystemZISD::MOV_STACKGUARD, SDLoc(SN), MVT::Other,
Ops);
8319 if (!SN->isTruncatingStore() &&
8335 Ops, MemVT, SN->getMemOperand());
8338 if (!SN->isTruncatingStore() &&
8341 Subtarget.hasVectorEnhancements2()) {
8343 ArrayRef<int> ShuffleMask = SVN->
getMask();
8351 Ops, MemVT, SN->getMemOperand());
8356 if (!SN->isTruncatingStore() &&
8359 N->getOperand(0).reachesChainWithoutSideEffects(
SDValue(Op1.
getNode(), 1))) {
8363 Ops, MemVT, SN->getMemOperand());
8373 SN->getChain(),
DL, HiPart, SN->getBasePtr(), SN->getPointerInfo(),
8374 SN->getBaseAlign(), SN->getMemOperand()->getFlags(), SN->getAAInfo());
8376 SN->getChain(),
DL, LoPart,
8378 SN->getPointerInfo().getWithOffset(8), SN->getBaseAlign(),
8379 SN->getMemOperand()->
getFlags(), SN->getAAInfo());
8397 auto FindReplicatedImm = [&](ConstantSDNode *
C,
unsigned TotBytes) {
8399 if (
C->getAPIntValue().getBitWidth() > 64 ||
C->isAllOnes() ||
8403 APInt Val =
C->getAPIntValue();
8406 assert(SN->isTruncatingStore() &&
8407 "Non-truncating store and immediate value does not fit?");
8408 Val = Val.
trunc(TotBytes * 8);
8411 SystemZVectorConstantInfo VCI(APInt(TotBytes * 8, Val.
getZExtValue()));
8412 if (VCI.isVectorConstantLegal(Subtarget) &&
8413 VCI.Opcode == SystemZISD::REPLICATE) {
8421 auto FindReplicatedReg = [&](
SDValue MulOp) {
8422 EVT MulVT = MulOp.getValueType();
8423 if (MulOp->getOpcode() ==
ISD::MUL &&
8424 (MulVT == MVT::i16 || MulVT == MVT::i32 || MulVT == MVT::i64)) {
8428 WordVT =
LHS->getOperand(0).getValueType();
8435 SystemZVectorConstantInfo VCI(
8437 if (VCI.isVectorConstantLegal(Subtarget) &&
8438 VCI.Opcode == SystemZISD::REPLICATE && VCI.OpVals[0] == 1 &&
8439 WordVT == VCI.VecVT.getScalarType())
8451 FindReplicatedReg(SplatVal);
8456 FindReplicatedReg(Op1);
8461 "Bad type handling");
8465 return DAG.
getStore(SN->getChain(), SDLoc(SN), SplatVal,
8466 SN->getBasePtr(), SN->getMemOperand());
8473SDValue SystemZTargetLowering::combineVECTOR_SHUFFLE(
8474 SDNode *
N, DAGCombinerInfo &DCI)
const {
8475 SelectionDAG &DAG = DCI.DAG;
8478 N->getOperand(0).hasOneUse() &&
8479 Subtarget.hasVectorEnhancements2()) {
8481 ArrayRef<int> ShuffleMask = SVN->
getMask();
8494 Ops,
LD->getMemoryVT(),
LD->getMemOperand());
8498 DCI.CombineTo(
N, ESLoad);
8502 DCI.CombineTo(
Load.getNode(), ESLoad, ESLoad.
getValue(1));
8512SDValue SystemZTargetLowering::combineEXTRACT_VECTOR_ELT(
8513 SDNode *
N, DAGCombinerInfo &DCI)
const {
8514 SelectionDAG &DAG = DCI.DAG;
8516 if (!Subtarget.hasVector())
8522 Op.getValueType().isVector() &&
8523 Op.getOperand(0).getValueType().isVector() &&
8524 Op.getValueType().getVectorNumElements() ==
8525 Op.getOperand(0).getValueType().getVectorNumElements())
8526 Op =
Op.getOperand(0);
8530 EVT VecVT =
Op.getValueType();
8533 Op.getOperand(0),
N->getOperand(1));
8534 DCI.AddToWorklist(
Op.getNode());
8536 if (EltVT !=
N->getValueType(0)) {
8537 DCI.AddToWorklist(
Op.getNode());
8547 if (canTreatAsByteVector(VecVT))
8548 return combineExtract(SDLoc(
N),
N->getValueType(0), VecVT, Op0,
8549 IndexN->getZExtValue(), DCI,
false);
8554SDValue SystemZTargetLowering::combineJOIN_DWORDS(
8555 SDNode *
N, DAGCombinerInfo &DCI)
const {
8556 SelectionDAG &DAG = DCI.DAG;
8558 if (
N->getOperand(0) ==
N->getOperand(1))
8559 return DAG.
getNode(SystemZISD::REPLICATE, SDLoc(
N),
N->getValueType(0),
8569 if (Chain1 == Chain2)
8577SDValue SystemZTargetLowering::combineFP_ROUND(
8578 SDNode *
N, DAGCombinerInfo &DCI)
const {
8580 if (!Subtarget.hasVector())
8589 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
8590 SelectionDAG &DAG = DCI.DAG;
8592 if (
N->getValueType(0) == MVT::f32 && Op0.
hasOneUse() &&
8598 for (
auto *U : Vec->
users()) {
8599 if (U != Op0.
getNode() &&
U->hasOneUse() &&
8601 U->getOperand(0) == Vec &&
8603 U->getConstantOperandVal(1) == 1) {
8605 if (OtherRound.
getOpcode() ==
N->getOpcode() &&
8609 if (
N->isStrictFPOpcode()) {
8613 VRound = DAG.
getNode(SystemZISD::STRICT_VROUND, SDLoc(
N),
8614 {MVT::v4f32, MVT::Other}, {Chain, Vec});
8617 VRound = DAG.
getNode(SystemZISD::VROUND, SDLoc(
N),
8619 DCI.AddToWorklist(VRound.
getNode());
8623 DCI.AddToWorklist(Extract1.
getNode());
8629 VRound, DAG.
getConstant(0, SDLoc(Op0), MVT::i32));
8632 N->getVTList(), Extract0, Chain);
8641SDValue SystemZTargetLowering::combineFP_EXTEND(
8642 SDNode *
N, DAGCombinerInfo &DCI)
const {
8644 if (!Subtarget.hasVector())
8653 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
8654 SelectionDAG &DAG = DCI.DAG;
8656 if (
N->getValueType(0) == MVT::f64 && Op0.
hasOneUse() &&
8662 for (
auto *U : Vec->
users()) {
8663 if (U != Op0.
getNode() &&
U->hasOneUse() &&
8665 U->getOperand(0) == Vec &&
8667 U->getConstantOperandVal(1) == 2) {
8669 if (OtherExtend.
getOpcode() ==
N->getOpcode() &&
8673 if (
N->isStrictFPOpcode()) {
8677 VExtend = DAG.
getNode(SystemZISD::STRICT_VEXTEND, SDLoc(
N),
8678 {MVT::v2f64, MVT::Other}, {Chain, Vec});
8681 VExtend = DAG.
getNode(SystemZISD::VEXTEND, SDLoc(
N),
8683 DCI.AddToWorklist(VExtend.
getNode());
8687 DCI.AddToWorklist(Extract1.
getNode());
8693 VExtend, DAG.
getConstant(0, SDLoc(Op0), MVT::i32));
8696 N->getVTList(), Extract0, Chain);
8705SDValue SystemZTargetLowering::combineINT_TO_FP(
8706 SDNode *
N, DAGCombinerInfo &DCI)
const {
8709 SelectionDAG &DAG = DCI.DAG;
8711 unsigned Opcode =
N->getOpcode();
8712 EVT OutVT =
N->getValueType(0);
8716 unsigned InScalarBits =
Op->getValueType(0).getScalarSizeInBits();
8722 if (OutLLVMTy->
isVectorTy() && OutScalarBits > InScalarBits &&
8723 OutScalarBits <= 64) {
8727 unsigned ExtOpcode =
8730 return DAG.
getNode(Opcode, SDLoc(
N), OutVT, ExtOp);
8735SDValue SystemZTargetLowering::combineFCOPYSIGN(
8736 SDNode *
N, DAGCombinerInfo &DCI)
const {
8737 SelectionDAG &DAG = DCI.DAG;
8738 EVT VT =
N->getValueType(0);
8751SDValue SystemZTargetLowering::combineBSWAP(
8752 SDNode *
N, DAGCombinerInfo &DCI)
const {
8753 SelectionDAG &DAG = DCI.DAG;
8756 N->getOperand(0).hasOneUse() &&
8757 canLoadStoreByteSwapped(
N->getValueType(0))) {
8766 EVT LoadVT =
N->getValueType(0);
8767 if (LoadVT == MVT::i16)
8772 Ops,
LD->getMemoryVT(),
LD->getMemOperand());
8776 if (
N->getValueType(0) == MVT::i16)
8781 DCI.CombineTo(
N, ResVal);
8785 DCI.CombineTo(
Load.getNode(), ResVal, BSLoad.
getValue(1));
8794 Op.getValueType().isVector() &&
8795 Op.getOperand(0).getValueType().isVector() &&
8796 Op.getValueType().getVectorNumElements() ==
8797 Op.getOperand(0).getValueType().getVectorNumElements())
8798 Op =
Op.getOperand(0);
8810 (canLoadStoreByteSwapped(
N->getValueType(0)) &&
8812 EVT VecVT =
N->getValueType(0);
8816 DCI.AddToWorklist(Vec.
getNode());
8820 DCI.AddToWorklist(Elt.
getNode());
8823 DCI.AddToWorklist(Vec.
getNode());
8825 DCI.AddToWorklist(Elt.
getNode());
8833 if (SV &&
Op.hasOneUse()) {
8841 EVT VecVT =
N->getValueType(0);
8844 DCI.AddToWorklist(Op0.
getNode());
8848 DCI.AddToWorklist(Op1.
getNode());
8851 DCI.AddToWorklist(Op0.
getNode());
8853 DCI.AddToWorklist(Op1.
getNode());
8861SDValue SystemZTargetLowering::combineSETCC(
8862 SDNode *
N, DAGCombinerInfo &DCI)
const {
8863 SelectionDAG &DAG = DCI.DAG;
8869 EVT VT =
N->getValueType(0);
8879 Src.getValueType().isFixedLengthVector() &&
8880 Src.getValueType().getScalarType() == MVT::i1) {
8881 EVT CmpVT = Src.getOperand(0).getValueType();
8898 unsigned Depth = 0) {
8906 case SystemZISD::IPM:
8911 case SystemZISD::SELECT_CCMASK: {
8913 if (Op4CCReg.
getOpcode() == SystemZISD::ICMP ||
8914 Op4CCReg.
getOpcode() == SystemZISD::TM) {
8917 return std::make_pair(OpCC, OpCCValid);
8922 int CCValidVal = CCValid->getZExtValue();
8923 return std::make_pair(Op4CCReg, CCValidVal);
8934 return std::make_pair(Op0CC, Op0CCValid);
8950 return {Val, Val, Val, Val};
8951 case SystemZISD::IPM: {
8956 for (
auto CC : {0, 1, 2, 3})
8959 return ShiftedCCVals;
8961 case SystemZISD::SELECT_CCMASK: {
8965 if (!CCValid || !CCMask)
8968 int CCValidVal = CCValid->getZExtValue();
8969 int CCMaskVal = CCMask->getZExtValue();
8979 if (TrueSDVals.empty() || FalseSDVals.empty())
8982 for (
auto &CCVal : {0, 1, 2, 3})
8983 MergedSDVals.
emplace_back(((CCMaskVal & (1 << (3 - CCVal))) != 0)
8985 : FalseSDVals[CCVal]);
8986 return MergedSDVals;
9003 if (Op0SDVals.empty() || Op1SDVals.empty())
9006 for (
auto CCVal : {0, 1, 2, 3})
9008 Opcode,
DL, Val.
getValueType(), Op0SDVals[CCVal], Op1SDVals[CCVal]));
9009 return BinaryOpSDVals;
9020 auto *CCNode = CCReg.
getNode();
9024 if (CCNode->getOpcode() == SystemZISD::TM) {
9027 auto emulateTMCCMask = [](
const SDValue &Op0Val,
const SDValue &Op1Val) {
9030 if (!Op0Node || !Op1Node)
9032 auto Op0APVal = Op0Node->getAPIntValue();
9033 auto Op1APVal = Op1Node->getAPIntValue();
9034 auto Result = Op0APVal & Op1APVal;
9035 bool AllOnes = Result == Op1APVal;
9036 bool AllZeros = Result == 0;
9037 bool IsLeftMostBitSet = Result[Op1APVal.getActiveBits() - 1] != 0;
9038 return AllZeros ? 0 :
AllOnes ? 3 : IsLeftMostBitSet ? 2 : 1;
9042 auto [Op0CC, Op0CCValid] =
findCCUse(Op0);
9047 if (Op0SDVals.empty() || Op1SDVals.empty())
9050 for (
auto CC : {0, 1, 2, 3}) {
9051 auto CCVal = emulateTMCCMask(Op0SDVals[
CC], Op1SDVals[
CC]);
9055 NewCCMask |= (CCMask & (1 << (3 - CCVal))) != 0;
9057 NewCCMask &= Op0CCValid;
9060 CCValid = Op0CCValid;
9063 if (CCNode->getOpcode() != SystemZISD::ICMP ||
9070 auto [Op0CC, Op0CCValid] =
findCCUse(CmpOp0);
9074 if (Op0SDVals.empty() || Op1SDVals.empty())
9078 auto CmpTypeVal = CmpType->getZExtValue();
9079 const auto compareCCSigned = [&CmpTypeVal](
const SDValue &Op0Val,
9083 if (!Op0Node || !Op1Node)
9085 auto Op0APVal = Op0Node->getAPIntValue();
9086 auto Op1APVal = Op1Node->getAPIntValue();
9088 return Op0APVal == Op1APVal ? 0 : Op0APVal.slt(Op1APVal) ? 1 : 2;
9089 return Op0APVal == Op1APVal ? 0 : Op0APVal.ult(Op1APVal) ? 1 : 2;
9092 for (
auto CC : {0, 1, 2, 3}) {
9093 auto CCVal = compareCCSigned(Op0SDVals[
CC], Op1SDVals[
CC]);
9097 NewCCMask |= (CCMask & (1 << (3 - CCVal))) != 0;
9099 NewCCMask &= Op0CCValid;
9102 CCValid = Op0CCValid;
9115 const auto isFlagOutOpCC = [](
const Value *V) {
9117 const Value *RHSVal;
9124 if (CB->isInlineAsm()) {
9126 return IA && IA->getConstraintString().contains(
"{@cc}");
9137 if (isFlagOutOpCC(Lhs) && isFlagOutOpCC(Rhs))
9140 return {-1, -1, -1};
9144 DAGCombinerInfo &DCI)
const {
9150 if (!CCValid || !CCMask)
9153 int CCValidVal = CCValid->getZExtValue();
9154 int CCMaskVal = CCMask->getZExtValue();
9161 if (
combineCCMask(CCReg, CCValidVal, CCMaskVal, DAG) && CCMaskVal != 0 &&
9162 CCMaskVal != CCValidVal)
9163 return DAG.
getNode(SystemZISD::BR_CCMASK,
SDLoc(
N),
N->getValueType(0),
9167 N->getOperand(3), CCReg);
9171SDValue SystemZTargetLowering::combineSELECT_CCMASK(
9172 SDNode *
N, DAGCombinerInfo &DCI)
const {
9178 if (!CCValid || !CCMask)
9181 int CCValidVal = CCValid->getZExtValue();
9182 int CCMaskVal = CCMask->getZExtValue();
9185 bool IsCombinedCCReg =
combineCCMask(CCReg, CCValidVal, CCMaskVal, DAG);
9189 const auto constructCCSDValsFromSELECT = [&CCReg](
SDValue &Val) {
9190 if (Val.getOpcode() == SystemZISD::SELECT_CCMASK) {
9192 if (Val.getOperand(4) != CCReg)
9199 int CCMaskVal = CCMask->getZExtValue();
9200 for (
auto &
CC : {0, 1, 2, 3})
9215 if (TrueSDVals.empty())
9216 TrueSDVals = constructCCSDValsFromSELECT(TrueVal);
9217 if (FalseSDVals.empty())
9218 FalseSDVals = constructCCSDValsFromSELECT(FalseVal);
9219 if (!TrueSDVals.empty() && !FalseSDVals.empty()) {
9220 SmallSet<SDValue, 4> MergedSDValsSet;
9222 for (
auto CC : {0, 1, 2, 3}) {
9223 if ((CCValidVal & ((1 << (3 -
CC)))) != 0)
9224 MergedSDValsSet.
insert(((CCMaskVal & (1 << (3 -
CC))) != 0)
9228 if (MergedSDValsSet.
size() == 1)
9229 return *MergedSDValsSet.
begin();
9230 if (MergedSDValsSet.
size() == 2) {
9231 auto BeginIt = MergedSDValsSet.
begin();
9232 SDValue NewTrueVal = *BeginIt, NewFalseVal = *next(BeginIt);
9233 if (NewTrueVal == FalseVal || NewFalseVal == TrueVal)
9236 for (
auto CC : {0, 1, 2, 3}) {
9238 NewCCMask |= ((CCMaskVal & (1 << (3 -
CC))) != 0)
9239 ? (TrueSDVals[
CC] == NewTrueVal)
9240 : (FalseSDVals[
CC] == NewTrueVal);
9242 CCMaskVal = NewCCMask;
9243 CCMaskVal &= CCValidVal;
9246 IsCombinedCCReg =
true;
9254 if (CCMaskVal == CCValidVal)
9257 if (IsCombinedCCReg)
9259 SystemZISD::SELECT_CCMASK, SDLoc(
N),
N->getValueType(0), TrueVal,
9266SDValue SystemZTargetLowering::combineGET_CCMASK(
9267 SDNode *
N, DAGCombinerInfo &DCI)
const {
9272 if (!CCValid || !CCMask)
9274 int CCValidVal = CCValid->getZExtValue();
9275 int CCMaskVal = CCMask->getZExtValue();
9280 if (
Select->getOpcode() != SystemZISD::SELECT_CCMASK)
9285 if (!SelectCCValid || !SelectCCMask)
9287 int SelectCCValidVal = SelectCCValid->getZExtValue();
9288 int SelectCCMaskVal = SelectCCMask->getZExtValue();
9292 if (!TrueVal || !FalseVal)
9296 else if (
TrueVal->getZExtValue() == 0 &&
FalseVal->getZExtValue() == 1)
9297 SelectCCMaskVal ^= SelectCCValidVal;
9301 if (SelectCCValidVal & ~CCValidVal)
9303 if (SelectCCMaskVal != (CCMaskVal & SelectCCValidVal))
9306 return Select->getOperand(4);
9309SDValue SystemZTargetLowering::combineIntDIVREM(
9310 SDNode *
N, DAGCombinerInfo &DCI)
const {
9311 SelectionDAG &DAG = DCI.DAG;
9312 EVT VT =
N->getValueType(0);
9329SDValue SystemZTargetLowering::combineShiftToMulAddHigh(
9330 SDNode *
N, DAGCombinerInfo &DCI)
const {
9331 SelectionDAG &DAG = DCI.DAG;
9335 "SRL or SRA node is required here!");
9337 if (!Subtarget.hasVector())
9347 SDValue ShiftOperand =
N->getOperand(0);
9367 if (!IsSignExt && !IsZeroExt)
9375 unsigned ActiveBits = IsSignExt
9376 ?
Constant->getAPIntValue().getSignificantBits()
9377 :
Constant->getAPIntValue().getActiveBits();
9378 if (ActiveBits > NarrowVTSize)
9394 unsigned ActiveBits = IsSignExt
9395 ?
Constant->getAPIntValue().getSignificantBits()
9396 :
Constant->getAPIntValue().getActiveBits();
9397 if (ActiveBits > NarrowVTSize)
9414 "Cannot have a multiply node with two different operand types.");
9416 "Cannot have an add node with two different operand types.");
9427 if (ShiftAmt != NarrowVTSize)
9431 if (!(NarrowVT == MVT::v16i8 || NarrowVT == MVT::v8i16 ||
9432 NarrowVT == MVT::v4i32 ||
9433 (Subtarget.hasVectorEnhancements3() &&
9434 (NarrowVT == MVT::v2i64 || NarrowVT == MVT::i128))))
9440 MulhRightOp, MulhAddOp);
9441 bool IsSigned =
N->getOpcode() ==
ISD::SRA;
9452 EVT VT =
Op.getValueType();
9461 Op =
Op.getOperand(0);
9462 if (
Op.getValueType().getVectorNumElements() == 2 * NumElts &&
9466 bool CanUseEven =
true, CanUseOdd =
true;
9467 for (
unsigned Elt = 0; Elt < NumElts; Elt++) {
9468 if (ShuffleMask[Elt] == -1)
9470 if (
unsigned(ShuffleMask[Elt]) != 2 * Elt)
9472 if (
unsigned(ShuffleMask[Elt]) != 2 * Elt + 1)
9475 Op =
Op.getOperand(0);
9477 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9479 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9485 if (VT == MVT::i128 &&
Subtarget.hasVectorEnhancements3() &&
9489 Op =
Op.getOperand(0);
9491 Op.getOperand(0).getValueType() == MVT::v2i64 &&
9493 unsigned Elem =
Op.getConstantOperandVal(1);
9494 Op =
Op.getOperand(0);
9496 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9498 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9505SDValue SystemZTargetLowering::combineMUL(
9506 SDNode *
N, DAGCombinerInfo &DCI)
const {
9507 SelectionDAG &DAG = DCI.DAG;
9514 if (OpcodeCand0 && OpcodeCand0 == OpcodeCand1)
9515 return DAG.
getNode(OpcodeCand0, SDLoc(
N),
N->getValueType(0), Op0, Op1);
9520SDValue SystemZTargetLowering::combineINTRINSIC(
9521 SDNode *
N, DAGCombinerInfo &DCI)
const {
9522 SelectionDAG &DAG = DCI.DAG;
9524 unsigned Id =
N->getConstantOperandVal(1);
9528 case Intrinsic::s390_vll:
9529 case Intrinsic::s390_vlrl:
9531 if (
C->getZExtValue() >= 15)
9532 return DAG.
getLoad(
N->getValueType(0), SDLoc(
N),
N->getOperand(0),
9533 N->getOperand(3), MachinePointerInfo());
9536 case Intrinsic::s390_vstl:
9537 case Intrinsic::s390_vstrl:
9539 if (
C->getZExtValue() >= 15)
9540 return DAG.
getStore(
N->getOperand(0), SDLoc(
N),
N->getOperand(2),
9541 N->getOperand(4), MachinePointerInfo());
9549 if (
N->getOpcode() == SystemZISD::PCREL_WRAPPER)
9556 switch(
N->getOpcode()) {
9561 case SystemZISD::MERGE_HIGH:
9562 case SystemZISD::MERGE_LOW:
return combineMERGE(
N, DCI);
9567 case SystemZISD::JOIN_DWORDS:
return combineJOIN_DWORDS(
N, DCI);
9577 case SystemZISD::BR_CCMASK:
return combineBR_CCMASK(
N, DCI);
9578 case SystemZISD::SELECT_CCMASK:
return combineSELECT_CCMASK(
N, DCI);
9581 case ISD::SRA:
return combineShiftToMulAddHigh(
N, DCI);
9582 case ISD::MUL:
return combineMUL(
N, DCI);
9586 case ISD::UREM:
return combineIntDIVREM(
N, DCI);
9598 EVT VT =
Op.getValueType();
9601 unsigned Opcode =
Op.getOpcode();
9603 unsigned Id =
Op.getConstantOperandVal(0);
9605 case Intrinsic::s390_vpksh:
9606 case Intrinsic::s390_vpksf:
9607 case Intrinsic::s390_vpksg:
9608 case Intrinsic::s390_vpkshs:
9609 case Intrinsic::s390_vpksfs:
9610 case Intrinsic::s390_vpksgs:
9611 case Intrinsic::s390_vpklsh:
9612 case Intrinsic::s390_vpklsf:
9613 case Intrinsic::s390_vpklsg:
9614 case Intrinsic::s390_vpklshs:
9615 case Intrinsic::s390_vpklsfs:
9616 case Intrinsic::s390_vpklsgs:
9618 SrcDemE = DemandedElts;
9621 SrcDemE = SrcDemE.
trunc(NumElts / 2);
9624 case Intrinsic::s390_vuphb:
9625 case Intrinsic::s390_vuphh:
9626 case Intrinsic::s390_vuphf:
9627 case Intrinsic::s390_vuplhb:
9628 case Intrinsic::s390_vuplhh:
9629 case Intrinsic::s390_vuplhf:
9630 SrcDemE =
APInt(NumElts * 2, 0);
9633 case Intrinsic::s390_vuplb:
9634 case Intrinsic::s390_vuplhw:
9635 case Intrinsic::s390_vuplf:
9636 case Intrinsic::s390_vupllb:
9637 case Intrinsic::s390_vupllh:
9638 case Intrinsic::s390_vupllf:
9639 SrcDemE =
APInt(NumElts * 2, 0);
9642 case Intrinsic::s390_vpdi: {
9644 SrcDemE =
APInt(NumElts, 0);
9645 if (!DemandedElts[OpNo - 1])
9647 unsigned Mask =
Op.getConstantOperandVal(3);
9648 unsigned MaskBit = ((OpNo - 1) ? 1 : 4);
9650 SrcDemE.
setBit((Mask & MaskBit)? 1 : 0);
9653 case Intrinsic::s390_vsldb: {
9655 assert(VT == MVT::v16i8 &&
"Unexpected type.");
9656 unsigned FirstIdx =
Op.getConstantOperandVal(3);
9657 assert (FirstIdx > 0 && FirstIdx < 16 &&
"Unused operand.");
9658 unsigned NumSrc0Els = 16 - FirstIdx;
9659 SrcDemE =
APInt(NumElts, 0);
9661 APInt DemEls = DemandedElts.
trunc(NumSrc0Els);
9664 APInt DemEls = DemandedElts.
lshr(NumSrc0Els);
9669 case Intrinsic::s390_vperm:
9678 case SystemZISD::JOIN_DWORDS:
9680 SrcDemE =
APInt(1, 1);
9682 case SystemZISD::SELECT_CCMASK:
9683 SrcDemE = DemandedElts;
9694 const APInt &DemandedElts,
9709 const APInt &DemandedElts,
9711 unsigned Depth)
const {
9715 unsigned Tmp0, Tmp1;
9717 Known.Zero.setBitsFrom(2);
9720 [[maybe_unused]]
EVT VT =
Op.getValueType();
9721 if (
Op.getResNo() != 0)
9724 "KnownBits does not match VT in bitwidth");
9727 "DemandedElts does not match VT number of elements");
9729 unsigned Opcode =
Op.getOpcode();
9731 bool IsLogical =
false;
9732 unsigned Id =
Op.getConstantOperandVal(0);
9734 case Intrinsic::s390_vpksh:
9735 case Intrinsic::s390_vpksf:
9736 case Intrinsic::s390_vpksg:
9737 case Intrinsic::s390_vpkshs:
9738 case Intrinsic::s390_vpksfs:
9739 case Intrinsic::s390_vpksgs:
9740 case Intrinsic::s390_vpklsh:
9741 case Intrinsic::s390_vpklsf:
9742 case Intrinsic::s390_vpklsg:
9743 case Intrinsic::s390_vpklshs:
9744 case Intrinsic::s390_vpklsfs:
9745 case Intrinsic::s390_vpklsgs:
9746 case Intrinsic::s390_vpdi:
9747 case Intrinsic::s390_vsldb:
9748 case Intrinsic::s390_vperm:
9751 case Intrinsic::s390_vuplhb:
9752 case Intrinsic::s390_vuplhh:
9753 case Intrinsic::s390_vuplhf:
9754 case Intrinsic::s390_vupllb:
9755 case Intrinsic::s390_vupllh:
9756 case Intrinsic::s390_vupllf:
9759 case Intrinsic::s390_vuphb:
9760 case Intrinsic::s390_vuphh:
9761 case Intrinsic::s390_vuphf:
9762 case Intrinsic::s390_vuplb:
9763 case Intrinsic::s390_vuplhw:
9764 case Intrinsic::s390_vuplf: {
9779 case SystemZISD::JOIN_DWORDS:
9780 case SystemZISD::SELECT_CCMASK:
9783 case SystemZISD::REPLICATE: {
9806 if (
LHS == 1)
return 1;
9809 if (
RHS == 1)
return 1;
9810 unsigned Common = std::min(
LHS,
RHS);
9811 unsigned SrcBitWidth =
Op.getOperand(OpNo).getScalarValueSizeInBits();
9812 EVT VT =
Op.getValueType();
9814 if (SrcBitWidth > VTBits) {
9815 unsigned SrcExtraBits = SrcBitWidth - VTBits;
9816 if (Common > SrcExtraBits)
9817 return (Common - SrcExtraBits);
9820 assert (SrcBitWidth == VTBits &&
"Expected operands of same bitwidth.");
9827 unsigned Depth)
const {
9828 if (
Op.getResNo() != 0)
9830 unsigned Opcode =
Op.getOpcode();
9832 unsigned Id =
Op.getConstantOperandVal(0);
9834 case Intrinsic::s390_vpksh:
9835 case Intrinsic::s390_vpksf:
9836 case Intrinsic::s390_vpksg:
9837 case Intrinsic::s390_vpkshs:
9838 case Intrinsic::s390_vpksfs:
9839 case Intrinsic::s390_vpksgs:
9840 case Intrinsic::s390_vpklsh:
9841 case Intrinsic::s390_vpklsf:
9842 case Intrinsic::s390_vpklsg:
9843 case Intrinsic::s390_vpklshs:
9844 case Intrinsic::s390_vpklsfs:
9845 case Intrinsic::s390_vpklsgs:
9846 case Intrinsic::s390_vpdi:
9847 case Intrinsic::s390_vsldb:
9848 case Intrinsic::s390_vperm:
9850 case Intrinsic::s390_vuphb:
9851 case Intrinsic::s390_vuphh:
9852 case Intrinsic::s390_vuphf:
9853 case Intrinsic::s390_vuplb:
9854 case Intrinsic::s390_vuplhw:
9855 case Intrinsic::s390_vuplf: {
9859 EVT VT =
Op.getValueType();
9869 case SystemZISD::SELECT_CCMASK:
9882 switch (
Op->getOpcode()) {
9883 case SystemZISD::PCREL_WRAPPER:
9884 case SystemZISD::PCREL_OFFSET:
9895 "Unexpected stack alignment");
9898 unsigned StackProbeSize =
9901 StackProbeSize &= ~(StackAlign - 1);
9902 return StackProbeSize ? StackProbeSize : StackAlign;
9919 Register Reg =
MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
9925 Register Reg =
MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
9941 if (
MI.readsRegister(SystemZ::CC,
nullptr))
9943 if (
MI.definesRegister(SystemZ::CC,
nullptr))
9949 if (miI ==
MBB->end()) {
9951 if (Succ->isLiveIn(SystemZ::CC))
9962 switch (
MI.getOpcode()) {
9963 case SystemZ::Select32:
9964 case SystemZ::Select64:
9965 case SystemZ::Select128:
9966 case SystemZ::SelectF32:
9967 case SystemZ::SelectF64:
9968 case SystemZ::SelectF128:
9969 case SystemZ::SelectVR32:
9970 case SystemZ::SelectVR64:
9971 case SystemZ::SelectVR128:
10003 for (
auto *
MI : Selects) {
10004 Register DestReg =
MI->getOperand(0).getReg();
10005 Register TrueReg =
MI->getOperand(1).getReg();
10006 Register FalseReg =
MI->getOperand(2).getReg();
10011 if (
MI->getOperand(4).getImm() == (CCValid ^ CCMask))
10014 if (
auto It = RegRewriteTable.
find(TrueReg); It != RegRewriteTable.
end())
10015 TrueReg = It->second.first;
10017 if (
auto It = RegRewriteTable.
find(FalseReg); It != RegRewriteTable.
end())
10018 FalseReg = It->second.second;
10021 BuildMI(*SinkMBB, SinkInsertionPoint,
DL,
TII->get(SystemZ::PHI), DestReg)
10026 RegRewriteTable[DestReg] = std::make_pair(TrueReg, FalseReg);
10037 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
10038 assert(TFL->hasReservedCallFrame(MF) &&
10039 "ADJSTACKDOWN and ADJSTACKUP should be no-ops");
10044 uint32_t NumBytes =
MI.getOperand(0).getImm();
10049 MI.eraseFromParent();
10058 const SystemZInstrInfo *
TII = Subtarget.getInstrInfo();
10060 unsigned CCValid =
MI.getOperand(3).getImm();
10061 unsigned CCMask =
MI.getOperand(4).getImm();
10066 SmallVector<MachineInstr*, 8> Selects;
10067 SmallVector<MachineInstr*, 8> DbgValues;
10069 unsigned Count = 0;
10073 assert(NextMI.getOperand(3).getImm() == CCValid &&
10074 "Bad CCValid operands since CC was not redefined.");
10075 if (NextMI.getOperand(4).getImm() == CCMask ||
10076 NextMI.getOperand(4).getImm() == (CCValid ^ CCMask)) {
10082 if (NextMI.definesRegister(SystemZ::CC,
nullptr) ||
10083 NextMI.usesCustomInsertionHook())
10086 for (
auto *SelMI : Selects)
10087 if (NextMI.readsVirtualRegister(SelMI->getOperand(0).getReg())) {
10091 if (NextMI.isDebugInstr()) {
10093 assert(NextMI.isDebugValue() &&
"Unhandled debug opcode.");
10096 }
else if (User || ++
Count > 20)
10100 MachineInstr *LastMI = Selects.back();
10101 bool CCKilled = (LastMI->
killsRegister(SystemZ::CC,
nullptr) ||
10103 MachineBasicBlock *StartMBB =
MBB;
10133 for (
auto *SelMI : Selects)
10134 SelMI->eraseFromParent();
10137 for (
auto *DbgMI : DbgValues)
10138 MBB->
splice(InsertPos, StartMBB, DbgMI);
10149 unsigned StoreOpcode,
10150 unsigned STOCOpcode,
10151 bool Invert)
const {
10152 const SystemZInstrInfo *
TII = Subtarget.getInstrInfo();
10154 Register SrcReg =
MI.getOperand(0).getReg();
10155 MachineOperand
Base =
MI.getOperand(1);
10156 int64_t Disp =
MI.getOperand(2).getImm();
10157 Register IndexReg =
MI.getOperand(3).getReg();
10158 unsigned CCValid =
MI.getOperand(4).getImm();
10159 unsigned CCMask =
MI.getOperand(5).getImm();
10162 StoreOpcode =
TII->getOpcodeForOffset(StoreOpcode, Disp);
10166 MachineMemOperand *MMO =
nullptr;
10167 for (
auto *
I :
MI.memoperands())
10168 if (
I->isStore()) {
10176 if (STOCOpcode && !IndexReg && Subtarget.hasLoadStoreOnCond()) {
10188 MI.eraseFromParent();
10196 MachineBasicBlock *StartMBB =
MBB;
10202 if (!
MI.killsRegister(SystemZ::CC,
nullptr) &&
10229 MI.eraseFromParent();
10239 const SystemZInstrInfo *
TII = Subtarget.getInstrInfo();
10247 MachineBasicBlock *StartMBB =
MBB;
10265 int HiOpcode =
Unsigned? SystemZ::VECLG : SystemZ::VECG;
10292 MI.eraseFromParent();
10303 bool Invert)
const {
10305 const SystemZInstrInfo *
TII = Subtarget.getInstrInfo();
10312 int64_t Disp =
MI.getOperand(2).getImm();
10314 Register BitShift =
MI.getOperand(4).getReg();
10315 Register NegBitShift =
MI.getOperand(5).getReg();
10316 unsigned BitSize =
MI.getOperand(6).getImm();
10320 unsigned LOpcode =
TII->getOpcodeForOffset(SystemZ::L, Disp);
10321 unsigned CSOpcode =
TII->getOpcodeForOffset(SystemZ::CS, Disp);
10322 assert(LOpcode && CSOpcode &&
"Displacement out of range");
10332 MachineBasicBlock *StartMBB =
MBB;
10368 .
addImm(-1U << (32 - BitSize))
10370 }
else if (BinOpcode)
10398 MI.eraseFromParent();
10409 unsigned KeepOldMask)
const {
10411 const SystemZInstrInfo *
TII = Subtarget.getInstrInfo();
10417 int64_t Disp =
MI.getOperand(2).getImm();
10419 Register BitShift =
MI.getOperand(4).getReg();
10420 Register NegBitShift =
MI.getOperand(5).getReg();
10421 unsigned BitSize =
MI.getOperand(6).getImm();
10425 unsigned LOpcode =
TII->getOpcodeForOffset(SystemZ::L, Disp);
10426 unsigned CSOpcode =
TII->getOpcodeForOffset(SystemZ::CS, Disp);
10427 assert(LOpcode && CSOpcode &&
"Displacement out of range");
10438 MachineBasicBlock *StartMBB =
MBB;
10506 MI.eraseFromParent();
10516 const SystemZInstrInfo *
TII = Subtarget.getInstrInfo();
10522 int64_t Disp =
MI.getOperand(2).getImm();
10523 Register CmpVal =
MI.getOperand(3).getReg();
10524 Register OrigSwapVal =
MI.getOperand(4).getReg();
10525 Register BitShift =
MI.getOperand(5).getReg();
10526 Register NegBitShift =
MI.getOperand(6).getReg();
10527 int64_t BitSize =
MI.getOperand(7).getImm();
10533 unsigned LOpcode =
TII->getOpcodeForOffset(SystemZ::L, Disp);
10534 unsigned CSOpcode =
TII->getOpcodeForOffset(SystemZ::CS, Disp);
10535 unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
10536 assert(LOpcode && CSOpcode &&
"Displacement out of range");
10548 MachineBasicBlock *StartMBB =
MBB;
10625 if (!
MI.registerDefIsDead(SystemZ::CC,
nullptr))
10628 MI.eraseFromParent();
10635 unsigned Opcode,
bool IsMemset)
const {
10637 const SystemZInstrInfo *
TII = Subtarget.getInstrInfo();
10642 uint64_t DestDisp =
MI.getOperand(1).getImm();
10647 auto foldDisplIfNeeded = [&](MachineOperand &
Base,
uint64_t &Disp) ->
void {
10650 unsigned Opcode =
TII->getOpcodeForOffset(SystemZ::LA, Disp);
10660 SrcDisp =
MI.getOperand(3).getImm();
10662 SrcBase = DestBase;
10663 SrcDisp = DestDisp++;
10664 foldDisplIfNeeded(DestBase, DestDisp);
10667 MachineOperand &LengthMO =
MI.getOperand(IsMemset ? 2 : 4);
10668 bool IsImmForm = LengthMO.
isImm();
10669 bool IsRegForm = !IsImmForm;
10672 auto insertMemMemOp = [&](MachineBasicBlock *InsMBB,
10674 MachineOperand DBase,
uint64_t DDisp,
10676 unsigned Length) ->
void {
10680 if (ByteMO.
isImm())
10695 bool NeedsLoop =
false;
10697 Register LenAdjReg = SystemZ::NoRegister;
10699 ImmLength = LengthMO.
getImm();
10700 ImmLength += IsMemset ? 2 : 1;
10701 if (ImmLength == 0) {
10702 MI.eraseFromParent();
10705 if (Opcode == SystemZ::CLC) {
10706 if (ImmLength > 3 * 256)
10716 }
else if (ImmLength > 6 * 256)
10724 LenAdjReg = LengthMO.
getReg();
10729 MachineBasicBlock *EndMBB =
10730 (Opcode == SystemZ::CLC && (ImmLength > 256 || NeedsLoop)
10738 TII->loadImmediate(*
MBB,
MI, StartCountReg, ImmLength / 256);
10748 auto loadZeroAddress = [&]() -> MachineOperand {
10753 if (DestBase.
isReg() && DestBase.
getReg() == SystemZ::NoRegister)
10754 DestBase = loadZeroAddress();
10755 if (SrcBase.
isReg() && SrcBase.
getReg() == SystemZ::NoRegister)
10756 SrcBase = HaveSingleBase ? DestBase : loadZeroAddress();
10758 MachineBasicBlock *StartMBB =
nullptr;
10759 MachineBasicBlock *LoopMBB =
nullptr;
10760 MachineBasicBlock *NextMBB =
nullptr;
10761 MachineBasicBlock *DoneMBB =
nullptr;
10762 MachineBasicBlock *AllDoneMBB =
nullptr;
10766 (HaveSingleBase ? StartSrcReg :
forceReg(
MI, DestBase,
TII));
10775 RC = &SystemZ::GR64BitRegClass;
10803 MBB = MemsetOneCheckMBB;
10814 MBB = MemsetOneMBB;
10846 if (EndMBB && !ImmLength)
10868 if (!HaveSingleBase)
10875 if (Opcode == SystemZ::MVC)
10902 if (!HaveSingleBase)
10927 Register RemDestReg = HaveSingleBase ? RemSrcReg
10932 if (!HaveSingleBase)
10940 MachineInstrBuilder EXRL_MIB =
10948 if (Opcode != SystemZ::MVC) {
10958 while (ImmLength > 0) {
10962 foldDisplIfNeeded(DestBase, DestDisp);
10963 foldDisplIfNeeded(SrcBase, SrcDisp);
10964 insertMemMemOp(
MBB,
MI, DestBase, DestDisp, SrcBase, SrcDisp, ThisLength);
10965 DestDisp += ThisLength;
10966 SrcDisp += ThisLength;
10967 ImmLength -= ThisLength;
10970 if (EndMBB && ImmLength > 0) {
10986 MI.eraseFromParent();
10993 const SystemZInstrInfo *
TII = Subtarget.getInstrInfo();
10999 assert(Len > 0 && Len <= 256 &&
"Memmove of of unsupported constant length.");
11028 MI.eraseFromParent();
11037 const SystemZInstrInfo *
TII = Subtarget.getInstrInfo();
11041 uint64_t End1Reg =
MI.getOperand(0).getReg();
11042 uint64_t Start1Reg =
MI.getOperand(1).getReg();
11043 uint64_t Start2Reg =
MI.getOperand(2).getReg();
11044 uint64_t CharReg =
MI.getOperand(3).getReg();
11051 MachineBasicBlock *StartMBB =
MBB;
11087 MI.eraseFromParent();
11094 bool NoFloat)
const {
11096 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
11097 const SystemZInstrInfo *
TII = Subtarget.getInstrInfo();
11100 MI.setDesc(
TII->get(Opcode));
11104 uint64_t Control =
MI.getOperand(2).getImm();
11105 static const unsigned GPRControlBit[16] = {
11106 0x8000, 0x8000, 0x4000, 0x4000, 0x2000, 0x2000, 0x1000, 0x1000,
11107 0x0800, 0x0800, 0x0400, 0x0400, 0x0200, 0x0200, 0x0100, 0x0100
11109 Control |= GPRControlBit[15];
11110 if (TFI->
hasFP(MF))
11111 Control |= GPRControlBit[11];
11112 MI.getOperand(2).setImm(Control);
11115 for (
int I = 0;
I < 16;
I++) {
11116 if ((Control & GPRControlBit[
I]) == 0) {
11123 if (!NoFloat && (Control & 4) != 0) {
11124 if (Subtarget.hasVector()) {
11142 const SystemZInstrInfo *
TII = Subtarget.getInstrInfo();
11145 Register SrcReg =
MI.getOperand(0).getReg();
11156 MI.eraseFromParent();
11165 const SystemZInstrInfo *
TII = Subtarget.getInstrInfo();
11168 Register DstReg =
MI.getOperand(0).getReg();
11169 Register SizeReg =
MI.getOperand(2).getReg();
11171 MachineBasicBlock *StartMBB =
MBB;
11258 MI.eraseFromParent();
11262SDValue SystemZTargetLowering::
11265 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
11277 const SystemZInstrInfo *
TII = Subtarget.getInstrInfo();
11282 .
addImm(
MI.getOperand(1).getImm());
11283 MI.eraseFromParent();
11289 switch (
MI.getOpcode()) {
11290 case SystemZ::ADJCALLSTACKDOWN:
11291 case SystemZ::ADJCALLSTACKUP:
11292 return emitAdjCallStack(
MI,
MBB);
11294 case SystemZ::Select32:
11295 case SystemZ::Select64:
11296 case SystemZ::Select128:
11297 case SystemZ::SelectF32:
11298 case SystemZ::SelectF64:
11299 case SystemZ::SelectF128:
11300 case SystemZ::SelectVR32:
11301 case SystemZ::SelectVR64:
11302 case SystemZ::SelectVR128:
11303 return emitSelect(
MI,
MBB);
11305 case SystemZ::CondStore8Mux:
11306 return emitCondStore(
MI,
MBB, SystemZ::STCMux, 0,
false);
11307 case SystemZ::CondStore8MuxInv:
11308 return emitCondStore(
MI,
MBB, SystemZ::STCMux, 0,
true);
11309 case SystemZ::CondStore16Mux:
11310 return emitCondStore(
MI,
MBB, SystemZ::STHMux, 0,
false);
11311 case SystemZ::CondStore16MuxInv:
11312 return emitCondStore(
MI,
MBB, SystemZ::STHMux, 0,
true);
11313 case SystemZ::CondStore32Mux:
11314 return emitCondStore(
MI,
MBB, SystemZ::STMux, SystemZ::STOCMux,
false);
11315 case SystemZ::CondStore32MuxInv:
11316 return emitCondStore(
MI,
MBB, SystemZ::STMux, SystemZ::STOCMux,
true);
11317 case SystemZ::CondStore8:
11318 return emitCondStore(
MI,
MBB, SystemZ::STC, 0,
false);
11319 case SystemZ::CondStore8Inv:
11320 return emitCondStore(
MI,
MBB, SystemZ::STC, 0,
true);
11321 case SystemZ::CondStore16:
11322 return emitCondStore(
MI,
MBB, SystemZ::STH, 0,
false);
11323 case SystemZ::CondStore16Inv:
11324 return emitCondStore(
MI,
MBB, SystemZ::STH, 0,
true);
11325 case SystemZ::CondStore32:
11326 return emitCondStore(
MI,
MBB, SystemZ::ST, SystemZ::STOC,
false);
11327 case SystemZ::CondStore32Inv:
11328 return emitCondStore(
MI,
MBB, SystemZ::ST, SystemZ::STOC,
true);
11329 case SystemZ::CondStore64:
11330 return emitCondStore(
MI,
MBB, SystemZ::STG, SystemZ::STOCG,
false);
11331 case SystemZ::CondStore64Inv:
11332 return emitCondStore(
MI,
MBB, SystemZ::STG, SystemZ::STOCG,
true);
11333 case SystemZ::CondStoreF32:
11334 return emitCondStore(
MI,
MBB, SystemZ::STE, 0,
false);
11335 case SystemZ::CondStoreF32Inv:
11336 return emitCondStore(
MI,
MBB, SystemZ::STE, 0,
true);
11337 case SystemZ::CondStoreF64:
11338 return emitCondStore(
MI,
MBB, SystemZ::STD, 0,
false);
11339 case SystemZ::CondStoreF64Inv:
11340 return emitCondStore(
MI,
MBB, SystemZ::STD, 0,
true);
11342 case SystemZ::SCmp128Hi:
11343 return emitICmp128Hi(
MI,
MBB,
false);
11344 case SystemZ::UCmp128Hi:
11345 return emitICmp128Hi(
MI,
MBB,
true);
11348 case SystemZ::ATOMIC_SWAPW:
11349 return emitAtomicLoadBinary(
MI,
MBB, 0);
11351 case SystemZ::ATOMIC_LOADW_AR:
11352 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::AR);
11353 case SystemZ::ATOMIC_LOADW_AFI:
11354 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::AFI);
11356 case SystemZ::ATOMIC_LOADW_SR:
11357 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::SR);
11359 case SystemZ::ATOMIC_LOADW_NR:
11360 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::NR);
11361 case SystemZ::ATOMIC_LOADW_NILH:
11362 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::NILH);
11364 case SystemZ::ATOMIC_LOADW_OR:
11365 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::OR);
11366 case SystemZ::ATOMIC_LOADW_OILH:
11367 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::OILH);
11369 case SystemZ::ATOMIC_LOADW_XR:
11370 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::XR);
11371 case SystemZ::ATOMIC_LOADW_XILF:
11372 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::XILF);
11374 case SystemZ::ATOMIC_LOADW_NRi:
11375 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::NR,
true);
11376 case SystemZ::ATOMIC_LOADW_NILHi:
11377 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::NILH,
true);
11379 case SystemZ::ATOMIC_LOADW_MIN:
11381 case SystemZ::ATOMIC_LOADW_MAX:
11383 case SystemZ::ATOMIC_LOADW_UMIN:
11385 case SystemZ::ATOMIC_LOADW_UMAX:
11388 case SystemZ::ATOMIC_CMP_SWAPW:
11389 return emitAtomicCmpSwapW(
MI,
MBB);
11390 case SystemZ::MVCImm:
11391 case SystemZ::MVCReg:
11392 return emitMemMemWrapper(
MI,
MBB, SystemZ::MVC);
11393 case SystemZ::NCImm:
11394 return emitMemMemWrapper(
MI,
MBB, SystemZ::NC);
11395 case SystemZ::OCImm:
11396 return emitMemMemWrapper(
MI,
MBB, SystemZ::OC);
11397 case SystemZ::XCImm:
11398 case SystemZ::XCReg:
11399 return emitMemMemWrapper(
MI,
MBB, SystemZ::XC);
11400 case SystemZ::CLCImm:
11401 case SystemZ::CLCReg:
11402 return emitMemMemWrapper(
MI,
MBB, SystemZ::CLC);
11403 case SystemZ::MemsetImmImm:
11404 case SystemZ::MemsetImmReg:
11405 case SystemZ::MemsetRegImm:
11406 case SystemZ::MemsetRegReg:
11407 return emitMemMemWrapper(
MI,
MBB, SystemZ::MVC,
true);
11408 case SystemZ::MemmoveImm:
11409 return emitMemmoveImm(
MI,
MBB);
11410 case SystemZ::CLSTLoop:
11411 return emitStringWrapper(
MI,
MBB, SystemZ::CLST);
11412 case SystemZ::MVSTLoop:
11413 return emitStringWrapper(
MI,
MBB, SystemZ::MVST);
11414 case SystemZ::SRSTLoop:
11415 return emitStringWrapper(
MI,
MBB, SystemZ::SRST);
11416 case SystemZ::TBEGIN:
11417 return emitTransactionBegin(
MI,
MBB, SystemZ::TBEGIN,
false);
11418 case SystemZ::TBEGIN_nofloat:
11419 return emitTransactionBegin(
MI,
MBB, SystemZ::TBEGIN,
true);
11420 case SystemZ::TBEGINC:
11421 return emitTransactionBegin(
MI,
MBB, SystemZ::TBEGINC,
true);
11422 case SystemZ::LTEBRCompare_Pseudo:
11423 return emitLoadAndTestCmp0(
MI,
MBB, SystemZ::LTEBR);
11424 case SystemZ::LTDBRCompare_Pseudo:
11425 return emitLoadAndTestCmp0(
MI,
MBB, SystemZ::LTDBR);
11426 case SystemZ::LTXBRCompare_Pseudo:
11427 return emitLoadAndTestCmp0(
MI,
MBB, SystemZ::LTXBR);
11429 case SystemZ::PROBED_ALLOCA:
11430 return emitProbedAlloca(
MI,
MBB);
11431 case SystemZ::EH_SjLj_SetJmp:
11433 case SystemZ::EH_SjLj_LongJmp:
11436 case TargetOpcode::STACKMAP:
11437 case TargetOpcode::PATCHPOINT:
11440 case SystemZ::MOV_STACKGUARD_DAG:
11441 return emitStackGuardPseudo(
MI,
MBB, SystemZ::MOV_STACKGUARD);
11443 case SystemZ::CMP_STACKGUARD_DAG:
11444 return emitStackGuardPseudo(
MI,
MBB, SystemZ::CMP_STACKGUARD);
11473 DAG.
getMachineNode(SystemZ::EFPC, dl, {MVT::i32, MVT::Other}, Chain), 0);
11493 EVT VT =
Op.getValueType();
11494 Op =
Op.getOperand(0);
11495 EVT OpVT =
Op.getValueType();
11497 assert(OpVT.
isVector() &&
"Operand type for VECREDUCE_ADD is not a vector.");
11529 if (Attrs.hasRetAttrs())
11530 OS << Attrs.getAsString(AttributeList::ReturnIndex) <<
" ";
11531 OS << *
F->getReturnType() <<
" @" <<
F->getName() <<
"(";
11532 for (
unsigned I = 0,
E = FT->getNumParams();
I !=
E; ++
I) {
11535 OS << *FT->getParamType(
I);
11537 for (
auto A : {Attribute::SExt, Attribute::ZExt, Attribute::NoExt})
11546bool SystemZTargetLowering::isInternal(
const Function *Fn)
const {
11547 std::map<const Function *, bool>::iterator Itr = IsInternalCache.find(Fn);
11548 if (Itr == IsInternalCache.end())
11549 Itr = IsInternalCache
11550 .insert(std::pair<const Function *, bool>(
11553 return Itr->second;
11556bool SystemZTargetLowering::enableNarrowIntArgsVerification()
const {
11557 if (!Subtarget.isTargetELF())
11570void SystemZTargetLowering::verifyNarrowIntegerArgs_Call(
11573 if (!enableNarrowIntArgsVerification())
11575 bool IsInternal =
false;
11576 const Function *CalleeFn =
nullptr;
11579 IsInternal = isInternal(CalleeFn);
11581 if (!verifyNarrowIntegerArgs(C_ABI, Outs, CalleeFn,
false)) {
11582 errs() <<
"Callee: ";
11583 if (CalleeFn !=
nullptr)
11587 errs() <<
"Caller: ";
11590 errs() <<
" " << *CLI.CB;
11593 LLVM_DEBUG(
dbgs() <<
"Outgoing call arguments verified as ABI compliant: ";
11595 else dbgs() <<
"-\n";);
11598void SystemZTargetLowering::verifyNarrowIntegerArgs_Ret(
11601 if (!enableNarrowIntArgsVerification())
11604 if (!verifyNarrowIntegerArgs(C_ABI, Outs,
F,
true)) {
11605 errs() <<
"Returning from function: ";
11609 LLVM_DEBUG(
dbgs() <<
"Return argument verified as ABI compliant : ";
11616bool SystemZTargetLowering::verifyNarrowIntegerArgs(
11618 const Function *FunHeader,
bool IsRet)
const {
11619 if (IsRet && Outs.
size() != 1)
11622 for (
unsigned i = 0; i < Outs.
size(); ++i) {
11623 EVT OrigArgVT = Outs[i].ArgVT;
11626 assert(Outs[i].VT == MVT::i32 &&
"Unexpected integer argument VT.");
11627 ISD::ArgFlagsTy
Flags = Outs[i].Flags;
11628 unsigned OrigIdx = Outs[i].OrigArgIndex;
11631 if (C_ABI && !
Flags.isSExt() && !
Flags.isZExt() && !
Flags.isNoExt()) {
11632 errs() <<
"ERROR: (C ABI violation) missing extension attribute on "
11633 << (IsRet ?
"return value" : Twine(
"arg ") + Twine(OrigIdx))
11640 const AttributeList &FunAttrs = FunHeader->
getAttributes();
11641 const AttributeSet
Attrs =
11642 IsRet ? FunAttrs.getRetAttrs() : FunAttrs.getParamAttrs(OrigIdx);
11643 if (
Attrs.hasAttribute(Attribute::SExt) && !
Flags.isSExt()) {
11644 errs() <<
"ERROR: Missing SExt on arg " << OrigIdx <<
".\n";
11647 if (
Attrs.hasAttribute(Attribute::ZExt) && !
Flags.isZExt()) {
11648 errs() <<
"ERROR: Missing ZExt on arg " << OrigIdx <<
".\n";
11659 StringRef GuardMode = M.getStackProtectorGuard();
11662 if (GuardMode ==
"tls" || GuardMode.
empty())
const AArch64Subtarget & Subtarget
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const AArch64InstrInfo & TII
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
AMDGPU Register Bank Select
static bool isZeroVector(SDValue N)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis false
Function Alias Analysis Results
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
Module.h This file contains the declarations for the Module class.
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isSelectPseudo(MachineInstr &MI)
static bool isUndef(const MachineInstr &MI)
Register const TargetRegisterInfo * TRI
Promote Memory to Register
uint64_t IntrinsicInst * II
static constexpr MCPhysReg SPReg
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
This file defines the SmallSet class.
static SDValue getI128Select(SelectionDAG &DAG, const SDLoc &DL, Comparison C, SDValue TrueOp, SDValue FalseOp)
static SmallVector< SDValue, 4 > simplifyAssumingCCVal(SDValue &Val, SDValue &CC, SelectionDAG &DAG)
static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static void adjustForLTGFR(Comparison &C)
static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0, SDValue Op1)
static cl::opt< bool > EnableIntArgExtCheck("argext-abi-check", cl::init(false), cl::desc("Verify that narrow int args are properly extended per the " "SystemZ ABI."))
static bool isOnlyUsedByStores(SDValue StoredVal, SelectionDAG &DAG)
static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT, unsigned Opcode, SDValue Op0, SDValue Op1, SDValue &Even, SDValue &Odd)
static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static SDValue lowerAddrSpaceCast(SDValue Op, SelectionDAG &DAG)
static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT, SDValue Value)
static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In)
static bool isSimpleShift(SDValue N, unsigned &ShiftVal)
static SDValue mergeHighParts(SelectionDAG &DAG, const SDLoc &DL, unsigned MergedBits, EVT VT, SDValue Op0, SDValue Op1)
static bool isI128MovedToParts(LoadSDNode *LD, SDNode *&LoPart, SDNode *&HiPart)
static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1)
static uint32_t findZeroVectorIdx(SDValue *Ops, unsigned Num)
static bool isVectorElementSwap(ArrayRef< int > M, EVT VT)
static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL, SDValue &AlignedAddr, SDValue &BitShift, SDValue &NegBitShift)
static bool isShlDoublePermute(const SmallVectorImpl< int > &Bytes, unsigned &StartIndex, unsigned &OpNo0, unsigned &OpNo1)
static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL, const Permute &P, SDValue Op0, SDValue Op1)
static SDNode * emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op, unsigned Opcode)
static SDValue getCCResult(SelectionDAG &DAG, SDValue CCReg)
static void adjustForStackGuardCompare(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode, unsigned &CCValid)
static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend, SDValue Op0, SDValue Op1, SDValue &Hi, SDValue &Lo)
static bool isF128MovedToParts(LoadSDNode *LD, SDNode *&LoPart, SDNode *&HiPart)
static void createPHIsForSelects(SmallVector< MachineInstr *, 8 > &Selects, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB, MachineBasicBlock *SinkMBB)
static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL, SDValue *Ops, const SmallVectorImpl< int > &Bytes)
static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, CmpMode Mode, bool &Invert)
static unsigned CCMaskForCondCode(ISD::CondCode CC)
static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static void adjustForFNeg(Comparison &C)
static bool isScalarToVector(SDValue Op)
static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg, unsigned CCValid, unsigned CCMask)
static bool matchPermute(const SmallVectorImpl< int > &Bytes, const Permute &P, unsigned &OpNo0, unsigned &OpNo1)
static bool isAddCarryChain(SDValue Carry)
static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static MachineOperand earlyUseOperand(MachineOperand Op)
static bool canUseSiblingCall(const CCState &ArgCCInfo, SmallVectorImpl< CCValAssign > &ArgLocs, SmallVectorImpl< ISD::OutputArg > &Outs)
static bool getzOSCalleeAndADA(SelectionDAG &DAG, SDValue &Callee, SDValue &ADA, SDLoc &DL, SDValue &Chain)
static SDValue convertToF16(SDValue Op, SelectionDAG &DAG)
static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask, SelectionDAG &DAG)
static bool shouldSwapCmpOperands(const Comparison &C)
static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType)
static SDValue getADAEntry(SelectionDAG &DAG, SDValue Val, SDLoc DL, unsigned Offset, bool LoadAdr=false)
static SDNode * emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op, unsigned Opcode)
static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static bool getVPermMask(SDValue ShuffleOp, SmallVectorImpl< int > &Bytes)
static const Permute PermuteForms[]
static bool isI128MovedFromParts(SDValue Val, SDValue &LoPart, SDValue &HiPart)
static std::pair< SDValue, int > findCCUse(const SDValue &Val, unsigned Depth=0)
static bool isSubBorrowChain(SDValue Carry)
static void adjustICmp128(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static bool analyzeArgSplit(const SmallVectorImpl< ArgTy > &Args, SmallVector< CCValAssign, 16 > &ArgLocs, unsigned I, MVT &PartVT, unsigned &NumParts)
static bool isInBoundsConstantVectorIndex(EVT VecVT, SDValue Idx)
static APInt getDemandedSrcElements(SDValue Op, const APInt &DemandedElts, unsigned OpNo)
static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op, bool IsNegative)
static unsigned computeNumSignBitsBinOp(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth, unsigned OpNo)
static SDValue expandBitCastI128ToF128(SelectionDAG &DAG, SDValue Src, const SDLoc &SL)
static SDValue tryBuildVectorShuffle(SelectionDAG &DAG, BuildVectorSDNode *BVN)
static SDValue convertFromF16(SDValue Op, SDLoc DL, SelectionDAG &DAG)
static unsigned getVectorComparison(ISD::CondCode CC, CmpMode Mode)
static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In)
static SDValue MergeInputChains(SDNode *N1, SDNode *N2)
static SDValue expandBitCastF128ToI128(SelectionDAG &DAG, SDValue Src, const SDLoc &SL)
static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask, uint64_t Mask, uint64_t CmpVal, unsigned ICmpType)
static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid)
static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL, SDValue Op, SDValue Chain)
static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1, ISD::CondCode Cond, const SDLoc &DL, SDValue Chain=SDValue(), bool IsSignaling=false)
static bool checkCCKill(MachineInstr &MI, MachineBasicBlock *MBB)
static Register forceReg(MachineInstr &MI, MachineOperand &Base, const SystemZInstrInfo *TII)
static bool is32Bit(EVT VT)
static std::pair< unsigned, const TargetRegisterClass * > parseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC, const unsigned *Map, unsigned Size)
static unsigned detectEvenOddMultiplyOperand(const SelectionDAG &DAG, const SystemZSubtarget &Subtarget, SDValue &Op)
static bool matchDoublePermute(const SmallVectorImpl< int > &Bytes, const Permute &P, SmallVectorImpl< int > &Transform)
static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode, SDValue Call, unsigned CCValid, uint64_t CC, ISD::CondCode Cond)
static Register getOrCreateADAVirtualRegister(MachineFunction &MF)
static SDValue buildFPVecFromScalars4(SelectionDAG &DAG, const SDLoc &DL, EVT VT, SmallVectorImpl< SDValue > &Elems, unsigned Pos)
static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg)
static AddressingMode getLoadStoreAddrMode(bool HasVector, Type *Ty)
static void printFunctionArgExts(const Function *F, raw_ostream &OS)
static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT, SDValue Op0, SDValue Op1)
static void computeKnownBitsBinOp(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth, unsigned OpNo)
static bool getShuffleInput(const SmallVectorImpl< int > &Bytes, unsigned Start, unsigned BytesPerElement, int &Base)
static AddressingMode supportedAddressingMode(Instruction *I, bool HasVector)
static bool isF128MovedFromParts(SDValue Val, SDValue &LoPart, SDValue &HiPart)
static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
unsigned getActiveBits() const
Compute the number of active bits in the value.
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool isSingleWord() const
Determine if this APInt just has one word to store value.
LLVM_ABI void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
an instruction that atomically reads a memory location, combines it with another value,...
BinOp getOperation() const
This class holds the attributes for a particular argument, parameter, function, or return value.
bool hasAttribute(Attribute::AttrKind Kind) const
Return true if the attribute exists in this set.
LLVM_ABI StringRef getValueAsString() const
Return the attribute's value as a string.
static LLVM_ABI StringRef getNameFromAttrKind(Attribute::AttrKind AttrKind)
LLVM Basic Block Representation.
A "pseudo-class" with methods for operating on BUILD_VECTORs.
LLVM_ABI bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
LLVM_ABI bool isConstant() const
CCState - This class holds information needed while lowering arguments and return values.
LLVM_ABI void AnalyzeCallResult(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeCallResult - Analyze the return values of a call, incorporating info about the passed values i...
LLVM_ABI bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
LLVM_ABI void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
LLVM_ABI void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
LLVM_ABI void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
This class represents a function call, abstracting a target machine's calling convention.
MachineConstantPoolValue * getMachineCPVal() const
bool isMachineConstantPoolEntry() const
const Constant * getConstVal() const
uint64_t getZExtValue() const
This is an important base class in LLVM.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
LLVM_ABI unsigned getPointerSize(unsigned AS=0) const
The pointer representation size in bytes, rounded up to a whole number of bytes.
iterator find(const_arg_type_t< KeyT > Val)
constexpr bool isScalar() const
Exactly one element.
bool hasAddressTaken(const User **=nullptr, bool IgnoreCallbackUses=false, bool IgnoreAssumeLikeCalls=true, bool IngoreLLVMUsed=false, bool IgnoreARCAttachedCall=false, bool IgnoreCastedDirectCall=false) const
hasAddressTaken - returns true if there are any uses of this function other than direct calls or invo...
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
uint64_t getFnAttributeAsParsedInteger(StringRef Kind, uint64_t Default=0) const
For a string attribute Kind, parse attribute as an integer.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
LLVM_ABI const GlobalObject * getAliaseeObject() const
bool hasLocalLinkage() const
bool hasPrivateLinkage() const
bool hasInternalLinkage() const
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
Tracks which library functions to use for a particular subtarget or function.
An instruction for reading from memory.
This class is used to represent ISD::LOAD nodes.
static auto integer_fixedlen_vector_valuetypes()
uint64_t getScalarSizeInBits() const
bool isVector() const
Return true if this is a vector value type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
LLVM_ABI iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setMaxCallFrameSize(uint64_t S)
LLVM_ABI int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void setAdjustsStack(bool V)
void setFrameAddressIsTaken(bool T)
uint64_t getMaxCallFrameSize() const
Return the maximum size of a call frame that must be allocated for an outgoing function call.
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void push_back(MachineBasicBlock *MBB)
reverse_iterator rbegin()
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineFunctionProperties & getProperties() const
Get the function properties.
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Representation of each machine instruction.
bool killsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr kills the specified register.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
Register getReg() const
getReg - Returns the register number.
LLVM_ABI bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
A Module instance is used to store all the information related to an LLVM module.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
iterator_range< user_iterator > users()
void setFlags(SDNodeFlags NewFlags)
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isMachineOpcode() const
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
ISD::CondCode getOperandCondCode(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getMachineOpcode() const
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI SDValue getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO)
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
LLVM_ABI bool isConstantIntBuildVectorOrConstantInt(SDValue N, bool AllowOpaques=true) const
Test whether the given value is a constant int or similar node.
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS, const SDNodeFlags Flags=SDNodeFlags())
Return an AddrSpaceCastSDNode.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
SDValue getGLOBAL_OFFSET_TABLE(EVT VT)
Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
LLVM_ABI SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
LLVM_ABI SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
const TargetLowering & getTargetLoweringInfo() const
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
const DataLayout & getDataLayout() const
SDValue getTargetFrameIndex(int FI, EVT VT)
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getSignedTargetConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
LLVM_ABI bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
LLVM_ABI SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
LLVM_ABI std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getRegisterMask(const uint32_t *RegMask)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
LLVM_ABI SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
ArrayRef< int > getMask() const
const_iterator begin() const
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
Represent a constant reference to a string, i.e.
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
Check if the string is empty.
StringRef slice(size_t Start, size_t End) const
Return a reference to the substring from [Start, End).
constexpr size_t size() const
Get the string size.
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
A SystemZ-specific class detailing special use registers particular for calling conventions.
virtual int getStackPointerBias()=0
virtual int getReturnFunctionAddressRegister()=0
virtual int getCallFrameSize()=0
virtual int getStackPointerRegister()=0
static SystemZConstantPoolValue * Create(const GlobalValue *GV, SystemZCP::SystemZCPModifier Modifier)
unsigned getVarArgsFrameIndex() const
void setVarArgsFrameIndex(unsigned FI)
void setRegSaveFrameIndex(unsigned FI)
void incNumLocalDynamicTLSAccesses()
Register getVarArgsFirstGPR() const
void setADAVirtualRegister(Register Reg)
void setVarArgsFirstGPR(Register GPR)
Register getADAVirtualRegister() const
void setSizeOfFnParams(unsigned Size)
void setVarArgsFirstFPR(Register FPR)
unsigned getRegSaveFrameIndex() const
Register getVarArgsFirstFPR() const
SystemZCallingConventionRegisters * getSpecialRegisters() const
AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool hasInlineStackProbe(const MachineFunction &MF) const override
Returns true if stack probing through inline assembly is requested.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
MachineBasicBlock * emitEHSjLjSetJmp(MachineInstr &MI, MachineBasicBlock *MBB) const
AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be cast by the IR-level AtomicExpand pass.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &, EVT) const override
Return the ValueType of the result of SETCC operations.
bool allowTruncateForTailCall(Type *, Type *) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag, const SDLoc &DL, const AsmOperandInfo &Constraint, SelectionDAG &DAG) const override
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
MachineBasicBlock * emitEHSjLjLongJmp(MachineInstr &MI, MachineBasicBlock *MBB) const
bool useSoftFloat() const override
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context, const Type *RetTy) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
std::pair< SDValue, SDValue > makeExternalCall(SDValue Chain, SelectionDAG &DAG, const char *CalleeName, EVT RetVT, ArrayRef< SDValue > Ops, CallingConv::ID CallConv, bool IsSigned, SDLoc DL, bool DoesNotReturn, bool IsReturnValueUsed) const
void insertSSPDeclarations(Module &M, const LibcallLoweringInfo &Libcalls) const override
Insert SSP declaration if global stack protector is used.
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, UndefPoisonKind Kind, unsigned Depth) const override
Return true if this function can prove that Op is never poison and, Kind can be used to track poison ...
SystemZTargetLowering(const TargetMachine &TM, const SystemZSubtarget &STI)
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
FPImmAction getFPImmAction(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns how the legalizer should materialize the specified FP immediate.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
bool isTypeDesirableForOp(SDNode *N, EVT VT) const override
Overload that takes the specific node being optimized.
TargetLowering::ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const override
Examine constraint string and operand type and determine a weight value.
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
Determine if the target supports unaligned memory accesses.
const MCPhysReg * getScratchRegisters(CallingConv::ID CC) const override
Returns a 0 terminated array of registers that can be safely used as scratch registers.
TargetLowering::ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint, return the type of constraint it is for this target.
Register getExceptionSelectorRegister(ExceptionHandling EH, const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isTruncateFree(Type *, Type *) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
SDValue useLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, MVT VT, SDValue Arg, SDLoc DL, SDValue Chain, bool IsStrict) const
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine the number of bits in the operation that are sign bits.
void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue LowerCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
CondMergingParams getJumpConditionMergingParams(Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, const Function *F) const override
TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
bool findOptimalMemOpLowering(LLVMContext &Context, std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes, EVT *LargestVT=nullptr) const override
Determines the optimal series of memory ops to replace the memset / memcpy.
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
Register getExceptionPointerRegister(ExceptionHandling EH, const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool aggressivelyPreferBuildVectorSources(EVT VecVT) const override
AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be cast by the IR-level AtomicExpand pass into.
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Return the register ID of the name passed in.
bool hasAndNot(SDValue Y) const override
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
unsigned getStackProbeSize(const MachineFunction &MF) const
XPLINK64 calling convention specific use registers Particular to z/OS when in 64 bit mode.
Information about stack frame layout on the target.
unsigned getStackAlignment() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
bool hasFP(const MachineFunction &MF) const
hasFP - Return true if the specified function should have a dedicated frame pointer register.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
virtual void insertSSPDeclarations(Module &M, const LibcallLoweringInfo &Libcalls) const
Inserts necessary declarations for SSP (stack protection) purpose.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
void setAtomicLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Let target indicate that an extending atomic load of the specified type is legal.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
FPImmAction
How the legalizer should materialize a floating-point immediate.
@ ConstantPool
Load the immediate from the constant pool.
@ Legal
The target can instruction select the immediate natively.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual const TargetRegisterClass * getRepRegClassFor(MVT VT) const
Return the 'representative' register class for the specified value type.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
virtual bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const
Returns true if arguments should be sign-extended in lib calls.
std::vector< ArgListEntry > ArgListTy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual bool findOptimalMemOpLowering(LLVMContext &Context, std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes, EVT *LargestVT=nullptr) const
Determines the optimal series of memory ops to replace the memset / memcpy.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
TargetLowering(const TargetLowering &)=delete
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
CodeModel::Model getCodeModel() const
Returns the code model.
unsigned VerifyArgABICompliance
When set to true, call/return argument extensions of narrow integers are verified in the target backe...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetInstrInfo * getInstrInfo() const
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
user_iterator user_begin()
bool hasOneUse() const
Return true if there is exactly one use of this value.
int getNumOccurrences() const
constexpr ScalarTy getFixedValue() const
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ C
The default llvm calling convention, compatible with C.
LLVM_ABI bool isConstantSplatVector(SDValue V, APInt &SplatValue)
Node predicates.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
LLVM_ABI bool isConstantSplatVectorAllZeros(SDValue V, bool BuildVectorOnly=false)
Return true if the specified value is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 ...
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ POISON
POISON - A poison node.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ PSEUDO_FMIN
PSEUDO_FMIN is strictly equivalent to op0 olt op1 ?
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
LLVM_ABI bool isBuildVectorAllZeros(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
auto m_Cmp()
Matches any compare instruction and ignore it.
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
bool match(Val *V, const Pattern &P)
auto m_Value()
Match an arbitrary value and ignore it.
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ System
Synchronized with respect to all concurrently executing threads.
@ MO_ADA_DATA_SYMBOL_ADDR
@ MO_ADA_DIRECT_FUNC_DESC
@ MO_ADA_INDIRECT_FUNC_DESC
const unsigned GR64Regs[16]
const unsigned VR128Regs[32]
const unsigned VR16Regs[32]
const unsigned GR128Regs[16]
const unsigned FP32Regs[16]
const unsigned FP16Regs[16]
const unsigned GR32Regs[16]
const unsigned FP64Regs[16]
const int64_t ELFCallFrameSize
const unsigned VR64Regs[32]
const unsigned FP128Regs[16]
const unsigned VR32Regs[32]
unsigned odd128(bool Is32bit)
const unsigned CCMASK_CMP_GE
static bool isImmHH(uint64_t Val)
const unsigned CCMASK_TEND
const unsigned CCMASK_CS_EQ
const unsigned CCMASK_TBEGIN
const MCPhysReg ELFArgFPRs[ELFNumArgFPRs]
MachineBasicBlock * splitBlockBefore(MachineBasicBlock::iterator MI, MachineBasicBlock *MBB)
const unsigned CCMASK_TM_SOME_1
const unsigned CCMASK_LOGICAL_CARRY
const unsigned TDCMASK_NORMAL_MINUS
const unsigned CCMASK_TDC
const unsigned CCMASK_FCMP
const unsigned CCMASK_TM_SOME_0
static bool isImmHL(uint64_t Val)
const unsigned TDCMASK_SUBNORMAL_MINUS
const unsigned TDCMASK_NORMAL_PLUS
const unsigned CCMASK_CMP_GT
const unsigned TDCMASK_QNAN_MINUS
const unsigned CCMASK_ANY
const unsigned CCMASK_ARITH
const unsigned CCMASK_TM_MIXED_MSB_0
const unsigned TDCMASK_SUBNORMAL_PLUS
static bool isImmLL(uint64_t Val)
const unsigned VectorBits
static bool isImmLH(uint64_t Val)
MachineBasicBlock * emitBlockAfter(MachineBasicBlock *MBB)
const unsigned TDCMASK_INFINITY_PLUS
unsigned reverseCCMask(unsigned CCMask)
const unsigned CCMASK_TM_ALL_0
const unsigned CCMASK_CMP_LE
const unsigned CCMASK_CMP_O
const unsigned CCMASK_CMP_EQ
const unsigned VectorBytes
const unsigned TDCMASK_INFINITY_MINUS
const unsigned CCMASK_ICMP
const unsigned CCMASK_VCMP_ALL
const unsigned CCMASK_VCMP_NONE
MachineBasicBlock * splitBlockAfter(MachineBasicBlock::iterator MI, MachineBasicBlock *MBB)
const unsigned CCMASK_VCMP
const unsigned CCMASK_TM_MIXED_MSB_1
const unsigned CCMASK_TM_MSB_0
const unsigned CCMASK_ARITH_OVERFLOW
const unsigned CCMASK_CS_NE
const unsigned TDCMASK_SNAN_PLUS
const unsigned CCMASK_NONE
const unsigned CCMASK_CMP_LT
const unsigned CCMASK_CMP_NE
const unsigned TDCMASK_ZERO_PLUS
const unsigned TDCMASK_QNAN_PLUS
const unsigned TDCMASK_ZERO_MINUS
unsigned even128(bool Is32bit)
const unsigned CCMASK_TM_ALL_1
const unsigned CCMASK_LOGICAL_BORROW
const unsigned ELFNumArgFPRs
const unsigned CCMASK_CMP_UO
const unsigned CCMASK_LOGICAL
const unsigned CCMASK_TM_MSB_1
const unsigned TDCMASK_SNAN_MINUS
initializer< Ty > init(const Ty &Val)
support::ulittle32_t Word
@ User
could "use" a pointer
NodeAddr< UseNode * > Use
NodeAddr< NodeBase * > Node
NodeAddr< CodeNode * > Code
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
@ Known
Known to have no common set bits.
@ Define
Register definition.
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
@ Load
The value being inserted comes from a load (InsertElement only).
testing::Matcher< const detail::ErrorHolder & > Failed()
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
constexpr T maskLeadingOnes(unsigned N)
Create a bitmask with the N left-most bits set to 1, and all other bits set to 0.
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
LLVM_ABI void dumpBytes(ArrayRef< uint8_t > Bytes, raw_ostream &OS)
Convert ‘Bytes’ to a hex string and output to ‘OS’.
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
LLVM_ABI bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
@ Success
The lock was released successfully.
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
@ Fast
Assign the register banks as fast as possible (default).
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
UndefPoisonKind
Enumeration to track whether we are interested in Undef, Poison, or both.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
@ End
Run the machine verifier once, at the end of the pipeline.
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
MCRegisterClass TargetRegisterClass
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
AddressingMode(bool LongDispl, bool IdxReg)
This struct is a compact representation of a valid (non-zero power of two) alignment.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isRound() const
Return true if the size is a power-of-two number of bytes.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isVectorOf(EVT EltVT) const
Return true if this is a vector with matching element type.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
SystemZVectorConstantInfo(APInt IntImm)
SmallVector< unsigned, 2 > OpVals
bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This contains information for each constraint that we are lowering.
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
CallLoweringInfo & setNoReturn(bool Value=true)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.