24#include "llvm/IR/IntrinsicsS390.h"
32#define DEBUG_TYPE "systemz-lower"
38 : Op0(Op0In), Op1(Op1In), Chain(ChainIn),
39 Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
89 if (Subtarget.hasHighWord())
95 if (Subtarget.hasVector()) {
102 if (Subtarget.hasVectorEnhancements1())
107 if (Subtarget.hasVector()) {
202 if (Subtarget.hasPopulationCount())
221 if (!Subtarget.hasFPExtension())
227 if (Subtarget.hasFPExtension())
232 if (Subtarget.hasFPExtension())
270 if (!Subtarget.hasFPExtension()) {
283 if (Subtarget.hasMiscellaneousExtensions3()) {
374 if (Subtarget.hasVectorEnhancements1())
403 if (Subtarget.hasVectorEnhancements1())
408 if (Subtarget.hasVector()) {
430 if (Subtarget.hasVectorEnhancements2()) {
460 if (Subtarget.hasFPExtension()) {
488 if (Subtarget.hasFPExtension()) {
499 if (Subtarget.hasVector()) {
543 if (Subtarget.hasVectorEnhancements1()) {
607 if (!Subtarget.hasVectorEnhancements1()) {
613 if (Subtarget.hasVectorEnhancements1())
623 if (Subtarget.hasVectorEnhancements1()) {
635 if (!Subtarget.hasVector()) {
691 return Subtarget.hasSoftFloat();
713 return Subtarget.hasVectorEnhancements1();
726 if (!Subtarget.hasVector() ||
727 (isFP128 && !Subtarget.hasVectorEnhancements1()))
749 if (SplatBitSize > 64)
755 if (isInt<16>(SignedValue)) {
764 if (
TII->isRxSBGMask(
Value, SplatBitSize, Start, End)) {
786 uint64_t Lower = SplatUndefZ & maskTrailingOnes<uint64_t>(LowerBits);
787 uint64_t Upper = SplatUndefZ & maskLeadingOnes<uint64_t>(UpperBits);
794 uint64_t Middle = SplatUndefZ & ~Upper & ~Lower;
795 return tryValue(SplatBitsZ | Middle);
810 unsigned HalfSize = Width / 2;
815 if (HighValue != LowValue || 8 > HalfSize)
818 SplatBits = HighValue;
822 SplatBitSize = Width;
830 BVN->
isConstantSplat(IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, 128,
834 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, 8,
839 bool ForCodeSize)
const {
841 if (Imm.isZero() || Imm.isNegZero())
858 return isInt<32>(Imm) || isUInt<32>(Imm);
863 return isUInt<32>(Imm) || isUInt<32>(-Imm);
885 LongDisplacement(LongDispl), IndexReg(IdxReg) {}
908 switch (II->getIntrinsicID()) {
910 case Intrinsic::memset:
911 case Intrinsic::memmove:
912 case Intrinsic::memcpy:
917 if (isa<LoadInst>(
I) &&
I->hasOneUse()) {
918 auto *SingleUser = cast<Instruction>(*
I->user_begin());
919 if (SingleUser->getParent() ==
I->getParent()) {
920 if (isa<ICmpInst>(SingleUser)) {
921 if (
auto *
C = dyn_cast<ConstantInt>(SingleUser->getOperand(1)))
922 if (
C->getBitWidth() <= 64 &&
923 (isInt<16>(
C->getSExtValue()) || isUInt<16>(
C->getZExtValue())))
926 }
else if (isa<StoreInst>(SingleUser))
930 }
else if (
auto *StoreI = dyn_cast<StoreInst>(
I)) {
931 if (
auto *LoadI = dyn_cast<LoadInst>(StoreI->getValueOperand()))
932 if (LoadI->hasOneUse() && LoadI->getParent() ==
I->getParent())
937 if (HasVector && (isa<LoadInst>(
I) || isa<StoreInst>(
I))) {
945 Type *MemAccessTy = (isa<LoadInst>(
I) ?
I->getType() :
946 I->getOperand(0)->getType());
948 bool IsVectorAccess = MemAccessTy->
isVectorTy();
952 if (!IsVectorAccess && isa<StoreInst>(
I)) {
953 Value *DataOp =
I->getOperand(0);
954 if (isa<ExtractElementInst>(DataOp))
955 IsVectorAccess =
true;
960 if (!IsVectorAccess && isa<LoadInst>(
I) &&
I->hasOneUse()) {
961 User *LoadUser = *
I->user_begin();
962 if (isa<InsertElementInst>(LoadUser))
963 IsVectorAccess =
true;
966 if (IsFPAccess || IsVectorAccess)
984 bool RequireD12 = Subtarget.hasVector() && Ty->
isVectorTy();
994 return AM.
Scale == 0;
1001 std::vector<EVT> &MemOps,
unsigned Limit,
const MemOp &Op,
unsigned DstAS,
1002 unsigned SrcAS,
const AttributeList &FuncAttributes)
const {
1003 const int MVCFastLen = 16;
1005 if (Limit != ~
unsigned(0)) {
1007 if (Op.isMemcpy() && Op.allowOverlap() && Op.size() <= MVCFastLen)
1009 if (Op.isMemset() && Op.size() - 1 <= MVCFastLen)
1011 if (Op.isZeroMemset())
1016 SrcAS, FuncAttributes);
1025 if (!FromType->isIntegerTy() || !ToType->
isIntegerTy())
1027 unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedValue();
1029 return FromBits > ToBits;
1037 return FromBits > ToBits;
1046 if (Constraint.
size() == 1) {
1047 switch (Constraint[0]) {
1073 }
else if (Constraint.
size() == 2 && Constraint[0] ==
'Z') {
1074 switch (Constraint[1]) {
1090 const char *constraint)
const {
1092 Value *CallOperandVal =
info.CallOperandVal;
1095 if (!CallOperandVal)
1099 switch (*constraint) {
1119 Subtarget.hasVector())
1124 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1125 if (isUInt<8>(
C->getZExtValue()))
1130 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1131 if (isUInt<12>(
C->getZExtValue()))
1136 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1137 if (isInt<16>(
C->getSExtValue()))
1142 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1143 if (isInt<20>(
C->getSExtValue()))
1148 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1149 if (
C->getZExtValue() == 0x7fffffff)
1159static std::pair<unsigned, const TargetRegisterClass *>
1161 const unsigned *Map,
unsigned Size) {
1162 assert(*(Constraint.
end()-1) ==
'}' &&
"Missing '}'");
1163 if (isdigit(Constraint[2])) {
1168 return std::make_pair(Map[
Index], RC);
1170 return std::make_pair(0U,
nullptr);
1173std::pair<unsigned, const TargetRegisterClass *>
1176 if (Constraint.
size() == 1) {
1178 switch (Constraint[0]) {
1183 return std::make_pair(0U, &SystemZ::GR64BitRegClass);
1185 return std::make_pair(0U, &SystemZ::GR128BitRegClass);
1186 return std::make_pair(0U, &SystemZ::GR32BitRegClass);
1190 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass);
1192 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass);
1193 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass);
1196 return std::make_pair(0U, &SystemZ::GRH32BitRegClass);
1201 return std::make_pair(0U, &SystemZ::FP64BitRegClass);
1203 return std::make_pair(0U, &SystemZ::FP128BitRegClass);
1204 return std::make_pair(0U, &SystemZ::FP32BitRegClass);
1208 if (Subtarget.hasVector()) {
1210 return std::make_pair(0U, &SystemZ::VR32BitRegClass);
1212 return std::make_pair(0U, &SystemZ::VR64BitRegClass);
1213 return std::make_pair(0U, &SystemZ::VR128BitRegClass);
1218 if (Constraint.
size() > 0 && Constraint[0] ==
'{') {
1223 if (Constraint[1] ==
'r') {
1233 if (Constraint[1] ==
'f') {
1235 return std::make_pair(
1246 if (Constraint[1] ==
'v') {
1247 if (!Subtarget.hasVector())
1248 return std::make_pair(
1281 std::vector<SDValue> &Ops,
1284 if (Constraint.length() == 1) {
1285 switch (Constraint[0]) {
1287 if (
auto *
C = dyn_cast<ConstantSDNode>(Op))
1288 if (isUInt<8>(
C->getZExtValue()))
1290 Op.getValueType()));
1294 if (
auto *
C = dyn_cast<ConstantSDNode>(Op))
1295 if (isUInt<12>(
C->getZExtValue()))
1297 Op.getValueType()));
1301 if (
auto *
C = dyn_cast<ConstantSDNode>(Op))
1302 if (isInt<16>(
C->getSExtValue()))
1304 Op.getValueType()));
1308 if (
auto *
C = dyn_cast<ConstantSDNode>(Op))
1309 if (isInt<20>(
C->getSExtValue()))
1311 Op.getValueType()));
1315 if (
auto *
C = dyn_cast<ConstantSDNode>(Op))
1316 if (
C->getZExtValue() == 0x7fffffff)
1318 Op.getValueType()));
1329#include "SystemZGenCallingConv.inc"
1333 static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D,
1339 Type *ToType)
const {
1356 for (
unsigned i = 0; i < Ins.size(); ++i)
1361 for (
unsigned i = 0; i < Outs.
size(); ++i)
1454 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
1458 (NumParts == 2 && PartVT ==
MVT::i64))) &&
1459 "Unknown handling of i128 value.");
1460 if (ValueVT ==
MVT::i128 && NumParts == 1) {
1470 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
1473 (NumParts == 2 && PartVT ==
MVT::i64))) &&
1474 "Unknown handling of i128 value.");
1475 if (ValueVT ==
MVT::i128 && NumParts == 1)
1494 if (Subtarget.hasVector())
1502 unsigned NumFixedGPRs = 0;
1503 unsigned NumFixedFPRs = 0;
1504 for (
unsigned I = 0,
E = ArgLocs.
size();
I !=
E; ++
I) {
1517 RC = &SystemZ::GR32BitRegClass;
1521 RC = &SystemZ::GR64BitRegClass;
1525 RC = &SystemZ::FP32BitRegClass;
1529 RC = &SystemZ::FP64BitRegClass;
1533 RC = &SystemZ::FP128BitRegClass;
1541 RC = &SystemZ::VR128BitRegClass;
1570 ArgValue = DAG.
getLoad(LocVT,
DL, Chain, FIN,
1581 unsigned ArgIndex = Ins[
I].OrigArgIndex;
1582 assert (Ins[
I].PartOffset == 0);
1583 while (
I + 1 !=
E && Ins[
I + 1].OrigArgIndex == ArgIndex) {
1585 unsigned PartOffset = Ins[
I + 1].PartOffset;
1609 int64_t RegSaveOffset =
1624 &SystemZ::FP64BitRegClass);
1647 for (
unsigned I = 0,
E = ArgLocs.
size();
I !=
E; ++
I) {
1654 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D)
1656 if (Outs[
I].
Flags.isSwiftSelf() || Outs[
I].Flags.isSwiftError())
1685 if (Subtarget.hasVector()) {
1708 NumBytes = std::max(64U, (
unsigned)
alignTo(NumBytes, 64));
1718 for (
unsigned I = 0,
E = ArgLocs.
size();
I !=
E; ++
I) {
1724 unsigned ArgIndex = Outs[
I].OrigArgIndex;
1726 if (
I + 1 !=
E && Outs[
I + 1].OrigArgIndex == ArgIndex) {
1728 Type *OrigArgType = CLI.
Args[Outs[
I].OrigArgIndex].Ty;
1734 SlotVT = Outs[
I].ArgVT;
1737 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
1743 assert (Outs[
I].PartOffset == 0);
1744 while (
I + 1 !=
E && Outs[
I + 1].OrigArgIndex == ArgIndex) {
1745 SDValue PartValue = OutVals[
I + 1];
1746 unsigned PartOffset = Outs[
I + 1].PartOffset;
1753 SlotVT.
getStoreSize()) &&
"Not enough space for argument part!");
1756 ArgValue = SpillSlot;
1773 if (!StackPtr.getNode())
1795 RegsToPass.
push_back(std::make_pair(SystemZ::R3D, ShadowArgValue));
1801 if (!MemOpChains.
empty())
1809 if (
auto *
G = dyn_cast<GlobalAddressSDNode>(
Callee)) {
1812 }
else if (
auto *
E = dyn_cast<ExternalSymbolSDNode>(
Callee)) {
1815 }
else if (IsTailCall) {
1822 for (
unsigned I = 0,
E = RegsToPass.
size();
I !=
E; ++
I) {
1824 RegsToPass[
I].second, Glue);
1835 for (
unsigned I = 0,
E = RegsToPass.
size();
I !=
E; ++
I)
1837 RegsToPass[
I].second.getValueType()));
1841 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
1842 assert(Mask &&
"Missing call preserved mask for calling convention");
1863 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx);
1867 for (
unsigned I = 0,
E = RetLocs.
size();
I !=
E; ++
I) {
1889 bool DoesNotReturn,
bool IsReturnValueUsed)
const {
1891 Args.reserve(Ops.
size());
1896 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.
getContext());
1899 Args.push_back(Entry);
1924 if (Subtarget.hasVector())
1929 for (
auto &Out : Outs)
1935 return RetCCInfo.
CheckReturn(Outs, RetCC_SystemZ);
1947 if (Subtarget.hasVector())
1956 if (RetLocs.
empty())
1966 for (
unsigned I = 0,
E = RetLocs.
size();
I !=
E; ++
I) {
1995 unsigned &CCValid) {
1996 unsigned Id = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
1998 case Intrinsic::s390_tbegin:
2003 case Intrinsic::s390_tbegin_nofloat:
2008 case Intrinsic::s390_tend:
2022 unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
2024 case Intrinsic::s390_vpkshs:
2025 case Intrinsic::s390_vpksfs:
2026 case Intrinsic::s390_vpksgs:
2031 case Intrinsic::s390_vpklshs:
2032 case Intrinsic::s390_vpklsfs:
2033 case Intrinsic::s390_vpklsgs:
2038 case Intrinsic::s390_vceqbs:
2039 case Intrinsic::s390_vceqhs:
2040 case Intrinsic::s390_vceqfs:
2041 case Intrinsic::s390_vceqgs:
2046 case Intrinsic::s390_vchbs:
2047 case Intrinsic::s390_vchhs:
2048 case Intrinsic::s390_vchfs:
2049 case Intrinsic::s390_vchgs:
2054 case Intrinsic::s390_vchlbs:
2055 case Intrinsic::s390_vchlhs:
2056 case Intrinsic::s390_vchlfs:
2057 case Intrinsic::s390_vchlgs:
2062 case Intrinsic::s390_vtm:
2067 case Intrinsic::s390_vfaebs:
2068 case Intrinsic::s390_vfaehs:
2069 case Intrinsic::s390_vfaefs:
2074 case Intrinsic::s390_vfaezbs:
2075 case Intrinsic::s390_vfaezhs:
2076 case Intrinsic::s390_vfaezfs:
2081 case Intrinsic::s390_vfeebs:
2082 case Intrinsic::s390_vfeehs:
2083 case Intrinsic::s390_vfeefs:
2088 case Intrinsic::s390_vfeezbs:
2089 case Intrinsic::s390_vfeezhs:
2090 case Intrinsic::s390_vfeezfs:
2095 case Intrinsic::s390_vfenebs:
2096 case Intrinsic::s390_vfenehs:
2097 case Intrinsic::s390_vfenefs:
2102 case Intrinsic::s390_vfenezbs:
2103 case Intrinsic::s390_vfenezhs:
2104 case Intrinsic::s390_vfenezfs:
2109 case Intrinsic::s390_vistrbs:
2110 case Intrinsic::s390_vistrhs:
2111 case Intrinsic::s390_vistrfs:
2116 case Intrinsic::s390_vstrcbs:
2117 case Intrinsic::s390_vstrchs:
2118 case Intrinsic::s390_vstrcfs:
2123 case Intrinsic::s390_vstrczbs:
2124 case Intrinsic::s390_vstrczhs:
2125 case Intrinsic::s390_vstrczfs:
2130 case Intrinsic::s390_vstrsb:
2131 case Intrinsic::s390_vstrsh:
2132 case Intrinsic::s390_vstrsf:
2137 case Intrinsic::s390_vstrszb:
2138 case Intrinsic::s390_vstrszh:
2139 case Intrinsic::s390_vstrszf:
2144 case Intrinsic::s390_vfcedbs:
2145 case Intrinsic::s390_vfcesbs:
2150 case Intrinsic::s390_vfchdbs:
2151 case Intrinsic::s390_vfchsbs:
2156 case Intrinsic::s390_vfchedbs:
2157 case Intrinsic::s390_vfchesbs:
2162 case Intrinsic::s390_vftcidb:
2163 case Intrinsic::s390_vftcisb:
2168 case Intrinsic::s390_tdc:
2182 unsigned NumOps = Op.getNumOperands();
2186 for (
unsigned I = 2;
I < NumOps; ++
I)
2189 assert(Op->getNumValues() == 2 &&
"Expected only CC result and chain");
2195 return Intr.getNode();
2202 unsigned NumOps = Op.getNumOperands();
2205 for (
unsigned I = 1;
I < NumOps; ++
I)
2209 return Intr.getNode();
2219 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
2220 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
2221 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
2246 auto *ConstOp1 = dyn_cast<ConstantSDNode>(
C.Op1.getNode());
2250 int64_t
Value = ConstOp1->getSExtValue();
2266 if (!
C.Op0.hasOneUse() ||
2272 auto *Load = cast<LoadSDNode>(
C.Op0);
2273 unsigned NumBits = Load->getMemoryVT().getSizeInBits();
2274 if ((NumBits != 8 && NumBits != 16) ||
2275 NumBits != Load->getMemoryVT().getStoreSizeInBits())
2280 auto *ConstOp1 = cast<ConstantSDNode>(
C.Op1);
2282 uint64_t Mask = (1 << NumBits) - 1;
2285 int64_t SignedValue = ConstOp1->getSExtValue();
2292 }
else if (NumBits == 8) {
2319 Load->getExtensionType() != ExtType) {
2321 Load->getBasePtr(), Load->getPointerInfo(),
2322 Load->getMemoryVT(), Load->getAlign(),
2323 Load->getMemOperand()->getFlags());
2330 Value != ConstOp1->getZExtValue())
2337 auto *Load = dyn_cast<LoadSDNode>(Op.getNode());
2340 if (Load->getMemoryVT() ==
MVT::i8)
2343 switch (Load->getExtensionType()) {
2366 if (isa<ConstantFPSDNode>(
C.Op1))
2371 auto *ConstOp1 = dyn_cast<ConstantSDNode>(
C.Op1);
2372 if (ConstOp1 && ConstOp1->getZExtValue() == 0)
2390 isUInt<16>(ConstOp1->getZExtValue()))
2395 isInt<16>(ConstOp1->getSExtValue()))
2401 unsigned Opcode0 =
C.Op0.getOpcode();
2409 cast<ConstantSDNode>(
C.Op0.getOperand(1))->getZExtValue() == 0xffffffff)
2424 ((
N->getOperand(0) ==
C.Op0 &&
N->getOperand(1) ==
C.Op1) ||
2425 (
N->getOperand(0) ==
C.Op1 &&
N->getOperand(1) ==
C.Op0))) {
2429 Flags.setNoSignedWrap(
false);
2430 Flags.setNoUnsignedWrap(
false);
2449 auto *C1 = dyn_cast<ConstantFPSDNode>(
C.Op1);
2450 if (C1 && C1->isZero()) {
2472 cast<ConstantSDNode>(
C.Op1)->getZExtValue() == 0) {
2473 auto *C1 = dyn_cast<ConstantSDNode>(
C.Op0.getOperand(1));
2474 if (C1 && C1->getZExtValue() == 32) {
2475 SDValue ShlOp0 =
C.Op0.getOperand(0);
2479 cast<VTSDNode>(
N->getOperand(1))->getVT() ==
MVT::i32) {
2494 C.Op0.getOperand(0).getOpcode() ==
ISD::LOAD &&
2496 cast<ConstantSDNode>(
C.Op1)->getZExtValue() == 0) {
2497 auto *L = cast<LoadSDNode>(
C.Op0.getOperand(0));
2498 if (L->getMemoryVT().getStoreSizeInBits().getFixedValue() <=
2499 C.Op0.getValueSizeInBits().getFixedValue()) {
2500 unsigned Type = L->getExtensionType();
2503 C.Op0 =
C.Op0.getOperand(0);
2513 auto *Shift = dyn_cast<ConstantSDNode>(
N.getOperand(1));
2517 uint64_t Amount = Shift->getZExtValue();
2518 if (Amount >=
N.getValueSizeInBits())
2533 unsigned ICmpType) {
2534 assert(Mask != 0 &&
"ANDs with zero should have been removed by now");
2556 if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <=
Low) {
2562 if (EffectivelyUnsigned && CmpVal <
Low) {
2570 if (CmpVal == Mask) {
2576 if (EffectivelyUnsigned && CmpVal >= Mask -
Low && CmpVal < Mask) {
2582 if (EffectivelyUnsigned && CmpVal > Mask -
Low && CmpVal <= Mask) {
2590 if (EffectivelyUnsigned && CmpVal >= Mask -
High && CmpVal <
High) {
2596 if (EffectivelyUnsigned && CmpVal > Mask -
High && CmpVal <=
High) {
2625 auto *ConstOp1 = dyn_cast<ConstantSDNode>(
C.Op1);
2628 uint64_t CmpVal = ConstOp1->getZExtValue();
2635 NewC.Op0 =
C.Op0.getOperand(0);
2636 NewC.Op1 =
C.Op0.getOperand(1);
2637 Mask = dyn_cast<ConstantSDNode>(NewC.Op1);
2640 MaskVal = Mask->getZExtValue();
2645 if (NewC.Op0.getValueType() !=
MVT::i64 ||
2660 MaskVal = -(CmpVal & -CmpVal);
2668 unsigned BitSize = NewC.Op0.getValueSizeInBits();
2669 unsigned NewCCMask, ShiftVal;
2671 NewC.Op0.getOpcode() ==
ISD::SHL &&
2673 (MaskVal >> ShiftVal != 0) &&
2674 ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
2676 MaskVal >> ShiftVal,
2679 NewC.Op0 = NewC.Op0.getOperand(0);
2680 MaskVal >>= ShiftVal;
2682 NewC.Op0.getOpcode() ==
ISD::SRL &&
2684 (MaskVal << ShiftVal != 0) &&
2685 ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
2687 MaskVal << ShiftVal,
2690 NewC.Op0 = NewC.Op0.getOperand(0);
2691 MaskVal <<= ShiftVal;
2702 if (Mask && Mask->getZExtValue() == MaskVal)
2707 C.CCMask = NewCCMask;
2717 auto *Mask = dyn_cast<ConstantSDNode>(
C.Op0.getOperand(1));
2721 if ((~Known.
Zero).getZExtValue() & ~Mask->getZExtValue())
2724 C.Op0 =
C.Op0.getOperand(0);
2736 C.CCValid = CCValid;
2739 C.CCMask =
CC < 4 ? 1 << (3 -
CC) : 0;
2742 C.CCMask =
CC < 4 ? ~(1 << (3 -
CC)) : -1;
2746 C.CCMask =
CC < 4 ? ~0U << (4 -
CC) : -1;
2749 C.CCMask =
CC < 4 ? ~(~0U << (4 -
CC)) : 0;
2753 C.CCMask =
CC < 4 ? ~0U << (3 -
CC) : -1;
2756 C.CCMask =
CC < 4 ? ~(~0U << (3 -
CC)) : 0;
2759 C.CCMask &= CCValid;
2767 bool IsSignaling =
false) {
2771 unsigned Opcode, CCValid;
2781 Comparison
C(CmpOp0, CmpOp1, Chain);
2783 if (
C.Op0.getValueType().isFloatingPoint()) {
2787 else if (!IsSignaling)
2809 C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
2829 if (!
C.Op1.getNode()) {
2831 switch (
C.Op0.getOpcode()) {
2853 return DAG.
getNode(
C.Opcode,
DL, VTs,
C.Chain,
C.Op0,
C.Op1);
2890 unsigned CCValid,
unsigned CCMask) {
2919 case CmpMode::Int:
return 0;
2939 case CmpMode::FP:
return 0;
2940 case CmpMode::StrictFP:
return 0;
2941 case CmpMode::SignalingFP:
return 0;
2973 int Mask[] = { Start, -1, Start + 1, -1 };
2993 !Subtarget.hasVectorEnhancements1()) {
3007 SDValue Ops[2] = { Res, NewChain };
3016 return DAG.
getNode(Opcode,
DL, VTs, Chain, CmpOp0, CmpOp1);
3018 return DAG.
getNode(Opcode,
DL, VT, CmpOp0, CmpOp1);
3031 bool IsSignaling)
const {
3034 assert (!IsSignaling || Chain);
3035 CmpMode Mode = IsSignaling ? CmpMode::SignalingFP :
3036 Chain ? CmpMode::StrictFP : IsFP ? CmpMode::FP : CmpMode::Int;
3037 bool Invert =
false;
3045 assert(IsFP &&
"Unexpected integer comparison");
3047 DL, VT, CmpOp1, CmpOp0, Chain);
3049 DL, VT, CmpOp0, CmpOp1, Chain);
3053 LT.getValue(1),
GE.getValue(1));
3062 assert(IsFP &&
"Unexpected integer comparison");
3064 DL, VT, CmpOp1, CmpOp0, Chain);
3066 DL, VT, CmpOp0, CmpOp1, Chain);
3070 LT.getValue(1),
GT.getValue(1));
3079 Cmp = getVectorCmp(DAG, Opcode,
DL, VT, CmpOp0, CmpOp1, Chain);
3083 Cmp = getVectorCmp(DAG, Opcode,
DL, VT, CmpOp1, CmpOp0, Chain);
3088 Chain =
Cmp.getValue(1);
3096 if (Chain && Chain.
getNode() !=
Cmp.getNode()) {
3109 EVT VT =
Op.getValueType();
3111 return lowerVectorSETCC(DAG,
DL, VT,
CC, CmpOp0, CmpOp1);
3113 Comparison
C(
getCmp(DAG, CmpOp0, CmpOp1,
CC,
DL));
3120 bool IsSignaling)
const {
3126 EVT VT =
Op.getNode()->getValueType(0);
3128 SDValue Res = lowerVectorSETCC(DAG,
DL, VT,
CC, CmpOp0, CmpOp1,
3129 Chain, IsSignaling);
3133 Comparison
C(
getCmp(DAG, CmpOp0, CmpOp1,
CC,
DL, Chain, IsSignaling));
3148 Comparison
C(
getCmp(DAG, CmpOp0, CmpOp1,
CC,
DL));
3161 cast<ConstantSDNode>(Neg.
getOperand(0))->getZExtValue() == 0 &&
3187 Comparison
C(
getCmp(DAG, CmpOp0, CmpOp1,
CC,
DL));
3196 cast<ConstantSDNode>(
C.Op1)->getZExtValue() == 0) {
3204 SDValue Ops[] = {TrueOp, FalseOp,
3275 Chain = DAG.
getCopyToReg(Chain,
DL, SystemZ::R2D, GOTOffset, Glue);
3282 Node->getValueType(0),
3294 assert(Mask &&
"Missing call preserved mask for calling convention");
3302 Chain = DAG.
getNode(Opcode,
DL, NodeTys, Ops);
3309SDValue SystemZTargetLowering::lowerThreadPointer(
const SDLoc &
DL,
3341 SDValue TP = lowerThreadPointer(
DL, DAG);
3449 if (
CP->isMachineConstantPoolEntry())
3468 unsigned Depth = cast<ConstantSDNode>(
Op.getOperand(0))->getZExtValue();
3475 int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
3496 unsigned Depth = cast<ConstantSDNode>(
Op.getOperand(0))->getZExtValue();
3513 EVT InVT =
In.getValueType();
3514 EVT ResVT =
Op.getValueType();
3519 if (
auto *LoadN = dyn_cast<LoadSDNode>(In))
3522 LoadN->getBasePtr(), LoadN->getMemOperand());
3530 if (Subtarget.hasHighWord()) {
3549 if (Subtarget.hasHighWord())
3563 return lowerVASTART_XPLINK(Op, DAG);
3565 return lowerVASTART_ELF(Op, DAG);
3580 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
3594 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
3598 const unsigned NumFields = 4;
3609 for (
unsigned I = 0;
I < NumFields; ++
I) {
3614 MemOps[
I] = DAG.
getStore(Chain,
DL, Fields[
I], FieldAddr,
3626 const Value *DstSV = cast<SrcValueSDNode>(
Op.getOperand(3))->getValue();
3627 const Value *SrcSV = cast<SrcValueSDNode>(
Op.getOperand(4))->getValue();
3633 Align(8),
false,
false,
3639SystemZTargetLowering::lowerDYNAMIC_STACKALLOC(
SDValue Op,
3642 return lowerDYNAMIC_STACKALLOC_XPLINK(Op, DAG);
3644 return lowerDYNAMIC_STACKALLOC_ELF(Op, DAG);
3648SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_XPLINK(
SDValue Op,
3661 (RealignOpt ? cast<ConstantSDNode>(
Align)->getZExtValue() : 0);
3664 uint64_t RequiredAlign = std::max(AlignVal, StackAlign);
3671 if (ExtraAlignSpace)
3675 bool IsSigned =
false;
3676 bool DoesNotReturn =
false;
3677 bool IsReturnValueUsed =
false;
3678 EVT VT =
Op.getValueType();
3689 Register SPReg = Regs.getStackPointerRegister();
3700 if (ExtraAlignSpace) {
3712SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_ELF(
SDValue Op,
3727 (RealignOpt ? cast<ConstantSDNode>(
Align)->getZExtValue() : 0);
3730 uint64_t RequiredAlign = std::max(AlignVal, StackAlign);
3746 if (ExtraAlignSpace)
3770 if (RequiredAlign > StackAlign) {
3780 Chain = DAG.
getStore(Chain,
DL, Backchain, getBackchainAddress(NewSP, DAG),
3787SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
3796 EVT VT =
Op.getValueType();
3803 Op.getOperand(1), Ops[1], Ops[0]);
3804 else if (Subtarget.hasMiscellaneousExtensions2())
3809 Op.getOperand(0),
Op.getOperand(1), Ops[1], Ops[0]);