125 using namespace llvm;
129 #define DEBUG_TYPE "isel" 137 cl::desc(
"Generate low-precision inline sequences " 138 "for some float libcalls"),
144 cl::desc(
"Set the case probability threshold for peeling the case from a " 145 "switch statement. A value greater than 100 will void this " 168 if (
auto *R = dyn_cast<ReturnInst>(V))
169 return R->getParent()->getParent()->getCallingConv();
171 if (
auto *CI = dyn_cast<CallInst>(V)) {
172 const bool IsInlineAsm = CI->isInlineAsm();
173 const bool IsIndirectFunctionCall =
174 !IsInlineAsm && !CI->getCalledFunction();
179 const bool IsInstrinsicCall =
180 !IsInlineAsm && !IsIndirectFunctionCall &&
183 if (!IsInlineAsm && !IsInstrinsicCall)
184 return CI->getCallingConv();
191 const SDValue *Parts,
unsigned NumParts,
201 const SDValue *Parts,
unsigned NumParts,
209 assert(NumParts > 0 &&
"No parts to assemble!");
220 unsigned RoundParts =
221 (NumParts & (NumParts - 1)) ? 1 <<
Log2_32(NumParts) : NumParts;
222 unsigned RoundBits = PartBits * RoundParts;
223 EVT RoundVT = RoundBits == ValueBits ?
229 if (RoundParts > 2) {
233 RoundParts / 2, PartVT, HalfVT, V);
244 if (RoundParts < NumParts) {
246 unsigned OddParts = NumParts - RoundParts;
277 !PartVT.
isVector() &&
"Unexpected split");
288 if (PartEVT == ValueVT)
292 ValueVT.
bitsLT(PartEVT)) {
305 if (ValueVT.
bitsLT(PartEVT)) {
309 if (AssertOp.hasValue())
310 Val = DAG.
getNode(*AssertOp, DL, PartEVT, Val,
330 ValueVT.
bitsLT(PartEVT)) {
339 const Twine &ErrMsg) {
340 const Instruction *
I = dyn_cast_or_null<Instruction>(V);
344 const char *AsmError =
", possible invalid constraint for vector type";
345 if (
const CallInst *CI = dyn_cast<CallInst>(I))
346 if (isa<InlineAsm>(CI->getCalledValue()))
347 return Ctx.
emitError(I, ErrMsg + AsmError);
358 const SDValue *Parts,
unsigned NumParts,
362 assert(NumParts > 0 &&
"No parts to assemble!");
363 const bool IsABIRegCopy = CallConv.
hasValue();
372 unsigned NumIntermediates;
378 NumIntermediates, RegisterVT);
382 NumIntermediates, RegisterVT);
385 assert(NumRegs == NumParts &&
"Part count doesn't match vector breakdown!");
387 assert(RegisterVT == PartVT &&
"Part type doesn't match vector breakdown!");
390 "Part type sizes don't match!");
394 if (NumIntermediates == NumParts) {
397 for (
unsigned i = 0; i != NumParts; ++i)
399 PartVT, IntermediateVT, V);
400 }
else if (NumParts > 0) {
403 assert(NumParts % NumIntermediates == 0 &&
404 "Must expand into a divisible number of parts!");
405 unsigned Factor = NumParts / NumIntermediates;
406 for (
unsigned i = 0; i != NumIntermediates; ++i)
408 PartVT, IntermediateVT, V);
417 : NumIntermediates));
420 DL, BuiltVectorTy, Ops);
426 if (PartEVT == ValueVT)
436 "Cannot narrow, it would be a lossy transformation");
447 "Cannot handle this kind of promotion");
477 *DAG.
getContext(), V,
"non-trivial scalar-to-vector conversion");
499 SDValue *Parts,
unsigned NumParts,
MVT PartVT,
511 unsigned OrigNumParts = NumParts;
513 "Copying to an illegal type!");
519 EVT PartEVT = PartVT;
520 if (PartEVT == ValueVT) {
521 assert(NumParts == 1 &&
"No-op copy with multiple parts!");
529 assert(NumParts == 1 &&
"Do not know what to promote to!");
540 "Unknown mismatch!");
542 Val = DAG.
getNode(ExtendKind, DL, ValueVT, Val);
548 assert(NumParts == 1 && PartEVT != ValueVT);
554 "Unknown mismatch!");
564 "Failed to tile the value with PartVT!");
567 if (PartEVT != ValueVT) {
569 "scalar-to-vector conversion failed");
578 if (NumParts & (NumParts - 1)) {
581 "Do not know what to expand to!");
582 unsigned RoundParts = 1 <<
Log2_32(NumParts);
583 unsigned RoundBits = RoundParts * PartBits;
584 unsigned OddParts = NumParts - RoundParts;
588 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
595 NumParts = RoundParts;
607 for (
unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
608 for (
unsigned i = 0; i < NumParts; i += StepSize) {
609 unsigned ThisBits = StepSize * PartBits / 2;
612 SDValue &Part1 = Parts[i+StepSize/2];
619 if (ThisBits == PartBits && ThisVT != PartVT) {
638 if (PartNumElts > ValueNumElts &&
646 for (
unsigned i = ValueNumElts,
e = PartNumElts; i !=
e; ++i)
665 const bool IsABIRegCopy = CallConv.
hasValue();
668 EVT PartEVT = PartVT;
669 if (PartEVT == ValueVT) {
690 "lossy conversion of vector to scalar type");
691 EVT IntermediateType =
706 unsigned NumIntermediates;
709 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
711 NumIntermediates, RegisterVT);
714 TLI.getVectorTypeBreakdown(*DAG.
getContext(), ValueVT, IntermediateVT,
715 NumIntermediates, RegisterVT);
718 assert(NumRegs == NumParts &&
"Part count doesn't match vector breakdown!");
720 assert(RegisterVT == PartVT &&
"Part type doesn't match vector breakdown!");
722 unsigned IntermediateNumElts = IntermediateVT.
isVector() ?
726 unsigned DestVectorNoElts = NumIntermediates * IntermediateNumElts;
731 if (ValueVT != BuiltVectorTy) {
740 for (
unsigned i = 0; i != NumIntermediates; ++i) {
743 DAG.
getConstant(i * IntermediateNumElts, DL, IdxVT));
752 if (NumParts == NumIntermediates) {
755 for (
unsigned i = 0; i != NumParts; ++i)
756 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
757 }
else if (NumParts > 0) {
760 assert(NumIntermediates != 0 &&
"division by zero");
761 assert(NumParts % NumIntermediates == 0 &&
762 "Must expand into a divisible number of parts!");
763 unsigned Factor = NumParts / NumIntermediates;
764 for (
unsigned i = 0; i != NumIntermediates; ++i)
765 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
772 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
773 RegCount(1, regs.
size()), CallConv(CC) {}
791 for (
unsigned i = 0; i != NumRegs; ++i)
793 RegVTs.push_back(RegisterVT);
822 for (
unsigned i = 0; i != NumRegs; ++i) {
849 if (NumZeroBits == RegSize) {
864 }
else if (NumSignBits > 1) {
908 NumParts, RegisterVT, V, CallConv, ExtendKind);
914 for (
unsigned i = 0; i != NumRegs; ++i) {
926 if (NumRegs == 1 || Flag)
937 Chain = Chains[NumRegs-1];
943 unsigned MatchingIdx,
const SDLoc &dl,
945 std::vector<SDValue> &Ops)
const {
970 "No 1:1 mapping from clobbers to regs?");
978 "If we clobbered the stack pointer, MFI should know about it.");
986 for (
unsigned i = 0; i != NumRegs; ++i) {
999 unsigned RegCount = std::get<0>(CountAndVT);
1000 MVT RegisterVT = std::get<1>(CountAndVT);
1002 for (
unsigned E = I + RegCount; I !=
E; ++
I)
1015 LPadToCallSiteMap.clear();
1021 UnusedArgNodeMap.clear();
1022 PendingLoads.clear();
1023 PendingExports.clear();
1025 HasTailCall =
false;
1026 SDNodeOrder = LowestSDNodeOrder;
1027 StatepointLowering.clear();
1031 DanglingDebugInfoMap.clear();
1035 if (PendingLoads.empty())
1038 if (PendingLoads.size() == 1) {
1039 SDValue Root = PendingLoads[0];
1041 PendingLoads.clear();
1047 PendingLoads.clear();
1055 if (PendingExports.empty())
1060 unsigned i = 0,
e = PendingExports.size();
1061 for (; i !=
e; ++i) {
1062 assert(PendingExports[i].getNode()->getNumOperands() > 1);
1063 if (PendingExports[i].getNode()->getOperand(0) == Root)
1068 PendingExports.push_back(Root);
1073 PendingExports.clear();
1081 HandlePHINodesInSuccessorBlocks(I.
getParent());
1085 if (!isa<DbgInfoIntrinsic>(I))
1092 if (
auto *FPMO = dyn_cast<FPMathOperator>(&I)) {
1098 if (
SDNode *Node = getNodeForIRValue(&I)) {
1101 if (!Node->getFlags().isDefined())
1102 Node->setFlags(IncomingFlags);
1104 Node->intersectFlagsWith(IncomingFlags);
1110 CopyToExportRegsIfNeeded(&I);
1115 void SelectionDAGBuilder::visitPHI(
const PHINode &) {
1125 #define HANDLE_INST(NUM, OPCODE, CLASS) \ 1126 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break; 1127 #include "llvm/IR/Instruction.def" 1133 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1137 if (DanglingVariable == Variable && Expr->
fragmentsOverlap(DanglingExpr)) {
1138 LLVM_DEBUG(
dbgs() <<
"Dropping dangling debug info for " << *DI <<
"\n");
1144 for (
auto &DDIMI : DanglingDebugInfoMap) {
1145 DanglingDebugInfoVector &DDIV = DDIMI.second;
1149 for (
auto &DDI : DDIV)
1150 if (isMatchingDbgValue(DDI))
1151 salvageUnresolvedDbgValue(DDI);
1153 DDIV.erase(
remove_if(DDIV, isMatchingDbgValue), DDIV.end());
1161 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1162 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1165 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1166 for (
auto &DDI : DDIV) {
1168 assert(DI &&
"Ill-formed DanglingDebugInfo");
1171 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1175 "Expected inlined-at fields to agree");
1184 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl,
false, Val)) {
1186 << DbgSDNodeOrder <<
"] for:\n " << *DI <<
"\n");
1193 <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " 1194 << ValSDNodeOrder <<
"\n");
1195 SDV = getDbgValue(Val, Variable, Expr, dl,
1196 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1200 <<
"in EmitFuncArgumentDbgValue\n");
1214 Value *V = DDI.getDI()->getValue();
1218 DebugLoc InstDL = DDI.getDI()->getDebugLoc();
1219 unsigned SDOrder = DDI.getSDNodeOrder();
1223 assert(isa<DbgValueInst>(DDI.getDI()));
1224 bool StackValue =
true;
1227 if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder))
1233 while (isa<Instruction>(V)) {
1248 if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder)) {
1250 << DDI.getDI() <<
"\nBy stripping back to:\n " << V);
1262 LLVM_DEBUG(
dbgs() <<
"Dropping debug value info for:\n " << DDI.getDI()
1264 LLVM_DEBUG(
dbgs() <<
" Last seen at:\n " << *DDI.getDI()->getOperand(0)
1273 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1274 isa<ConstantPointerNull>(V)) {
1282 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1283 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1284 if (
SI != FuncInfo.StaticAllocaMap.end()) {
1287 false, dl, SDNodeOrder);
1298 if (!N.
getNode() && isa<Argument>(V))
1299 N = UnusedArgNodeMap[V];
1301 if (EmitFuncArgumentDbgValue(V, Var, Expr, dl,
false, N))
1303 SDV = getDbgValue(N, Var, Expr, dl, SDNodeOrder);
1312 bool IsParamOfFunc = isa<Argument>(V) && Var->
isParameter() &&
1314 if (!IsParamOfFunc) {
1318 auto VMI = FuncInfo.ValueMap.find(V);
1319 if (VMI != FuncInfo.ValueMap.end()) {
1320 unsigned Reg = VMI->second;
1325 if (RFV.occupiesMultipleRegs()) {
1327 unsigned BitsToDescribe = 0;
1329 BitsToDescribe = *VarSize;
1331 BitsToDescribe = Fragment->SizeInBits;
1332 for (
auto RegAndSize : RFV.getRegsAndSizes()) {
1333 unsigned RegisterSize = RegAndSize.second;
1335 if (Offset >= BitsToDescribe)
1337 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1338 ? BitsToDescribe - Offset
1341 Expr, Offset, FragmentSize);
1345 false, dl, SDNodeOrder);
1347 Offset += RegisterSize;
1362 for (
auto &Pair : DanglingDebugInfoMap)
1363 for (
auto &DDI : Pair.second)
1364 salvageUnresolvedDbgValue(DDI);
1365 clearDanglingDebugInfo();
1374 if (It != FuncInfo.ValueMap.
end()) {
1375 unsigned InReg = It->second;
1381 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain,
nullptr,
1383 resolveDanglingDebugInfo(V, Result);
1403 SDValue Val = getValueImpl(V);
1405 resolveDanglingDebugInfo(V, Val);
1411 return (NodeMap.find(V) != NodeMap.end()) ||
1412 (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
1421 if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(
N)) {
1432 SDValue Val = getValueImpl(V);
1434 resolveDanglingDebugInfo(V, Val);
1443 if (
const Constant *
C = dyn_cast<Constant>(V)) {
1452 if (isa<ConstantPointerNull>(
C)) {
1458 if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(
C))
1465 visit(CE->getOpcode(), *CE);
1467 assert(N1.getNode() &&
"visit didn't populate the NodeMap!");
1471 if (isa<ConstantStruct>(
C) || isa<ConstantArray>(
C)) {
1488 dyn_cast<ConstantDataSequential>(
C)) {
1490 for (
unsigned i = 0,
e = CDS->getNumElements(); i !=
e; ++i) {
1491 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1498 if (isa<ArrayType>(CDS->getType()))
1504 assert((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(
C)) &&
1505 "Unknown struct or array constant!");
1509 unsigned NumElts = ValueVTs.
size();
1513 for (
unsigned i = 0; i != NumElts; ++i) {
1514 EVT EltVT = ValueVTs[i];
1515 if (isa<UndefValue>(
C))
1516 Constants[i] = DAG.
getUNDEF(EltVT);
1520 Constants[i] = DAG.
getConstant(0, getCurSDLoc(), EltVT);
1536 for (
unsigned i = 0; i != NumElements; ++i)
1537 Ops.
push_back(getValue(CV->getOperand(i)));
1539 assert(isa<ConstantAggregateZero>(
C) &&
"Unknown vector constant!");
1548 Ops.
assign(NumElements, Op);
1557 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1559 FuncInfo.StaticAllocaMap.find(AI);
1560 if (SI != FuncInfo.StaticAllocaMap.end())
1566 if (
const Instruction *Inst = dyn_cast<Instruction>(V)) {
1567 unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1572 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain,
nullptr, V);
1578 void SelectionDAGBuilder::visitCatchPad(
const CatchPadInst &
I) {
1588 if (IsMSVCCXX || IsCoreCLR)
1606 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1619 if (isa<ConstantTokenNone>(ParentPad))
1620 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1622 SuccessorColor = cast<Instruction>(ParentPad)->
getParent();
1623 assert(SuccessorColor &&
"No parent funclet for catchret!");
1625 assert(SuccessorColorMBB &&
"No MBB for SuccessorColor!");
1634 void SelectionDAGBuilder::visitCleanupPad(
const CleanupPadInst &CPI) {
1637 FuncInfo.MBB->setIsEHScopeEntry();
1640 FuncInfo.MBB->setIsEHFuncletEntry();
1641 FuncInfo.MBB->setIsCleanupFuncletEntry();
1656 if (isa<CleanupPadInst>(Pad)) {
1658 UnwindDests.emplace_back(FuncInfo.
MBBMap[EHPadBB], Prob);
1659 UnwindDests.back().first->setIsEHScopeEntry();
1661 }
else if (
auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1664 for (
const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1665 UnwindDests.emplace_back(FuncInfo.
MBBMap[CatchPadBB], Prob);
1666 UnwindDests.back().first->setIsEHScopeEntry();
1697 assert(UnwindDests.size() <= 1 &&
1698 "There should be at most one unwind destination for wasm");
1705 if (isa<LandingPadInst>(Pad)) {
1707 UnwindDests.emplace_back(FuncInfo.
MBBMap[EHPadBB], Prob);
1709 }
else if (isa<CleanupPadInst>(Pad)) {
1712 UnwindDests.emplace_back(FuncInfo.
MBBMap[EHPadBB], Prob);
1713 UnwindDests.
back().first->setIsEHScopeEntry();
1714 UnwindDests.back().first->setIsEHFuncletEntry();
1716 }
else if (
auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1718 for (
const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1719 UnwindDests.emplace_back(FuncInfo.
MBBMap[CatchPadBB], Prob);
1721 if (IsMSVCCXX || IsCoreCLR)
1722 UnwindDests.back().first->setIsEHFuncletEntry();
1724 UnwindDests.back().first->setIsEHScopeEntry();
1726 NewEHPadBB = CatchSwitch->getUnwindDest();
1732 if (BPI && NewEHPadBB)
1734 EHPadBB = NewEHPadBB;
1748 for (
auto &UnwindDest : UnwindDests) {
1749 UnwindDest.first->setIsEHPad();
1750 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1752 FuncInfo.MBB->normalizeSuccProbs();
1760 void SelectionDAGBuilder::visitCatchSwitch(
const CatchSwitchInst &CSI) {
1764 void SelectionDAGBuilder::visitRet(
const ReturnInst &I) {
1767 SDValue Chain = getControlRoot();
1779 LowerDeoptimizingReturn();
1783 if (!FuncInfo.CanLowerReturn) {
1784 unsigned DemoteReg = FuncInfo.DemoteRegister;
1797 DemoteReg, PtrValueVTs[0]);
1804 unsigned NumValues = ValueVTs.size();
1807 for (
unsigned i = 0; i != NumValues; ++i) {
1813 if (MemVTs[i] != ValueVTs[i])
1815 Chains[i] = DAG.
getStore(Chain, getCurSDLoc(), Val,
1825 unsigned NumValues = ValueVTs.
size();
1844 bool RetInReg = F->getAttributes().hasAttribute(
1847 for (
unsigned j = 0; j != NumValues; ++j) {
1848 EVT VT = ValueVTs[j];
1860 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1873 if (NeedsRegBlock) {
1875 if (j == NumValues - 1)
1885 for (
unsigned i = 0; i < NumParts; ++i) {
1900 assert(SwiftError.getFunctionArg() &&
"Need a swift error argument");
1902 Flags.setSwiftError();
1910 &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
1918 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1922 "LowerReturn didn't return a valid chain!");
1937 if (VMI != FuncInfo.ValueMap.
end()) {
1938 assert(!V->
use_empty() &&
"Unused value assigned virtual registers!");
1939 CopyValueToVirtualRegister(V, VMI->second);
1948 if (!isa<Instruction>(V) && !isa<Argument>(V))
return;
1951 if (FuncInfo.isExportedInst(V))
return;
1953 unsigned Reg = FuncInfo.InitializeRegForValue(V);
1954 CopyValueToVirtualRegister(V, Reg);
1963 if (
VI->getParent() == FromBB)
1967 return FuncInfo.isExportedInst(V);
1972 if (isa<Argument>(V)) {
1977 return FuncInfo.isExportedInst(V);
1994 auto SuccSize = std::max<uint32_t>(
succ_size(SrcBB), 1);
2007 Prob = getEdgeProbability(Src, Dst);
2013 if (
const Instruction *I = dyn_cast<Instruction>(V))
2034 if (
const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2038 if (CurBB == SwitchBB ||
2039 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2040 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2042 if (
const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2044 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2051 if (
TM.Options.NoNaNsFPMath)
2055 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1),
nullptr,
2056 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2057 SL->SwitchCases.push_back(CB);
2065 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2066 SL->SwitchCases.push_back(CB);
2083 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2098 if (BOpc == Instruction::And)
2099 BOpc = Instruction::Or;
2100 else if (BOpc == Instruction::Or)
2101 BOpc = Instruction::And;
2106 if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
2107 BOpc !=
unsigned(Opc) || !BOp->
hasOneUse() ||
2111 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2112 TProb, FProb, InvertCond);
2122 if (Opc == Instruction::Or) {
2143 auto NewTrueProb = TProb / 2;
2144 auto NewFalseProb = TProb / 2 + FProb;
2146 FindMergedConditions(BOp->
getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
2147 NewTrueProb, NewFalseProb, InvertCond);
2153 FindMergedConditions(BOp->
getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
2154 Probs[0], Probs[1], InvertCond);
2156 assert(Opc == Instruction::And &&
"Unknown merge op!");
2176 auto NewTrueProb = TProb + FProb / 2;
2177 auto NewFalseProb = FProb / 2;
2179 FindMergedConditions(BOp->
getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
2180 NewTrueProb, NewFalseProb, InvertCond);
2186 FindMergedConditions(BOp->
getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
2187 Probs[0], Probs[1], InvertCond);
2196 if (Cases.size() != 2)
return true;
2200 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2201 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2202 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2203 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2209 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2210 Cases[0].CC == Cases[1].CC &&
2211 isa<Constant>(Cases[0].CmpRHS) &&
2212 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2213 if (Cases[0].CC ==
ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2215 if (Cases[0].CC ==
ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2222 void SelectionDAGBuilder::visitBr(
const BranchInst &I) {
2262 if (
const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
2266 (Opcode == Instruction::And || Opcode == Instruction::Or)) {
2267 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
2269 getEdgeProbability(BrMBB, Succ0MBB),
2270 getEdgeProbability(BrMBB, Succ1MBB),
2275 assert(SL->SwitchCases[0].ThisBB == BrMBB &&
"Unexpected lowering!");
2278 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2279 for (
unsigned i = 1,
e = SL->SwitchCases.size(); i !=
e; ++i) {
2280 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2281 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2285 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2286 SL->SwitchCases.erase(SL->SwitchCases.begin());
2292 for (
unsigned i = 1,
e = SL->SwitchCases.size(); i !=
e; ++i)
2293 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2295 SL->SwitchCases.clear();
2301 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2305 visitSwitchCase(CB, BrMBB);
2313 SDValue CondLHS = getValue(CB.CmpLHS);
2318 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2320 if (CB.TrueBB != NextBlock(SwitchBB)) {
2328 EVT MemVT = TLI.getMemValueType(DAG.
getDataLayout(), CB.CmpLHS->getType());
2342 SDValue CondRHS = getValue(CB.CmpRHS);
2356 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2357 const APInt&
High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2359 SDValue CmpOp = getValue(CB.CmpMHS);
2362 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(
true)) {
2374 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2377 if (CB.TrueBB != CB.FalseBB)
2378 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2383 if (CB.TrueBB == NextBlock(SwitchBB)) {
2405 assert(JT.
Reg != -1U &&
"Should lower JT Header first!");
2419 JumpTableHeader &JTH,
2421 SDLoc dl = getCurSDLoc();
2424 SDValue SwitchOp = getValue(JTH.SValue);
2437 unsigned JumpTableReg =
2440 JumpTableReg, SwitchOp);
2441 JT.
Reg = JumpTableReg;
2443 if (!JTH.OmitRangeCheck) {
2457 if (JT.
MBB != NextBlock(SwitchBB))
2464 if (JT.
MBB != NextBlock(SwitchBB))
2482 DAG.
getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2491 if (PtrTy != PtrMemTy)
2514 SDLoc dl = getCurSDLoc();
2538 Entry.
Node = GuardVal;
2540 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
2542 Args.push_back(Entry);
2547 .setCallee(GuardCheckFn->getCallingConv(), FnTy->
getReturnType(),
2548 getValue(GuardCheckFn), std::move(Args));
2550 std::pair<SDValue, SDValue> Result = TLI.
LowerCallTo(CLI);
2562 SDValue GuardPtr = getValue(IRGuard);
2564 Guard = DAG.
getLoad(PtrMemTy, dl, Chain, GuardPtr,
2606 None, CallOptions, getCurSDLoc()).second;
2610 if (
TM.getTargetTriple().isPS4CPU())
2620 SDLoc dl = getCurSDLoc();
2623 SDValue SwitchOp = getValue(B.SValue);
2630 bool UsePtrType =
false;
2634 for (
unsigned i = 0,
e = B.Cases.size(); i !=
e; ++i)
2649 B.Reg = FuncInfo.CreateReg(B.RegVT);
2654 if (!B.OmitRangeCheck)
2655 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2656 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2660 if (!B.OmitRangeCheck) {
2673 if (MBB != NextBlock(SwitchBB))
2686 SDLoc dl = getCurSDLoc();
2692 if (PopCount == 1) {
2699 }
else if (PopCount == BB.Range) {
2719 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2721 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2732 if (NextMBB != NextBlock(SwitchBB))
2739 void SelectionDAGBuilder::visitInvoke(
const InvokeInst &I) {
2750 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2751 "Cannot lower invokes with arbitrary operand bundles yet!");
2755 if (isa<InlineAsm>(
Callee))
2757 else if (Fn && Fn->isIntrinsic()) {
2758 switch (Fn->getIntrinsicID()) {
2761 case Intrinsic::donothing:
2764 case Intrinsic::experimental_patchpoint_void:
2765 case Intrinsic::experimental_patchpoint_i64:
2766 visitPatchpoint(&I, EHPadBB);
2768 case Intrinsic::experimental_gc_statepoint:
2771 case Intrinsic::wasm_rethrow_in_catch: {
2791 LowerCallSiteWithDeoptBundle(&I, getValue(
Callee), EHPadBB);
2793 LowerCallTo(&I, getValue(
Callee),
false, EHPadBB);
2801 CopyToExportRegsIfNeeded(&I);
2812 addSuccessorWithProb(InvokeMBB, Return);
2813 for (
auto &UnwindDest : UnwindDests) {
2814 UnwindDest.first->setIsEHPad();
2815 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2824 void SelectionDAGBuilder::visitCallBr(
const CallBrInst &I) {
2830 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2831 "Cannot lower callbrs with arbitrary operand bundles yet!");
2834 "Only know how to handle inlineasm callbr");
2841 addSuccessorWithProb(CallBrMBB, Return);
2844 addSuccessorWithProb(CallBrMBB, Target);
2854 void SelectionDAGBuilder::visitResume(
const ResumeInst &RI) {
2855 llvm_unreachable(
"SelectionDAGBuilder shouldn't visit resume instructions!");
2858 void SelectionDAGBuilder::visitLandingPad(
const LandingPadInst &LP) {
2859 assert(FuncInfo.MBB->isEHPad() &&
2860 "Call to landingpad not in landing pad!");
2865 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2866 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2867 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2878 SDLoc dl = getCurSDLoc();
2880 assert(ValueVTs.
size() == 2 &&
"Only two-valued landingpads are supported");
2885 if (FuncInfo.ExceptionPointerVirtReg) {
2888 FuncInfo.ExceptionPointerVirtReg,
2896 FuncInfo.ExceptionSelectorVirtReg,
2909 for (
unsigned i = 0,
e = SL->JTCases.size(); i !=
e; ++i)
2910 if (SL->JTCases[i].first.HeaderBB == First)
2911 SL->JTCases[i].first.HeaderBB = Last;
2914 for (
unsigned i = 0,
e = SL->BitTestCases.size(); i !=
e; ++i)
2915 if (SL->BitTestCases[i].Parent == First)
2916 SL->BitTestCases[i].Parent = Last;
2919 void SelectionDAGBuilder::visitIndirectBr(
const IndirectBrInst &I) {
2926 bool Inserted = Done.
insert(BB).second;
2931 addSuccessorWithProb(IndirectBrMBB, Succ);
2940 void SelectionDAGBuilder::visitUnreachable(
const UnreachableInst &I) {
2947 if (&I != &BB.
front()) {
2950 if (
const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2951 if (Call->doesNotReturn())
2960 void SelectionDAGBuilder::visitFSub(
const User &I) {
2985 case Instruction::Mul:
2986 case Instruction::And:
2987 case Instruction::Or:
2988 case Instruction::Xor:
2990 case Instruction::FAdd:
2991 case Instruction::FMul:
2992 if (
const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2993 if (FPOp->getFastMathFlags().isFast())
3005 unsigned ElemNumToReduce = ElemNum;
3028 bool ReduxExtracted =
false;
3030 while (!UsersToVisit.empty()) {
3031 auto User = UsersToVisit.back();
3032 UsersToVisit.pop_back();
3041 if (Inst->
getOpcode() == OpCode || isa<PHINode>(U)) {
3042 if (
const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
3043 if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().isFast())
3045 UsersToVisit.push_back(U);
3047 dyn_cast<ShuffleVectorInst>(U)) {
3054 if (ResultElements < ElemNum)
3057 if (ElemNumToReduce == 1)
3059 if (!isa<UndefValue>(U->getOperand(1)))
3061 for (
unsigned i = 0; i < ElemNumToReduce / 2; ++i)
3062 if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
3064 for (
unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
3065 if (ShufInst->getMaskValue(i) != -1)
3070 if (!U->hasOneUse())
3074 if (!U2 || U2->getOpcode() != OpCode)
3078 if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
3079 (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
3080 UsersToVisit.push_back(U2);
3081 ElemNumToReduce /= 2;
3084 }
else if (isa<ExtractElementInst>(U)) {
3086 if (ElemNumToReduce != 1)
3090 if (!Val || !Val->
isZero())
3093 ReduxExtracted =
true;
3098 return ReduxExtracted;
3101 void SelectionDAGBuilder::visitUnary(
const User &I,
unsigned Opcode) {
3107 setValue(&I, UnNodeValue);
3110 void SelectionDAGBuilder::visitBinary(
const User &I,
unsigned Opcode) {
3112 if (
auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3116 if (
auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) {
3117 Flags.
setExact(ExactOp->isExact());
3121 LLVM_DEBUG(
dbgs() <<
"Detected a reduction operation:" << I <<
"\n");
3128 setValue(&I, BinNodeValue);
3131 void SelectionDAGBuilder::visitShift(
const User &I,
unsigned Opcode) {
3141 unsigned Op2Size = Op2.getValueSizeInBits();
3142 SDLoc DL = getCurSDLoc();
3145 if (ShiftSize > Op2Size)
3152 else if (ShiftSize >=
Log2_32_Ceil(Op2.getValueSizeInBits()))
3167 dyn_cast<const OverflowingBinaryOperator>(&I)) {
3168 nuw = OFBinOp->hasNoUnsignedWrap();
3169 nsw = OFBinOp->hasNoSignedWrap();
3172 dyn_cast<const PossiblyExactOperator>(&I))
3173 exact = ExactOp->isExact();
3184 void SelectionDAGBuilder::visitSDiv(
const User &I) {
3189 Flags.
setExact(isa<PossiblyExactOperator>(&I) &&
3190 cast<PossiblyExactOperator>(&I)->isExact());
3195 void SelectionDAGBuilder::visitICmp(
const User &I) {
3197 if (
const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3198 predicate = IC->getPredicate();
3199 else if (
const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3212 if (Op1.getValueType() != MemVT) {
3219 setValue(&I, DAG.
getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3222 void SelectionDAGBuilder::visitFCmp(
const User &I) {
3224 if (
const FCmpInst *
FC = dyn_cast<FCmpInst>(&I))
3225 predicate =
FC->getPredicate();
3233 if ((FPMO && FPMO->hasNoNaNs()) ||
TM.Options.NoNaNsFPMath)
3238 setValue(&I, DAG.
getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3245 return isa<SelectInst>(V);
3249 void SelectionDAGBuilder::visitSelect(
const User &I) {
3253 unsigned NumValues = ValueVTs.
size();
3254 if (NumValues == 0)
return;
3260 auto BaseOps = {Cond};
3264 bool IsUnaryAbs =
false;
3268 EVT VT = ValueVTs[0];
3275 VT = TLI.getTypeToTransformTo(Ctx, VT);
3280 bool UseScalarMinMax = VT.
isVector() &&
3286 switch (SPR.Flavor) {
3292 switch (SPR.NaNBehavior) {
3301 else if (UseScalarMinMax)
3309 switch (SPR.NaNBehavior) {
3319 else if (UseScalarMinMax)
3335 (TLI.isOperationLegalOrCustom(Opc, VT) ||
3343 LHSVal = getValue(LHS);
3344 RHSVal = getValue(RHS);
3350 LHSVal = getValue(LHS);
3356 for (
unsigned i = 0; i != NumValues; ++i) {
3358 DAG.
getNode(OpCode, getCurSDLoc(),
3359 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i),
3360 SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3363 for (
unsigned i = 0; i != NumValues; ++i) {
3366 Ops.push_back(
SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3368 OpCode, getCurSDLoc(),
3369 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops);
3377 void SelectionDAGBuilder::visitTrunc(
const User &I) {
3385 void SelectionDAGBuilder::visitZExt(
const User &I) {
3394 void SelectionDAGBuilder::visitSExt(
const User &I) {
3403 void SelectionDAGBuilder::visitFPTrunc(
const User &I) {
3406 SDLoc dl = getCurSDLoc();
3414 void SelectionDAGBuilder::visitFPExt(
const User &I) {
3422 void SelectionDAGBuilder::visitFPToUI(
const User &I) {
3430 void SelectionDAGBuilder::visitFPToSI(
const User &I) {
3438 void SelectionDAGBuilder::visitUIToFP(
const User &I) {
3446 void SelectionDAGBuilder::visitSIToFP(
const User &I) {
3454 void SelectionDAGBuilder::visitPtrToInt(
const User &I) {
3468 void SelectionDAGBuilder::visitIntToPtr(
const User &I) {
3480 void SelectionDAGBuilder::visitBitCast(
const User &I) {
3482 SDLoc dl = getCurSDLoc();
3496 setValue(&I, DAG.
getConstant(
C->getValue(), dl, DestVT,
false,
3502 void SelectionDAGBuilder::visitAddrSpaceCast(
const User &I) {
3517 void SelectionDAGBuilder::visitInsertElement(
const User &I) {
3525 InVec, InVal, InIdx));
3528 void SelectionDAGBuilder::visitExtractElement(
const User &I) {
3538 void SelectionDAGBuilder::visitShuffleVector(
const User &I) {
3542 SDLoc DL = getCurSDLoc();
3565 unsigned MaskNumElts = Mask.size();
3567 if (SrcNumElts == MaskNumElts) {
3573 if (SrcNumElts < MaskNumElts) {
3577 if (MaskNumElts % SrcNumElts == 0) {
3581 unsigned NumConcat = MaskNumElts / SrcNumElts;
3582 bool IsConcat =
true;
3584 for (
unsigned i = 0; i != MaskNumElts; ++i) {
3590 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3591 (ConcatSrcs[i / SrcNumElts] >= 0 &&
3592 ConcatSrcs[i / SrcNumElts] != (
int)(Idx / SrcNumElts))) {
3597 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3604 for (
auto Src : ConcatSrcs) {
3617 unsigned PaddedMaskNumElts =
alignTo(MaskNumElts, SrcNumElts);
3618 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3635 for (
unsigned i = 0; i != MaskNumElts; ++i) {
3637 if (Idx >= (
int)SrcNumElts)
3638 Idx -= SrcNumElts - PaddedMaskNumElts;
3646 if (MaskNumElts != PaddedMaskNumElts)
3651 setValue(&I, Result);
3655 if (SrcNumElts > MaskNumElts) {
3658 int StartIdx[2] = { -1, -1 };
3659 bool CanExtract =
true;
3660 for (
int Idx : Mask) {
3665 if (Idx >= (
int)SrcNumElts) {
3673 int NewStartIdx =
alignDown(Idx, MaskNumElts);
3674 if (NewStartIdx + MaskNumElts > SrcNumElts ||
3675 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3679 StartIdx[Input] = NewStartIdx;
3682 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3688 for (
unsigned Input = 0; Input < 2; ++Input) {
3689 SDValue &Src = Input == 0 ? Src1 : Src2;
3690 if (StartIdx[Input] < 0)
3702 for (
int &Idx : MappedOps) {
3703 if (Idx >= (
int)SrcNumElts)
3704 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3720 for (
int Idx : Mask) {
3726 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3727 if (Idx >= (
int)SrcNumElts) Idx -= SrcNumElts;
3739 void SelectionDAGBuilder::visitInsertValue(
const User &I) {
3742 Indices = IV->getIndices();
3744 Indices = cast<ConstantExpr>(&
I)->getIndices();
3750 bool IntoUndef = isa<UndefValue>(Op0);
3751 bool FromUndef = isa<UndefValue>(Op1);
3761 unsigned NumAggValues = AggValueVTs.
size();
3762 unsigned NumValValues = ValValueVTs.size();
3766 if (!NumAggValues) {
3774 for (; i != LinearIndex; ++i)
3775 Values[i] = IntoUndef ? DAG.
getUNDEF(AggValueVTs[i]) :
3780 for (; i != LinearIndex + NumValValues; ++i)
3781 Values[i] = FromUndef ? DAG.
getUNDEF(AggValueVTs[i]) :
3785 for (; i != NumAggValues; ++i)
3786 Values[i] = IntoUndef ? DAG.
getUNDEF(AggValueVTs[i]) :
3793 void SelectionDAGBuilder::visitExtractValue(
const User &I) {
3796 Indices = EV->getIndices();
3798 Indices = cast<ConstantExpr>(&
I)->getIndices();
3803 bool OutOfUndef = isa<UndefValue>(Op0);
3811 unsigned NumValValues = ValValueVTs.
size();
3814 if (!NumValValues) {
3823 for (
unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3824 Values[i - LinearIndex] =
3833 void SelectionDAGBuilder::visitGetElementPtr(
const User &I) {
3839 SDLoc dl = getCurSDLoc();
3857 const Value *Idx = GTI.getOperand();
3858 if (
StructType *StTy = GTI.getStructTypeOrNull()) {
3859 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3862 uint64_t
Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3867 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3876 APInt ElementSize(IdxSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3882 C =
C->getSplatValue();
3884 if (
const auto *CI = dyn_cast_or_null<ConstantInt>(
C)) {
3889 SDValue OffsVal = VectorWidth ?
3919 if (ElementSize != 1) {
3921 unsigned Amt = ElementSize.
logBase2();
3938 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
3944 void SelectionDAGBuilder::visitAlloca(
const AllocaInst &I) {
3947 if (FuncInfo.StaticAllocaMap.count(&I))
3950 SDLoc dl = getCurSDLoc();
3954 uint64_t TySize = DL.getTypeAllocSize(Ty);
3961 if (AllocSize.getValueType() != IntPtr)
3971 unsigned StackAlign =
3973 if (Align <= StackAlign)
3981 AllocSize = DAG.
getNode(
ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
3982 DAG.
getConstant(StackAlign - 1, dl, IntPtr), Flags);
3987 DAG.
getConstant(~(uint64_t)(StackAlign - 1), dl, IntPtr));
3995 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
3998 void SelectionDAGBuilder::visitLoad(
const LoadInst &I) {
4000 return visitAtomicLoad(I);
4007 if (
const Argument *
Arg = dyn_cast<Argument>(SV)) {
4008 if (
Arg->hasSwiftErrorAttr())
4009 return visitLoadFromSwiftError(I);
4012 if (
const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4013 if (Alloca->isSwiftError())
4014 return visitLoadFromSwiftError(I);
4023 bool isNonTemporal = I.
hasMetadata(LLVMContext::MD_nontemporal);
4024 bool isInvariant = I.
hasMetadata(LLVMContext::MD_invariant_load);
4025 bool isDereferenceable =
4036 unsigned NumValues = ValueVTs.size();
4041 bool ConstantMemory =
false;
4042 if (isVolatile || NumValues > MaxParallelChains)
4052 ConstantMemory =
true;
4058 SDLoc dl = getCurSDLoc();
4071 unsigned ChainI = 0;
4072 for (
unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4079 if (ChainI == MaxParallelChains) {
4080 assert(PendingLoads.empty() &&
"PendingLoads must be serialized first");
4097 if (isDereferenceable)
4103 MMOFlags, AAInfo, Ranges);
4106 if (MemVTs[i] != ValueVTs[i])
4112 if (!ConstantMemory) {
4118 PendingLoads.push_back(Chain);
4125 void SelectionDAGBuilder::visitStoreToSwiftError(
const StoreInst &I) {
4127 "call visitStoreToSwiftError when backend supports swifterror");
4134 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4135 "expect a single EVT for swifterror");
4144 SDValue(Src.getNode(), Src.getResNo()));
4148 void SelectionDAGBuilder::visitLoadFromSwiftError(
const LoadInst &I) {
4150 "call visitLoadFromSwiftError when backend supports swifterror");
4155 "Support volatile, non temporal, invariant for load_from_swift_error");
4166 "load_from_swift_error should not be constant memory");
4172 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4173 "expect a single EVT for swifterror");
4177 getRoot(), getCurSDLoc(),
4178 SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4183 void SelectionDAGBuilder::visitStore(
const StoreInst &I) {
4185 return visitAtomicStore(I);
4194 if (
const Argument *
Arg = dyn_cast<Argument>(PtrV)) {
4195 if (
Arg->hasSwiftErrorAttr())
4196 return visitStoreToSwiftError(I);
4199 if (
const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4200 if (Alloca->isSwiftError())
4201 return visitStoreToSwiftError(I);
4209 unsigned NumValues = ValueVTs.
size();
4221 SDLoc dl = getCurSDLoc();
4239 unsigned ChainI = 0;
4240 for (
unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4242 if (ChainI == MaxParallelChains) {
4251 if (MemVTs[i] != ValueVTs[i])
4255 Alignment, MMOFlags, AAInfo);
4256 Chains[ChainI] = St;
4264 void SelectionDAGBuilder::visitMaskedStore(
const CallInst &I,
4265 bool IsCompressing) {
4266 SDLoc sdl = getCurSDLoc();
4269 unsigned& Alignment) {
4273 Alignment = cast<ConstantInt>(I.
getArgOperand(2))->getZExtValue();
4277 unsigned& Alignment) {
4285 Value *PtrOperand, *MaskOperand, *Src0Operand;
4288 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4290 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4292 SDValue Ptr = getValue(PtrOperand);
4293 SDValue Src0 = getValue(Src0Operand);
4294 SDValue Mask = getValue(MaskOperand);
4312 setValue(&I, StoreNode);
4341 const Value *GEPPtr = GEP->getPointerOperand();
4347 unsigned FinalIndex = GEP->getNumOperands() - 1;
4348 Value *IndexVal = GEP->getOperand(FinalIndex);
4351 for (
unsigned i = 1; i < FinalIndex; ++i) {
4356 C =
C->getSplatValue();
4357 auto *CI = dyn_cast_or_null<ConstantInt>(
C);
4358 if (!CI || !CI->isZero())
4376 unsigned GEPWidth = GEP->getType()->getVectorNumElements();
4383 void SelectionDAGBuilder::visitMaskedScatter(
const CallInst &I) {
4384 SDLoc sdl = getCurSDLoc();
4391 unsigned Alignment = (cast<ConstantInt>(I.
getArgOperand(2)))->getZExtValue();
4403 const Value *BasePtr = Ptr;
4404 bool UniformBase =
getUniformBase(BasePtr, Base, Index, IndexType, Scale,
4407 const Value *MemOpBasePtr = UniformBase ? BasePtr :
nullptr;
4414 Index = getValue(Ptr);
4420 Ops, MMO, IndexType);
4422 setValue(&I, Scatter);
4425 void SelectionDAGBuilder::visitMaskedLoad(
const CallInst &I,
bool IsExpanding) {
4426 SDLoc sdl = getCurSDLoc();
4429 unsigned& Alignment) {
4432 Alignment = cast<ConstantInt>(I.
getArgOperand(1))->getZExtValue();
4437 unsigned& Alignment) {
4445 Value *PtrOperand, *MaskOperand, *Src0Operand;
4448 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4450 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4452 SDValue Ptr = getValue(PtrOperand);
4453 SDValue Src0 = getValue(Src0Operand);
4454 SDValue Mask = getValue(MaskOperand);
4477 Alignment, AAInfo, Ranges);
4482 PendingLoads.push_back(Load.getValue(1));
4486 void SelectionDAGBuilder::visitMaskedGather(
const CallInst &I) {
4487 SDLoc sdl = getCurSDLoc();
4496 unsigned Alignment = (cast<ConstantInt>(I.
getArgOperand(1)))->getZExtValue();
4509 const Value *BasePtr = Ptr;
4510 bool UniformBase =
getUniformBase(BasePtr, Base, Index, IndexType, Scale,
4512 bool ConstantMemory =
false;
4513 if (UniformBase && AA &&
4514 AA->pointsToConstantMemory(
4521 ConstantMemory =
true;
4528 Alignment, AAInfo, Ranges);
4532 Index = getValue(Ptr);
4538 Ops, MMO, IndexType);
4541 if (!ConstantMemory)
4542 PendingLoads.push_back(OutChain);
4543 setValue(&I, Gather);
4547 SDLoc dl = getCurSDLoc();
4568 AAMDNodes(),
nullptr, SSID, SuccessOrdering,
4572 dl, MemVT, VTs, InChain,
4583 void SelectionDAGBuilder::visitAtomicRMW(
const AtomicRMWInst &I) {
4584 SDLoc dl = getCurSDLoc();
4607 auto MemVT = getValue(I.
getValOperand()).getSimpleValueType();
4618 MemVT.getStoreSize(), Alignment,
AAMDNodes(),
4619 nullptr, SSID, Ordering);
4632 void SelectionDAGBuilder::visitFence(
const FenceInst &I) {
4633 SDLoc dl = getCurSDLoc();
4644 void SelectionDAGBuilder::visitAtomicLoad(
const LoadInst &I) {
4645 SDLoc dl = getCurSDLoc();
4662 if (I.
hasMetadata(LLVMContext::MD_invariant_load))
4694 PendingLoads.push_back(OutChain);
4709 void SelectionDAGBuilder::visitAtomicStore(
const StoreInst &I) {
4710 SDLoc dl = getCurSDLoc();
4733 nullptr, SSID, Ordering);
4756 void SelectionDAGBuilder::visitTargetIntrinsic(
const CallInst &I,
4757 unsigned Intrinsic) {
4777 TargetLowering::IntrinsicInfo
Info;
4799 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
4800 assert(CI->getBitWidth() <= 64 &&
4801 "large intrinsic immediates not handled");
4819 if (IsTgtIntrinsic) {
4824 Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT,
4826 Info.align ? Info.align->value() : 0, Info.flags, Info.size, AAInfo);
4827 }
else if (!HasChain) {
4838 PendingLoads.push_back(Chain);
4848 Result = lowerRangeToAssertZExt(DAG, I, Result);
4850 setValue(&I, Result);
4909 SDValue TwoToFractionalPartOfX;
4910 if (LimitFloatPrecision <= 6) {
4925 }
else if (LimitFloatPrecision <= 12) {
4987 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5011 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5025 if (LimitFloatPrecision <= 6) {
5040 }
else if (LimitFloatPrecision <= 12) {