76 #include "llvm/IR/IntrinsicsAArch64.h"
77 #include "llvm/IR/IntrinsicsWebAssembly.h"
104 using namespace llvm;
105 using namespace PatternMatch;
106 using namespace SwitchCG;
108 #define DEBUG_TYPE "isel"
116 cl::desc(
"Insert the experimental `assertalign` node."),
121 cl::desc(
"Generate low-precision inline sequences "
122 "for some float libcalls"),
128 cl::desc(
"Set the case probability threshold for peeling the case from a "
129 "switch statement. A value greater than 100 will void this "
149 const SDValue *Parts,
unsigned NumParts,
159 const SDValue *Parts,
unsigned NumParts,
166 PartVT, ValueVT, CC))
173 assert(NumParts > 0 &&
"No parts to assemble!");
183 unsigned RoundParts =
184 (NumParts & (NumParts - 1)) ? 1 <<
Log2_32(NumParts) : NumParts;
185 unsigned RoundBits = PartBits * RoundParts;
186 EVT RoundVT = RoundBits == ValueBits ?
192 if (RoundParts > 2) {
196 RoundParts / 2, PartVT, HalfVT, V);
207 if (RoundParts < NumParts) {
209 unsigned OddParts = NumParts - RoundParts;
240 !PartVT.
isVector() &&
"Unexpected split");
251 if (PartEVT == ValueVT)
255 ValueVT.
bitsLT(PartEVT)) {
268 if (ValueVT.
bitsLT(PartEVT)) {
273 Val = DAG.
getNode(*AssertOp,
DL, PartEVT, Val,
293 ValueVT.
bitsLT(PartEVT)) {
302 const Twine &ErrMsg) {
303 const Instruction *
I = dyn_cast_or_null<Instruction>(V);
307 const char *AsmError =
", possible invalid constraint for vector type";
308 if (
const CallInst *CI = dyn_cast<CallInst>(
I))
309 if (CI->isInlineAsm())
321 const SDValue *Parts,
unsigned NumParts,
325 assert(NumParts > 0 &&
"No parts to assemble!");
326 const bool IsABIRegCopy = CallConv.
has_value();
335 unsigned NumIntermediates;
340 *DAG.
getContext(), *CallConv, ValueVT, IntermediateVT,
341 NumIntermediates, RegisterVT);
345 NumIntermediates, RegisterVT);
348 assert(NumRegs == NumParts &&
"Part count doesn't match vector breakdown!");
350 assert(RegisterVT == PartVT &&
"Part type doesn't match vector breakdown!");
353 "Part type sizes don't match!");
357 if (NumIntermediates == NumParts) {
360 for (
unsigned i = 0;
i != NumParts; ++
i)
362 PartVT, IntermediateVT, V, CallConv);
363 }
else if (NumParts > 0) {
366 assert(NumParts % NumIntermediates == 0 &&
367 "Must expand into a divisible number of parts!");
368 unsigned Factor = NumParts / NumIntermediates;
369 for (
unsigned i = 0;
i != NumIntermediates; ++
i)
371 PartVT, IntermediateVT, V, CallConv);
386 DL, BuiltVectorTy, Ops);
392 if (PartEVT == ValueVT)
409 "Cannot narrow, it would be a lossy transformation");
415 if (PartEVT == ValueVT)
434 }
else if (ValueVT.
bitsLT(PartEVT)) {
443 *DAG.
getContext(), V,
"non-trivial scalar-to-vector conversion");
470 SDValue *Parts,
unsigned NumParts,
MVT PartVT,
487 unsigned OrigNumParts = NumParts;
489 "Copying to an illegal type!");
495 EVT PartEVT = PartVT;
496 if (PartEVT == ValueVT) {
497 assert(NumParts == 1 &&
"No-op copy with multiple parts!");
505 assert(NumParts == 1 &&
"Do not know what to promote to!");
516 "Unknown mismatch!");
518 Val = DAG.
getNode(ExtendKind,
DL, ValueVT, Val);
524 assert(NumParts == 1 && PartEVT != ValueVT);
530 "Unknown mismatch!");
540 "Failed to tile the value with PartVT!");
543 if (PartEVT != ValueVT) {
545 "scalar-to-vector conversion failed");
554 if (NumParts & (NumParts - 1)) {
557 "Do not know what to expand to!");
558 unsigned RoundParts = 1 <<
Log2_32(NumParts);
559 unsigned RoundBits = RoundParts * PartBits;
560 unsigned OddParts = NumParts - RoundParts;
571 NumParts = RoundParts;
583 for (
unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
584 for (
unsigned i = 0;
i < NumParts;
i += StepSize) {
585 unsigned ThisBits = StepSize * PartBits / 2;
588 SDValue &Part1 = Parts[
i+StepSize/2];
595 if (ThisBits == PartBits && ThisVT != PartVT) {
635 Ops.
append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
650 const bool IsABIRegCopy = CallConv.
has_value();
653 EVT PartEVT = PartVT;
654 if (PartEVT == ValueVT) {
687 "lossy conversion of vector to scalar type");
702 unsigned NumIntermediates;
707 NumIntermediates, RegisterVT);
711 NumIntermediates, RegisterVT);
714 assert(NumRegs == NumParts &&
"Part count doesn't match vector breakdown!");
716 assert(RegisterVT == PartVT &&
"Part type doesn't match vector breakdown!");
719 "Mixing scalable and fixed vectors when copying in parts");
731 if (ValueVT == BuiltVectorTy) {
755 for (
unsigned i = 0;
i != NumIntermediates; ++
i) {
770 if (NumParts == NumIntermediates) {
773 for (
unsigned i = 0;
i != NumParts; ++
i)
775 }
else if (NumParts > 0) {
778 assert(NumIntermediates != 0 &&
"division by zero");
779 assert(NumParts % NumIntermediates == 0 &&
780 "Must expand into a divisible number of parts!");
781 unsigned Factor = NumParts / NumIntermediates;
782 for (
unsigned i = 0;
i != NumIntermediates; ++
i)
790 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
791 RegCount(1, regs.
size()), CallConv(CC) {}
809 for (
unsigned i = 0;
i != NumRegs; ++
i)
811 RegVTs.push_back(RegisterVT);
840 for (
unsigned i = 0;
i != NumRegs; ++
i) {
846 *
Flag =
P.getValue(2);
849 Chain =
P.getValue(1);
882 }
else if (NumSignBits > 1) {
912 unsigned NumRegs =
Regs.size();
926 NumParts, RegisterVT, V,
CallConv, ExtendKind);
932 for (
unsigned i = 0;
i != NumRegs; ++
i) {
944 if (NumRegs == 1 ||
Flag)
955 Chain = Chains[NumRegs-1];
961 unsigned MatchingIdx,
const SDLoc &dl,
963 std::vector<SDValue> &Ops)
const {
988 "No 1:1 mapping from clobbers to regs?");
996 "If we clobbered the stack pointer, MFI should know about it.");
1005 for (
unsigned i = 0;
i != NumRegs; ++
i) {
1006 assert(
Reg <
Regs.size() &&
"Mismatch in # registers expected");
1007 unsigned TheReg =
Regs[
Reg++];
1008 Ops.push_back(DAG.
getRegister(TheReg, RegisterVT));
1018 unsigned RegCount = std::get<0>(CountAndVT);
1019 MVT RegisterVT = std::get<1>(CountAndVT);
1022 OutVec.push_back(std::make_pair(
Regs[
I], RegisterSize));
1039 UnusedArgNodeMap.clear();
1041 PendingExports.clear();
1042 PendingConstrainedFP.clear();
1043 PendingConstrainedFPStrict.clear();
1051 DanglingDebugInfoMap.clear();
1058 if (Pending.empty())
1064 unsigned i = 0,
e = Pending.size();
1065 for (;
i !=
e; ++
i) {
1066 assert(Pending[
i].getNode()->getNumOperands() > 1);
1067 if (Pending[
i].getNode()->getOperand(0) == Root)
1072 Pending.push_back(Root);
1075 if (Pending.size() == 1)
1094 PendingConstrainedFP.size() +
1095 PendingConstrainedFPStrict.size());
1097 PendingConstrainedFP.end());
1098 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1099 PendingConstrainedFPStrict.end());
1100 PendingConstrainedFP.clear();
1101 PendingConstrainedFPStrict.clear();
1108 PendingExports.append(PendingConstrainedFPStrict.begin(),
1109 PendingConstrainedFPStrict.end());
1110 PendingConstrainedFPStrict.clear();
1111 return updateRoot(PendingExports);
1116 if (
I.isTerminator()) {
1117 HandlePHINodesInSuccessorBlocks(
I.getParent());
1121 if (!isa<DbgInfoIntrinsic>(
I))
1129 !isa<GCStatepointInst>(
I))
1135 void SelectionDAGBuilder::visitPHI(
const PHINode &) {
1145 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1146 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1147 #include "llvm/IR/Instruction.def"
1164 false,
DL, Order,
true);
1172 "DbgValueInst without an ArgList should have a single location "
1174 DanglingDebugInfoMap[DI->
getValue(0)].emplace_back(DI,
DL, Order);
1180 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1184 if (DanglingVariable == Variable && Expr->
fragmentsOverlap(DanglingExpr)) {
1185 LLVM_DEBUG(
dbgs() <<
"Dropping dangling debug info for " << *DI <<
"\n");
1191 for (
auto &DDIMI : DanglingDebugInfoMap) {
1192 DanglingDebugInfoVector &DDIV = DDIMI.second;
1196 for (
auto &DDI : DDIV)
1197 if (isMatchingDbgValue(DDI))
1200 erase_if(DDIV, isMatchingDbgValue);
1208 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1209 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1212 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1213 for (
auto &DDI : DDIV) {
1216 assert(DI &&
"Ill-formed DanglingDebugInfo");
1219 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1223 "Expected inlined-at fields to agree");
1232 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl,
1233 FuncArgumentDbgValueKind::Value, Val)) {
1235 << DbgSDNodeOrder <<
"] for:\n " << *DI <<
"\n");
1242 <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to "
1243 << ValSDNodeOrder <<
"\n");
1244 SDV = getDbgValue(Val, Variable, Expr, dl,
1245 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1249 <<
"in EmitFuncArgumentDbgValue\n");
1266 assert(!DDI.getDI()->hasArgList() &&
1267 "Not implemented for variadic dbg_values");
1268 Value *V = DDI.getDI()->getValue(0);
1272 DebugLoc InstDL = DDI.getDI()->getDebugLoc();
1273 unsigned SDOrder = DDI.getSDNodeOrder();
1276 assert(isa<DbgValueInst>(DDI.getDI()));
1277 bool StackValue =
true;
1286 while (isa<Instruction>(V)) {
1301 if (!AdditionalValues.empty())
1312 << DDI.getDI() <<
"\nBy stripping back to:\n " << V);
1324 LLVM_DEBUG(
dbgs() <<
"Dropping debug value info for:\n " << DDI.getDI()
1326 LLVM_DEBUG(
dbgs() <<
" Last seen at:\n " << *DDI.getDI()->getOperand(0)
1339 for (
const Value *V : Values) {
1341 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1342 isa<ConstantPointerNull>(V)) {
1349 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1360 if (!
N.getNode() && isa<Argument>(V))
1361 N = UnusedArgNodeMap[V];
1365 EmitFuncArgumentDbgValue(V, Var, Expr, dl,
1366 FuncArgumentDbgValueKind::Value,
N))
1368 if (
auto *FISDN = dyn_cast<FrameIndexSDNode>(
N.getNode())) {
1379 Dependencies.push_back(
N.getNode());
1393 bool IsParamOfFunc =
1403 unsigned Reg = VMI->second;
1407 V->getType(),
None);
1412 unsigned Offset = 0;
1413 unsigned BitsToDescribe = 0;
1415 BitsToDescribe = *VarSize;
1417 BitsToDescribe = Fragment->SizeInBits;
1420 if (Offset >= BitsToDescribe)
1423 unsigned RegisterSize = RegAndSize.second;
1424 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1425 ? BitsToDescribe - Offset
1428 Expr, Offset, FragmentSize);
1432 Var, *FragmentExpr, RegAndSize.first,
false, dl, SDNodeOrder);
1434 Offset += RegisterSize;
1448 assert(!LocationOps.empty());
1451 false, dl, SDNodeOrder, IsVariadic);
1458 for (
auto &Pair : DanglingDebugInfoMap)
1459 for (
auto &DDI : Pair.second)
1491 if (
N.getNode())
return N;
1511 if (isa<ConstantSDNode>(
N) || isa<ConstantFPSDNode>(
N)) {
1533 if (
const Constant *
C = dyn_cast<Constant>(V)) {
1542 if (isa<ConstantPointerNull>(
C)) {
1551 if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(
C))
1558 visit(CE->getOpcode(), *CE);
1560 assert(N1.
getNode() &&
"visit didn't populate the NodeMap!");
1564 if (isa<ConstantStruct>(
C) || isa<ConstantArray>(
C)) {
1566 for (
const Use &U :
C->operands()) {
1580 dyn_cast<ConstantDataSequential>(
C)) {
1582 for (
unsigned i = 0,
e = CDS->getNumElements();
i !=
e; ++
i) {
1590 if (isa<ArrayType>(CDS->getType()))
1595 if (
C->getType()->isStructTy() ||
C->getType()->isArrayTy()) {
1596 assert((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(
C)) &&
1597 "Unknown struct or array constant!");
1601 unsigned NumElts = ValueVTs.size();
1605 for (
unsigned i = 0;
i != NumElts; ++
i) {
1606 EVT EltVT = ValueVTs[
i];
1607 if (isa<UndefValue>(
C))
1621 if (
const auto *Equiv = dyn_cast<DSOLocalEquivalent>(
C))
1622 return getValue(Equiv->getGlobalValue());
1624 if (
const auto *
NC = dyn_cast<NoCFIValue>(
C))
1633 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1634 for (
unsigned i = 0;
i != NumElements; ++
i)
1635 Ops.push_back(
getValue(CV->getOperand(
i)));
1640 if (isa<ConstantAggregateZero>(
C)) {
1650 if (isa<ScalableVectorType>(VecTy))
1654 Ops.
assign(cast<FixedVectorType>(VecTy)->getNumElements(),
Op);
1663 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1672 if (
const Instruction *Inst = dyn_cast<Instruction>(V)) {
1676 Inst->getType(),
None);
1684 if (
const auto *
BB = dyn_cast<BasicBlock>(V))
1690 void SelectionDAGBuilder::visitCatchPad(
const CatchPadInst &
I) {
1699 if (IsMSVCCXX || IsCoreCLR)
1726 Value *ParentPad =
I.getCatchSwitchParentPad();
1728 if (isa<ConstantTokenNone>(ParentPad))
1731 SuccessorColor = cast<Instruction>(ParentPad)->
getParent();
1732 assert(SuccessorColor &&
"No parent funclet for catchret!");
1734 assert(SuccessorColorMBB &&
"No MBB for SuccessorColor!");
1743 void SelectionDAGBuilder::visitCleanupPad(
const CleanupPadInst &CPI) {
1787 if (isa<CleanupPadInst>(Pad)) {
1789 UnwindDests.emplace_back(FuncInfo.
MBBMap[EHPadBB], Prob);
1790 UnwindDests.back().first->setIsEHScopeEntry();
1792 }
else if (
auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1795 for (
const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1796 UnwindDests.emplace_back(FuncInfo.
MBBMap[CatchPadBB], Prob);
1797 UnwindDests.back().first->setIsEHScopeEntry();
1828 assert(UnwindDests.size() <= 1 &&
1829 "There should be at most one unwind destination for wasm");
1836 if (isa<LandingPadInst>(Pad)) {
1838 UnwindDests.emplace_back(FuncInfo.
MBBMap[EHPadBB], Prob);
1840 }
else if (isa<CleanupPadInst>(Pad)) {
1843 UnwindDests.emplace_back(FuncInfo.
MBBMap[EHPadBB], Prob);
1844 UnwindDests.
back().first->setIsEHScopeEntry();
1845 UnwindDests.back().first->setIsEHFuncletEntry();
1847 }
else if (
auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1849 for (
const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1850 UnwindDests.emplace_back(FuncInfo.
MBBMap[CatchPadBB], Prob);
1852 if (IsMSVCCXX || IsCoreCLR)
1853 UnwindDests.back().first->setIsEHFuncletEntry();
1855 UnwindDests.back().first->setIsEHScopeEntry();
1857 NewEHPadBB = CatchSwitch->getUnwindDest();
1863 if (BPI && NewEHPadBB)
1865 EHPadBB = NewEHPadBB;
1872 auto UnwindDest =
I.getUnwindDest();
1879 for (
auto &UnwindDest : UnwindDests) {
1880 UnwindDest.first->setIsEHPad();
1881 addSuccessorWithProb(
FuncInfo.
MBB, UnwindDest.first, UnwindDest.second);
1891 void SelectionDAGBuilder::visitCatchSwitch(
const CatchSwitchInst &CSI) {
1895 void SelectionDAGBuilder::visitRet(
const ReturnInst &
I) {
1909 if (
I.getParent()->getTerminatingDeoptimizeCall()) {
1916 const Function *
F =
I.getParent()->getParent();
1923 F->getReturnType()->getPointerTo(
1935 unsigned NumValues = ValueVTs.size();
1938 Align BaseAlign =
DL.getPrefTypeAlign(
I.getOperand(0)->getType());
1939 for (
unsigned i = 0;
i != NumValues; ++
i) {
1946 if (MemVTs[
i] != ValueVTs[
i])
1957 }
else if (
I.getNumOperands() != 0) {
1960 unsigned NumValues = ValueVTs.size();
1964 const Function *
F =
I.getParent()->getParent();
1967 I.getOperand(0)->getType(),
F->getCallingConv(),
1971 if (
F->getAttributes().hasRetAttr(Attribute::SExt))
1973 else if (
F->getAttributes().hasRetAttr(Attribute::ZExt))
1977 bool RetInReg =
F->getAttributes().hasRetAttr(Attribute::InReg);
1979 for (
unsigned j = 0;
j != NumValues; ++
j) {
1980 EVT VT = ValueVTs[
j];
1992 &Parts[0], NumParts, PartVT, &
I, CC, ExtendKind);
1999 if (
I.getOperand(0)->getType()->isPointerTy()) {
2002 cast<PointerType>(
I.getOperand(0)->getType())->getAddressSpace());
2005 if (NeedsRegBlock) {
2007 if (
j == NumValues - 1)
2017 for (
unsigned i = 0;
i < NumParts; ++
i) {
2019 Parts[
i].getValueType().getSimpleVT(),
2021 OutVals.push_back(Parts[
i]);
2030 const Function *
F =
I.getParent()->getParent();
2032 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2054 "LowerReturn didn't return a valid chain!");
2070 assert(!V->
use_empty() &&
"Unused value assigned virtual registers!");
2080 if (!isa<Instruction>(V) && !isa<Argument>(V))
return;
2095 if (
VI->getParent() == FromBB)
2104 if (isa<Argument>(V)) {
2121 const BasicBlock *SrcBB = Src->getBasicBlock();
2122 const BasicBlock *DstBB = Dst->getBasicBlock();
2126 auto SuccSize = std::max<uint32_t>(
succ_size(SrcBB), 1);
2136 Src->addSuccessorWithoutProb(Dst);
2139 Prob = getEdgeProbability(Src, Dst);
2140 Src->addSuccessor(Dst, Prob);
2146 return I->getParent() ==
BB;
2166 if (
const CmpInst *BOp = dyn_cast<CmpInst>(
Cond)) {
2170 if (CurBB == SwitchBB ||
2176 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2181 InvertCond ?
FC->getInversePredicate() :
FC->getPredicate();
2187 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1),
nullptr,
2189 SL->SwitchCases.push_back(CB);
2197 nullptr, TBB, FBB, CurBB,
getCurSDLoc(), TProb, FProb);
2198 SL->SwitchCases.push_back(CB);
2221 const Value *BOpOp0, *BOpOp1;
2235 if (BOpc == Instruction::And)
2236 BOpc = Instruction::Or;
2237 else if (BOpc == Instruction::Or)
2238 BOpc = Instruction::And;
2244 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->
hasOneUse();
2249 TProb, FProb, InvertCond);
2259 if (Opc == Instruction::Or) {
2280 auto NewTrueProb = TProb / 2;
2281 auto NewFalseProb = TProb / 2 + FProb;
2284 NewFalseProb, InvertCond);
2291 Probs[1], InvertCond);
2293 assert(Opc == Instruction::And &&
"Unknown merge op!");
2313 auto NewTrueProb = TProb + FProb / 2;
2314 auto NewFalseProb = FProb / 2;
2317 NewFalseProb, InvertCond);
2324 Probs[1], InvertCond);
2333 if (Cases.size() != 2)
return true;
2337 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2338 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2339 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2340 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2346 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2347 Cases[0].CC == Cases[1].CC &&
2348 isa<Constant>(Cases[0].CmpRHS) &&
2349 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2350 if (Cases[0].CC ==
ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2352 if (Cases[0].CC ==
ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2359 void SelectionDAGBuilder::visitBr(
const BranchInst &
I) {
2365 if (
I.isUnconditional()) {
2381 const Value *CondVal =
I.getCondition();
2401 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2403 BOp->
hasOneUse() && !
I.hasMetadata(LLVMContext::MD_unpredictable)) {
2405 const Value *BOp0, *BOp1;
2408 Opcode = Instruction::And;
2410 Opcode = Instruction::Or;
2415 getEdgeProbability(BrMBB, Succ0MBB),
2416 getEdgeProbability(BrMBB, Succ1MBB),
2421 assert(
SL->SwitchCases[0].ThisBB == BrMBB &&
"Unexpected lowering!");
2425 for (
unsigned i = 1,
e =
SL->SwitchCases.size();
i !=
e; ++
i) {
2432 SL->SwitchCases.erase(
SL->SwitchCases.begin());
2438 for (
unsigned i = 1,
e =
SL->SwitchCases.size();
i !=
e; ++
i)
2441 SL->SwitchCases.clear();
2447 nullptr, Succ0MBB, Succ1MBB, BrMBB,
getCurSDLoc());
2464 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2466 if (CB.TrueBB != NextBlock(SwitchBB)) {
2502 const APInt&
Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2503 const APInt&
High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2508 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(
true)) {
2520 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2523 if (CB.TrueBB != CB.FalseBB)
2524 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2529 if (CB.TrueBB == NextBlock(SwitchBB)) {
2551 assert(
JT.Reg != -1U &&
"Should lower JT Header first!");
2565 JumpTableHeader &JTH,
2583 unsigned JumpTableReg =
2586 JumpTableReg, SwitchOp);
2587 JT.Reg = JumpTableReg;
2589 if (!JTH.FallthroughUnreachable) {
2603 if (
JT.MBB != NextBlock(SwitchBB))
2610 if (
JT.MBB != NextBlock(SwitchBB))
2637 if (PtrTy != PtrMemTy)
2685 Entry.Node = GuardVal;
2687 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
2688 Entry.IsInReg =
true;
2689 Args.push_back(Entry);
2697 std::pair<SDValue, SDValue> Result = TLI.
LowerCallTo(CLI);
2711 Guard =
DAG.
getLoad(PtrMemTy, dl, Chain, GuardPtr,
2778 bool UsePtrType =
false;
2782 for (
unsigned i = 0,
e =
B.Cases.size();
i !=
e; ++
i)
2802 if (!
B.FallthroughUnreachable)
2803 addSuccessorWithProb(SwitchBB,
B.Default,
B.DefaultProb);
2804 addSuccessorWithProb(SwitchBB,
MBB,
B.Prob);
2808 if (!
B.FallthroughUnreachable) {
2821 if (
MBB != NextBlock(SwitchBB))
2840 if (PopCount == 1) {
2847 }
else if (PopCount ==
BB.Range) {
2867 addSuccessorWithProb(SwitchBB,
B.TargetBB,
B.ExtraProb);
2869 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2880 if (NextMBB != NextBlock(SwitchBB))
2887 void SelectionDAGBuilder::visitInvoke(
const InvokeInst &
I) {
2897 assert(!
I.hasOperandBundlesOtherThan(
2898 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
2899 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
2900 LLVMContext::OB_cfguardtarget,
2901 LLVMContext::OB_clang_arc_attachedcall}) &&
2902 "Cannot lower invokes with arbitrary operand bundles yet!");
2905 const Function *Fn = dyn_cast<Function>(Callee);
2906 if (isa<InlineAsm>(Callee))
2907 visitInlineAsm(
I, EHPadBB);
2912 case Intrinsic::donothing:
2914 case Intrinsic::seh_try_begin:
2915 case Intrinsic::seh_scope_begin:
2916 case Intrinsic::seh_try_end:
2917 case Intrinsic::seh_scope_end:
2919 case Intrinsic::experimental_patchpoint_void:
2920 case Intrinsic::experimental_patchpoint_i64:
2921 visitPatchpoint(
I, EHPadBB);
2923 case Intrinsic::experimental_gc_statepoint:
2926 case Intrinsic::wasm_rethrow: {
2955 if (!isa<GCStatepointInst>(
I)) {
2967 addSuccessorWithProb(InvokeMBB, Return);
2968 for (
auto &UnwindDest : UnwindDests) {
2969 UnwindDest.first->setIsEHPad();
2970 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2979 void SelectionDAGBuilder::visitCallBr(
const CallBrInst &
I) {
2984 assert(!
I.hasOperandBundlesOtherThan(
2985 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2986 "Cannot lower callbrs with arbitrary operand bundles yet!");
2988 assert(
I.isInlineAsm() &&
"Only know how to handle inlineasm callbr");
2997 for (
unsigned i = 0,
e =
I.getNumIndirectDests();
i <
e; ++
i) {
3000 Target->setIsInlineAsmBrIndirectTarget();
3010 void SelectionDAGBuilder::visitResume(
const ResumeInst &RI) {
3011 llvm_unreachable(
"SelectionDAGBuilder shouldn't visit resume instructions!");
3014 void SelectionDAGBuilder::visitLandingPad(
const LandingPadInst &LP) {
3016 "Call to landingpad not in landing pad!");
3036 assert(ValueVTs.size() == 2 &&
"Only two-valued landingpads are supported");
3066 if (JTB.first.HeaderBB ==
First)
3067 JTB.first.HeaderBB = Last;
3070 for (BitTestBlock &BTB :
SL->BitTestCases)
3071 if (BTB.Parent ==
First)
3080 for (
unsigned i = 0,
e =
I.getNumSuccessors();
i !=
e; ++
i) {
3082 bool Inserted = Done.
insert(
BB).second;
3087 addSuccessorWithProb(IndirectBrMBB, Succ);
3103 if (&
I != &
BB.front()) {
3106 if (
const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
3107 if (
Call->doesNotReturn())
3116 void SelectionDAGBuilder::visitUnary(
const User &
I,
unsigned Opcode) {
3118 if (
auto *FPOp = dyn_cast<FPMathOperator>(&
I))
3127 void SelectionDAGBuilder::visitBinary(
const User &
I,
unsigned Opcode) {
3129 if (
auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&
I)) {
3133 if (
auto *ExactOp = dyn_cast<PossiblyExactOperator>(&
I))
3134 Flags.
setExact(ExactOp->isExact());
3135 if (
auto *FPOp = dyn_cast<FPMathOperator>(&
I))
3145 void SelectionDAGBuilder::visitShift(
const User &
I,
unsigned Opcode) {
3154 if (!
I.getType()->isVectorTy() && Op2.
getValueType() != ShiftTy) {
3156 "Unexpected shift type");
3167 dyn_cast<const OverflowingBinaryOperator>(&
I)) {
3168 nuw = OFBinOp->hasNoUnsignedWrap();
3169 nsw = OFBinOp->hasNoSignedWrap();
3172 dyn_cast<const PossiblyExactOperator>(&
I))
3173 exact = ExactOp->isExact();
3184 void SelectionDAGBuilder::visitSDiv(
const User &
I) {
3189 Flags.
setExact(isa<PossiblyExactOperator>(&
I) &&
3190 cast<PossiblyExactOperator>(&
I)->isExact());
3195 void SelectionDAGBuilder::visitICmp(
const User &
I) {
3197 if (
const ICmpInst *IC = dyn_cast<ICmpInst>(&
I))
3198 predicate = IC->getPredicate();
3199 else if (
const ConstantExpr *IC = dyn_cast<ConstantExpr>(&
I))
3222 void SelectionDAGBuilder::visitFCmp(
const User &
I) {
3225 predicate =
FC->getPredicate();
3232 auto *FPMO = cast<FPMathOperator>(&
I);
3249 return isa<SelectInst>(V);
3253 void SelectionDAGBuilder::visitSelect(
const User &
I) {
3257 unsigned NumValues = ValueVTs.size();
3258 if (NumValues == 0)
return;
3268 bool IsUnaryAbs =
false;
3269 bool Negate =
false;
3272 if (
auto *FPOp = dyn_cast<FPMathOperator>(&
I))
3277 EVT VT = ValueVTs[0];
3289 bool UseScalarMinMax = VT.
isVector() &&
3295 switch (SPR.Flavor) {
3301 switch (SPR.NaNBehavior) {
3310 else if (UseScalarMinMax)
3318 switch (SPR.NaNBehavior) {
3328 else if (UseScalarMinMax)
3366 for (
unsigned i = 0;
i != NumValues; ++
i) {
3376 for (
unsigned i = 0;
i != NumValues; ++
i) {
3390 void SelectionDAGBuilder::visitTrunc(
const User &
I) {
3398 void SelectionDAGBuilder::visitZExt(
const User &
I) {
3407 void SelectionDAGBuilder::visitSExt(
const User &
I) {
3416 void SelectionDAGBuilder::visitFPTrunc(
const User &
I) {
3427 void SelectionDAGBuilder::visitFPExt(
const User &
I) {
3435 void SelectionDAGBuilder::visitFPToUI(
const User &
I) {
3443 void SelectionDAGBuilder::visitFPToSI(
const User &
I) {
3451 void SelectionDAGBuilder::visitUIToFP(
const User &
I) {
3459 void SelectionDAGBuilder::visitSIToFP(
const User &
I) {
3467 void SelectionDAGBuilder::visitPtrToInt(
const User &
I) {
3481 void SelectionDAGBuilder::visitIntToPtr(
const User &
I) {
3493 void SelectionDAGBuilder::visitBitCast(
const User &
I) {
3501 if (DestVT !=
N.getValueType())
3508 else if(
ConstantInt *
C = dyn_cast<ConstantInt>(
I.getOperand(0)))
3515 void SelectionDAGBuilder::visitAddrSpaceCast(
const User &
I) {
3517 const Value *SV =
I.getOperand(0);
3522 unsigned DestAS =
I.getType()->getPointerAddressSpace();
3530 void SelectionDAGBuilder::visitInsertElement(
const User &
I) {
3538 InVec, InVal, InIdx));
3541 void SelectionDAGBuilder::visitExtractElement(
const User &
I) {
3551 void SelectionDAGBuilder::visitShuffleVector(
const User &
I) {
3555 if (
auto *SVI = dyn_cast<ShuffleVectorInst>(&
I))
3556 Mask = SVI->getShuffleMask();
3558 Mask = cast<ConstantExpr>(
I).getShuffleMask();
3564 if (
all_of(
Mask, [](
int Elem) {
return Elem == 0; }) &&
3580 unsigned MaskNumElts =
Mask.size();
3582 if (SrcNumElts == MaskNumElts) {
3588 if (SrcNumElts < MaskNumElts) {
3592 if (MaskNumElts % SrcNumElts == 0) {
3596 unsigned NumConcat = MaskNumElts / SrcNumElts;
3597 bool IsConcat =
true;
3599 for (
unsigned i = 0;
i != MaskNumElts; ++
i) {
3605 if ((Idx % SrcNumElts != (
i % SrcNumElts)) ||
3606 (ConcatSrcs[
i / SrcNumElts] >= 0 &&
3607 ConcatSrcs[
i / SrcNumElts] != (
int)(Idx / SrcNumElts))) {
3612 ConcatSrcs[
i / SrcNumElts] = Idx / SrcNumElts;
3619 for (
auto Src : ConcatSrcs) {
3623 ConcatOps.push_back(Src1);
3625 ConcatOps.push_back(Src2);
3632 unsigned PaddedMaskNumElts =
alignTo(MaskNumElts, SrcNumElts);
3633 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3650 for (
unsigned i = 0;
i != MaskNumElts; ++
i) {
3652 if (Idx >= (
int)SrcNumElts)
3653 Idx -= SrcNumElts - PaddedMaskNumElts;
3661 if (MaskNumElts != PaddedMaskNumElts)
3669 if (SrcNumElts > MaskNumElts) {
3672 int StartIdx[2] = { -1, -1 };
3673 bool CanExtract =
true;
3674 for (
int Idx :
Mask) {
3679 if (Idx >= (
int)SrcNumElts) {
3687 int NewStartIdx =
alignDown(Idx, MaskNumElts);
3688 if (NewStartIdx + MaskNumElts > SrcNumElts ||
3689 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3693 StartIdx[
Input] = NewStartIdx;
3696 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3702 for (
unsigned Input = 0;
Input < 2; ++
Input) {
3704 if (StartIdx[Input] < 0)
3714 for (
int &Idx : MappedOps) {
3715 if (Idx >= (
int)SrcNumElts)
3716 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3731 for (
int Idx :
Mask) {
3737 SDValue &Src = Idx < (
int)SrcNumElts ? Src1 : Src2;
3738 if (Idx >= (
int)SrcNumElts) Idx -= SrcNumElts;
3750 void SelectionDAGBuilder::visitInsertValue(
const User &
I) {
3753 Indices =
IV->getIndices();
3755 Indices = cast<ConstantExpr>(&
I)->getIndices();
3757 const Value *Op0 =
I.getOperand(0);
3758 const Value *Op1 =
I.getOperand(1);
3759 Type *AggTy =
I.getType();
3761 bool IntoUndef = isa<UndefValue>(Op0);
3762 bool FromUndef = isa<UndefValue>(Op1);
3772 unsigned NumAggValues = AggValueVTs.size();
3773 unsigned NumValValues = ValValueVTs.size();
3777 if (!NumAggValues) {
3785 for (;
i != LinearIndex; ++
i)
3791 for (;
i != LinearIndex + NumValValues; ++
i)
3796 for (;
i != NumAggValues; ++
i)
3806 const Value *Op0 =
I.getOperand(0);
3808 Type *ValTy =
I.getType();
3809 bool OutOfUndef = isa<UndefValue>(Op0);
3817 unsigned NumValValues = ValValueVTs.size();
3820 if (!NumValValues) {
3829 for (
unsigned i = LinearIndex;
i != LinearIndex + NumValValues; ++
i)
3830 Values[
i - LinearIndex] =
3839 void SelectionDAGBuilder::visitGetElementPtr(
const User &
I) {
3840 Value *Op0 =
I.getOperand(0);
3850 bool IsVectorGEP =
I.getType()->isVectorTy();
3852 IsVectorGEP ? cast<VectorType>(
I.getType())->getElementCount()
3855 if (IsVectorGEP && !
N.getValueType().isVector()) {
3866 const Value *Idx = GTI.getOperand();
3867 if (
StructType *StTy = GTI.getStructTypeOrNull()) {
3868 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3877 if (int64_t(Offset) >= 0 && cast<GEPOperator>(
I).isInBounds())
3894 bool ElementScalable = ElementSize.
isScalable();
3898 const auto *
C = dyn_cast<Constant>(Idx);
3899 if (
C && isa<VectorType>(
C->getType()))
3900 C =
C->getSplatValue();
3902 const auto *CI = dyn_cast_or_null<ConstantInt>(
C);
3903 if (CI && CI->isZero())
3905 if (CI && !ElementScalable) {
3932 VectorElementCount);
3943 if (ElementScalable) {
3944 EVT VScaleTy =
N.getValueType().getScalarType();
3954 if (ElementMul != 1) {
3955 if (ElementMul.isPowerOf2()) {
3956 unsigned Amt = ElementMul.logBase2();
3958 N.getValueType(), IdxN,
3964 N.getValueType(), IdxN, Scale);
3970 N.getValueType(),
N, IdxN);
3981 if (PtrMemTy != PtrTy && !cast<GEPOperator>(
I).isInBounds())
3987 void SelectionDAGBuilder::visitAlloca(
const AllocaInst &
I) {
3994 Type *Ty =
I.getAllocatedType();
4047 void SelectionDAGBuilder::visitLoad(
const LoadInst &
I) {
4049 return visitAtomicLoad(
I);
4052 const Value *SV =
I.getOperand(0);
4056 if (
const Argument *
Arg = dyn_cast<Argument>(SV)) {
4057 if (
Arg->hasSwiftErrorAttr())
4058 return visitLoadFromSwiftError(
I);
4061 if (
const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4062 if (Alloca->isSwiftError())
4063 return visitLoadFromSwiftError(
I);
4069 Type *Ty =
I.getType();
4073 const MDNode *Ranges =
I.getMetadata(LLVMContext::MD_range);
4078 unsigned NumValues = ValueVTs.size();
4082 bool isVolatile =
I.isVolatile();
4085 bool ConstantMemory =
false;
4098 ConstantMemory =
true;
4121 unsigned ChainI = 0;
4122 for (
unsigned i = 0;
i != NumValues; ++
i, ++ChainI) {
4143 MMOFlags, AAInfo, Ranges);
4146 if (MemVTs[
i] != ValueVTs[
i])
4152 if (!ConstantMemory) {
4165 void SelectionDAGBuilder::visitStoreToSwiftError(
const StoreInst &
I) {
4167 "call visitStoreToSwiftError when backend supports swifterror");
4171 const Value *SrcV =
I.getOperand(0);
4175 "expect a single EVT for swifterror");
4184 SDValue(Src.getNode(), Src.getResNo()));
4188 void SelectionDAGBuilder::visitLoadFromSwiftError(
const LoadInst &
I) {
4190 "call visitLoadFromSwiftError when backend supports swifterror");
4193 !
I.hasMetadata(LLVMContext::MD_nontemporal) &&
4194 !
I.hasMetadata(LLVMContext::MD_invariant_load) &&
4195 "Support volatile, non temporal, invariant for load_from_swift_error");
4197 const Value *SV =
I.getOperand(0);
4198 Type *Ty =
I.getType();
4203 I.getAAMetadata()))) &&
4204 "load_from_swift_error should not be constant memory");
4211 "expect a single EVT for swifterror");
4221 void SelectionDAGBuilder::visitStore(
const StoreInst &
I) {
4223 return visitAtomicStore(
I);
4225 const Value *SrcV =
I.getOperand(0);
4226 const Value *PtrV =
I.getOperand(1);
4232 if (
const Argument *
Arg = dyn_cast<Argument>(PtrV)) {
4233 if (
Arg->hasSwiftErrorAttr())
4234 return visitStoreToSwiftError(
I);
4237 if (
const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4238 if (Alloca->isSwiftError())
4239 return visitStoreToSwiftError(
I);
4247 unsigned NumValues = ValueVTs.size();
4270 unsigned ChainI = 0;
4271 for (
unsigned i = 0;
i != NumValues; ++
i, ++ChainI) {
4282 if (MemVTs[
i] != ValueVTs[
i])
4286 Alignment, MMOFlags, AAInfo);
4287 Chains[ChainI] = St;
4295 void SelectionDAGBuilder::visitMaskedStore(
const CallInst &
I,
4296 bool IsCompressing) {
4302 Src0 =
I.getArgOperand(0);
4303 Ptr =
I.getArgOperand(1);
4304 Alignment = cast<ConstantInt>(
I.getArgOperand(2))->getMaybeAlignValue();
4305 Mask =
I.getArgOperand(3);
4310 Src0 =
I.getArgOperand(0);
4311 Ptr =
I.getArgOperand(1);
4312 Mask =
I.getArgOperand(2);
4316 Value *PtrOperand, *MaskOperand, *Src0Operand;
4319 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4321 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4368 if (
auto *
C = dyn_cast<Constant>(Ptr)) {
4369 C =
C->getSplatValue();
4384 if (!
GEP ||
GEP->getParent() != CurBB)
4387 if (
GEP->getNumOperands() != 2)
4390 const Value *BasePtr =
GEP->getPointerOperand();
4391 const Value *IndexVal =
GEP->getOperand(
GEP->getNumOperands() - 1);