LLVM  4.0.0
SelectionDAGISel.cpp
Go to the documentation of this file.
1 //===-- SelectionDAGISel.cpp - Implement the SelectionDAGISel class -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements the SelectionDAGISel class.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "ScheduleDAGSDNodes.h"
16 #include "SelectionDAGBuilder.h"
18 #include "llvm/ADT/Statistic.h"
21 #include "llvm/Analysis/CFG.h"
24 #include "llvm/CodeGen/FastISel.h"
38 #include "llvm/IR/Constants.h"
39 #include "llvm/IR/DebugInfo.h"
40 #include "llvm/IR/Function.h"
41 #include "llvm/IR/InlineAsm.h"
42 #include "llvm/IR/Instructions.h"
43 #include "llvm/IR/IntrinsicInst.h"
44 #include "llvm/IR/Intrinsics.h"
45 #include "llvm/IR/LLVMContext.h"
46 #include "llvm/IR/Module.h"
47 #include "llvm/MC/MCAsmInfo.h"
48 #include "llvm/Support/Compiler.h"
49 #include "llvm/Support/Debug.h"
51 #include "llvm/Support/Timer.h"
61 #include <algorithm>
62 
63 using namespace llvm;
64 
65 #define DEBUG_TYPE "isel"
66 
67 STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on");
68 STATISTIC(NumFastIselSuccess, "Number of instructions fast isel selected");
69 STATISTIC(NumFastIselBlocks, "Number of blocks selected entirely by fast isel");
70 STATISTIC(NumDAGBlocks, "Number of blocks selected using DAG");
71 STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path");
72 STATISTIC(NumEntryBlocks, "Number of entry blocks encountered");
73 STATISTIC(NumFastIselFailLowerArguments,
74  "Number of entry blocks where fast isel failed to lower arguments");
75 
76 #ifndef NDEBUG
77 static cl::opt<bool>
78 EnableFastISelVerbose2("fast-isel-verbose2", cl::Hidden,
79  cl::desc("Enable extra verbose messages in the \"fast\" "
80  "instruction selector"));
81 
82  // Terminators
83 STATISTIC(NumFastIselFailRet,"Fast isel fails on Ret");
84 STATISTIC(NumFastIselFailBr,"Fast isel fails on Br");
85 STATISTIC(NumFastIselFailSwitch,"Fast isel fails on Switch");
86 STATISTIC(NumFastIselFailIndirectBr,"Fast isel fails on IndirectBr");
87 STATISTIC(NumFastIselFailInvoke,"Fast isel fails on Invoke");
88 STATISTIC(NumFastIselFailResume,"Fast isel fails on Resume");
89 STATISTIC(NumFastIselFailUnreachable,"Fast isel fails on Unreachable");
90 
91  // Standard binary operators...
92 STATISTIC(NumFastIselFailAdd,"Fast isel fails on Add");
93 STATISTIC(NumFastIselFailFAdd,"Fast isel fails on FAdd");
94 STATISTIC(NumFastIselFailSub,"Fast isel fails on Sub");
95 STATISTIC(NumFastIselFailFSub,"Fast isel fails on FSub");
96 STATISTIC(NumFastIselFailMul,"Fast isel fails on Mul");
97 STATISTIC(NumFastIselFailFMul,"Fast isel fails on FMul");
98 STATISTIC(NumFastIselFailUDiv,"Fast isel fails on UDiv");
99 STATISTIC(NumFastIselFailSDiv,"Fast isel fails on SDiv");
100 STATISTIC(NumFastIselFailFDiv,"Fast isel fails on FDiv");
101 STATISTIC(NumFastIselFailURem,"Fast isel fails on URem");
102 STATISTIC(NumFastIselFailSRem,"Fast isel fails on SRem");
103 STATISTIC(NumFastIselFailFRem,"Fast isel fails on FRem");
104 
105  // Logical operators...
106 STATISTIC(NumFastIselFailAnd,"Fast isel fails on And");
107 STATISTIC(NumFastIselFailOr,"Fast isel fails on Or");
108 STATISTIC(NumFastIselFailXor,"Fast isel fails on Xor");
109 
110  // Memory instructions...
111 STATISTIC(NumFastIselFailAlloca,"Fast isel fails on Alloca");
112 STATISTIC(NumFastIselFailLoad,"Fast isel fails on Load");
113 STATISTIC(NumFastIselFailStore,"Fast isel fails on Store");
114 STATISTIC(NumFastIselFailAtomicCmpXchg,"Fast isel fails on AtomicCmpXchg");
115 STATISTIC(NumFastIselFailAtomicRMW,"Fast isel fails on AtomicRWM");
116 STATISTIC(NumFastIselFailFence,"Fast isel fails on Frence");
117 STATISTIC(NumFastIselFailGetElementPtr,"Fast isel fails on GetElementPtr");
118 
119  // Convert instructions...
120 STATISTIC(NumFastIselFailTrunc,"Fast isel fails on Trunc");
121 STATISTIC(NumFastIselFailZExt,"Fast isel fails on ZExt");
122 STATISTIC(NumFastIselFailSExt,"Fast isel fails on SExt");
123 STATISTIC(NumFastIselFailFPTrunc,"Fast isel fails on FPTrunc");
124 STATISTIC(NumFastIselFailFPExt,"Fast isel fails on FPExt");
125 STATISTIC(NumFastIselFailFPToUI,"Fast isel fails on FPToUI");
126 STATISTIC(NumFastIselFailFPToSI,"Fast isel fails on FPToSI");
127 STATISTIC(NumFastIselFailUIToFP,"Fast isel fails on UIToFP");
128 STATISTIC(NumFastIselFailSIToFP,"Fast isel fails on SIToFP");
129 STATISTIC(NumFastIselFailIntToPtr,"Fast isel fails on IntToPtr");
130 STATISTIC(NumFastIselFailPtrToInt,"Fast isel fails on PtrToInt");
131 STATISTIC(NumFastIselFailBitCast,"Fast isel fails on BitCast");
132 
133  // Other instructions...
134 STATISTIC(NumFastIselFailICmp,"Fast isel fails on ICmp");
135 STATISTIC(NumFastIselFailFCmp,"Fast isel fails on FCmp");
136 STATISTIC(NumFastIselFailPHI,"Fast isel fails on PHI");
137 STATISTIC(NumFastIselFailSelect,"Fast isel fails on Select");
138 STATISTIC(NumFastIselFailCall,"Fast isel fails on Call");
139 STATISTIC(NumFastIselFailShl,"Fast isel fails on Shl");
140 STATISTIC(NumFastIselFailLShr,"Fast isel fails on LShr");
141 STATISTIC(NumFastIselFailAShr,"Fast isel fails on AShr");
142 STATISTIC(NumFastIselFailVAArg,"Fast isel fails on VAArg");
143 STATISTIC(NumFastIselFailExtractElement,"Fast isel fails on ExtractElement");
144 STATISTIC(NumFastIselFailInsertElement,"Fast isel fails on InsertElement");
145 STATISTIC(NumFastIselFailShuffleVector,"Fast isel fails on ShuffleVector");
146 STATISTIC(NumFastIselFailExtractValue,"Fast isel fails on ExtractValue");
147 STATISTIC(NumFastIselFailInsertValue,"Fast isel fails on InsertValue");
148 STATISTIC(NumFastIselFailLandingPad,"Fast isel fails on LandingPad");
149 
150 // Intrinsic instructions...
151 STATISTIC(NumFastIselFailIntrinsicCall, "Fast isel fails on Intrinsic call");
152 STATISTIC(NumFastIselFailSAddWithOverflow,
153  "Fast isel fails on sadd.with.overflow");
154 STATISTIC(NumFastIselFailUAddWithOverflow,
155  "Fast isel fails on uadd.with.overflow");
156 STATISTIC(NumFastIselFailSSubWithOverflow,
157  "Fast isel fails on ssub.with.overflow");
158 STATISTIC(NumFastIselFailUSubWithOverflow,
159  "Fast isel fails on usub.with.overflow");
160 STATISTIC(NumFastIselFailSMulWithOverflow,
161  "Fast isel fails on smul.with.overflow");
162 STATISTIC(NumFastIselFailUMulWithOverflow,
163  "Fast isel fails on umul.with.overflow");
164 STATISTIC(NumFastIselFailFrameaddress, "Fast isel fails on Frameaddress");
165 STATISTIC(NumFastIselFailSqrt, "Fast isel fails on sqrt call");
166 STATISTIC(NumFastIselFailStackMap, "Fast isel fails on StackMap call");
167 STATISTIC(NumFastIselFailPatchPoint, "Fast isel fails on PatchPoint call");
168 #endif
169 
170 static cl::opt<bool>
171 EnableFastISelVerbose("fast-isel-verbose", cl::Hidden,
172  cl::desc("Enable verbose messages in the \"fast\" "
173  "instruction selector"));
175  "fast-isel-abort", cl::Hidden,
176  cl::desc("Enable abort calls when \"fast\" instruction selection "
177  "fails to lower an instruction: 0 disable the abort, 1 will "
178  "abort but for args, calls and terminators, 2 will also "
179  "abort for argument lowering, and 3 will never fallback "
180  "to SelectionDAG."));
181 
182 static cl::opt<bool>
183 UseMBPI("use-mbpi",
184  cl::desc("use Machine Branch Probability Info"),
185  cl::init(true), cl::Hidden);
186 
187 #ifndef NDEBUG
189 FilterDAGBasicBlockName("filter-view-dags", cl::Hidden,
190  cl::desc("Only display the basic block whose name "
191  "matches this for all view-*-dags options"));
192 static cl::opt<bool>
193 ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden,
194  cl::desc("Pop up a window to show dags before the first "
195  "dag combine pass"));
196 static cl::opt<bool>
197 ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden,
198  cl::desc("Pop up a window to show dags before legalize types"));
199 static cl::opt<bool>
200 ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,
201  cl::desc("Pop up a window to show dags before legalize"));
202 static cl::opt<bool>
203 ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden,
204  cl::desc("Pop up a window to show dags before the second "
205  "dag combine pass"));
206 static cl::opt<bool>
207 ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden,
208  cl::desc("Pop up a window to show dags before the post legalize types"
209  " dag combine pass"));
210 static cl::opt<bool>
211 ViewISelDAGs("view-isel-dags", cl::Hidden,
212  cl::desc("Pop up a window to show isel dags as they are selected"));
213 static cl::opt<bool>
214 ViewSchedDAGs("view-sched-dags", cl::Hidden,
215  cl::desc("Pop up a window to show sched dags as they are processed"));
216 static cl::opt<bool>
217 ViewSUnitDAGs("view-sunit-dags", cl::Hidden,
218  cl::desc("Pop up a window to show SUnit dags after they are processed"));
219 #else
220 static const bool ViewDAGCombine1 = false,
221  ViewLegalizeTypesDAGs = false, ViewLegalizeDAGs = false,
222  ViewDAGCombine2 = false,
223  ViewDAGCombineLT = false,
224  ViewISelDAGs = false, ViewSchedDAGs = false,
225  ViewSUnitDAGs = false;
226 #endif
227 
228 //===---------------------------------------------------------------------===//
229 ///
230 /// RegisterScheduler class - Track the registration of instruction schedulers.
231 ///
232 //===---------------------------------------------------------------------===//
234 
235 //===---------------------------------------------------------------------===//
236 ///
237 /// ISHeuristic command line option for instruction schedulers.
238 ///
239 //===---------------------------------------------------------------------===//
242 ISHeuristic("pre-RA-sched",
244  cl::desc("Instruction schedulers available (before register"
245  " allocation):"));
246 
247 static RegisterScheduler
248 defaultListDAGScheduler("default", "Best scheduler for the target",
250 
251 namespace llvm {
252  //===--------------------------------------------------------------------===//
253  /// \brief This class is used by SelectionDAGISel to temporarily override
254  /// the optimization level on a per-function basis.
256  SelectionDAGISel &IS;
257  CodeGenOpt::Level SavedOptLevel;
258  bool SavedFastISel;
259 
260  public:
262  CodeGenOpt::Level NewOptLevel) : IS(ISel) {
263  SavedOptLevel = IS.OptLevel;
264  if (NewOptLevel == SavedOptLevel)
265  return;
266  IS.OptLevel = NewOptLevel;
267  IS.TM.setOptLevel(NewOptLevel);
268  DEBUG(dbgs() << "\nChanging optimization level for Function "
269  << IS.MF->getFunction()->getName() << "\n");
270  DEBUG(dbgs() << "\tBefore: -O" << SavedOptLevel
271  << " ; After: -O" << NewOptLevel << "\n");
272  SavedFastISel = IS.TM.Options.EnableFastISel;
273  if (NewOptLevel == CodeGenOpt::None) {
275  DEBUG(dbgs() << "\tFastISel is "
276  << (IS.TM.Options.EnableFastISel ? "enabled" : "disabled")
277  << "\n");
278  }
279  }
280 
282  if (IS.OptLevel == SavedOptLevel)
283  return;
284  DEBUG(dbgs() << "\nRestoring optimization level for Function "
285  << IS.MF->getFunction()->getName() << "\n");
286  DEBUG(dbgs() << "\tBefore: -O" << IS.OptLevel
287  << " ; After: -O" << SavedOptLevel << "\n");
288  IS.OptLevel = SavedOptLevel;
289  IS.TM.setOptLevel(SavedOptLevel);
290  IS.TM.setFastISel(SavedFastISel);
291  }
292  };
293 
294  //===--------------------------------------------------------------------===//
295  /// createDefaultScheduler - This creates an instruction scheduler appropriate
296  /// for the target.
298  CodeGenOpt::Level OptLevel) {
299  const TargetLowering *TLI = IS->TLI;
300  const TargetSubtargetInfo &ST = IS->MF->getSubtarget();
301 
302  // Try first to see if the Target has its own way of selecting a scheduler
303  if (auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) {
304  return SchedulerCtor(IS, OptLevel);
305  }
306 
307  if (OptLevel == CodeGenOpt::None ||
310  return createSourceListDAGScheduler(IS, OptLevel);
312  return createBURRListDAGScheduler(IS, OptLevel);
314  return createHybridListDAGScheduler(IS, OptLevel);
315  if (TLI->getSchedulingPreference() == Sched::VLIW)
316  return createVLIWDAGScheduler(IS, OptLevel);
318  "Unknown sched type!");
319  return createILPListDAGScheduler(IS, OptLevel);
320  }
321 } // end namespace llvm
322 
323 // EmitInstrWithCustomInserter - This method should be implemented by targets
324 // that mark instructions with the 'usesCustomInserter' flag. These
325 // instructions are special in various ways, which require special support to
326 // insert. The specified MachineInstr is created but not inserted into any
327 // basic blocks, and this method is called to expand it into a sequence of
328 // instructions, potentially also creating new basic blocks and control flow.
329 // When new basic blocks are inserted and the edges from MBB to its successors
330 // are modified, the method should insert pairs of <OldSucc, NewSucc> into the
331 // DenseMap.
334  MachineBasicBlock *MBB) const {
335 #ifndef NDEBUG
336  dbgs() << "If a target marks an instruction with "
337  "'usesCustomInserter', it must implement "
338  "TargetLowering::EmitInstrWithCustomInserter!";
339 #endif
340  llvm_unreachable(nullptr);
341 }
342 
344  SDNode *Node) const {
345  assert(!MI.hasPostISelHook() &&
346  "If a target marks an instruction with 'hasPostISelHook', "
347  "it must implement TargetLowering::AdjustInstrPostInstrSelection!");
348 }
349 
350 //===----------------------------------------------------------------------===//
351 // SelectionDAGISel code
352 //===----------------------------------------------------------------------===//
353 
355  CodeGenOpt::Level OL) :
356  MachineFunctionPass(ID), TM(tm),
357  FuncInfo(new FunctionLoweringInfo()),
358  CurDAG(new SelectionDAG(tm, OL)),
359  SDB(new SelectionDAGBuilder(*CurDAG, *FuncInfo, OL)),
360  GFI(),
361  OptLevel(OL),
362  DAGSize(0) {
369  }
370 
372  delete SDB;
373  delete CurDAG;
374  delete FuncInfo;
375 }
376 
387 }
388 
389 /// SplitCriticalSideEffectEdges - Look for critical edges with a PHI value that
390 /// may trap on it. In this case we have to split the edge so that the path
391 /// through the predecessor block that doesn't go to the phi block doesn't
392 /// execute the possibly trapping instruction.
393 ///
394 /// This is required for correctness, so it must be done at -O0.
395 ///
397  // Loop for blocks with phi nodes.
398  for (BasicBlock &BB : Fn) {
399  PHINode *PN = dyn_cast<PHINode>(BB.begin());
400  if (!PN) continue;
401 
402  ReprocessBlock:
403  // For each block with a PHI node, check to see if any of the input values
404  // are potentially trapping constant expressions. Constant expressions are
405  // the only potentially trapping value that can occur as the argument to a
406  // PHI.
407  for (BasicBlock::iterator I = BB.begin(); (PN = dyn_cast<PHINode>(I)); ++I)
408  for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
410  if (!CE || !CE->canTrap()) continue;
411 
412  // The only case we have to worry about is when the edge is critical.
413  // Since this block has a PHI Node, we assume it has multiple input
414  // edges: check to see if the pred has multiple successors.
415  BasicBlock *Pred = PN->getIncomingBlock(i);
416  if (Pred->getTerminator()->getNumSuccessors() == 1)
417  continue;
418 
419  // Okay, we have to split this edge.
421  Pred->getTerminator(), GetSuccessorNumber(Pred, &BB),
423  goto ReprocessBlock;
424  }
425  }
426 }
427 
429  // If we already selected that function, we do not need to run SDISel.
430  if (mf.getProperties().hasProperty(
432  return false;
433  // Do some sanity-checking on the command-line options.
435  "-fast-isel-verbose requires -fast-isel");
437  "-fast-isel-abort > 0 requires -fast-isel");
438 
439  const Function &Fn = *mf.getFunction();
440  MF = &mf;
441 
442  // Reset the target options before resetting the optimization
443  // level below.
444  // FIXME: This is a horrible hack and should be processed via
445  // codegen looking at the optimization level explicitly when
446  // it wants to look at it.
448  // Reset OptLevel to None for optnone functions.
449  CodeGenOpt::Level NewOptLevel = OptLevel;
450  if (OptLevel != CodeGenOpt::None && skipFunction(Fn))
451  NewOptLevel = CodeGenOpt::None;
452  OptLevelChanger OLC(*this, NewOptLevel);
453 
456  RegInfo = &MF->getRegInfo();
457  AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
458  LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
459  GFI = Fn.hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) : nullptr;
460 
461  DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n");
462 
463  SplitCriticalSideEffectEdges(const_cast<Function &>(Fn));
464 
465  CurDAG->init(*MF);
466  FuncInfo->set(Fn, *MF, CurDAG);
467 
469  FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
470  else
471  FuncInfo->BPI = nullptr;
472 
473  SDB->init(GFI, *AA, LibInfo);
474 
475  MF->setHasInlineAsm(false);
476 
477  FuncInfo->SplitCSR = false;
478 
479  // We split CSR if the target supports it for the given function
480  // and the function has only return exits.
482  FuncInfo->SplitCSR = true;
483 
484  // Collect all the return blocks.
485  for (const BasicBlock &BB : Fn) {
486  if (!succ_empty(&BB))
487  continue;
488 
489  const TerminatorInst *Term = BB.getTerminator();
490  if (isa<UnreachableInst>(Term) || isa<ReturnInst>(Term))
491  continue;
492 
493  // Bail out if the exit block is not Return nor Unreachable.
494  FuncInfo->SplitCSR = false;
495  break;
496  }
497  }
498 
499  MachineBasicBlock *EntryMBB = &MF->front();
500  if (FuncInfo->SplitCSR)
501  // This performs initialization so lowering for SplitCSR will be correct.
502  TLI->initializeSplitCSR(EntryMBB);
503 
504  SelectAllBasicBlocks(Fn);
505 
506  // If the first basic block in the function has live ins that need to be
507  // copied into vregs, emit the copies into the top of the block before
508  // emitting the code for the block.
510  RegInfo->EmitLiveInCopies(EntryMBB, TRI, *TII);
511 
512  // Insert copies in the entry block and the return blocks.
513  if (FuncInfo->SplitCSR) {
515  // Collect all the return blocks.
516  for (MachineBasicBlock &MBB : mf) {
517  if (!MBB.succ_empty())
518  continue;
519 
521  if (Term != MBB.end() && Term->isReturn()) {
522  Returns.push_back(&MBB);
523  continue;
524  }
525  }
526  TLI->insertCopiesSplitCSR(EntryMBB, Returns);
527  }
528 
530  if (!FuncInfo->ArgDbgValues.empty())
532  E = RegInfo->livein_end(); LI != E; ++LI)
533  if (LI->second)
534  LiveInMap.insert(std::make_pair(LI->first, LI->second));
535 
536  // Insert DBG_VALUE instructions for function arguments to the entry block.
537  for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) {
539  bool hasFI = MI->getOperand(0).isFI();
540  unsigned Reg =
541  hasFI ? TRI.getFrameRegister(*MF) : MI->getOperand(0).getReg();
543  EntryMBB->insert(EntryMBB->begin(), MI);
544  else {
546  if (Def) {
547  MachineBasicBlock::iterator InsertPos = Def;
548  // FIXME: VR def may not be in entry block.
549  Def->getParent()->insert(std::next(InsertPos), MI);
550  } else
551  DEBUG(dbgs() << "Dropping debug info for dead vreg"
552  << TargetRegisterInfo::virtReg2Index(Reg) << "\n");
553  }
554 
555  // If Reg is live-in then update debug info to track its copy in a vreg.
556  DenseMap<unsigned, unsigned>::iterator LDI = LiveInMap.find(Reg);
557  if (LDI != LiveInMap.end()) {
558  assert(!hasFI && "There's no handling of frame pointer updating here yet "
559  "- add if needed");
560  MachineInstr *Def = RegInfo->getVRegDef(LDI->second);
561  MachineBasicBlock::iterator InsertPos = Def;
562  const MDNode *Variable = MI->getDebugVariable();
563  const MDNode *Expr = MI->getDebugExpression();
564  DebugLoc DL = MI->getDebugLoc();
565  bool IsIndirect = MI->isIndirectDebugValue();
566  unsigned Offset = IsIndirect ? MI->getOperand(1).getImm() : 0;
567  assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&
568  "Expected inlined-at fields to agree");
569  // Def is never a terminator here, so it is ok to increment InsertPos.
570  BuildMI(*EntryMBB, ++InsertPos, DL, TII->get(TargetOpcode::DBG_VALUE),
571  IsIndirect, LDI->second, Offset, Variable, Expr);
572 
573  // If this vreg is directly copied into an exported register then
574  // that COPY instructions also need DBG_VALUE, if it is the only
575  // user of LDI->second.
576  MachineInstr *CopyUseMI = nullptr;
578  UI = RegInfo->use_instr_begin(LDI->second),
579  E = RegInfo->use_instr_end(); UI != E; ) {
580  MachineInstr *UseMI = &*(UI++);
581  if (UseMI->isDebugValue()) continue;
582  if (UseMI->isCopy() && !CopyUseMI && UseMI->getParent() == EntryMBB) {
583  CopyUseMI = UseMI; continue;
584  }
585  // Otherwise this is another use or second copy use.
586  CopyUseMI = nullptr; break;
587  }
588  if (CopyUseMI) {
589  // Use MI's debug location, which describes where Variable was
590  // declared, rather than whatever is attached to CopyUseMI.
591  MachineInstr *NewMI =
592  BuildMI(*MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
593  CopyUseMI->getOperand(0).getReg(), Offset, Variable, Expr);
594  MachineBasicBlock::iterator Pos = CopyUseMI;
595  EntryMBB->insertAfter(Pos, NewMI);
596  }
597  }
598  }
599 
600  // Determine if there are any calls in this machine function.
601  MachineFrameInfo &MFI = MF->getFrameInfo();
602  for (const auto &MBB : *MF) {
603  if (MFI.hasCalls() && MF->hasInlineAsm())
604  break;
605 
606  for (const auto &MI : MBB) {
607  const MCInstrDesc &MCID = TII->get(MI.getOpcode());
608  if ((MCID.isCall() && !MCID.isReturn()) ||
609  MI.isStackAligningInlineAsm()) {
610  MFI.setHasCalls(true);
611  }
612  if (MI.isInlineAsm()) {
613  MF->setHasInlineAsm(true);
614  }
615  }
616  }
617 
618  // Determine if there is a call to setjmp in the machine function.
619  MF->setExposesReturnsTwice(Fn.callsFunctionThatReturnsTwice());
620 
621  // Replace forward-declared registers with the registers containing
622  // the desired value.
623  MachineRegisterInfo &MRI = MF->getRegInfo();
626  I != E; ++I) {
627  unsigned From = I->first;
628  unsigned To = I->second;
629  // If To is also scheduled to be replaced, find what its ultimate
630  // replacement is.
631  for (;;) {
633  if (J == E) break;
634  To = J->second;
635  }
636  // Make sure the new register has a sufficiently constrained register class.
639  MRI.constrainRegClass(To, MRI.getRegClass(From));
640  // Replace it.
641 
642 
643  // Replacing one register with another won't touch the kill flags.
644  // We need to conservatively clear the kill flags as a kill on the old
645  // register might dominate existing uses of the new register.
646  if (!MRI.use_empty(To))
647  MRI.clearKillFlags(From);
648  MRI.replaceRegWith(From, To);
649  }
650 
653 
654  // Freeze the set of reserved registers now that MachineFrameInfo has been
655  // set up. All the information required by getReservedRegs() should be
656  // available now.
657  MRI.freezeReservedRegs(*MF);
658 
659  // Release function-specific state. SDB and CurDAG are already cleared
660  // at this point.
661  FuncInfo->clear();
662 
663  DEBUG(dbgs() << "*** MachineFunction at end of ISel ***\n");
664  DEBUG(MF->print(dbgs()));
665 
666  return true;
667 }
668 
669 void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin,
671  bool &HadTailCall) {
672  // Lower the instructions. If a call is emitted as a tail call, cease emitting
673  // nodes for this block.
674  for (BasicBlock::const_iterator I = Begin; I != End && !SDB->HasTailCall; ++I)
675  SDB->visit(*I);
676 
677  // Make sure the root of the DAG is up-to-date.
679  HadTailCall = SDB->HasTailCall;
680  SDB->clear();
681 
682  // Final step, emit the lowered DAG as machine code.
683  CodeGenAndEmitDAG();
684 }
685 
686 void SelectionDAGISel::ComputeLiveOutVRegInfo() {
687  SmallPtrSet<SDNode*, 16> VisitedNodes;
688  SmallVector<SDNode*, 128> Worklist;
689 
690  Worklist.push_back(CurDAG->getRoot().getNode());
691 
692  APInt KnownZero;
693  APInt KnownOne;
694 
695  do {
696  SDNode *N = Worklist.pop_back_val();
697 
698  // If we've already seen this node, ignore it.
699  if (!VisitedNodes.insert(N).second)
700  continue;
701 
702  // Otherwise, add all chain operands to the worklist.
703  for (const SDValue &Op : N->op_values())
704  if (Op.getValueType() == MVT::Other)
705  Worklist.push_back(Op.getNode());
706 
707  // If this is a CopyToReg with a vreg dest, process it.
708  if (N->getOpcode() != ISD::CopyToReg)
709  continue;
710 
711  unsigned DestReg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
713  continue;
714 
715  // Ignore non-scalar or non-integer values.
716  SDValue Src = N->getOperand(2);
717  EVT SrcVT = Src.getValueType();
718  if (!SrcVT.isInteger() || SrcVT.isVector())
719  continue;
720 
721  unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src);
722  CurDAG->computeKnownBits(Src, KnownZero, KnownOne);
723  FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, KnownZero, KnownOne);
724  } while (!Worklist.empty());
725 }
726 
727 void SelectionDAGISel::CodeGenAndEmitDAG() {
728  StringRef GroupName = "sdag";
729  StringRef GroupDescription = "Instruction Selection and Scheduling";
730  std::string BlockName;
731  int BlockNumber = -1;
732  (void)BlockNumber;
733  bool MatchFilterBB = false; (void)MatchFilterBB;
734 #ifndef NDEBUG
735  MatchFilterBB = (FilterDAGBasicBlockName.empty() ||
738 #endif
739 #ifdef NDEBUG
743 #endif
744  {
745  BlockNumber = FuncInfo->MBB->getNumber();
746  BlockName =
747  (MF->getName() + ":" + FuncInfo->MBB->getBasicBlock()->getName()).str();
748  }
749  DEBUG(dbgs() << "Initial selection DAG: BB#" << BlockNumber
750  << " '" << BlockName << "'\n"; CurDAG->dump());
751 
752  if (ViewDAGCombine1 && MatchFilterBB)
753  CurDAG->viewGraph("dag-combine1 input for " + BlockName);
754 
755  // Run the DAG combiner in pre-legalize mode.
756  {
757  NamedRegionTimer T("combine1", "DAG Combining 1", GroupName,
758  GroupDescription, TimePassesIsEnabled);
760  }
761 
762  DEBUG(dbgs() << "Optimized lowered selection DAG: BB#" << BlockNumber
763  << " '" << BlockName << "'\n"; CurDAG->dump());
764 
765  // Second step, hack on the DAG until it only uses operations and types that
766  // the target supports.
767  if (ViewLegalizeTypesDAGs && MatchFilterBB)
768  CurDAG->viewGraph("legalize-types input for " + BlockName);
769 
770  bool Changed;
771  {
772  NamedRegionTimer T("legalize_types", "Type Legalization", GroupName,
773  GroupDescription, TimePassesIsEnabled);
774  Changed = CurDAG->LegalizeTypes();
775  }
776 
777  DEBUG(dbgs() << "Type-legalized selection DAG: BB#" << BlockNumber
778  << " '" << BlockName << "'\n"; CurDAG->dump());
779 
781 
782  if (Changed) {
783  if (ViewDAGCombineLT && MatchFilterBB)
784  CurDAG->viewGraph("dag-combine-lt input for " + BlockName);
785 
786  // Run the DAG combiner in post-type-legalize mode.
787  {
788  NamedRegionTimer T("combine_lt", "DAG Combining after legalize types",
789  GroupName, GroupDescription, TimePassesIsEnabled);
791  }
792 
793  DEBUG(dbgs() << "Optimized type-legalized selection DAG: BB#" << BlockNumber
794  << " '" << BlockName << "'\n"; CurDAG->dump());
795 
796  }
797 
798  {
799  NamedRegionTimer T("legalize_vec", "Vector Legalization", GroupName,
800  GroupDescription, TimePassesIsEnabled);
801  Changed = CurDAG->LegalizeVectors();
802  }
803 
804  if (Changed) {
805  {
806  NamedRegionTimer T("legalize_types2", "Type Legalization 2", GroupName,
807  GroupDescription, TimePassesIsEnabled);
809  }
810 
811  if (ViewDAGCombineLT && MatchFilterBB)
812  CurDAG->viewGraph("dag-combine-lv input for " + BlockName);
813 
814  // Run the DAG combiner in post-type-legalize mode.
815  {
816  NamedRegionTimer T("combine_lv", "DAG Combining after legalize vectors",
817  GroupName, GroupDescription, TimePassesIsEnabled);
819  }
820 
821  DEBUG(dbgs() << "Optimized vector-legalized selection DAG: BB#"
822  << BlockNumber << " '" << BlockName << "'\n"; CurDAG->dump());
823  }
824 
825  if (ViewLegalizeDAGs && MatchFilterBB)
826  CurDAG->viewGraph("legalize input for " + BlockName);
827 
828  {
829  NamedRegionTimer T("legalize", "DAG Legalization", GroupName,
830  GroupDescription, TimePassesIsEnabled);
831  CurDAG->Legalize();
832  }
833 
834  DEBUG(dbgs() << "Legalized selection DAG: BB#" << BlockNumber
835  << " '" << BlockName << "'\n"; CurDAG->dump());
836 
837  if (ViewDAGCombine2 && MatchFilterBB)
838  CurDAG->viewGraph("dag-combine2 input for " + BlockName);
839 
840  // Run the DAG combiner in post-legalize mode.
841  {
842  NamedRegionTimer T("combine2", "DAG Combining 2", GroupName,
843  GroupDescription, TimePassesIsEnabled);
845  }
846 
847  DEBUG(dbgs() << "Optimized legalized selection DAG: BB#" << BlockNumber
848  << " '" << BlockName << "'\n"; CurDAG->dump());
849 
850  if (OptLevel != CodeGenOpt::None)
851  ComputeLiveOutVRegInfo();
852 
853  if (ViewISelDAGs && MatchFilterBB)
854  CurDAG->viewGraph("isel input for " + BlockName);
855 
856  // Third, instruction select all of the operations to machine code, adding the
857  // code to the MachineBasicBlock.
858  {
859  NamedRegionTimer T("isel", "Instruction Selection", GroupName,
860  GroupDescription, TimePassesIsEnabled);
861  DoInstructionSelection();
862  }
863 
864  DEBUG(dbgs() << "Selected selection DAG: BB#" << BlockNumber
865  << " '" << BlockName << "'\n"; CurDAG->dump());
866 
867  if (ViewSchedDAGs && MatchFilterBB)
868  CurDAG->viewGraph("scheduler input for " + BlockName);
869 
870  // Schedule machine code.
871  ScheduleDAGSDNodes *Scheduler = CreateScheduler();
872  {
873  NamedRegionTimer T("sched", "Instruction Scheduling", GroupName,
874  GroupDescription, TimePassesIsEnabled);
875  Scheduler->Run(CurDAG, FuncInfo->MBB);
876  }
877 
878  if (ViewSUnitDAGs && MatchFilterBB)
879  Scheduler->viewGraph();
880 
881  // Emit machine code to BB. This can change 'BB' to the last block being
882  // inserted into.
883  MachineBasicBlock *FirstMBB = FuncInfo->MBB, *LastMBB;
884  {
885  NamedRegionTimer T("emit", "Instruction Creation", GroupName,
886  GroupDescription, TimePassesIsEnabled);
887 
888  // FuncInfo->InsertPt is passed by reference and set to the end of the
889  // scheduled instructions.
890  LastMBB = FuncInfo->MBB = Scheduler->EmitSchedule(FuncInfo->InsertPt);
891  }
892 
893  // If the block was split, make sure we update any references that are used to
894  // update PHI nodes later on.
895  if (FirstMBB != LastMBB)
896  SDB->UpdateSplitBlock(FirstMBB, LastMBB);
897 
898  // Free the scheduler state.
899  {
900  NamedRegionTimer T("cleanup", "Instruction Scheduling Cleanup", GroupName,
901  GroupDescription, TimePassesIsEnabled);
902  delete Scheduler;
903  }
904 
905  // Free the SelectionDAG state, now that we're finished with it.
906  CurDAG->clear();
907 }
908 
909 namespace {
910 /// ISelUpdater - helper class to handle updates of the instruction selection
911 /// graph.
912 class ISelUpdater : public SelectionDAG::DAGUpdateListener {
913  SelectionDAG::allnodes_iterator &ISelPosition;
914 public:
915  ISelUpdater(SelectionDAG &DAG, SelectionDAG::allnodes_iterator &isp)
916  : SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}
917 
918  /// NodeDeleted - Handle nodes deleted from the graph. If the node being
919  /// deleted is the current ISelPosition node, update ISelPosition.
920  ///
921  void NodeDeleted(SDNode *N, SDNode *E) override {
922  if (ISelPosition == SelectionDAG::allnodes_iterator(N))
923  ++ISelPosition;
924  }
925 };
926 } // end anonymous namespace
927 
928 void SelectionDAGISel::DoInstructionSelection() {
929  DEBUG(dbgs() << "===== Instruction selection begins: BB#"
930  << FuncInfo->MBB->getNumber()
931  << " '" << FuncInfo->MBB->getName() << "'\n");
932 
934 
935  // Select target instructions for the DAG.
936  {
937  // Number all nodes with a topological order and set DAGSize.
939 
940  // Create a dummy node (which is not added to allnodes), that adds
941  // a reference to the root node, preventing it from being deleted,
942  // and tracking any changes of the root.
945  ++ISelPosition;
946 
947  // Make sure that ISelPosition gets properly updated when nodes are deleted
948  // in calls made from this function.
949  ISelUpdater ISU(*CurDAG, ISelPosition);
950 
951  // The AllNodes list is now topological-sorted. Visit the
952  // nodes by starting at the end of the list (the root of the
953  // graph) and preceding back toward the beginning (the entry
954  // node).
955  while (ISelPosition != CurDAG->allnodes_begin()) {
956  SDNode *Node = &*--ISelPosition;
957  // Skip dead nodes. DAGCombiner is expected to eliminate all dead nodes,
958  // but there are currently some corner cases that it misses. Also, this
959  // makes it theoretically possible to disable the DAGCombiner.
960  if (Node->use_empty())
961  continue;
962 
963  Select(Node);
964  }
965 
966  CurDAG->setRoot(Dummy.getValue());
967  }
968 
969  DEBUG(dbgs() << "===== Instruction selection ends:\n");
970 
972 }
973 
975  for (const User *U : CPI->users()) {
976  if (const IntrinsicInst *EHPtrCall = dyn_cast<IntrinsicInst>(U)) {
977  Intrinsic::ID IID = EHPtrCall->getIntrinsicID();
978  if (IID == Intrinsic::eh_exceptionpointer ||
979  IID == Intrinsic::eh_exceptioncode)
980  return true;
981  }
982  }
983  return false;
984 }
985 
986 /// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and
987 /// do other setup for EH landing-pad blocks.
988 bool SelectionDAGISel::PrepareEHLandingPad() {
990  const Constant *PersonalityFn = FuncInfo->Fn->getPersonalityFn();
991  const BasicBlock *LLVMBB = MBB->getBasicBlock();
992  const TargetRegisterClass *PtrRC =
994 
995  // Catchpads have one live-in register, which typically holds the exception
996  // pointer or code.
997  if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI())) {
999  // Get or create the virtual register to hold the pointer or code. Mark
1000  // the live in physreg and copy into the vreg.
1001  MCPhysReg EHPhysReg = TLI->getExceptionPointerRegister(PersonalityFn);
1002  assert(EHPhysReg && "target lacks exception pointer register");
1003  MBB->addLiveIn(EHPhysReg);
1004  unsigned VReg = FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC);
1006  TII->get(TargetOpcode::COPY), VReg)
1007  .addReg(EHPhysReg, RegState::Kill);
1008  }
1009  return true;
1010  }
1011 
1012  if (!LLVMBB->isLandingPad())
1013  return true;
1014 
1015  // Add a label to mark the beginning of the landing pad. Deletion of the
1016  // landing pad can thus be detected via the MachineModuleInfo.
1017  MCSymbol *Label = MF->addLandingPad(MBB);
1018 
1019  // Assign the call site to the landing pad's begin label.
1021 
1022  const MCInstrDesc &II = TII->get(TargetOpcode::EH_LABEL);
1023  BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II)
1024  .addSym(Label);
1025 
1026  // Mark exception register as live in.
1027  if (unsigned Reg = TLI->getExceptionPointerRegister(PersonalityFn))
1029 
1030  // Mark exception selector register as live in.
1031  if (unsigned Reg = TLI->getExceptionSelectorRegister(PersonalityFn))
1033 
1034  return true;
1035 }
1036 
1037 /// isFoldedOrDeadInstruction - Return true if the specified instruction is
1038 /// side-effect free and is either dead or folded into a generated instruction.
1039 /// Return false if it needs to be emitted.
1041  FunctionLoweringInfo *FuncInfo) {
1042  return !I->mayWriteToMemory() && // Side-effecting instructions aren't folded.
1043  !isa<TerminatorInst>(I) && // Terminators aren't folded.
1044  !isa<DbgInfoIntrinsic>(I) && // Debug instructions aren't folded.
1045  !I->isEHPad() && // EH pad instructions aren't folded.
1046  !FuncInfo->isExportedInst(I); // Exported instrs must be computed.
1047 }
1048 
1049 #ifndef NDEBUG
1050 // Collect per Instruction statistics for fast-isel misses. Only those
1051 // instructions that cause the bail are accounted for. It does not account for
1052 // instructions higher in the block. Thus, summing the per instructions stats
1053 // will not add up to what is reported by NumFastIselFailures.
1054 static void collectFailStats(const Instruction *I) {
1055  switch (I->getOpcode()) {
1056  default: assert (0 && "<Invalid operator> ");
1057 
1058  // Terminators
1059  case Instruction::Ret: NumFastIselFailRet++; return;
1060  case Instruction::Br: NumFastIselFailBr++; return;
1061  case Instruction::Switch: NumFastIselFailSwitch++; return;
1062  case Instruction::IndirectBr: NumFastIselFailIndirectBr++; return;
1063  case Instruction::Invoke: NumFastIselFailInvoke++; return;
1064  case Instruction::Resume: NumFastIselFailResume++; return;
1065  case Instruction::Unreachable: NumFastIselFailUnreachable++; return;
1066 
1067  // Standard binary operators...
1068  case Instruction::Add: NumFastIselFailAdd++; return;
1069  case Instruction::FAdd: NumFastIselFailFAdd++; return;
1070  case Instruction::Sub: NumFastIselFailSub++; return;
1071  case Instruction::FSub: NumFastIselFailFSub++; return;
1072  case Instruction::Mul: NumFastIselFailMul++; return;
1073  case Instruction::FMul: NumFastIselFailFMul++; return;
1074  case Instruction::UDiv: NumFastIselFailUDiv++; return;
1075  case Instruction::SDiv: NumFastIselFailSDiv++; return;
1076  case Instruction::FDiv: NumFastIselFailFDiv++; return;
1077  case Instruction::URem: NumFastIselFailURem++; return;
1078  case Instruction::SRem: NumFastIselFailSRem++; return;
1079  case Instruction::FRem: NumFastIselFailFRem++; return;
1080 
1081  // Logical operators...
1082  case Instruction::And: NumFastIselFailAnd++; return;
1083  case Instruction::Or: NumFastIselFailOr++; return;
1084  case Instruction::Xor: NumFastIselFailXor++; return;
1085 
1086  // Memory instructions...
1087  case Instruction::Alloca: NumFastIselFailAlloca++; return;
1088  case Instruction::Load: NumFastIselFailLoad++; return;
1089  case Instruction::Store: NumFastIselFailStore++; return;
1090  case Instruction::AtomicCmpXchg: NumFastIselFailAtomicCmpXchg++; return;
1091  case Instruction::AtomicRMW: NumFastIselFailAtomicRMW++; return;
1092  case Instruction::Fence: NumFastIselFailFence++; return;
1093  case Instruction::GetElementPtr: NumFastIselFailGetElementPtr++; return;
1094 
1095  // Convert instructions...
1096  case Instruction::Trunc: NumFastIselFailTrunc++; return;
1097  case Instruction::ZExt: NumFastIselFailZExt++; return;
1098  case Instruction::SExt: NumFastIselFailSExt++; return;
1099  case Instruction::FPTrunc: NumFastIselFailFPTrunc++; return;
1100  case Instruction::FPExt: NumFastIselFailFPExt++; return;
1101  case Instruction::FPToUI: NumFastIselFailFPToUI++; return;
1102  case Instruction::FPToSI: NumFastIselFailFPToSI++; return;
1103  case Instruction::UIToFP: NumFastIselFailUIToFP++; return;
1104  case Instruction::SIToFP: NumFastIselFailSIToFP++; return;
1105  case Instruction::IntToPtr: NumFastIselFailIntToPtr++; return;
1106  case Instruction::PtrToInt: NumFastIselFailPtrToInt++; return;
1107  case Instruction::BitCast: NumFastIselFailBitCast++; return;
1108 
1109  // Other instructions...
1110  case Instruction::ICmp: NumFastIselFailICmp++; return;
1111  case Instruction::FCmp: NumFastIselFailFCmp++; return;
1112  case Instruction::PHI: NumFastIselFailPHI++; return;
1113  case Instruction::Select: NumFastIselFailSelect++; return;
1114  case Instruction::Call: {
1115  if (auto const *Intrinsic = dyn_cast<IntrinsicInst>(I)) {
1116  switch (Intrinsic->getIntrinsicID()) {
1117  default:
1118  NumFastIselFailIntrinsicCall++; return;
1119  case Intrinsic::sadd_with_overflow:
1120  NumFastIselFailSAddWithOverflow++; return;
1121  case Intrinsic::uadd_with_overflow:
1122  NumFastIselFailUAddWithOverflow++; return;
1123  case Intrinsic::ssub_with_overflow:
1124  NumFastIselFailSSubWithOverflow++; return;
1125  case Intrinsic::usub_with_overflow:
1126  NumFastIselFailUSubWithOverflow++; return;
1127  case Intrinsic::smul_with_overflow:
1128  NumFastIselFailSMulWithOverflow++; return;
1129  case Intrinsic::umul_with_overflow:
1130  NumFastIselFailUMulWithOverflow++; return;
1131  case Intrinsic::frameaddress:
1132  NumFastIselFailFrameaddress++; return;
1133  case Intrinsic::sqrt:
1134  NumFastIselFailSqrt++; return;
1135  case Intrinsic::experimental_stackmap:
1136  NumFastIselFailStackMap++; return;
1137  case Intrinsic::experimental_patchpoint_void: // fall-through
1138  case Intrinsic::experimental_patchpoint_i64:
1139  NumFastIselFailPatchPoint++; return;
1140  }
1141  }
1142  NumFastIselFailCall++;
1143  return;
1144  }
1145  case Instruction::Shl: NumFastIselFailShl++; return;
1146  case Instruction::LShr: NumFastIselFailLShr++; return;
1147  case Instruction::AShr: NumFastIselFailAShr++; return;
1148  case Instruction::VAArg: NumFastIselFailVAArg++; return;
1149  case Instruction::ExtractElement: NumFastIselFailExtractElement++; return;
1150  case Instruction::InsertElement: NumFastIselFailInsertElement++; return;
1151  case Instruction::ShuffleVector: NumFastIselFailShuffleVector++; return;
1152  case Instruction::ExtractValue: NumFastIselFailExtractValue++; return;
1153  case Instruction::InsertValue: NumFastIselFailInsertValue++; return;
1154  case Instruction::LandingPad: NumFastIselFailLandingPad++; return;
1155  }
1156 }
1157 #endif // NDEBUG
1158 
1159 /// Set up SwiftErrorVals by going through the function. If the function has
1160 /// swifterror argument, it will be the first entry.
1161 static void setupSwiftErrorVals(const Function &Fn, const TargetLowering *TLI,
1162  FunctionLoweringInfo *FuncInfo) {
1163  if (!TLI->supportSwiftError())
1164  return;
1165 
1166  FuncInfo->SwiftErrorVals.clear();
1167  FuncInfo->SwiftErrorVRegDefMap.clear();
1168  FuncInfo->SwiftErrorVRegUpwardsUse.clear();
1169  FuncInfo->SwiftErrorArg = nullptr;
1170 
1171  // Check if function has a swifterror argument.
1172  bool HaveSeenSwiftErrorArg = false;
1173  for (Function::const_arg_iterator AI = Fn.arg_begin(), AE = Fn.arg_end();
1174  AI != AE; ++AI)
1175  if (AI->hasSwiftErrorAttr()) {
1176  assert(!HaveSeenSwiftErrorArg &&
1177  "Must have only one swifterror parameter");
1178  (void)HaveSeenSwiftErrorArg; // silence warning.
1179  HaveSeenSwiftErrorArg = true;
1180  FuncInfo->SwiftErrorArg = &*AI;
1181  FuncInfo->SwiftErrorVals.push_back(&*AI);
1182  }
1183 
1184  for (const auto &LLVMBB : Fn)
1185  for (const auto &Inst : LLVMBB) {
1186  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(&Inst))
1187  if (Alloca->isSwiftError())
1188  FuncInfo->SwiftErrorVals.push_back(Alloca);
1189  }
1190 }
1191 
1193  const TargetLowering *TLI,
1194  const TargetInstrInfo *TII,
1195  const BasicBlock *LLVMBB,
1196  SelectionDAGBuilder *SDB) {
1197  if (!TLI->supportSwiftError())
1198  return;
1199 
1200  // We only need to do this when we have swifterror parameter or swifterror
1201  // alloc.
1202  if (FuncInfo->SwiftErrorVals.empty())
1203  return;
1204 
1205  if (pred_begin(LLVMBB) == pred_end(LLVMBB)) {
1206  auto &DL = FuncInfo->MF->getDataLayout();
1207  auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL));
1208  for (const auto *SwiftErrorVal : FuncInfo->SwiftErrorVals) {
1209  // We will always generate a copy from the argument. It is always used at
1210  // least by the 'return' of the swifterror.
1211  if (FuncInfo->SwiftErrorArg && FuncInfo->SwiftErrorArg == SwiftErrorVal)
1212  continue;
1213  unsigned VReg = FuncInfo->MF->getRegInfo().createVirtualRegister(RC);
1214  // Assign Undef to Vreg. We construct MI directly to make sure it works
1215  // with FastISel.
1216  BuildMI(*FuncInfo->MBB, FuncInfo->MBB->getFirstNonPHI(),
1217  SDB->getCurDebugLoc(), TII->get(TargetOpcode::IMPLICIT_DEF),
1218  VReg);
1219  FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB, SwiftErrorVal, VReg);
1220  }
1221  }
1222 }
1223 
1224 /// Propagate swifterror values through the machine function CFG.
1226  auto *TLI = FuncInfo->TLI;
1227  if (!TLI->supportSwiftError())
1228  return;
1229 
1230  // We only need to do this when we have swifterror parameter or swifterror
1231  // alloc.
1232  if (FuncInfo->SwiftErrorVals.empty())
1233  return;
1234 
1235  // For each machine basic block in reverse post order.
1238  It = RPOT.begin(),
1239  E = RPOT.end();
1240  It != E; ++It) {
1241  MachineBasicBlock *MBB = *It;
1242 
1243  // For each swifterror value in the function.
1244  for(const auto *SwiftErrorVal : FuncInfo->SwiftErrorVals) {
1245  auto Key = std::make_pair(MBB, SwiftErrorVal);
1246  auto UUseIt = FuncInfo->SwiftErrorVRegUpwardsUse.find(Key);
1247  auto VRegDefIt = FuncInfo->SwiftErrorVRegDefMap.find(Key);
1248  bool UpwardsUse = UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end();
1249  unsigned UUseVReg = UpwardsUse ? UUseIt->second : 0;
1250  bool DownwardDef = VRegDefIt != FuncInfo->SwiftErrorVRegDefMap.end();
1251  assert(!(UpwardsUse && !DownwardDef) &&
1252  "We can't have an upwards use but no downwards def");
1253 
1254  // If there is no upwards exposed use and an entry for the swifterror in
1255  // the def map for this value we don't need to do anything: We already
1256  // have a downward def for this basic block.
1257  if (!UpwardsUse && DownwardDef)
1258  continue;
1259 
1260  // Otherwise we either have an upwards exposed use vreg that we need to
1261  // materialize or need to forward the downward def from predecessors.
1262 
1263  // Check whether we have a single vreg def from all predecessors.
1264  // Otherwise we need a phi.
1267  for (auto *Pred : MBB->predecessors()) {
1268  if (!Visited.insert(Pred).second)
1269  continue;
1270  VRegs.push_back(std::make_pair(
1271  Pred, FuncInfo->getOrCreateSwiftErrorVReg(Pred, SwiftErrorVal)));
1272  if (Pred != MBB)
1273  continue;
1274  // We have a self-edge.
1275  // If there was no upwards use in this basic block there is now one: the
1276  // phi needs to use it self.
1277  if (!UpwardsUse) {
1278  UpwardsUse = true;
1279  UUseIt = FuncInfo->SwiftErrorVRegUpwardsUse.find(Key);
1280  assert(UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end());
1281  UUseVReg = UUseIt->second;
1282  }
1283  }
1284 
1285  // We need a phi node if we have more than one predecessor with different
1286  // downward defs.
1287  bool needPHI =
1288  VRegs.size() >= 1 &&
1289  std::find_if(
1290  VRegs.begin(), VRegs.end(),
1291  [&](const std::pair<const MachineBasicBlock *, unsigned> &V)
1292  -> bool { return V.second != VRegs[0].second; }) !=
1293  VRegs.end();
1294 
1295  // If there is no upwards exposed used and we don't need a phi just
1296  // forward the swifterror vreg from the predecessor(s).
1297  if (!UpwardsUse && !needPHI) {
1298  assert(!VRegs.empty() &&
1299  "No predecessors? The entry block should bail out earlier");
1300  // Just forward the swifterror vreg from the predecessor(s).
1301  FuncInfo->setCurrentSwiftErrorVReg(MBB, SwiftErrorVal, VRegs[0].second);
1302  continue;
1303  }
1304 
1305  auto DLoc = isa<Instruction>(SwiftErrorVal)
1306  ? dyn_cast<Instruction>(SwiftErrorVal)->getDebugLoc()
1307  : DebugLoc();
1308  const auto *TII = FuncInfo->MF->getSubtarget().getInstrInfo();
1309 
1310  // If we don't need a phi create a copy to the upward exposed vreg.
1311  if (!needPHI) {
1312  assert(UpwardsUse);
1313  unsigned DestReg = UUseVReg;
1314  BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc, TII->get(TargetOpcode::COPY),
1315  DestReg)
1316  .addReg(VRegs[0].second);
1317  continue;
1318  }
1319 
1320  // We need a phi: if there is an upwards exposed use we already have a
1321  // destination virtual register number otherwise we generate a new one.
1322  auto &DL = FuncInfo->MF->getDataLayout();
1323  auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL));
1324  unsigned PHIVReg =
1325  UpwardsUse ? UUseVReg
1326  : FuncInfo->MF->getRegInfo().createVirtualRegister(RC);
1327  MachineInstrBuilder SwiftErrorPHI =
1328  BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc,
1329  TII->get(TargetOpcode::PHI), PHIVReg);
1330  for (auto BBRegPair : VRegs) {
1331  SwiftErrorPHI.addReg(BBRegPair.second).addMBB(BBRegPair.first);
1332  }
1333 
1334  // We did not have a definition in this block before: store the phi's vreg
1335  // as this block downward exposed def.
1336  if (!UpwardsUse)
1337  FuncInfo->setCurrentSwiftErrorVReg(MBB, SwiftErrorVal, PHIVReg);
1338  }
1339  }
1340 }
1341 
1342 void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
1343  // Initialize the Fast-ISel state, if needed.
1344  FastISel *FastIS = nullptr;
1346  FastIS = TLI->createFastISel(*FuncInfo, LibInfo);
1347 
1349 
1350  // Iterate over all basic blocks in the function.
1353  I = RPOT.begin(), E = RPOT.end(); I != E; ++I) {
1354  const BasicBlock *LLVMBB = *I;
1355 
1356  if (OptLevel != CodeGenOpt::None) {
1357  bool AllPredsVisited = true;
1358  for (const_pred_iterator PI = pred_begin(LLVMBB), PE = pred_end(LLVMBB);
1359  PI != PE; ++PI) {
1360  if (!FuncInfo->VisitedBBs.count(*PI)) {
1361  AllPredsVisited = false;
1362  break;
1363  }
1364  }
1365 
1366  if (AllPredsVisited) {
1367  for (BasicBlock::const_iterator I = LLVMBB->begin();
1368  const PHINode *PN = dyn_cast<PHINode>(I); ++I)
1370  } else {
1371  for (BasicBlock::const_iterator I = LLVMBB->begin();
1372  const PHINode *PN = dyn_cast<PHINode>(I); ++I)
1374  }
1375 
1376  FuncInfo->VisitedBBs.insert(LLVMBB);
1377  }
1378 
1379  BasicBlock::const_iterator const Begin =
1380  LLVMBB->getFirstNonPHI()->getIterator();
1381  BasicBlock::const_iterator const End = LLVMBB->end();
1383 
1384  FuncInfo->MBB = FuncInfo->MBBMap[LLVMBB];
1385  if (!FuncInfo->MBB)
1386  continue; // Some blocks like catchpads have no code or MBB.
1389 
1390  // Setup an EH landing-pad block.
1393  if (LLVMBB->isEHPad())
1394  if (!PrepareEHLandingPad())
1395  continue;
1396 
1397  // Before doing SelectionDAG ISel, see if FastISel has been requested.
1398  if (FastIS) {
1399  FastIS->startNewBlock();
1400 
1401  // Emit code for any incoming arguments. This must happen before
1402  // beginning FastISel on the entry block.
1403  if (LLVMBB == &Fn.getEntryBlock()) {
1404  ++NumEntryBlocks;
1405 
1406  // Lower any arguments needed in this block if this is the entry block.
1407  if (!FastIS->lowerArguments()) {
1408  // Fast isel failed to lower these arguments
1409  ++NumFastIselFailLowerArguments;
1410  if (EnableFastISelAbort > 1)
1411  report_fatal_error("FastISel didn't lower all arguments");
1412 
1413  // Use SelectionDAG argument lowering
1414  LowerArguments(Fn);
1416  SDB->clear();
1417  CodeGenAndEmitDAG();
1418  }
1419 
1420  // If we inserted any instructions at the beginning, make a note of
1421  // where they are, so we can be sure to emit subsequent instructions
1422  // after them.
1423  if (FuncInfo->InsertPt != FuncInfo->MBB->begin())
1424  FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));
1425  else
1426  FastIS->setLastLocalValue(nullptr);
1427  }
1428 
1429  unsigned NumFastIselRemaining = std::distance(Begin, End);
1430  // Do FastISel on as many instructions as possible.
1431  for (; BI != Begin; --BI) {
1432  const Instruction *Inst = &*std::prev(BI);
1433 
1434  // If we no longer require this instruction, skip it.
1435  if (isFoldedOrDeadInstruction(Inst, FuncInfo)) {
1436  --NumFastIselRemaining;
1437  continue;
1438  }
1439 
1440  // Bottom-up: reset the insert pos at the top, after any local-value
1441  // instructions.
1442  FastIS->recomputeInsertPt();
1443 
1444  // Try to select the instruction with FastISel.
1445  if (FastIS->selectInstruction(Inst)) {
1446  --NumFastIselRemaining;
1447  ++NumFastIselSuccess;
1448  // If fast isel succeeded, skip over all the folded instructions, and
1449  // then see if there is a load right before the selected instructions.
1450  // Try to fold the load if so.
1451  const Instruction *BeforeInst = Inst;
1452  while (BeforeInst != &*Begin) {
1453  BeforeInst = &*std::prev(BasicBlock::const_iterator(BeforeInst));
1454  if (!isFoldedOrDeadInstruction(BeforeInst, FuncInfo))
1455  break;
1456  }
1457  if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&
1458  BeforeInst->hasOneUse() &&
1459  FastIS->tryToFoldLoad(cast<LoadInst>(BeforeInst), Inst)) {
1460  // If we succeeded, don't re-select the load.
1461  BI = std::next(BasicBlock::const_iterator(BeforeInst));
1462  --NumFastIselRemaining;
1463  ++NumFastIselSuccess;
1464  }
1465  continue;
1466  }
1467 
1468 #ifndef NDEBUG
1470  collectFailStats(Inst);
1471 #endif
1472 
1473  // Then handle certain instructions as single-LLVM-Instruction blocks.
1474  if (isa<CallInst>(Inst)) {
1475 
1477  dbgs() << "FastISel missed call: ";
1478  Inst->dump();
1479  }
1480  if (EnableFastISelAbort > 2)
1481  // FastISel selector couldn't handle something and bailed.
1482  // For the purpose of debugging, just abort.
1483  report_fatal_error("FastISel didn't select the entire block");
1484 
1485  if (!Inst->getType()->isVoidTy() && !Inst->getType()->isTokenTy() &&
1486  !Inst->use_empty()) {
1487  unsigned &R = FuncInfo->ValueMap[Inst];
1488  if (!R)
1489  R = FuncInfo->CreateRegs(Inst->getType());
1490  }
1491 
1492  bool HadTailCall = false;
1494  SelectBasicBlock(Inst->getIterator(), BI, HadTailCall);
1495 
1496  // If the call was emitted as a tail call, we're done with the block.
1497  // We also need to delete any previously emitted instructions.
1498  if (HadTailCall) {
1499  FastIS->removeDeadCode(SavedInsertPt, FuncInfo->MBB->end());
1500  --BI;
1501  break;
1502  }
1503 
1504  // Recompute NumFastIselRemaining as Selection DAG instruction
1505  // selection may have handled the call, input args, etc.
1506  unsigned RemainingNow = std::distance(Begin, BI);
1507  NumFastIselFailures += NumFastIselRemaining - RemainingNow;
1508  NumFastIselRemaining = RemainingNow;
1509  continue;
1510  }
1511 
1512  bool ShouldAbort = EnableFastISelAbort;
1514  if (isa<TerminatorInst>(Inst)) {
1515  // Use a different message for terminator misses.
1516  dbgs() << "FastISel missed terminator: ";
1517  // Don't abort unless for terminator unless the level is really high
1518  ShouldAbort = (EnableFastISelAbort > 2);
1519  } else {
1520  dbgs() << "FastISel miss: ";
1521  }
1522  Inst->dump();
1523  }
1524  if (ShouldAbort)
1525  // FastISel selector couldn't handle something and bailed.
1526  // For the purpose of debugging, just abort.
1527  report_fatal_error("FastISel didn't select the entire block");
1528 
1529  NumFastIselFailures += NumFastIselRemaining;
1530  break;
1531  }
1532 
1533  FastIS->recomputeInsertPt();
1534  } else {
1535  // Lower any arguments needed in this block if this is the entry block.
1536  if (LLVMBB == &Fn.getEntryBlock()) {
1537  ++NumEntryBlocks;
1538  LowerArguments(Fn);
1539  }
1540  }
1541  if (getAnalysis<StackProtector>().shouldEmitSDCheck(*LLVMBB)) {
1542  bool FunctionBasedInstrumentation =
1544  SDB->SPDescriptor.initialize(LLVMBB, FuncInfo->MBBMap[LLVMBB],
1545  FunctionBasedInstrumentation);
1546  }
1547 
1548  if (Begin != BI)
1549  ++NumDAGBlocks;
1550  else
1551  ++NumFastIselBlocks;
1552 
1553  if (Begin != BI) {
1554  // Run SelectionDAG instruction selection on the remainder of the block
1555  // not handled by FastISel. If FastISel is not run, this is the entire
1556  // block.
1557  bool HadTailCall;
1558  SelectBasicBlock(Begin, BI, HadTailCall);
1559  }
1560 
1561  FinishBasicBlock();
1562  FuncInfo->PHINodesToUpdate.clear();
1563  }
1564 
1566 
1567  delete FastIS;
1569  SDB->SPDescriptor.resetPerFunctionState();
1570 }
1571 
1572 /// Given that the input MI is before a partial terminator sequence TSeq, return
1573 /// true if M + TSeq also a partial terminator sequence.
1574 ///
1575 /// A Terminator sequence is a sequence of MachineInstrs which at this point in
1576 /// lowering copy vregs into physical registers, which are then passed into
1577 /// terminator instructors so we can satisfy ABI constraints. A partial
1578 /// terminator sequence is an improper subset of a terminator sequence (i.e. it
1579 /// may be the whole terminator sequence).
1581  // If we do not have a copy or an implicit def, we return true if and only if
1582  // MI is a debug value.
1583  if (!MI.isCopy() && !MI.isImplicitDef())
1584  // Sometimes DBG_VALUE MI sneak in between the copies from the vregs to the
1585  // physical registers if there is debug info associated with the terminator
1586  // of our mbb. We want to include said debug info in our terminator
1587  // sequence, so we return true in that case.
1588  return MI.isDebugValue();
1589 
1590  // We have left the terminator sequence if we are not doing one of the
1591  // following:
1592  //
1593  // 1. Copying a vreg into a physical register.
1594  // 2. Copying a vreg into a vreg.
1595  // 3. Defining a register via an implicit def.
1596 
1597  // OPI should always be a register definition...
1599  if (!OPI->isReg() || !OPI->isDef())
1600  return false;
1601 
1602  // Defining any register via an implicit def is always ok.
1603  if (MI.isImplicitDef())
1604  return true;
1605 
1606  // Grab the copy source...
1608  ++OPI2;
1609  assert(OPI2 != MI.operands_end()
1610  && "Should have a copy implying we should have 2 arguments.");
1611 
1612  // Make sure that the copy dest is not a vreg when the copy source is a
1613  // physical register.
1614  if (!OPI2->isReg() ||
1617  return false;
1618 
1619  return true;
1620 }
1621 
1622 /// Find the split point at which to splice the end of BB into its success stack
1623 /// protector check machine basic block.
1624 ///
1625 /// On many platforms, due to ABI constraints, terminators, even before register
1626 /// allocation, use physical registers. This creates an issue for us since
1627 /// physical registers at this point can not travel across basic
1628 /// blocks. Luckily, selectiondag always moves physical registers into vregs
1629 /// when they enter functions and moves them through a sequence of copies back
1630 /// into the physical registers right before the terminator creating a
1631 /// ``Terminator Sequence''. This function is searching for the beginning of the
1632 /// terminator sequence so that we can ensure that we splice off not just the
1633 /// terminator, but additionally the copies that move the vregs into the
1634 /// physical registers.
1638  //
1639  if (SplitPoint == BB->begin())
1640  return SplitPoint;
1641 
1642  MachineBasicBlock::iterator Start = BB->begin();
1643  MachineBasicBlock::iterator Previous = SplitPoint;
1644  --Previous;
1645 
1646  while (MIIsInTerminatorSequence(*Previous)) {
1647  SplitPoint = Previous;
1648  if (Previous == Start)
1649  break;
1650  --Previous;
1651  }
1652 
1653  return SplitPoint;
1654 }
1655 
1656 void
1657 SelectionDAGISel::FinishBasicBlock() {
1658  DEBUG(dbgs() << "Total amount of phi nodes to update: "
1659  << FuncInfo->PHINodesToUpdate.size() << "\n";
1660  for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i)
1661  dbgs() << "Node " << i << " : ("
1662  << FuncInfo->PHINodesToUpdate[i].first
1663  << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n");
1664 
1665  // Next, now that we know what the last MBB the LLVM BB expanded is, update
1666  // PHI nodes in successors.
1667  for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
1669  assert(PHI->isPHI() &&
1670  "This is not a machine PHI node that we are updating!");
1671  if (!FuncInfo->MBB->isSuccessor(PHI->getParent()))
1672  continue;
1673  PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB);
1674  }
1675 
1676  // Handle stack protector.
1677  if (SDB->SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
1678  // The target provides a guard check function. There is no need to
1679  // generate error handling code or to split current basic block.
1680  MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1681 
1682  // Add load and check to the basicblock.
1683  FuncInfo->MBB = ParentMBB;
1684  FuncInfo->InsertPt =
1687  CurDAG->setRoot(SDB->getRoot());
1688  SDB->clear();
1689  CodeGenAndEmitDAG();
1690 
1691  // Clear the Per-BB State.
1692  SDB->SPDescriptor.resetPerBBState();
1693  } else if (SDB->SPDescriptor.shouldEmitStackProtector()) {
1694  MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1695  MachineBasicBlock *SuccessMBB = SDB->SPDescriptor.getSuccessMBB();
1696 
1697  // Find the split point to split the parent mbb. At the same time copy all
1698  // physical registers used in the tail of parent mbb into virtual registers
1699  // before the split point and back into physical registers after the split
1700  // point. This prevents us needing to deal with Live-ins and many other
1701  // register allocation issues caused by us splitting the parent mbb. The
1702  // register allocator will clean up said virtual copies later on.
1703  MachineBasicBlock::iterator SplitPoint =
1705 
1706  // Splice the terminator of ParentMBB into SuccessMBB.
1707  SuccessMBB->splice(SuccessMBB->end(), ParentMBB,
1708  SplitPoint,
1709  ParentMBB->end());
1710 
1711  // Add compare/jump on neq/jump to the parent BB.
1712  FuncInfo->MBB = ParentMBB;
1713  FuncInfo->InsertPt = ParentMBB->end();
1715  CurDAG->setRoot(SDB->getRoot());
1716  SDB->clear();
1717  CodeGenAndEmitDAG();
1718 
1719  // CodeGen Failure MBB if we have not codegened it yet.
1720  MachineBasicBlock *FailureMBB = SDB->SPDescriptor.getFailureMBB();
1721  if (FailureMBB->empty()) {
1722  FuncInfo->MBB = FailureMBB;
1723  FuncInfo->InsertPt = FailureMBB->end();
1725  CurDAG->setRoot(SDB->getRoot());
1726  SDB->clear();
1727  CodeGenAndEmitDAG();
1728  }
1729 
1730  // Clear the Per-BB State.
1731  SDB->SPDescriptor.resetPerBBState();
1732  }
1733 
1734  // Lower each BitTestBlock.
1735  for (auto &BTB : SDB->BitTestCases) {
1736  // Lower header first, if it wasn't already lowered
1737  if (!BTB.Emitted) {
1738  // Set the current basic block to the mbb we wish to insert the code into
1739  FuncInfo->MBB = BTB.Parent;
1740  FuncInfo->InsertPt = FuncInfo->MBB->end();
1741  // Emit the code
1743  CurDAG->setRoot(SDB->getRoot());
1744  SDB->clear();
1745  CodeGenAndEmitDAG();
1746  }
1747 
1748  BranchProbability UnhandledProb = BTB.Prob;
1749  for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
1750  UnhandledProb -= BTB.Cases[j].ExtraProb;
1751  // Set the current basic block to the mbb we wish to insert the code into
1752  FuncInfo->MBB = BTB.Cases[j].ThisBB;
1753  FuncInfo->InsertPt = FuncInfo->MBB->end();
1754  // Emit the code
1755 
1756  // If all cases cover a contiguous range, it is not necessary to jump to
1757  // the default block after the last bit test fails. This is because the
1758  // range check during bit test header creation has guaranteed that every
1759  // case here doesn't go outside the range. In this case, there is no need
1760  // to perform the last bit test, as it will always be true. Instead, make
1761  // the second-to-last bit-test fall through to the target of the last bit
1762  // test, and delete the last bit test.
1763 
1764  MachineBasicBlock *NextMBB;
1765  if (BTB.ContiguousRange && j + 2 == ej) {
1766  // Second-to-last bit-test with contiguous range: fall through to the
1767  // target of the final bit test.
1768  NextMBB = BTB.Cases[j + 1].TargetBB;
1769  } else if (j + 1 == ej) {
1770  // For the last bit test, fall through to Default.
1771  NextMBB = BTB.Default;
1772  } else {
1773  // Otherwise, fall through to the next bit test.
1774  NextMBB = BTB.Cases[j + 1].ThisBB;
1775  }
1776 
1777  SDB->visitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j],
1778  FuncInfo->MBB);
1779 
1780  CurDAG->setRoot(SDB->getRoot());
1781  SDB->clear();
1782  CodeGenAndEmitDAG();
1783 
1784  if (BTB.ContiguousRange && j + 2 == ej) {
1785  // Since we're not going to use the final bit test, remove it.
1786  BTB.Cases.pop_back();
1787  break;
1788  }
1789  }
1790 
1791  // Update PHI Nodes
1792  for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
1793  pi != pe; ++pi) {
1794  MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);
1795  MachineBasicBlock *PHIBB = PHI->getParent();
1796  assert(PHI->isPHI() &&
1797  "This is not a machine PHI node that we are updating!");
1798  // This is "default" BB. We have two jumps to it. From "header" BB and
1799  // from last "case" BB, unless the latter was skipped.
1800  if (PHIBB == BTB.Default) {
1801  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(BTB.Parent);
1802  if (!BTB.ContiguousRange) {
1803  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
1804  .addMBB(BTB.Cases.back().ThisBB);
1805  }
1806  }
1807  // One of "cases" BB.
1808  for (unsigned j = 0, ej = BTB.Cases.size();
1809  j != ej; ++j) {
1810  MachineBasicBlock* cBB = BTB.Cases[j].ThisBB;
1811  if (cBB->isSuccessor(PHIBB))
1812  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(cBB);
1813  }
1814  }
1815  }
1816  SDB->BitTestCases.clear();
1817 
1818  // If the JumpTable record is filled in, then we need to emit a jump table.
1819  // Updating the PHI nodes is tricky in this case, since we need to determine
1820  // whether the PHI is a successor of the range check MBB or the jump table MBB
1821  for (unsigned i = 0, e = SDB->JTCases.size(); i != e; ++i) {
1822  // Lower header first, if it wasn't already lowered
1823  if (!SDB->JTCases[i].first.Emitted) {
1824  // Set the current basic block to the mbb we wish to insert the code into
1825  FuncInfo->MBB = SDB->JTCases[i].first.HeaderBB;
1826  FuncInfo->InsertPt = FuncInfo->MBB->end();
1827  // Emit the code
1828  SDB->visitJumpTableHeader(SDB->JTCases[i].second, SDB->JTCases[i].first,
1829  FuncInfo->MBB);
1830  CurDAG->setRoot(SDB->getRoot());
1831  SDB->clear();
1832  CodeGenAndEmitDAG();
1833  }
1834 
1835  // Set the current basic block to the mbb we wish to insert the code into
1836  FuncInfo->MBB = SDB->JTCases[i].second.MBB;
1837  FuncInfo->InsertPt = FuncInfo->MBB->end();
1838  // Emit the code
1839  SDB->visitJumpTable(SDB->JTCases[i].second);
1840  CurDAG->setRoot(SDB->getRoot());
1841  SDB->clear();
1842  CodeGenAndEmitDAG();
1843 
1844  // Update PHI Nodes
1845  for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
1846  pi != pe; ++pi) {
1847  MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);
1848  MachineBasicBlock *PHIBB = PHI->getParent();
1849  assert(PHI->isPHI() &&
1850  "This is not a machine PHI node that we are updating!");
1851  // "default" BB. We can go there only from header BB.
1852  if (PHIBB == SDB->JTCases[i].second.Default)
1853  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
1854  .addMBB(SDB->JTCases[i].first.HeaderBB);
1855  // JT BB. Just iterate over successors here
1856  if (FuncInfo->MBB->isSuccessor(PHIBB))
1857  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(FuncInfo->MBB);
1858  }
1859  }
1860  SDB->JTCases.clear();
1861 
1862  // If we generated any switch lowering information, build and codegen any
1863  // additional DAGs necessary.
1864  for (unsigned i = 0, e = SDB->SwitchCases.size(); i != e; ++i) {
1865  // Set the current basic block to the mbb we wish to insert the code into
1866  FuncInfo->MBB = SDB->SwitchCases[i].ThisBB;
1867  FuncInfo->InsertPt = FuncInfo->MBB->end();
1868 
1869  // Determine the unique successors.
1871  Succs.push_back(SDB->SwitchCases[i].TrueBB);
1872  if (SDB->SwitchCases[i].TrueBB != SDB->SwitchCases[i].FalseBB)
1873  Succs.push_back(SDB->SwitchCases[i].FalseBB);
1874 
1875  // Emit the code. Note that this could result in FuncInfo->MBB being split.
1877  CurDAG->setRoot(SDB->getRoot());
1878  SDB->clear();
1879  CodeGenAndEmitDAG();
1880 
1881  // Remember the last block, now that any splitting is done, for use in
1882  // populating PHI nodes in successors.
1883  MachineBasicBlock *ThisBB = FuncInfo->MBB;
1884 
1885  // Handle any PHI nodes in successors of this chunk, as if we were coming
1886  // from the original BB before switch expansion. Note that PHI nodes can
1887  // occur multiple times in PHINodesToUpdate. We have to be very careful to
1888  // handle them the right number of times.
1889  for (unsigned i = 0, e = Succs.size(); i != e; ++i) {
1890  FuncInfo->MBB = Succs[i];
1891  FuncInfo->InsertPt = FuncInfo->MBB->end();
1892  // FuncInfo->MBB may have been removed from the CFG if a branch was
1893  // constant folded.
1894  if (ThisBB->isSuccessor(FuncInfo->MBB)) {
1896  MBBI = FuncInfo->MBB->begin(), MBBE = FuncInfo->MBB->end();
1897  MBBI != MBBE && MBBI->isPHI(); ++MBBI) {
1898  MachineInstrBuilder PHI(*MF, MBBI);
1899  // This value for this PHI node is recorded in PHINodesToUpdate.
1900  for (unsigned pn = 0; ; ++pn) {
1901  assert(pn != FuncInfo->PHINodesToUpdate.size() &&
1902  "Didn't find PHI entry!");
1903  if (FuncInfo->PHINodesToUpdate[pn].first == PHI) {
1904  PHI.addReg(FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB);
1905  break;
1906  }
1907  }
1908  }
1909  }
1910  }
1911  }
1912  SDB->SwitchCases.clear();
1913 }
1914 
1915 /// Create the scheduler. If a specific scheduler was specified
1916 /// via the SchedulerRegistry, use it, otherwise select the
1917 /// one preferred by the target.
1918 ///
1919 ScheduleDAGSDNodes *SelectionDAGISel::CreateScheduler() {
1920  return ISHeuristic(this, OptLevel);
1921 }
1922 
1923 //===----------------------------------------------------------------------===//
1924 // Helper functions used by the generated instruction selector.
1925 //===----------------------------------------------------------------------===//
1926 // Calls to these methods are generated by tblgen.
1927 
1928 /// CheckAndMask - The isel is trying to match something like (and X, 255). If
1929 /// the dag combiner simplified the 255, we still want to match. RHS is the
1930 /// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value
1931 /// specified in the .td file (e.g. 255).
1933  int64_t DesiredMaskS) const {
1934  const APInt &ActualMask = RHS->getAPIntValue();
1935  const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
1936 
1937  // If the actual mask exactly matches, success!
1938  if (ActualMask == DesiredMask)
1939  return true;
1940 
1941  // If the actual AND mask is allowing unallowed bits, this doesn't match.
1942  if (ActualMask.intersects(~DesiredMask))
1943  return false;
1944 
1945  // Otherwise, the DAG Combiner may have proven that the value coming in is
1946  // either already zero or is not demanded. Check for known zero input bits.
1947  APInt NeededMask = DesiredMask & ~ActualMask;
1948  if (CurDAG->MaskedValueIsZero(LHS, NeededMask))
1949  return true;
1950 
1951  // TODO: check to see if missing bits are just not demanded.
1952 
1953  // Otherwise, this pattern doesn't match.
1954  return false;
1955 }
1956 
1957 /// CheckOrMask - The isel is trying to match something like (or X, 255). If
1958 /// the dag combiner simplified the 255, we still want to match. RHS is the
1959 /// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value
1960 /// specified in the .td file (e.g. 255).
1962  int64_t DesiredMaskS) const {
1963  const APInt &ActualMask = RHS->getAPIntValue();
1964  const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
1965 
1966  // If the actual mask exactly matches, success!
1967  if (ActualMask == DesiredMask)
1968  return true;
1969 
1970  // If the actual AND mask is allowing unallowed bits, this doesn't match.
1971  if (ActualMask.intersects(~DesiredMask))
1972  return false;
1973 
1974  // Otherwise, the DAG Combiner may have proven that the value coming in is
1975  // either already zero or is not demanded. Check for known zero input bits.
1976  APInt NeededMask = DesiredMask & ~ActualMask;
1977 
1978  APInt KnownZero, KnownOne;
1979  CurDAG->computeKnownBits(LHS, KnownZero, KnownOne);
1980 
1981  // If all the missing bits in the or are already known to be set, match!
1982  if ((NeededMask & KnownOne) == NeededMask)
1983  return true;
1984 
1985  // TODO: check to see if missing bits are just not demanded.
1986 
1987  // Otherwise, this pattern doesn't match.
1988  return false;
1989 }
1990 
1991 /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
1992 /// by tblgen. Others should not call it.
1994  const SDLoc &DL) {
1995  std::vector<SDValue> InOps;
1996  std::swap(InOps, Ops);
1997 
1998  Ops.push_back(InOps[InlineAsm::Op_InputChain]); // 0
1999  Ops.push_back(InOps[InlineAsm::Op_AsmString]); // 1
2000  Ops.push_back(InOps[InlineAsm::Op_MDNode]); // 2, !srcloc
2001  Ops.push_back(InOps[InlineAsm::Op_ExtraInfo]); // 3 (SideEffect, AlignStack)
2002 
2003  unsigned i = InlineAsm::Op_FirstOperand, e = InOps.size();
2004  if (InOps[e-1].getValueType() == MVT::Glue)
2005  --e; // Don't process a glue operand if it is here.
2006 
2007  while (i != e) {
2008  unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue();
2009  if (!InlineAsm::isMemKind(Flags)) {
2010  // Just skip over this operand, copying the operands verbatim.
2011  Ops.insert(Ops.end(), InOps.begin()+i,
2012  InOps.begin()+i+InlineAsm::getNumOperandRegisters(Flags) + 1);
2013  i += InlineAsm::getNumOperandRegisters(Flags) + 1;
2014  } else {
2016  "Memory operand with multiple values?");
2017 
2018  unsigned TiedToOperand;
2019  if (InlineAsm::isUseOperandTiedToDef(Flags, TiedToOperand)) {
2020  // We need the constraint ID from the operand this is tied to.
2021  unsigned CurOp = InlineAsm::Op_FirstOperand;
2022  Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
2023  for (; TiedToOperand; --TiedToOperand) {
2024  CurOp += InlineAsm::getNumOperandRegisters(Flags)+1;
2025  Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
2026  }
2027  }
2028 
2029  // Otherwise, this is a memory operand. Ask the target to select it.
2030  std::vector<SDValue> SelOps;
2031  unsigned ConstraintID = InlineAsm::getMemoryConstraintID(Flags);
2032  if (SelectInlineAsmMemoryOperand(InOps[i+1], ConstraintID, SelOps))
2033  report_fatal_error("Could not match memory address. Inline asm"
2034  " failure!");
2035 
2036  // Add this to the output node.
2037  unsigned NewFlags =
2039  NewFlags = InlineAsm::getFlagWordForMem(NewFlags, ConstraintID);
2040  Ops.push_back(CurDAG->getTargetConstant(NewFlags, DL, MVT::i32));
2041  Ops.insert(Ops.end(), SelOps.begin(), SelOps.end());
2042  i += 2;
2043  }
2044  }
2045 
2046  // Add the glue input back if present.
2047  if (e != InOps.size())
2048  Ops.push_back(InOps.back());
2049 }
2050 
2051 /// findGlueUse - Return use of MVT::Glue value produced by the specified
2052 /// SDNode.
2053 ///
2055  unsigned FlagResNo = N->getNumValues()-1;
2056  for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
2057  SDUse &Use = I.getUse();
2058  if (Use.getResNo() == FlagResNo)
2059  return Use.getUser();
2060  }
2061  return nullptr;
2062 }
2063 
2064 /// findNonImmUse - Return true if "Use" is a non-immediate use of "Def".
2065 /// This function recursively traverses up the operand chain, ignoring
2066 /// certain nodes.
2067 static bool findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse,
2068  SDNode *Root, SmallPtrSetImpl<SDNode*> &Visited,
2069  bool IgnoreChains) {
2070  // The NodeID's are given uniques ID's where a node ID is guaranteed to be
2071  // greater than all of its (recursive) operands. If we scan to a point where
2072  // 'use' is smaller than the node we're scanning for, then we know we will
2073  // never find it.
2074  //
2075  // The Use may be -1 (unassigned) if it is a newly allocated node. This can
2076  // happen because we scan down to newly selected nodes in the case of glue
2077  // uses.
2078  if ((Use->getNodeId() < Def->getNodeId() && Use->getNodeId() != -1))
2079  return false;
2080 
2081  // Don't revisit nodes if we already scanned it and didn't fail, we know we
2082  // won't fail if we scan it again.
2083  if (!Visited.insert(Use).second)
2084  return false;
2085 
2086  for (const SDValue &Op : Use->op_values()) {
2087  // Ignore chain uses, they are validated by HandleMergeInputChains.
2088  if (Op.getValueType() == MVT::Other && IgnoreChains)
2089  continue;
2090 
2091  SDNode *N = Op.getNode();
2092  if (N == Def) {
2093  if (Use == ImmedUse || Use == Root)
2094  continue; // We are not looking for immediate use.
2095  assert(N != Root);
2096  return true;
2097  }
2098 
2099  // Traverse up the operand chain.
2100  if (findNonImmUse(N, Def, ImmedUse, Root, Visited, IgnoreChains))
2101  return true;
2102  }
2103  return false;
2104 }
2105 
2106 /// IsProfitableToFold - Returns true if it's profitable to fold the specific
2107 /// operand node N of U during instruction selection that starts at Root.
2109  SDNode *Root) const {
2110  if (OptLevel == CodeGenOpt::None) return false;
2111  return N.hasOneUse();
2112 }
2113 
2114 /// IsLegalToFold - Returns true if the specific operand node N of
2115 /// U can be folded during instruction selection that starts at Root.
2117  CodeGenOpt::Level OptLevel,
2118  bool IgnoreChains) {
2119  if (OptLevel == CodeGenOpt::None) return false;
2120 
2121  // If Root use can somehow reach N through a path that that doesn't contain
2122  // U then folding N would create a cycle. e.g. In the following
2123  // diagram, Root can reach N through X. If N is folded into into Root, then
2124  // X is both a predecessor and a successor of U.
2125  //
2126  // [N*] //
2127  // ^ ^ //
2128  // / \ //
2129  // [U*] [X]? //
2130  // ^ ^ //
2131  // \ / //
2132  // \ / //
2133  // [Root*] //
2134  //
2135  // * indicates nodes to be folded together.
2136  //
2137  // If Root produces glue, then it gets (even more) interesting. Since it
2138  // will be "glued" together with its glue use in the scheduler, we need to
2139  // check if it might reach N.
2140  //
2141  // [N*] //
2142  // ^ ^ //
2143  // / \ //
2144  // [U*] [X]? //
2145  // ^ ^ //
2146  // \ \ //
2147  // \ | //
2148  // [Root*] | //
2149  // ^ | //
2150  // f | //
2151  // | / //
2152  // [Y] / //
2153  // ^ / //
2154  // f / //
2155  // | / //
2156  // [GU] //
2157  //
2158  // If GU (glue use) indirectly reaches N (the load), and Root folds N
2159  // (call it Fold), then X is a predecessor of GU and a successor of
2160  // Fold. But since Fold and GU are glued together, this will create
2161  // a cycle in the scheduling graph.
2162 
2163  // If the node has glue, walk down the graph to the "lowest" node in the
2164  // glueged set.
2165  EVT VT = Root->getValueType(Root->getNumValues()-1);
2166  while (VT == MVT::Glue) {
2167  SDNode *GU = findGlueUse(Root);
2168  if (!GU)
2169  break;
2170  Root = GU;
2171  VT = Root->getValueType(Root->getNumValues()-1);
2172 
2173  // If our query node has a glue result with a use, we've walked up it. If
2174  // the user (which has already been selected) has a chain or indirectly uses
2175  // the chain, our WalkChainUsers predicate will not consider it. Because of
2176  // this, we cannot ignore chains in this predicate.
2177  IgnoreChains = false;
2178  }
2179 
2180 
2181  SmallPtrSet<SDNode*, 16> Visited;
2182  return !findNonImmUse(Root, N.getNode(), U, Root, Visited, IgnoreChains);
2183 }
2184 
2185 void SelectionDAGISel::Select_INLINEASM(SDNode *N) {
2186  SDLoc DL(N);
2187 
2188  std::vector<SDValue> Ops(N->op_begin(), N->op_end());
2190 
2191  const EVT VTs[] = {MVT::Other, MVT::Glue};
2192  SDValue New = CurDAG->getNode(ISD::INLINEASM, DL, VTs, Ops);
2193  New->setNodeId(-1);
2194  ReplaceUses(N, New.getNode());
2195  CurDAG->RemoveDeadNode(N);
2196 }
2197 
2198 void SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) {
2199  SDLoc dl(Op);
2201  const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));
2202  unsigned Reg =
2203  TLI->getRegisterByName(RegStr->getString().data(), Op->getValueType(0),
2204  *CurDAG);
2205  SDValue New = CurDAG->getCopyFromReg(
2206  Op->getOperand(0), dl, Reg, Op->getValueType(0));
2207  New->setNodeId(-1);
2208  ReplaceUses(Op, New.getNode());
2209  CurDAG->RemoveDeadNode(Op);
2210 }
2211 
2212 void SelectionDAGISel::Select_WRITE_REGISTER(SDNode *Op) {
2213  SDLoc dl(Op);
2215  const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));
2216  unsigned Reg = TLI->getRegisterByName(RegStr->getString().data(),
2217  Op->getOperand(2).getValueType(),
2218  *CurDAG);
2219  SDValue New = CurDAG->getCopyToReg(
2220  Op->getOperand(0), dl, Reg, Op->getOperand(2));
2221  New->setNodeId(-1);
2222  ReplaceUses(Op, New.getNode());
2223  CurDAG->RemoveDeadNode(Op);
2224 }
2225 
2226 void SelectionDAGISel::Select_UNDEF(SDNode *N) {
2227  CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0));
2228 }
2229 
2230 /// GetVBR - decode a vbr encoding whose top bit is set.
2231 LLVM_ATTRIBUTE_ALWAYS_INLINE static inline uint64_t
2232 GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) {
2233  assert(Val >= 128 && "Not a VBR");
2234  Val &= 127; // Remove first vbr bit.
2235 
2236  unsigned Shift = 7;
2237  uint64_t NextBits;
2238  do {
2239  NextBits = MatcherTable[Idx++];
2240  Val |= (NextBits&127) << Shift;
2241  Shift += 7;
2242  } while (NextBits & 128);
2243 
2244  return Val;
2245 }
2246 
2247 /// When a match is complete, this method updates uses of interior chain results
2248 /// to use the new results.
2249 void SelectionDAGISel::UpdateChains(
2250  SDNode *NodeToMatch, SDValue InputChain,
2251  SmallVectorImpl<SDNode *> &ChainNodesMatched, bool isMorphNodeTo) {
2252  SmallVector<SDNode*, 4> NowDeadNodes;
2253 
2254  // Now that all the normal results are replaced, we replace the chain and
2255  // glue results if present.
2256  if (!ChainNodesMatched.empty()) {
2257  assert(InputChain.getNode() &&
2258  "Matched input chains but didn't produce a chain");
2259  // Loop over all of the nodes we matched that produced a chain result.
2260  // Replace all the chain results with the final chain we ended up with.
2261  for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
2262  SDNode *ChainNode = ChainNodesMatched[i];
2263  // If ChainNode is null, it's because we replaced it on a previous
2264  // iteration and we cleared it out of the map. Just skip it.
2265  if (!ChainNode)
2266  continue;
2267 
2268  assert(ChainNode->getOpcode() != ISD::DELETED_NODE &&
2269  "Deleted node left in chain");
2270 
2271  // Don't replace the results of the root node if we're doing a
2272  // MorphNodeTo.
2273  if (ChainNode == NodeToMatch && isMorphNodeTo)
2274  continue;
2275 
2276  SDValue ChainVal = SDValue(ChainNode, ChainNode->getNumValues()-1);
2277  if (ChainVal.getValueType() == MVT::Glue)
2278  ChainVal = ChainVal.getValue(ChainVal->getNumValues()-2);
2279  assert(ChainVal.getValueType() == MVT::Other && "Not a chain?");
2281  *CurDAG, [&](SDNode *N, SDNode *E) {
2282  std::replace(ChainNodesMatched.begin(), ChainNodesMatched.end(), N,
2283  static_cast<SDNode *>(nullptr));
2284  });
2285  CurDAG->ReplaceAllUsesOfValueWith(ChainVal, InputChain);
2286 
2287  // If the node became dead and we haven't already seen it, delete it.
2288  if (ChainNode != NodeToMatch && ChainNode->use_empty() &&
2289  !std::count(NowDeadNodes.begin(), NowDeadNodes.end(), ChainNode))
2290  NowDeadNodes.push_back(ChainNode);
2291  }
2292  }
2293 
2294  if (!NowDeadNodes.empty())
2295  CurDAG->RemoveDeadNodes(NowDeadNodes);
2296 
2297  DEBUG(dbgs() << "ISEL: Match complete!\n");
2298 }
2299 
2304 };
2305 
2306 /// WalkChainUsers - Walk down the users of the specified chained node that is
2307 /// part of the pattern we're matching, looking at all of the users we find.
2308 /// This determines whether something is an interior node, whether we have a
2309 /// non-pattern node in between two pattern nodes (which prevent folding because
2310 /// it would induce a cycle) and whether we have a TokenFactor node sandwiched
2311 /// between pattern nodes (in which case the TF becomes part of the pattern).
2312 ///
2313 /// The walk we do here is guaranteed to be small because we quickly get down to
2314 /// already selected nodes "below" us.
2315 static ChainResult
2316 WalkChainUsers(const SDNode *ChainedNode,
2317  SmallVectorImpl<SDNode *> &ChainedNodesInPattern,
2318  DenseMap<const SDNode *, ChainResult> &TokenFactorResult,
2319  SmallVectorImpl<SDNode *> &InteriorChainedNodes) {
2320  ChainResult Result = CR_Simple;
2321 
2322  for (SDNode::use_iterator UI = ChainedNode->use_begin(),
2323  E = ChainedNode->use_end(); UI != E; ++UI) {
2324  // Make sure the use is of the chain, not some other value we produce.
2325  if (UI.getUse().getValueType() != MVT::Other) continue;
2326 
2327  SDNode *User = *UI;
2328 
2329  if (User->getOpcode() == ISD::HANDLENODE) // Root of the graph.
2330  continue;
2331 
2332  // If we see an already-selected machine node, then we've gone beyond the
2333  // pattern that we're selecting down into the already selected chunk of the
2334  // DAG.
2335  unsigned UserOpcode = User->getOpcode();
2336  if (User->isMachineOpcode() ||
2337  UserOpcode == ISD::CopyToReg ||
2338  UserOpcode == ISD::CopyFromReg ||
2339  UserOpcode == ISD::INLINEASM ||
2340  UserOpcode == ISD::EH_LABEL ||
2341  UserOpcode == ISD::LIFETIME_START ||
2342  UserOpcode == ISD::LIFETIME_END) {
2343  // If their node ID got reset to -1 then they've already been selected.
2344  // Treat them like a MachineOpcode.
2345  if (User->getNodeId() == -1)
2346  continue;
2347  }
2348 
2349  // If we have a TokenFactor, we handle it specially.
2350  if (User->getOpcode() != ISD::TokenFactor) {
2351  // If the node isn't a token factor and isn't part of our pattern, then it
2352  // must be a random chained node in between two nodes we're selecting.
2353  // This happens when we have something like:
2354  // x = load ptr
2355  // call
2356  // y = x+4
2357  // store y -> ptr
2358  // Because we structurally match the load/store as a read/modify/write,
2359  // but the call is chained between them. We cannot fold in this case
2360  // because it would induce a cycle in the graph.
2361  if (!std::count(ChainedNodesInPattern.begin(),
2362  ChainedNodesInPattern.end(), User))
2363  return CR_InducesCycle;
2364 
2365  // Otherwise we found a node that is part of our pattern. For example in:
2366  // x = load ptr
2367  // y = x+4
2368  // store y -> ptr
2369  // This would happen when we're scanning down from the load and see the
2370  // store as a user. Record that there is a use of ChainedNode that is
2371  // part of the pattern and keep scanning uses.
2372  Result = CR_LeadsToInteriorNode;
2373  InteriorChainedNodes.push_back(User);
2374  continue;
2375  }
2376 
2377  // If we found a TokenFactor, there are two cases to consider: first if the
2378  // TokenFactor is just hanging "below" the pattern we're matching (i.e. no
2379  // uses of the TF are in our pattern) we just want to ignore it. Second,
2380  // the TokenFactor can be sandwiched in between two chained nodes, like so:
2381  // [Load chain]
2382  // ^
2383  // |
2384  // [Load]
2385  // ^ ^
2386  // | \ DAG's like cheese
2387  // / \ do you?
2388  // / |
2389  // [TokenFactor] [Op]
2390  // ^ ^
2391  // | |
2392  // \ /
2393  // \ /
2394  // [Store]
2395  //
2396  // In this case, the TokenFactor becomes part of our match and we rewrite it
2397  // as a new TokenFactor.
2398  //
2399  // To distinguish these two cases, do a recursive walk down the uses.
2400  auto MemoizeResult = TokenFactorResult.find(User);
2401  bool Visited = MemoizeResult != TokenFactorResult.end();
2402  // Recursively walk chain users only if the result is not memoized.
2403  if (!Visited) {
2404  auto Res = WalkChainUsers(User, ChainedNodesInPattern, TokenFactorResult,
2405  InteriorChainedNodes);
2406  MemoizeResult = TokenFactorResult.insert(std::make_pair(User, Res)).first;
2407  }
2408  switch (MemoizeResult->second) {
2409  case CR_Simple:
2410  // If the uses of the TokenFactor are just already-selected nodes, ignore
2411  // it, it is "below" our pattern.
2412  continue;
2413  case CR_InducesCycle:
2414  // If the uses of the TokenFactor lead to nodes that are not part of our
2415  // pattern that are not selected, folding would turn this into a cycle,
2416  // bail out now.
2417  return CR_InducesCycle;
2419  break; // Otherwise, keep processing.
2420  }
2421 
2422  // Okay, we know we're in the interesting interior case. The TokenFactor
2423  // is now going to be considered part of the pattern so that we rewrite its
2424  // uses (it may have uses that are not part of the pattern) with the
2425  // ultimate chain result of the generated code. We will also add its chain
2426  // inputs as inputs to the ultimate TokenFactor we create.
2427  Result = CR_LeadsToInteriorNode;
2428  if (!Visited) {
2429  ChainedNodesInPattern.push_back(User);
2430  InteriorChainedNodes.push_back(User);
2431  }
2432  }
2433 
2434  return Result;
2435 }
2436 
2437 /// HandleMergeInputChains - This implements the OPC_EmitMergeInputChains
2438 /// operation for when the pattern matched at least one node with a chains. The
2439 /// input vector contains a list of all of the chained nodes that we match. We
2440 /// must determine if this is a valid thing to cover (i.e. matching it won't
2441 /// induce cycles in the DAG) and if so, creating a TokenFactor node. that will
2442 /// be used as the input node chain for the generated nodes.
2443 static SDValue
2445  SelectionDAG *CurDAG) {
2446  // Used for memoization. Without it WalkChainUsers could take exponential
2447  // time to run.
2448  DenseMap<const SDNode *, ChainResult> TokenFactorResult;
2449  // Walk all of the chained nodes we've matched, recursively scanning down the
2450  // users of the chain result. This adds any TokenFactor nodes that are caught
2451  // in between chained nodes to the chained and interior nodes list.
2452  SmallVector<SDNode*, 3> InteriorChainedNodes;
2453  for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
2454  if (WalkChainUsers(ChainNodesMatched[i], ChainNodesMatched,
2455  TokenFactorResult,
2456  InteriorChainedNodes) == CR_InducesCycle)
2457  return SDValue(); // Would induce a cycle.
2458  }
2459 
2460  // Okay, we have walked all the matched nodes and collected TokenFactor nodes
2461  // that we are interested in. Form our input TokenFactor node.
2462  SmallVector<SDValue, 3> InputChains;
2463  for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
2464  // Add the input chain of this node to the InputChains list (which will be
2465  // the operands of the generated TokenFactor) if it's not an interior node.
2466  SDNode *N = ChainNodesMatched[i];
2467  if (N->getOpcode() != ISD::TokenFactor) {
2468  if (std::count(InteriorChainedNodes.begin(),InteriorChainedNodes.end(),N))
2469  continue;
2470 
2471  // Otherwise, add the input chain.
2472  SDValue InChain = ChainNodesMatched[i]->getOperand(0);
2473  assert(InChain.getValueType() == MVT::Other && "Not a chain");
2474  InputChains.push_back(InChain);
2475  continue;
2476  }
2477 
2478  // If we have a token factor, we want to add all inputs of the token factor
2479  // that are not part of the pattern we're matching.
2480  for (const SDValue &Op : N->op_values()) {
2481  if (!std::count(ChainNodesMatched.begin(), ChainNodesMatched.end(),
2482  Op.getNode()))
2483  InputChains.push_back(Op);
2484  }
2485  }
2486 
2487  if (InputChains.size() == 1)
2488  return InputChains[0];
2489  return CurDAG->getNode(ISD::TokenFactor, SDLoc(ChainNodesMatched[0]),
2490  MVT::Other, InputChains);
2491 }
2492 
2493 /// MorphNode - Handle morphing a node in place for the selector.
2494 SDNode *SelectionDAGISel::
2495 MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
2496  ArrayRef<SDValue> Ops, unsigned EmitNodeInfo) {
2497  // It is possible we're using MorphNodeTo to replace a node with no
2498  // normal results with one that has a normal result (or we could be
2499  // adding a chain) and the input could have glue and chains as well.
2500  // In this case we need to shift the operands down.
2501  // FIXME: This is a horrible hack and broken in obscure cases, no worse
2502  // than the old isel though.
2503  int OldGlueResultNo = -1, OldChainResultNo = -1;
2504 
2505  unsigned NTMNumResults = Node->getNumValues();
2506  if (Node->getValueType(NTMNumResults-1) == MVT::Glue) {
2507  OldGlueResultNo = NTMNumResults-1;
2508  if (NTMNumResults != 1 &&
2509  Node->getValueType(NTMNumResults-2) == MVT::Other)
2510  OldChainResultNo = NTMNumResults-2;
2511  } else if (Node->getValueType(NTMNumResults-1) == MVT::Other)
2512  OldChainResultNo = NTMNumResults-1;
2513 
2514  // Call the underlying SelectionDAG routine to do the transmogrification. Note
2515  // that this deletes operands of the old node that become dead.
2516  SDNode *Res = CurDAG->MorphNodeTo(Node, ~TargetOpc, VTList, Ops);
2517 
2518  // MorphNodeTo can operate in two ways: if an existing node with the
2519  // specified operands exists, it can just return it. Otherwise, it
2520  // updates the node in place to have the requested operands.
2521  if (Res == Node) {
2522  // If we updated the node in place, reset the node ID. To the isel,
2523  // this should be just like a newly allocated machine node.
2524  Res->setNodeId(-1);
2525  }
2526 
2527  unsigned ResNumResults = Res->getNumValues();
2528  // Move the glue if needed.
2529  if ((EmitNodeInfo & OPFL_GlueOutput) && OldGlueResultNo != -1 &&
2530  (unsigned)OldGlueResultNo != ResNumResults-1)
2531  CurDAG->ReplaceAllUsesOfValueWith(SDValue(Node, OldGlueResultNo),
2532  SDValue(Res, ResNumResults-1));
2533 
2534  if ((EmitNodeInfo & OPFL_GlueOutput) != 0)
2535  --ResNumResults;
2536 
2537  // Move the chain reference if needed.
2538  if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 &&
2539  (unsigned)OldChainResultNo != ResNumResults-1)
2540  CurDAG->ReplaceAllUsesOfValueWith(SDValue(Node, OldChainResultNo),
2541  SDValue(Res, ResNumResults-1));
2542 
2543  // Otherwise, no replacement happened because the node already exists. Replace
2544  // Uses of the old node with the new one.
2545  if (Res != Node) {
2546  CurDAG->ReplaceAllUsesWith(Node, Res);
2547  CurDAG->RemoveDeadNode(Node);
2548  }
2549 
2550  return Res;
2551 }
2552 
2553 /// CheckSame - Implements OP_CheckSame.
2554 LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
2555 CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2556  SDValue N,
2557  const SmallVectorImpl<std::pair<SDValue, SDNode*> > &RecordedNodes) {
2558  // Accept if it is exactly the same as a previously recorded node.
2559  unsigned RecNo = MatcherTable[MatcherIndex++];
2560  assert(RecNo < RecordedNodes.size() && "Invalid CheckSame");
2561  return N == RecordedNodes[RecNo].first;
2562 }
2563 
2564 /// CheckChildSame - Implements OP_CheckChildXSame.
2565 LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
2566 CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2567  SDValue N,
2568  const SmallVectorImpl<std::pair<SDValue, SDNode*> > &RecordedNodes,
2569  unsigned ChildNo) {
2570  if (ChildNo >= N.getNumOperands())
2571  return false; // Match fails if out of range child #.
2572  return ::CheckSame(MatcherTable, MatcherIndex, N.getOperand(ChildNo),
2573  RecordedNodes);
2574 }
2575 
2576 /// CheckPatternPredicate - Implements OP_CheckPatternPredicate.
2577 LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
2578 CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2579  const SelectionDAGISel &SDISel) {
2580  return SDISel.CheckPatternPredicate(MatcherTable[MatcherIndex++]);
2581 }
2582 
2583 /// CheckNodePredicate - Implements OP_CheckNodePredicate.
2584 LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
2585 CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2586  const SelectionDAGISel &SDISel, SDNode *N) {
2587  return SDISel.CheckNodePredicate(N, MatcherTable[MatcherIndex++]);
2588 }
2589 
2590 LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
2591 CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2592  SDNode *N) {
2593  uint16_t Opc = MatcherTable[MatcherIndex++];
2594  Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
2595  return N->getOpcode() == Opc;
2596 }
2597 
2598 LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
2599 CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2600  const TargetLowering *TLI, const DataLayout &DL) {
2601  MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2602  if (N.getValueType() == VT) return true;
2603 
2604  // Handle the case when VT is iPTR.
2605  return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(DL);
2606 }
2607 
2608 LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
2609 CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2610  SDValue N, const TargetLowering *TLI, const DataLayout &DL,
2611  unsigned ChildNo) {
2612  if (ChildNo >= N.getNumOperands())
2613  return false; // Match fails if out of range child #.
2614  return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI,
2615  DL);
2616 }
2617 
2618 LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
2619 CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2620  SDValue N) {
2621  return cast<CondCodeSDNode>(N)->get() ==
2622  (ISD::CondCode)MatcherTable[MatcherIndex++];
2623 }
2624 
2625 LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
2626 CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2627  SDValue N, const TargetLowering *TLI, const DataLayout &DL) {
2628  MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2629  if (cast<VTSDNode>(N)->getVT() == VT)
2630  return true;
2631 
2632  // Handle the case when VT is iPTR.
2633  return VT == MVT::iPTR && cast<VTSDNode>(N)->getVT() == TLI->getPointerTy(DL);
2634 }
2635 
2636 LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
2637 CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2638  SDValue N) {
2639  int64_t Val = MatcherTable[MatcherIndex++];
2640  if (Val & 128)
2641  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2642 
2644  return C && C->getSExtValue() == Val;
2645 }
2646 
2647 LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
2648 CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2649  SDValue N, unsigned ChildNo) {
2650  if (ChildNo >= N.getNumOperands())
2651  return false; // Match fails if out of range child #.
2652  return ::CheckInteger(MatcherTable, MatcherIndex, N.getOperand(ChildNo));
2653 }
2654 
2655 LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
2656 CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2657  SDValue N, const SelectionDAGISel &SDISel) {
2658  int64_t Val = MatcherTable[MatcherIndex++];
2659  if (Val & 128)
2660  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2661 
2662  if (N->getOpcode() != ISD::AND) return false;
2663 
2665  return C && SDISel.CheckAndMask(N.getOperand(0), C, Val);
2666 }
2667 
2668 LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
2669 CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2670  SDValue N, const SelectionDAGISel &SDISel) {
2671  int64_t Val = MatcherTable[MatcherIndex++];
2672  if (Val & 128)
2673  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2674 
2675  if (N->getOpcode() != ISD::OR) return false;
2676 
2678  return C && SDISel.CheckOrMask(N.getOperand(0), C, Val);
2679 }
2680 
2681 /// IsPredicateKnownToFail - If we know how and can do so without pushing a
2682 /// scope, evaluate the current node. If the current predicate is known to
2683 /// fail, set Result=true and return anything. If the current predicate is
2684 /// known to pass, set Result=false and return the MatcherIndex to continue
2685 /// with. If the current predicate is unknown, set Result=false and return the
2686 /// MatcherIndex to continue with.
2687 static unsigned IsPredicateKnownToFail(const unsigned char *Table,
2688  unsigned Index, SDValue N,
2689  bool &Result,
2690  const SelectionDAGISel &SDISel,
2691  SmallVectorImpl<std::pair<SDValue, SDNode*> > &RecordedNodes) {
2692  switch (Table[Index++]) {
2693  default:
2694  Result = false;
2695  return Index-1; // Could not evaluate this predicate.
2697  Result = !::CheckSame(Table, Index, N, RecordedNodes);
2698  return Index;
2703  Result = !::CheckChildSame(Table, Index, N, RecordedNodes,
2704  Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Same);
2705  return Index;
2707  Result = !::CheckPatternPredicate(Table, Index, SDISel);
2708  return Index;
2710  Result = !::CheckNodePredicate(Table, Index, SDISel, N.getNode());
2711  return Index;
2713  Result = !::CheckOpcode(Table, Index, N.getNode());
2714  return Index;
2716  Result = !::CheckType(Table, Index, N, SDISel.TLI,
2717  SDISel.CurDAG->getDataLayout());
2718  return Index;
2727  Result = !::CheckChildType(
2728  Table, Index, N, SDISel.TLI, SDISel.CurDAG->getDataLayout(),
2729  Table[Index - 1] - SelectionDAGISel::OPC_CheckChild0Type);
2730  return Index;
2732  Result = !::CheckCondCode(Table, Index, N);
2733  return Index;
2735  Result = !::CheckValueType(Table, Index, N, SDISel.TLI,
2736  SDISel.CurDAG->getDataLayout());
2737  return Index;
2739  Result = !::CheckInteger(Table, Index, N);
2740  return Index;
2746  Result = !::CheckChildInteger(Table, Index, N,
2747  Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Integer);
2748  return Index;
2750  Result = !::CheckAndImm(Table, Index, N, SDISel);
2751  return Index;
2753  Result = !::CheckOrImm(Table, Index, N, SDISel);
2754  return Index;
2755  }
2756 }
2757 
2758 namespace {
2759 struct MatchScope {
2760  /// FailIndex - If this match fails, this is the index to continue with.
2761  unsigned FailIndex;
2762 
2763  /// NodeStack - The node stack when the scope was formed.
2764  SmallVector<SDValue, 4> NodeStack;
2765 
2766  /// NumRecordedNodes - The number of recorded nodes when the scope was formed.
2767  unsigned NumRecordedNodes;
2768 
2769  /// NumMatchedMemRefs - The number of matched memref entries.
2770  unsigned NumMatchedMemRefs;
2771 
2772  /// InputChain/InputGlue - The current chain/glue
2773  SDValue InputChain, InputGlue;
2774 
2775  /// HasChainNodesMatched - True if the ChainNodesMatched list is non-empty.
2776  bool HasChainNodesMatched;
2777 };
2778 
2779 /// \\brief A DAG update listener to keep the matching state
2780 /// (i.e. RecordedNodes and MatchScope) uptodate if the target is allowed to
2781 /// change the DAG while matching. X86 addressing mode matcher is an example
2782 /// for this.
2783 class MatchStateUpdater : public SelectionDAG::DAGUpdateListener
2784 {
2785  SDNode **NodeToMatch;
2787  SmallVectorImpl<MatchScope> &MatchScopes;
2788 public:
2789  MatchStateUpdater(SelectionDAG &DAG, SDNode **NodeToMatch,
2790  SmallVectorImpl<std::pair<SDValue, SDNode *>> &RN,
2792  : SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch),
2793  RecordedNodes(RN), MatchScopes(MS) {}
2794 
2795  void NodeDeleted(SDNode *N, SDNode *E) override {
2796  // Some early-returns here to avoid the search if we deleted the node or
2797  // if the update comes from MorphNodeTo (MorphNodeTo is the last thing we
2798  // do, so it's unnecessary to update matching state at that point).
2799  // Neither of these can occur currently because we only install this
2800  // update listener during matching a complex patterns.
2801  if (!E || E->isMachineOpcode())
2802  return;
2803  // Check if NodeToMatch was updated.
2804  if (N == *NodeToMatch)
2805  *NodeToMatch = E;
2806  // Performing linear search here does not matter because we almost never
2807  // run this code. You'd have to have a CSE during complex pattern
2808  // matching.
2809  for (auto &I : RecordedNodes)
2810  if (I.first.getNode() == N)
2811  I.first.setNode(E);
2812 
2813  for (auto &I : MatchScopes)
2814  for (auto &J : I.NodeStack)
2815  if (J.getNode() == N)
2816  J.setNode(E);
2817  }
2818 };
2819 } // end anonymous namespace
2820 
2822  const unsigned char *MatcherTable,
2823  unsigned TableSize) {
2824  // FIXME: Should these even be selected? Handle these cases in the caller?
2825  switch (NodeToMatch->getOpcode()) {
2826  default:
2827  break;
2828  case ISD::EntryToken: // These nodes remain the same.
2829  case ISD::BasicBlock:
2830  case ISD::Register:
2831  case ISD::RegisterMask:
2832  case ISD::HANDLENODE:
2833  case ISD::MDNODE_SDNODE:
2834  case ISD::TargetConstant:
2835  case ISD::TargetConstantFP:
2837  case ISD::TargetFrameIndex:
2839  case ISD::MCSymbol:
2841  case ISD::TargetJumpTable:
2844  case ISD::TokenFactor:
2845  case ISD::CopyFromReg:
2846  case ISD::CopyToReg:
2847  case ISD::EH_LABEL:
2848  case ISD::LIFETIME_START:
2849  case ISD::LIFETIME_END:
2850  NodeToMatch->setNodeId(-1); // Mark selected.
2851  return;
2852  case ISD::AssertSext:
2853  case ISD::AssertZext:
2854  CurDAG->ReplaceAllUsesOfValueWith(SDValue(NodeToMatch, 0),
2855  NodeToMatch->getOperand(0));
2856  CurDAG->RemoveDeadNode(NodeToMatch);
2857  return;
2858  case ISD::INLINEASM:
2859  Select_INLINEASM(NodeToMatch);
2860  return;
2861  case ISD::READ_REGISTER:
2862  Select_READ_REGISTER(NodeToMatch);
2863  return;
2864  case ISD::WRITE_REGISTER:
2865  Select_WRITE_REGISTER(NodeToMatch);
2866  return;
2867  case ISD::UNDEF:
2868  Select_UNDEF(NodeToMatch);
2869  return;
2870  }
2871 
2872  assert(!NodeToMatch->isMachineOpcode() && "Node already selected!");
2873 
2874  // Set up the node stack with NodeToMatch as the only node on the stack.
2875  SmallVector<SDValue, 8> NodeStack;
2876  SDValue N = SDValue(NodeToMatch, 0);
2877  NodeStack.push_back(N);
2878 
2879  // MatchScopes - Scopes used when matching, if a match failure happens, this
2880  // indicates where to continue checking.
2881  SmallVector<MatchScope, 8> MatchScopes;
2882 
2883  // RecordedNodes - This is the set of nodes that have been recorded by the
2884  // state machine. The second value is the parent of the node, or null if the
2885  // root is recorded.
2886  SmallVector<std::pair<SDValue, SDNode*>, 8> RecordedNodes;
2887 
2888  // MatchedMemRefs - This is the set of MemRef's we've seen in the input
2889  // pattern.
2890  SmallVector<MachineMemOperand*, 2> MatchedMemRefs;
2891 
2892  // These are the current input chain and glue for use when generating nodes.
2893  // Various Emit operations change these. For example, emitting a copytoreg
2894  // uses and updates these.
2895  SDValue InputChain, InputGlue;
2896 
2897  // ChainNodesMatched - If a pattern matches nodes that have input/output
2898  // chains, the OPC_EmitMergeInputChains operation is emitted which indicates
2899  // which ones they are. The result is captured into this list so that we can
2900  // update the chain results when the pattern is complete.
2901  SmallVector<SDNode*, 3> ChainNodesMatched;
2902 
2903  DEBUG(dbgs() << "ISEL: Starting pattern match on root node: ";
2904  NodeToMatch->dump(CurDAG);
2905  dbgs() << '\n');
2906 
2907  // Determine where to start the interpreter. Normally we start at opcode #0,
2908  // but if the state machine starts with an OPC_SwitchOpcode, then we
2909  // accelerate the first lookup (which is guaranteed to be hot) with the
2910  // OpcodeOffset table.
2911  unsigned MatcherIndex = 0;
2912 
2913  if (!OpcodeOffset.empty()) {
2914  // Already computed the OpcodeOffset table, just index into it.
2915  if (N.getOpcode() < OpcodeOffset.size())
2916  MatcherIndex = OpcodeOffset[N.getOpcode()];
2917  DEBUG(dbgs() << " Initial Opcode index to " << MatcherIndex << "\n");
2918 
2919  } else if (MatcherTable[0] == OPC_SwitchOpcode) {
2920  // Otherwise, the table isn't computed, but the state machine does start
2921  // with an OPC_SwitchOpcode instruction. Populate the table now, since this
2922  // is the first time we're selecting an instruction.
2923  unsigned Idx = 1;
2924  while (1) {
2925  // Get the size of this case.
2926  unsigned CaseSize = MatcherTable[Idx++];
2927  if (CaseSize & 128)
2928  CaseSize = GetVBR(CaseSize, MatcherTable, Idx);
2929  if (CaseSize == 0) break;
2930 
2931  // Get the opcode, add the index to the table.
2932  uint16_t Opc = MatcherTable[Idx++];
2933  Opc |= (unsigned short)MatcherTable[Idx++] << 8;
2934  if (Opc >= OpcodeOffset.size())
2935  OpcodeOffset.resize((Opc+1)*2);
2936  OpcodeOffset[Opc] = Idx;
2937  Idx += CaseSize;
2938  }
2939 
2940  // Okay, do the lookup for the first opcode.
2941  if (N.getOpcode() < OpcodeOffset.size())
2942  MatcherIndex = OpcodeOffset[N.getOpcode()];
2943  }
2944 
2945  while (1) {
2946  assert(MatcherIndex < TableSize && "Invalid index");
2947 #ifndef NDEBUG
2948  unsigned CurrentOpcodeIndex = MatcherIndex;
2949 #endif
2950  BuiltinOpcodes Opcode = (BuiltinOpcodes)MatcherTable[MatcherIndex++];
2951  switch (Opcode) {
2952  case OPC_Scope: {
2953  // Okay, the semantics of this operation are that we should push a scope
2954  // then evaluate the first child. However, pushing a scope only to have
2955  // the first check fail (which then pops it) is inefficient. If we can
2956  // determine immediately that the first check (or first several) will
2957  // immediately fail, don't even bother pushing a scope for them.
2958  unsigned FailIndex;
2959 
2960  while (1) {
2961  unsigned NumToSkip = MatcherTable[MatcherIndex++];
2962  if (NumToSkip & 128)
2963  NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
2964  // Found the end of the scope with no match.
2965  if (NumToSkip == 0) {
2966  FailIndex = 0;
2967  break;
2968  }
2969 
2970  FailIndex = MatcherIndex+NumToSkip;
2971 
2972  unsigned MatcherIndexOfPredicate = MatcherIndex;
2973  (void)MatcherIndexOfPredicate; // silence warning.
2974 
2975  // If we can't evaluate this predicate without pushing a scope (e.g. if
2976  // it is a 'MoveParent') or if the predicate succeeds on this node, we
2977  // push the scope and evaluate the full predicate chain.
2978  bool Result;
2979  MatcherIndex = IsPredicateKnownToFail(MatcherTable, MatcherIndex, N,
2980  Result, *this, RecordedNodes);
2981  if (!Result)
2982  break;
2983 
2984  DEBUG(dbgs() << " Skipped scope entry (due to false predicate) at "
2985  << "index " << MatcherIndexOfPredicate
2986  << ", continuing at " << FailIndex << "\n");
2987  ++NumDAGIselRetries;
2988 
2989  // Otherwise, we know that this case of the Scope is guaranteed to fail,
2990  // move to the next case.
2991  MatcherIndex = FailIndex;
2992  }
2993 
2994  // If the whole scope failed to match, bail.
2995  if (FailIndex == 0) break;
2996 
2997  // Push a MatchScope which indicates where to go if the first child fails
2998  // to match.
2999  MatchScope NewEntry;
3000  NewEntry.FailIndex = FailIndex;
3001  NewEntry.NodeStack.append(NodeStack.begin(), NodeStack.end());
3002  NewEntry.NumRecordedNodes = RecordedNodes.size();
3003  NewEntry.NumMatchedMemRefs = MatchedMemRefs.size();
3004  NewEntry.InputChain = InputChain;
3005  NewEntry.InputGlue = InputGlue;
3006  NewEntry.HasChainNodesMatched = !ChainNodesMatched.empty();
3007  MatchScopes.push_back(NewEntry);
3008  continue;
3009  }
3010  case OPC_RecordNode: {
3011  // Remember this node, it may end up being an operand in the pattern.
3012  SDNode *Parent = nullptr;
3013  if (NodeStack.size() > 1)
3014  Parent = NodeStack[NodeStack.size()-2].getNode();
3015  RecordedNodes.push_back(std::make_pair(N, Parent));
3016  continue;
3017  }
3018 
3022  case OPC_RecordChild6: case OPC_RecordChild7: {
3023  unsigned ChildNo = Opcode-OPC_RecordChild0;
3024  if (ChildNo >= N.getNumOperands())
3025  break; // Match fails if out of range child #.
3026 
3027  RecordedNodes.push_back(std::make_pair(N->getOperand(ChildNo),
3028  N.getNode()));
3029  continue;
3030  }
3031  case OPC_RecordMemRef:
3032  MatchedMemRefs.push_back(cast<MemSDNode>(N)->getMemOperand());
3033  continue;
3034 
3035  case OPC_CaptureGlueInput:
3036  // If the current node has an input glue, capture it in InputGlue.
3037  if (N->getNumOperands() != 0 &&
3038  N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue)
3039  InputGlue = N->getOperand(N->getNumOperands()-1);
3040  continue;
3041 
3042  case OPC_MoveChild: {
3043  unsigned ChildNo = MatcherTable[MatcherIndex++];
3044  if (ChildNo >= N.getNumOperands())
3045  break; // Match fails if out of range child #.
3046  N = N.getOperand(ChildNo);
3047  NodeStack.push_back(N);
3048  continue;
3049  }
3050 
3051  case OPC_MoveChild0: case OPC_MoveChild1:
3052  case OPC_MoveChild2: case OPC_MoveChild3:
3053  case OPC_MoveChild4: case OPC_MoveChild5:
3054  case OPC_MoveChild6: case OPC_MoveChild7: {
3055  unsigned ChildNo = Opcode-OPC_MoveChild0;
3056  if (ChildNo >= N.getNumOperands())
3057  break; // Match fails if out of range child #.
3058  N = N.getOperand(ChildNo);
3059  NodeStack.push_back(N);
3060  continue;
3061  }
3062 
3063  case OPC_MoveParent:
3064  // Pop the current node off the NodeStack.
3065  NodeStack.pop_back();
3066  assert(!NodeStack.empty() && "Node stack imbalance!");
3067  N = NodeStack.back();
3068  continue;
3069 
3070  case OPC_CheckSame:
3071  if (!::CheckSame(MatcherTable, MatcherIndex, N, RecordedNodes)) break;
3072  continue;
3073 
3076  if (!::CheckChildSame(MatcherTable, MatcherIndex, N, RecordedNodes,
3077  Opcode-OPC_CheckChild0Same))
3078  break;
3079  continue;
3080 
3082  if (!::CheckPatternPredicate(MatcherTable, MatcherIndex, *this)) break;
3083  continue;
3084  case OPC_CheckPredicate:
3085  if (!::CheckNodePredicate(MatcherTable, MatcherIndex, *this,
3086  N.getNode()))
3087  break;
3088  continue;
3089  case OPC_CheckComplexPat: {
3090  unsigned CPNum = MatcherTable[MatcherIndex++];
3091  unsigned RecNo = MatcherTable[MatcherIndex++];
3092  assert(RecNo < RecordedNodes.size() && "Invalid CheckComplexPat");
3093 
3094  // If target can modify DAG during matching, keep the matching state
3095  // consistent.
3096  std::unique_ptr<MatchStateUpdater> MSU;
3098  MSU.reset(new MatchStateUpdater(*CurDAG, &NodeToMatch, RecordedNodes,
3099  MatchScopes));
3100 
3101  if (!CheckComplexPattern(NodeToMatch, RecordedNodes[RecNo].second,
3102  RecordedNodes[RecNo].first, CPNum,
3103  RecordedNodes))
3104  break;
3105  continue;
3106  }
3107  case OPC_CheckOpcode:
3108  if (!::CheckOpcode(MatcherTable, MatcherIndex, N.getNode())) break;
3109  continue;
3110 
3111  case OPC_CheckType:
3112  if (!::CheckType(MatcherTable, MatcherIndex, N, TLI,
3113  CurDAG->getDataLayout()))
3114  break;
3115  continue;
3116 
3117  case OPC_SwitchOpcode: {
3118  unsigned CurNodeOpcode = N.getOpcode();
3119  unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3120  unsigned CaseSize;
3121  while (1) {
3122  // Get the size of this case.
3123  CaseSize = MatcherTable[MatcherIndex++];
3124  if (CaseSize & 128)
3125  CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3126  if (CaseSize == 0) break;
3127 
3128  uint16_t Opc = MatcherTable[MatcherIndex++];
3129  Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3130 
3131  // If the opcode matches, then we will execute this case.
3132  if (CurNodeOpcode == Opc)
3133  break;
3134 
3135  // Otherwise, skip over this case.
3136  MatcherIndex += CaseSize;
3137  }
3138 
3139  // If no cases matched, bail out.
3140  if (CaseSize == 0) break;
3141 
3142  // Otherwise, execute the case we found.
3143  DEBUG(dbgs() << " OpcodeSwitch from " << SwitchStart
3144  << " to " << MatcherIndex << "\n");
3145  continue;
3146  }
3147 
3148  case OPC_SwitchType: {
3149  MVT CurNodeVT = N.getSimpleValueType();
3150  unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3151  unsigned CaseSize;
3152  while (1) {
3153  // Get the size of this case.
3154  CaseSize = MatcherTable[MatcherIndex++];
3155  if (CaseSize & 128)
3156  CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3157  if (CaseSize == 0) break;
3158 
3159  MVT CaseVT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3160  if (CaseVT == MVT::iPTR)
3161  CaseVT = TLI->getPointerTy(CurDAG->getDataLayout());
3162 
3163  // If the VT matches, then we will execute this case.
3164  if (CurNodeVT == CaseVT)
3165  break;
3166 
3167  // Otherwise, skip over this case.
3168  MatcherIndex += CaseSize;
3169  }
3170 
3171  // If no cases matched, bail out.
3172  if (CaseSize == 0) break;
3173 
3174  // Otherwise, execute the case we found.
3175  DEBUG(dbgs() << " TypeSwitch[" << EVT(CurNodeVT).getEVTString()
3176  << "] from " << SwitchStart << " to " << MatcherIndex<<'\n');
3177  continue;
3178  }
3183  if (!::CheckChildType(MatcherTable, MatcherIndex, N, TLI,
3184  CurDAG->getDataLayout(),
3185  Opcode - OPC_CheckChild0Type))
3186  break;
3187  continue;
3188  case OPC_CheckCondCode:
3189  if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break;
3190  continue;
3191  case OPC_CheckValueType:
3192  if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI,
3193  CurDAG->getDataLayout()))
3194  break;
3195  continue;
3196  case OPC_CheckInteger:
3197  if (!::CheckInteger(MatcherTable, MatcherIndex, N)) break;
3198  continue;
3202  if (!::CheckChildInteger(MatcherTable, MatcherIndex, N,
3203  Opcode-OPC_CheckChild0Integer)) break;
3204  continue;
3205  case OPC_CheckAndImm:
3206  if (!::CheckAndImm(MatcherTable, MatcherIndex, N, *this)) break;
3207  continue;
3208  case OPC_CheckOrImm:
3209  if (!::CheckOrImm(MatcherTable, MatcherIndex, N, *this)) break;
3210  continue;
3211 
3213  assert(NodeStack.size() != 1 && "No parent node");
3214  // Verify that all intermediate nodes between the root and this one have
3215  // a single use.
3216  bool HasMultipleUses = false;
3217  for (unsigned i = 1, e = NodeStack.size()-1; i != e; ++i)
3218  if (!NodeStack[i].hasOneUse()) {
3219  HasMultipleUses = true;
3220  break;
3221  }
3222  if (HasMultipleUses) break;
3223 
3224  // Check to see that the target thinks this is profitable to fold and that
3225  // we can fold it without inducing cycles in the graph.
3226  if (!IsProfitableToFold(N, NodeStack[NodeStack.size()-2].getNode(),
3227  NodeToMatch) ||
3228  !IsLegalToFold(N, NodeStack[NodeStack.size()-2].getNode(),
3229  NodeToMatch, OptLevel,
3230  true/*We validate our own chains*/))
3231  break;
3232 
3233  continue;
3234  }
3235  case OPC_EmitInteger: {
3237  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3238  int64_t Val = MatcherTable[MatcherIndex++];
3239  if (Val & 128)
3240  Val = GetVBR(Val, MatcherTable, MatcherIndex);
3241  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3242  CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch),
3243  VT), nullptr));
3244  continue;
3245  }
3246  case OPC_EmitRegister: {
3248  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3249  unsigned RegNo = MatcherTable[MatcherIndex++];
3250  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3251  CurDAG->getRegister(RegNo, VT), nullptr));
3252  continue;
3253  }
3254  case OPC_EmitRegister2: {
3255  // For targets w/ more than 256 register names, the register enum
3256  // values are stored in two bytes in the matcher table (just like
3257  // opcodes).
3259  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3260  unsigned RegNo = MatcherTable[MatcherIndex++];
3261  RegNo |= MatcherTable[MatcherIndex++] << 8;
3262  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3263  CurDAG->getRegister(RegNo, VT), nullptr));
3264  continue;
3265  }
3266 
3267  case OPC_EmitConvertToTarget: {
3268  // Convert from IMM/FPIMM to target version.
3269  unsigned RecNo = MatcherTable[MatcherIndex++];
3270  assert(RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget");
3271  SDValue Imm = RecordedNodes[RecNo].first;
3272 
3273  if (Imm->getOpcode() == ISD::Constant) {
3274  const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();
3275  Imm = CurDAG->getTargetConstant(*Val, SDLoc(NodeToMatch),
3276  Imm.getValueType());
3277  } else if (Imm->getOpcode() == ISD::ConstantFP) {
3278  const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();
3279  Imm = CurDAG->getTargetConstantFP(*Val, SDLoc(NodeToMatch),
3280  Imm.getValueType());
3281  }
3282 
3283  RecordedNodes.push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
3284  continue;
3285  }
3286 
3287  case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0
3288  case OPC_EmitMergeInputChains1_1: // OPC_EmitMergeInputChains, 1, 1
3289  case OPC_EmitMergeInputChains1_2: { // OPC_EmitMergeInputChains, 1, 2
3290  // These are space-optimized forms of OPC_EmitMergeInputChains.
3291  assert(!InputChain.getNode() &&
3292  "EmitMergeInputChains should be the first chain producing node");
3293  assert(ChainNodesMatched.empty() &&
3294  "Should only have one EmitMergeInputChains per match");
3295 
3296  // Read all of the chained nodes.
3297  unsigned RecNo = Opcode - OPC_EmitMergeInputChains1_0;
3298  assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains");
3299  ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3300 
3301  // FIXME: What if other value results of the node have uses not matched
3302  // by this pattern?
3303  if (ChainNodesMatched.back() != NodeToMatch &&
3304  !RecordedNodes[RecNo].first.hasOneUse()) {
3305  ChainNodesMatched.clear();
3306  break;
3307  }
3308 
3309  // Merge the input chains if they are not intra-pattern references.
3310  InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3311 
3312  if (!InputChain.getNode())
3313  break; // Failed to merge.
3314  continue;
3315  }
3316 
3317  case OPC_EmitMergeInputChains: {
3318  assert(!InputChain.getNode() &&
3319  "EmitMergeInputChains should be the first chain producing node");
3320  // This node gets a list of nodes we matched in the input that have
3321  // chains. We want to token factor all of the input chains to these nodes
3322  // together. However, if any of the input chains is actually one of the
3323  // nodes matched in this pattern, then we have an intra-match reference.
3324  // Ignore these because the newly token factored chain should not refer to
3325  // the old nodes.
3326  unsigned NumChains = MatcherTable[MatcherIndex++];
3327  assert(NumChains != 0 && "Can't TF zero chains");
3328 
3329  assert(ChainNodesMatched.empty() &&
3330  "Should only have one EmitMergeInputChains per match");
3331 
3332  // Read all of the chained nodes.
3333  for (unsigned i = 0; i != NumChains; ++i) {
3334  unsigned RecNo = MatcherTable[MatcherIndex++];
3335  assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains");
3336  ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3337 
3338  // FIXME: What if other value results of the node have uses not matched
3339  // by this pattern?
3340  if (ChainNodesMatched.back() != NodeToMatch &&
3341  !RecordedNodes[RecNo].first.hasOneUse()) {
3342  ChainNodesMatched.clear();
3343  break;
3344  }
3345  }
3346 
3347  // If the inner loop broke out, the match fails.
3348  if (ChainNodesMatched.empty())
3349  break;
3350 
3351  // Merge the input chains if they are not intra-pattern references.
3352  InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3353 
3354  if (!InputChain.getNode())
3355  break; // Failed to merge.
3356 
3357  continue;
3358  }
3359 
3360  case OPC_EmitCopyToReg: {
3361  unsigned RecNo = MatcherTable[MatcherIndex++];
3362  assert(RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg");
3363  unsigned DestPhysReg = MatcherTable[MatcherIndex++];
3364 
3365  if (!InputChain.getNode())
3366  InputChain = CurDAG->getEntryNode();
3367 
3368  InputChain = CurDAG->getCopyToReg(InputChain, SDLoc(NodeToMatch),
3369  DestPhysReg, RecordedNodes[RecNo].first,
3370  InputGlue);
3371 
3372  InputGlue = InputChain.getValue(1);
3373  continue;
3374  }
3375 
3376  case OPC_EmitNodeXForm: {
3377  unsigned XFormNo = MatcherTable[MatcherIndex++];
3378  unsigned RecNo = MatcherTable[MatcherIndex++];
3379  assert(RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm");
3380  SDValue Res = RunSDNodeXForm(RecordedNodes[RecNo].first, XFormNo);
3381  RecordedNodes.push_back(std::pair<SDValue,SDNode*>(Res, nullptr));
3382  continue;
3383  }
3384 
3385  case OPC_EmitNode: case OPC_MorphNodeTo:
3386  case OPC_EmitNode0: case OPC_EmitNode1: case OPC_EmitNode2:
3388  uint16_t TargetOpc = MatcherTable[MatcherIndex++];
3389  TargetOpc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3390  unsigned EmitNodeInfo = MatcherTable[MatcherIndex++];
3391  // Get the result VT list.
3392  unsigned NumVTs;
3393  // If this is one of the compressed forms, get the number of VTs based
3394  // on the Opcode. Otherwise read the next byte from the table.
3395  if (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2)
3396  NumVTs = Opcode - OPC_MorphNodeTo0;
3397  else if (Opcode >= OPC_EmitNode0 && Opcode <= OPC_EmitNode2)
3398  NumVTs = Opcode - OPC_EmitNode0;
3399  else
3400  NumVTs = MatcherTable[MatcherIndex++];
3401  SmallVector<EVT, 4> VTs;
3402  for (unsigned i = 0; i != NumVTs; ++i) {
3404  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3405  if (VT == MVT::iPTR)
3406  VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy;
3407  VTs.push_back(VT);
3408  }
3409 
3410  if (EmitNodeInfo & OPFL_Chain)
3411  VTs.push_back(MVT::Other);
3412  if (EmitNodeInfo & OPFL_GlueOutput)
3413  VTs.push_back(MVT::Glue);
3414 
3415  // This is hot code, so optimize the two most common cases of 1 and 2
3416  // results.
3417  SDVTList VTList;
3418  if (VTs.size() == 1)
3419  VTList = CurDAG->getVTList(VTs[0]);
3420  else if (VTs.size() == 2)
3421  VTList = CurDAG->getVTList(VTs[0], VTs[1]);
3422  else
3423  VTList = CurDAG->getVTList(VTs);
3424 
3425  // Get the operand list.
3426  unsigned NumOps = MatcherTable[MatcherIndex++];
3428  for (unsigned i = 0; i != NumOps; ++i) {
3429  unsigned RecNo = MatcherTable[MatcherIndex++];
3430  if (RecNo & 128)
3431  RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex);
3432 
3433  assert(RecNo < RecordedNodes.size() && "Invalid EmitNode");
3434  Ops.push_back(RecordedNodes[RecNo].first);
3435  }
3436 
3437  // If there are variadic operands to add, handle them now.
3438  if (EmitNodeInfo & OPFL_VariadicInfo) {
3439  // Determine the start index to copy from.
3440  unsigned FirstOpToCopy = getNumFixedFromVariadicInfo(EmitNodeInfo);
3441  FirstOpToCopy += (EmitNodeInfo & OPFL_Chain) ? 1 : 0;
3442  assert(NodeToMatch->getNumOperands() >= FirstOpToCopy &&
3443  "Invalid variadic node");
3444  // Copy all of the variadic operands, not including a potential glue
3445  // input.
3446  for (unsigned i = FirstOpToCopy, e = NodeToMatch->getNumOperands();
3447  i != e; ++i) {
3448  SDValue V = NodeToMatch->getOperand(i);
3449  if (V.getValueType() == MVT::Glue) break;
3450  Ops.push_back(V);
3451  }
3452  }
3453 
3454  // If this has chain/glue inputs, add them.
3455  if (EmitNodeInfo & OPFL_Chain)
3456  Ops.push_back(InputChain);
3457  if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr)
3458  Ops.push_back(InputGlue);
3459 
3460  // Create the node.
3461  SDNode *Res = nullptr;
3462  bool IsMorphNodeTo = Opcode == OPC_MorphNodeTo ||
3463  (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2);
3464  if (!IsMorphNodeTo) {
3465  // If this is a normal EmitNode command, just create the new node and
3466  // add the results to the RecordedNodes list.
3467  Res = CurDAG->getMachineNode(TargetOpc, SDLoc(NodeToMatch),
3468  VTList, Ops);
3469 
3470  // Add all the non-glue/non-chain results to the RecordedNodes list.
3471  for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
3472  if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue) break;
3473  RecordedNodes.push_back(std::pair<SDValue,SDNode*>(SDValue(Res, i),
3474  nullptr));
3475  }
3476 
3477  } else {
3478  assert(NodeToMatch->getOpcode() != ISD::DELETED_NODE &&
3479  "NodeToMatch was removed partway through selection");
3481  SDNode *E) {
3482  auto &Chain = ChainNodesMatched;
3483  assert((!E || !is_contained(Chain, N)) &&
3484  "Chain node replaced during MorphNode");
3485  Chain.erase(std::remove(Chain.begin(), Chain.end(), N), Chain.end());
3486  });
3487  Res = MorphNode(NodeToMatch, TargetOpc, VTList, Ops, EmitNodeInfo);
3488  }
3489 
3490  // If the node had chain/glue results, update our notion of the current
3491  // chain and glue.
3492  if (EmitNodeInfo & OPFL_GlueOutput) {
3493  InputGlue = SDValue(Res, VTs.size()-1);
3494  if (EmitNodeInfo & OPFL_Chain)
3495  InputChain = SDValue(Res, VTs.size()-2);
3496  } else if (EmitNodeInfo & OPFL_Chain)
3497  InputChain = SDValue(Res, VTs.size()-1);
3498 
3499  // If the OPFL_MemRefs glue is set on this node, slap all of the
3500  // accumulated memrefs onto it.
3501  //
3502  // FIXME: This is vastly incorrect for patterns with multiple outputs
3503  // instructions that access memory and for ComplexPatterns that match
3504  // loads.
3505  if (EmitNodeInfo & OPFL_MemRefs) {
3506  // Only attach load or store memory operands if the generated
3507  // instruction may load or store.
3508  const MCInstrDesc &MCID = TII->get(TargetOpc);
3509  bool mayLoad = MCID.mayLoad();
3510  bool mayStore = MCID.mayStore();
3511 
3512  unsigned NumMemRefs = 0;
3514  MatchedMemRefs.begin(), E = MatchedMemRefs.end(); I != E; ++I) {
3515  if ((*I)->isLoad()) {
3516  if (mayLoad)
3517  ++NumMemRefs;
3518  } else if ((*I)->isStore()) {
3519  if (mayStore)
3520  ++NumMemRefs;
3521  } else {
3522  ++NumMemRefs;
3523  }
3524  }
3525 
3526  MachineSDNode::mmo_iterator MemRefs =
3527  MF->allocateMemRefsArray(NumMemRefs);
3528 
3529  MachineSDNode::mmo_iterator MemRefsPos = MemRefs;
3531  MatchedMemRefs.begin(), E = MatchedMemRefs.end(); I != E; ++I) {
3532  if ((*I)->isLoad()) {
3533  if (mayLoad)
3534  *MemRefsPos++ = *I;
3535  } else if ((*I)->isStore()) {
3536  if (mayStore)
3537  *MemRefsPos++ = *I;
3538  } else {
3539  *MemRefsPos++ = *I;
3540  }
3541  }
3542 
3543  cast<MachineSDNode>(Res)
3544  ->setMemRefs(MemRefs, MemRefs + NumMemRefs);
3545  }
3546 
3547  DEBUG(dbgs() << " "
3548  << (IsMorphNodeTo ? "Morphed" : "Created")
3549  << " node: "; Res->dump(CurDAG); dbgs() << "\n");
3550 
3551  // If this was a MorphNodeTo then we're completely done!
3552  if (IsMorphNodeTo) {
3553  // Update chain uses.
3554  UpdateChains(Res, InputChain, ChainNodesMatched, true);
3555  return;
3556  }
3557  continue;
3558  }
3559 
3560  case OPC_CompleteMatch: {
3561  // The match has been completed, and any new nodes (if any) have been
3562  // created. Patch up references to the matched dag to use the newly
3563  // created nodes.
3564  unsigned NumResults = MatcherTable[MatcherIndex++];
3565 
3566  for (unsigned i = 0; i != NumResults; ++i) {
3567  unsigned ResSlot = MatcherTable[MatcherIndex++];
3568  if (ResSlot & 128)
3569  ResSlot = GetVBR(ResSlot, MatcherTable, MatcherIndex);
3570 
3571  assert(ResSlot < RecordedNodes.size() && "Invalid CompleteMatch");
3572  SDValue Res = RecordedNodes[ResSlot].first;
3573 
3574  assert(i < NodeToMatch->getNumValues() &&
3575  NodeToMatch->getValueType(i) != MVT::Other &&
3576  NodeToMatch->getValueType(i) != MVT::Glue &&
3577  "Invalid number of results to complete!");
3578  assert((NodeToMatch->getValueType(i) == Res.getValueType() ||
3579  NodeToMatch->getValueType(i) == MVT::iPTR ||
3580  Res.getValueType() == MVT::iPTR ||
3581  NodeToMatch->getValueType(i).getSizeInBits() ==
3582  Res.getValueSizeInBits()) &&
3583  "invalid replacement");
3584  CurDAG->ReplaceAllUsesOfValueWith(SDValue(NodeToMatch, i), Res);
3585  }
3586 
3587  // Update chain uses.
3588  UpdateChains(NodeToMatch, InputChain, ChainNodesMatched, false);
3589 
3590  // If the root node defines glue, we need to update it to the glue result.
3591  // TODO: This never happens in our tests and I think it can be removed /
3592  // replaced with an assert, but if we do it this the way the change is
3593  // NFC.
3594  if (NodeToMatch->getValueType(NodeToMatch->getNumValues() - 1) ==
3595  MVT::Glue &&
3596  InputGlue.getNode())
3598  SDValue(NodeToMatch, NodeToMatch->getNumValues() - 1), InputGlue);
3599 
3600  assert(NodeToMatch->use_empty() &&
3601  "Didn't replace all uses of the node?");
3602  CurDAG->RemoveDeadNode(NodeToMatch);
3603 
3604  return;
3605  }
3606  }
3607 
3608  // If the code reached this point, then the match failed. See if there is
3609  // another child to try in the current 'Scope', otherwise pop it until we
3610  // find a case to check.
3611  DEBUG(dbgs() << " Match failed at index " << CurrentOpcodeIndex << "\n");
3612  ++NumDAGIselRetries;
3613  while (1) {
3614  if (MatchScopes.empty()) {
3615  CannotYetSelect(NodeToMatch);
3616  return;
3617  }
3618 
3619  // Restore the interpreter state back to the point where the scope was
3620  // formed.
3621  MatchScope &LastScope = MatchScopes.back();
3622  RecordedNodes.resize(LastScope.NumRecordedNodes);
3623  NodeStack.clear();
3624  NodeStack.append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());
3625  N = NodeStack.back();
3626 
3627  if (LastScope.NumMatchedMemRefs != MatchedMemRefs.size())
3628  MatchedMemRefs.resize(LastScope.NumMatchedMemRefs);
3629  MatcherIndex = LastScope.FailIndex;
3630 
3631  DEBUG(dbgs() << " Continuing at " << MatcherIndex << "\n");
3632 
3633  InputChain = LastScope.InputChain;
3634  InputGlue = LastScope.InputGlue;
3635  if (!LastScope.HasChainNodesMatched)
3636  ChainNodesMatched.clear();
3637 
3638  // Check to see what the offset is at the new MatcherIndex. If it is zero
3639  // we have reached the end of this scope, otherwise we have another child
3640  // in the current scope to try.
3641  unsigned NumToSkip = MatcherTable[MatcherIndex++];
3642  if (NumToSkip & 128)
3643  NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3644 
3645  // If we have another child in this scope to match, update FailIndex and
3646  // try it.
3647  if (NumToSkip != 0) {
3648  LastScope.FailIndex = MatcherIndex+NumToSkip;
3649  break;
3650  }
3651 
3652  // End of this scope, pop it and try the next child in the containing
3653  // scope.
3654  MatchScopes.pop_back();
3655  }
3656  }
3657 }
3658 
3659 void SelectionDAGISel::CannotYetSelect(SDNode *N) {
3660  std::string msg;
3661  raw_string_ostream Msg(msg);
3662  Msg << "Cannot select: ";
3663 
3664  if (N->getOpcode() != ISD::INTRINSIC_W_CHAIN &&
3666  N->getOpcode() != ISD::INTRINSIC_VOID) {
3667  N->printrFull(Msg, CurDAG);
3668  Msg << "\nIn function: " << MF->getName();
3669  } else {
3670  bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other;
3671  unsigned iid =
3672  cast<ConstantSDNode>(N->getOperand(HasInputChain))->getZExtValue();
3673  if (iid < Intrinsic::num_intrinsics)
3674  Msg << "intrinsic %" << Intrinsic::getName((Intrinsic::ID)iid, None);
3675  else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo())
3676  Msg << "target intrinsic %" << TII->getName(iid);
3677  else
3678  Msg << "unknown intrinsic #" << iid;
3679  }
3680  report_fatal_error(Msg.str());
3681 }
3682 
3683 char SelectionDAGISel::ID = 0;
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
bool use_empty() const
Return true if there are no uses of this node.
std::vector< BitTestBlock > BitTestCases
BitTestCases - Vector of BitTestBlock structures used to communicate SwitchInst code generation infor...
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
SelectionDAGBuilder * SDB
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:102
llvm::DenseMap< std::pair< const MachineBasicBlock *, const Value * >, unsigned > SwiftErrorVRegUpwardsUse
A list of upward exposed vreg uses that need to be satisfied by either a copy def or a phi node at th...
mop_iterator operands_end()
Definition: MachineInstr.h:296
SDValue getValue(unsigned R) const
bool hasPostISelHook(QueryType Type=IgnoreBundle) const
Return true if this instruction requires adjustment after instruction selection by calling a target h...
Definition: MachineInstr.h:671
static SDNode * findGlueUse(SDNode *N)
findGlueUse - Return use of MVT::Glue value produced by the specified SDNode.
void EmitLiveInCopies(MachineBasicBlock *EntryMBB, const TargetRegisterInfo &TRI, const TargetInstrInfo &TII)
EmitLiveInCopies - Emit copies to initialize livein virtual registers into the given entry block...
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
virtual bool enableMachineScheduler() const
True if the subtarget should run MachineScheduler after aggressive coalescing.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
static int getNumFixedFromVariadicInfo(unsigned Flags)
getNumFixedFromVariadicInfo - Transform an EmitNode flags word into the number of fixed arity values ...
bool LegalizeTypes()
This transforms the SelectionDAG into a SelectionDAG that only uses types natively supported by the t...
GCFunctionInfo * GFI
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
virtual bool hasCopyImplyingStackAdjustment(MachineFunction *MF) const
Return true if the MachineFunction contains a COPY which would imply HasCopyImplyingStackAdjustment.
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:42
STATISTIC(NumFunctions,"Total number of functions")
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR...
Definition: ISDOpcodes.h:645
static unsigned virtReg2Index(unsigned Reg)
Convert a virtual register number to a 0-based index.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
size_t i
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
livein_iterator livein_end() const
Various leaf nodes.
Definition: ISDOpcodes.h:60
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad. Returns the label ID for the landing pad entry.
bool hasProperty(Property P) const
static cl::opt< bool > ViewISelDAGs("view-isel-dags", cl::Hidden, cl::desc("Pop up a window to show isel dags as they are selected"))
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:39
unsigned getCatchPadExceptionPointerVReg(const Value *CPI, const TargetRegisterClass *RC)
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
static void propagateSwiftErrorVRegs(FunctionLoweringInfo *FuncInfo)
Propagate swifterror values through the machine function CFG.
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
virtual void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const
Insert explicit copies in entry and exit blocks.
unsigned createVirtualRegister(const TargetRegisterClass *RegClass)
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
std::error_code remove(const Twine &path, bool IgnoreNonExisting=true)
Remove path.
ScheduleDAGSDNodes *(* FunctionPassCtor)(SelectionDAGISel *, CodeGenOpt::Level)
bool isReturn() const
Return true if the instruction is a return.
Definition: MCInstrDesc.h:236
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
bool mayStore() const
Return true if this instruction could possibly modify memory.
Definition: MCInstrDesc.h:384
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
static unsigned getFlagWord(unsigned Kind, unsigned NumOps)
Definition: InlineAsm.h:268
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &)
Clients of various APIs that cause global effects on the DAG can optionally implement this interface...
Definition: SelectionDAG.h:215
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in the KnownZero/KnownO...
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel)
CheckPatternPredicate - Implements OP_CheckPatternPredicate.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
void visitJumpTableHeader(JumpTable &JT, JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
Definition: SelectionDAG.h:251
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:131
static const MCPhysReg VRegs[32]
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:192
unsigned getNumOperands() const
Return the number of values used by this operation.
bool isEHPad() const
Return true if this basic block is an exception handling block.
Definition: BasicBlock.h:315
unsigned getNumOperands() const
arg_iterator arg_end()
Definition: Function.h:559
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
A debug info location.
Definition: DebugLoc.h:34
Metadata node.
Definition: Metadata.h:830
const SDValue & getOperand(unsigned Num) const
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
RegisterPassParser class - Handle the addition of new machine passes.
void setNodeId(int Id)
Set unique node id.
static MachineBasicBlock::iterator FindSplitPointForStackProtector(MachineBasicBlock *BB)
Find the split point at which to splice the end of BB into its success stack protector check machine ...
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:387
virtual unsigned getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
void viewGraph(const std::string &Title)
Pop up a GraphViz/gv window with the DAG rendered using 'dot'.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL, unsigned ChildNo)
DebugLoc getCurDebugLoc() const
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:45
void setCurrentSwiftErrorVReg(const MachineBasicBlock *MBB, const Value *, unsigned)
Set the swifterror virtual register in the SwiftErrorVRegDefMap for this basic block.
static bool isUseOperandTiedToDef(unsigned Flag, unsigned &Idx)
isUseOperandTiedToDef - Return true if the flag of the inline asm operand indicates it is an use oper...
Definition: InlineAsm.h:341
MachineFunction * MF
const TargetLibraryInfo * LibInfo
void set(const Function &Fn, MachineFunction &MF, SelectionDAG *DAG)
set - Initialize this FunctionLoweringInfo with the given Function and its associated MachineFunction...
static void setupSwiftErrorVals(const Function &Fn, const TargetLowering *TLI, FunctionLoweringInfo *FuncInfo)
Set up SwiftErrorVals by going through the function.
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:159
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:191
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:345
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:228
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1338
static bool isFoldedOrDeadInstruction(const Instruction *I, FunctionLoweringInfo *FuncInfo)
isFoldedOrDeadInstruction - Return true if the specified instruction is side-effect free and is eithe...
StackProtectorDescriptor SPDescriptor
A StackProtectorDescriptor structure used to communicate stack protector information in between Selec...
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:172
bool isVector() const
isVector - Return true if this is a vector value type.
Definition: ValueTypes.h:133
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
SDValue getRoot()
getRoot - Return the current virtual root of the Selection DAG, flushing any PendingLoad items...
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block...
AnalysisUsage & addRequired()
void setLastLocalValue(MachineInstr *I)
Update the position of the last instruction emitted for materializing constants for use in the curren...
Definition: FastISel.h:229
A description of a memory reference used in the backend.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:250
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:555
static bool findNonImmUse(SDNode *Use, SDNode *Def, SDNode *ImmedUse, SDNode *Root, SmallPtrSetImpl< SDNode * > &Visited, bool IgnoreChains)
findNonImmUse - Return true if "Use" is a non-immediate use of "Def".
bool CheckAndMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckAndMask - The isel is trying to match something like (and X, 255).
void visitSwitchCase(CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
struct fuzzer::@269 Flags
Option class for critical edge splitting.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOpt::Level OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
This class is basically a combination of TimeRegion and Timer.
Definition: Timer.h:160
void AddLiveOutRegInfo(unsigned Reg, unsigned NumSignBits, const APInt &KnownZero, const APInt &KnownOne)
AddLiveOutRegInfo - Adds LiveOutInfo for a register.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
static SDValue HandleMergeInputChains(SmallVectorImpl< SDNode * > &ChainNodesMatched, SelectionDAG *CurDAG)
HandleMergeInputChains - This implements the OPC_EmitMergeInputChains operation for when the pattern ...
const Value * SwiftErrorArg
The swifterror argument of the current function.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s), MachineInstr opcode, and operands.
void visitJumpTable(JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
DenseMap< const Value *, unsigned > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
use_instr_iterator use_instr_begin(unsigned RegNo) const
const TargetLowering * TLI
#define LLVM_ATTRIBUTE_ALWAYS_INLINE
LLVM_ATTRIBUTE_ALWAYS_INLINE - On compilers where we have a directive to do so, mark a method "always...
Definition: Compiler.h:204
bool isCall() const
Return true if the instruction is a call.
Definition: MCInstrDesc.h:242
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
bool isReg() const
isReg - Tests if this is a MO_Register operand.
StringRef getName(unsigned Opcode) const
Returns the name for the instructions with the given opcode.
Definition: MCInstrInfo.h:51
CopyToReg - This node has three operands: a chain, a register number to set to this value...
Definition: ISDOpcodes.h:170
Instruction * getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:180
static MachinePassRegistry Registry
RegisterScheduler class - Track the registration of instruction schedulers.
const TargetRegisterClass * getRegClass(unsigned Reg) const
Return the register class of the specified virtual register.
const MachineFunctionProperties & getProperties() const
Get the function properties.
void freezeReservedRegs(const MachineFunction &)
freezeReservedRegs - Called by the register allocator to freeze the set of reserved registers before ...
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1218
Reg
All possible values of the reg field in the ModR/M byte.
virtual void Select(SDNode *N)=0
Main hook for targets to transform nodes into machine nodes.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
An analysis pass which caches information about the entire Module.
Definition: GCMetadata.h:155
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDNode *N)
virtual unsigned getFrameRegister(const MachineFunction &MF) const =0
Debug information queries.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
Definition: ValueTypes.h:123
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:589
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:60
unsigned DAGSize
DAGSize - Size of DAG being instruction selected.
void initializeAAResultsWrapperPassPass(PassRegistry &)
static cl::opt< bool > EnableFastISelVerbose("fast-isel-verbose", cl::Hidden, cl::desc("Enable verbose messages in the \"fast\" ""instruction selector"))
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
defusechain_iterator - This class provides iterator support for machine operands in the function that...
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
const MachineBasicBlock & front() const
static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI)
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:31
A constant value that is initialized with an expression using other constant values.
Definition: Constants.h:873
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
ScheduleDAGSDNodes * createHybridListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createHybridListDAGScheduler - This creates a bottom up register pressure aware list scheduler that m...
MachineBasicBlock * MBB
#define T
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
virtual void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag...
void setHasCopyImplyingStackAdjustment(bool B)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:842
const APInt & getAPIntValue() const
auto count(R &&Range, const E &Element) -> typename std::iterator_traits< decltype(std::begin(Range))>::difference_type
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:791
MachinePassRegistry - Track the registration of machine passes.
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification, or lowering of the constant.
Definition: ISDOpcodes.h:125
int64_t getImm() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:487
static LLVM_ATTRIBUTE_ALWAYS_INLINE uint64_t GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx)
GetVBR - decode a vbr encoding whose top bit is set.
ScheduleDAGSDNodes * createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createBURRListDAGScheduler - This creates a bottom up list scheduler that schedules nodes in source c...
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
void init(MachineFunction &mf)
Prepare this SelectionDAG to process code in the given MachineFunction.
int Switch(int a)
Definition: Switch2Test.cpp:11
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:151
SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT)
Definition: SelectionDAG.h:515
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:328
const TargetRegisterClass * constrainRegClass(unsigned Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
Legacy analysis pass which computes BranchProbabilityInfo.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
UNDEF - An undefined node.
Definition: ISDOpcodes.h:178
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
unsigned getNumIncomingValues() const
Return the number of incoming edges.
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:131
TargetInstrInfo - Interface to description of machine instruction set.
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:743
MachineRegisterInfo * RegInfo
void clear()
clear - Clear out all the function-specific state.
virtual void initializeSplitCSR(MachineBasicBlock *Entry) const
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
bool isDebugValue() const
Definition: MachineInstr.h:777
bool isImplicitDef() const
Definition: MachineInstr.h:788
unsigned getNumSuccessors() const
Return the number of successors that this terminator has.
Definition: InstrTypes.h:74
SDNode * getNode() const
get the SDNode which holds the desired result
static cl::opt< RegisterScheduler::FunctionPassCtor, false, RegisterPassParser< RegisterScheduler > > ISHeuristic("pre-RA-sched", cl::init(&createDefaultScheduler), cl::Hidden, cl::desc("Instruction schedulers available (before register"" allocation):"))
ISHeuristic command line option for instruction schedulers.
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
BasicBlock * SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions())
If this edge is a critical edge, insert a new node to split the critical edge.
CriticalEdgeSplittingOptions & setMergeIdenticalEdges()
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:395
virtual FastISel * createFastISel(FunctionLoweringInfo &, const TargetLibraryInfo *) const
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry &)
llvm::DenseMap< std::pair< const MachineBasicBlock *, const Value * >, unsigned > SwiftErrorVRegDefMap
A map from swifterror value in a basic block to the virtual register it is currently represented by...
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:166
CodeGenOpt::Level OptLevel
void addLiveIn(MCPhysReg PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:85
static cl::opt< int > EnableFastISelAbort("fast-isel-abort", cl::Hidden, cl::desc("Enable abort calls when \"fast\" instruction selection ""fails to lower an instruction: 0 disable the abort, 1 will ""abort but for args, calls and terminators, 2 will also ""abort for argument lowering, and 3 will never fallback ""to SelectionDAG."))
Subclasses of this class are all able to terminate a basic block.
Definition: InstrTypes.h:52
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
unsigned const MachineRegisterInfo * MRI
virtual bool CheckNodePredicate(SDNode *N, unsigned PredNo) const
CheckNodePredicate - This function is generated by tblgen in the target.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1147
ScheduleDAGSDNodes * createVLIWDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createVLIWDAGScheduler - Scheduler for VLIW targets.
bool HasTailCall
HasTailCall - This is set to true if a call in the current block has been translated as a tail call...
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
ScheduleDAGSDNodes * createDefaultScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createDefaultScheduler - This creates an instruction scheduler appropriate for the target...
MVT - Machine Value Type.
LLVM Basic Block Representation.
Definition: BasicBlock.h:51
const SDValue & getOperand(unsigned i) const
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
bool canTrap() const
Return true if evaluation of this constant could trap.
Definition: Constants.cpp:359
ScheduleDAGSDNodes * createBURRListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createBURRListDAGScheduler - This creates a bottom up register usage reduction list scheduler...
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineInstrBuilder & UseMI
void Combine(CombineLevel Level, AliasAnalysis &AA, CodeGenOpt::Level OptLevel)
This iterates over the nodes in the SelectionDAG, folding certain types of nodes together, or eliminating superfluous nodes.
virtual bool supportSplitCSR(MachineFunction *MF) const
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
This is an important base class in LLVM.
Definition: Constant.h:42
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
Definition: FastISel.cpp:354
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:115
bool isIndirectDebugValue() const
A DBG_VALUE is indirect iff the first operand is a register and the second operand is an immediate...
Definition: MachineInstr.h:780
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:36
This file contains the declarations for the subclasses of Constant, which represent the different fla...
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:279
bool succ_empty(const BasicBlock *BB)
Definition: IR/CFG.h:140
static ChainResult WalkChainUsers(const SDNode *ChainedNode, SmallVectorImpl< SDNode * > &ChainedNodesInPattern, DenseMap< const SDNode *, ChainResult > &TokenFactorResult, SmallVectorImpl< SDNode * > &InteriorChainedNodes)
WalkChainUsers - Walk down the users of the specified chained node that is part of the pattern we're ...
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
Definition: APInt.h:1947
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:269
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:368
APInt Xor(const APInt &LHS, const APInt &RHS)
Bitwise XOR function for APInt.
Definition: APInt.h:1952
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:116
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag...
Definition: InlineAsm.h:335
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
bool isCopy() const
Definition: MachineInstr.h:807
Represent the analysis usage information of a pass.
This class provides iterator support for SDUse operands that use a specific SDNode.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We're checking to see if we can fold LI into FoldInst.
Definition: FastISel.cpp:2054
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, unsigned ChildNo)
static unsigned getMemoryConstraintID(unsigned Flag)
Definition: InlineAsm.h:328
uint32_t Offset
void printrFull(raw_ostream &O, const SelectionDAG *G=nullptr) const
Print a SelectionDAG node and all children down to the leaves.
std::vector< NodeRef >::reverse_iterator rpo_iterator
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:110
virtual unsigned getRegisterByName(const char *RegName, EVT VT, SelectionDAG &DAG) const
Return the register ID of the name passed in.
static const unsigned End
static void collectFailStats(const Instruction *I)
unsigned getOpcode() const
static cl::opt< bool > UseMBPI("use-mbpi", cl::desc("use Machine Branch Probability Info"), cl::init(true), cl::Hidden)
Interval::pred_iterator pred_end(Interval *I)
Definition: Interval.h:119
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:57
arg_iterator arg_begin()
Definition: Function.h:550
void setHasInlineAsm(bool B)
Set a flag that indicates that the function contains inline assembly.
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection...
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
static void createSwiftErrorEntriesInEntryBlock(FunctionLoweringInfo *FuncInfo, const TargetLowering *TLI, const TargetInstrInfo *TII, const BasicBlock *LLVMBB, SelectionDAGBuilder *SDB)
self_iterator getIterator()
Definition: ilist_node.h:81
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:149
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:80
static cl::opt< bool > ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the second ""dag combine pass"))
iterator_range< pred_iterator > predecessors()
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
static unsigned getFlagWordForMem(unsigned InputFlag, unsigned Constraint)
Augment an existing flag word returned by getFlagWord with the constraint code for a memory constrain...
Definition: InlineAsm.h:311
const DIExpression * getDebugExpression() const
Return the complex address expression referenced by this DBG_VALUE instruction.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
unsigned ExceptionPointerVirtReg
If the current MBB is a landing pad, the exception pointer and exception selector registers are copie...
SmallPtrSet< const BasicBlock *, 4 > VisitedBBs
VisitedBBs - The set of basic blocks visited thus far by instruction selection.
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:225
static bool isMemKind(unsigned Flag)
Definition: InlineAsm.h:276
EVT - Extended Value Type.
Definition: ValueTypes.h:31
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
void ComputePHILiveOutRegInfo(const PHINode *)
ComputePHILiveOutRegInfo - Compute LiveOutInfo for a PHI's destination register based on the LiveOutI...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
HANDLENODE node - Used as a handle for various purposes.
Definition: ISDOpcodes.h:659
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:378
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:45
MachineBasicBlock * MBB
MBB - The current block.
bool mayWriteToMemory() const
Return true if this instruction may modify memory.
virtual bool enableMachineSchedDefaultSched() const
True if the machine scheduler should disable the TLI preference for preRA scheduling with the source ...
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:594
void dump() const
Support for debugging, callable in GDB: V->dump()
Definition: AsmWriter.cpp:3540
TargetIntrinsicInfo - Interface to description of machine instruction set.
static cl::opt< bool > ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the first ""dag combine pass"))
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
Definition: FastISel.cpp:340
virtual Value * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
bool hasCalls() const
Return true if the current function has any function calls.
bool SplitCSR
True if part of the CSRs will be handled via explicit copies.
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1034
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:425
This is the shared class of boolean and integer constants.
Definition: Constants.h:88
ScheduleDAGSDNodes * createILPListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createILPListDAGScheduler - This creates a bottom up register pressure aware list scheduler that trie...
void dump() const
Dump this node, for debugging.
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:361
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
iterator end()
Definition: BasicBlock.h:230
DenseMap< unsigned, unsigned > RegFixups
RegFixups - Registers which need to be replaced after isel is done.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:166
SDNode * SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type...
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
SmallVector< MachineInstr *, 8 > ArgDbgValues
ArgDbgValues - A list of DBG_VALUE instructions created during isel for function arguments that are i...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:230
void clear()
clear - Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuil...
virtual const TargetLowering * getTargetLowering() const
void setFastISel(bool Enable)
SelectionDAGISel(TargetMachine &tm, CodeGenOpt::Level OL=CodeGenOpt::Default)
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
void visit(const Instruction &I)
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:382
bool mayLoad() const
Return true if this instruction could possibly read memory.
Definition: MCInstrDesc.h:378
livein_iterator livein_begin() const
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel, SDNode *N)
CheckNodePredicate - Implements OP_CheckNodePredicate.
Represents one node in the SelectionDAG.
const BasicBlock & getEntryBlock() const
Definition: Function.h:519
virtual bool CheckComplexPattern(SDNode *Root, SDNode *Parent, SDValue N, unsigned PatternNo, SmallVectorImpl< std::pair< SDValue, SDNode * > > &Result)
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
UpdateSplitBlock - When an MBB was split during scheduling, update the references that need to refer ...
static cl::opt< bool > ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize types"))
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:584
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
unsigned CreateRegs(Type *Ty)
CreateRegs - Allocate the appropriate number of virtual registers of the correctly promoted or expand...
void startNewBlock()
Set the current block to which generated machine instructions will be appended, and clear the local C...
Definition: FastISel.cpp:98
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:586
static cl::opt< bool > ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize"))
void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef< unsigned > Sites)
Map the landing pad's EH symbol to the call site indexes.
StringRef getName() const
Return the name of the corresponding LLVM basic block, or "(null)".
Class for arbitrary precision integers.
Definition: APInt.h:77
static unsigned IsPredicateKnownToFail(const unsigned char *Table, unsigned Index, SDValue N, bool &Result, const SelectionDAGISel &SDISel, SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
IsPredicateKnownToFail - If we know how and can do so without pushing a scope, evaluate the current n...
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
void visitBitTestCase(BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, unsigned Reg, BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
BranchProbabilityInfo * BPI
machine Machine Instruction Scheduler
int64_t getSExtValue() const
op_iterator op_begin() const
This file defines the FastISel class.
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
static use_iterator use_end()
iterator_range< user_iterator > users()
Definition: Value.h:370
std::vector< JumpTableBlock > JTCases
JTCases - Vector of JumpTable structures used to communicate SwitchInst code generation information...
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:250
void replaceRegWith(unsigned FromReg, unsigned ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes, unsigned ChildNo)
CheckChildSame - Implements OP_CheckChildXSame.
iterator_range< value_op_iterator > op_values() const
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
Definition: APInt.h:1942
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
TargetSubtargetInfo - Generic base class for all target subtargets.
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
Definition: Instruction.h:453
Representation of each machine instruction.
Definition: MachineInstr.h:52
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
Represents a use of a SDNode.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:119
unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ)
Search for the specified successor of basic block BB and return its position in the terminator instru...
Definition: CFG.cpp:72
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:333
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:610
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
iterator begin()
Definition: DenseMap.h:65
virtual void viewGraph(const Twine &Name, const Twine &Title)
viewGraph - Pop up a GraphViz/gv window with the ScheduleDAG rendered using 'dot'.
static bool MIIsInTerminatorSequence(const MachineInstr &MI)
Given that the input MI is before a partial terminator sequence TSeq, return true if M + TSeq also a ...
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:256
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
void initializeGCModuleInfoPass(PassRegistry &)
#define I(x, y, z)
Definition: MD5.cpp:54
static cl::opt< bool > ViewSUnitDAGs("view-sunit-dags", cl::Hidden, cl::desc("Pop up a window to show SUnit dags after they are processed"))
#define N
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:124
bool isLandingPad() const
Return true if this basic block is a landing pad.
Definition: BasicBlock.cpp:436
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:383
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
bool callsFunctionThatReturnsTwice() const
callsFunctionThatReturnsTwice - Return true if the function has a call to setjmp or other function th...
Definition: Function.cpp:1207
void resetTargetOptions(const Function &F) const
Reset the target options based on the function's attributes.
iterator end()
Definition: DenseMap.h:69
void clearKillFlags(unsigned Reg) const
clearKillFlags - Iterate over all the uses of the given register and clear the kill flag from the Mac...
iterator find(const KeyT &Val)
Definition: DenseMap.h:127
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
op_iterator op_end() const
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:287
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
void Run(SelectionDAG *dag, MachineBasicBlock *bb)
Run - perform scheduling.
OptLevelChanger(SelectionDAGISel &ISel, CodeGenOpt::Level NewOptLevel)
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
void setOptLevel(CodeGenOpt::Level Level)
Overrides the optimization level.
bool CheckOrMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckOrMask - The isel is trying to match something like (or X, 255).
int getNodeId() const
Return the unique node id.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:175
EVT getValueType() const
Return the ValueType of the referenced return value.
MachineInstr * getVRegDef(unsigned Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode...
const DILocalVariable * getDebugVariable() const
Return the debug variable referenced by this DBG_VALUE instruction.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
std::vector< std::pair< unsigned, unsigned > >::const_iterator livein_iterator
unsigned getReg() const
getReg - Returns the register number.
const MDNode * getMD() const
bool use_empty() const
Definition: Value.h:299
virtual bool CheckPatternPredicate(unsigned PredNo) const
CheckPatternPredicate - This function is generated by tblgen in the target.
This class is used to form a handle around another node that is persistent and is updated across invo...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:463
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void InvalidatePHILiveOutRegInfo(const PHINode *PN)
InvalidatePHILiveOutRegInfo - Invalidates a PHI's LiveOutInfo, to be called when a block is visited b...
virtual const TargetInstrInfo * getInstrInfo() const
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:537
SDValue getRegister(unsigned Reg, EVT VT)
mop_iterator operands_begin()
Definition: MachineInstr.h:295
void init(GCFunctionInfo *gfi, AliasAnalysis &aa, const TargetLibraryInfo *li)
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:111
void SelectInlineAsmMemoryOperands(std::vector< SDValue > &Ops, const SDLoc &DL)
SelectInlineAsmMemoryOperands - Calls to this are automatically generated by tblgen.
virtual MachineBasicBlock * EmitSchedule(MachineBasicBlock::iterator &InsertPos)
EmitSchedule - Insert MachineInstrs into the MachineBasicBlock according to the order specified in Se...
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
static use_instr_iterator use_instr_end()
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
CheckSame - Implements OP_CheckSame.
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
static cl::opt< bool > ViewSchedDAGs("view-sched-dags", cl::Hidden, cl::desc("Pop up a window to show sched dags as they are processed"))
std::vector< CaseBlock > SwitchCases
SwitchCases - Vector of CaseBlock structures used to communicate SwitchInst code generation informati...
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.Val alone...
#define DEBUG(X)
Definition: Debug.h:100
Primary interface to the complete machine description for the target machine.
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
IRTranslator LLVM IR MI
SDValue getControlRoot()
getControlRoot - Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports items.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:47
A single uniqued string.
Definition: Metadata.h:586
void clearDanglingDebugInfo()
clearDanglingDebugInfo - Clear the dangling debug information map.
SwiftErrorValues SwiftErrorVals
A function can only have a single swifterror argument.
const TargetLowering * TLI
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
static cl::opt< bool > EnableFastISelVerbose2("fast-isel-verbose2", cl::Hidden, cl::desc("Enable extra verbose messages in the \"fast\" ""instruction selector"))
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
LPadToCallSiteMap - Map a landing pad to the call site indexes.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:381
bool TimePassesIsEnabled
If the user specifies the -time-passes argument on an LLVM tool command line then the value of this b...
Definition: IRReader.cpp:26
SDNode * getUser()
This returns the SDNode that contains this Use.
auto find_if(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range))
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:764
virtual RegisterScheduler::FunctionPassCtor getDAGScheduler(CodeGenOpt::Level) const
Target can subclass this hook to select a different DAG scheduler.
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
static cl::opt< std::string > FilterDAGBasicBlockName("filter-view-dags", cl::Hidden, cl::desc("Only display the basic block whose name ""matches this for all view-*-dags options"))
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const TargetInstrInfo * TII
virtual unsigned getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num)
allocateMemRefsArray - Allocate an array to hold MachineMemOperand pointers.
void visitBitTestHeader(BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests" ...
static void SplitCriticalSideEffectEdges(Function &Fn)
SplitCriticalSideEffectEdges - Look for critical edges with a PHI value that may trap on it...
bool isExportedInst(const Value *V)
isExportedInst - Return true if the specified value is an instruction exported from its block...
unsigned getResNo() const
Convenience function for get().getResNo().
FunctionLoweringInfo * FuncInfo
static cl::opt< bool > ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the post legalize types"" dag combine pass"))
static RegisterScheduler defaultListDAGScheduler("default","Best scheduler for the target", createDefaultScheduler)
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:44
This file describes how to lower LLVM code to machine code.
virtual SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo)
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
an instruction to allocate memory on the stack
Definition: Instructions.h:60
bool use_empty(unsigned RegNo) const
use_empty - Return true if there are no instructions using the specified register.
This class is used by SelectionDAGISel to temporarily override the optimization level on a per-functi...
void SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, unsigned TableSize)
void resize(size_type N)
Definition: SmallVector.h:352
unsigned getOrCreateSwiftErrorVReg(const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register in SwiftErrorVRegDefMap for this basic block...
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:783