LLVM  3.7.0
SelectionDAGISel.cpp
Go to the documentation of this file.
1 //===-- SelectionDAGISel.cpp - Implement the SelectionDAGISel class -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements the SelectionDAGISel class.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "ScheduleDAGSDNodes.h"
16 #include "SelectionDAGBuilder.h"
18 #include "llvm/ADT/Statistic.h"
21 #include "llvm/Analysis/CFG.h"
23 #include "llvm/CodeGen/Analysis.h"
24 #include "llvm/CodeGen/FastISel.h"
37 #include "llvm/IR/Constants.h"
38 #include "llvm/IR/DebugInfo.h"
39 #include "llvm/IR/Function.h"
40 #include "llvm/IR/InlineAsm.h"
41 #include "llvm/IR/Instructions.h"
42 #include "llvm/IR/IntrinsicInst.h"
43 #include "llvm/IR/Intrinsics.h"
44 #include "llvm/IR/LLVMContext.h"
45 #include "llvm/IR/Module.h"
46 #include "llvm/MC/MCAsmInfo.h"
47 #include "llvm/Support/Compiler.h"
48 #include "llvm/Support/Debug.h"
50 #include "llvm/Support/Timer.h"
60 #include <algorithm>
61 using namespace llvm;
62 
63 #define DEBUG_TYPE "isel"
64 
65 STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on");
66 STATISTIC(NumFastIselSuccess, "Number of instructions fast isel selected");
67 STATISTIC(NumFastIselBlocks, "Number of blocks selected entirely by fast isel");
68 STATISTIC(NumDAGBlocks, "Number of blocks selected using DAG");
69 STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path");
70 STATISTIC(NumEntryBlocks, "Number of entry blocks encountered");
71 STATISTIC(NumFastIselFailLowerArguments,
72  "Number of entry blocks where fast isel failed to lower arguments");
73 
74 #ifndef NDEBUG
75 static cl::opt<bool>
76 EnableFastISelVerbose2("fast-isel-verbose2", cl::Hidden,
77  cl::desc("Enable extra verbose messages in the \"fast\" "
78  "instruction selector"));
79 
80  // Terminators
81 STATISTIC(NumFastIselFailRet,"Fast isel fails on Ret");
82 STATISTIC(NumFastIselFailBr,"Fast isel fails on Br");
83 STATISTIC(NumFastIselFailSwitch,"Fast isel fails on Switch");
84 STATISTIC(NumFastIselFailIndirectBr,"Fast isel fails on IndirectBr");
85 STATISTIC(NumFastIselFailInvoke,"Fast isel fails on Invoke");
86 STATISTIC(NumFastIselFailResume,"Fast isel fails on Resume");
87 STATISTIC(NumFastIselFailUnreachable,"Fast isel fails on Unreachable");
88 
89  // Standard binary operators...
90 STATISTIC(NumFastIselFailAdd,"Fast isel fails on Add");
91 STATISTIC(NumFastIselFailFAdd,"Fast isel fails on FAdd");
92 STATISTIC(NumFastIselFailSub,"Fast isel fails on Sub");
93 STATISTIC(NumFastIselFailFSub,"Fast isel fails on FSub");
94 STATISTIC(NumFastIselFailMul,"Fast isel fails on Mul");
95 STATISTIC(NumFastIselFailFMul,"Fast isel fails on FMul");
96 STATISTIC(NumFastIselFailUDiv,"Fast isel fails on UDiv");
97 STATISTIC(NumFastIselFailSDiv,"Fast isel fails on SDiv");
98 STATISTIC(NumFastIselFailFDiv,"Fast isel fails on FDiv");
99 STATISTIC(NumFastIselFailURem,"Fast isel fails on URem");
100 STATISTIC(NumFastIselFailSRem,"Fast isel fails on SRem");
101 STATISTIC(NumFastIselFailFRem,"Fast isel fails on FRem");
102 
103  // Logical operators...
104 STATISTIC(NumFastIselFailAnd,"Fast isel fails on And");
105 STATISTIC(NumFastIselFailOr,"Fast isel fails on Or");
106 STATISTIC(NumFastIselFailXor,"Fast isel fails on Xor");
107 
108  // Memory instructions...
109 STATISTIC(NumFastIselFailAlloca,"Fast isel fails on Alloca");
110 STATISTIC(NumFastIselFailLoad,"Fast isel fails on Load");
111 STATISTIC(NumFastIselFailStore,"Fast isel fails on Store");
112 STATISTIC(NumFastIselFailAtomicCmpXchg,"Fast isel fails on AtomicCmpXchg");
113 STATISTIC(NumFastIselFailAtomicRMW,"Fast isel fails on AtomicRWM");
114 STATISTIC(NumFastIselFailFence,"Fast isel fails on Frence");
115 STATISTIC(NumFastIselFailGetElementPtr,"Fast isel fails on GetElementPtr");
116 
117  // Convert instructions...
118 STATISTIC(NumFastIselFailTrunc,"Fast isel fails on Trunc");
119 STATISTIC(NumFastIselFailZExt,"Fast isel fails on ZExt");
120 STATISTIC(NumFastIselFailSExt,"Fast isel fails on SExt");
121 STATISTIC(NumFastIselFailFPTrunc,"Fast isel fails on FPTrunc");
122 STATISTIC(NumFastIselFailFPExt,"Fast isel fails on FPExt");
123 STATISTIC(NumFastIselFailFPToUI,"Fast isel fails on FPToUI");
124 STATISTIC(NumFastIselFailFPToSI,"Fast isel fails on FPToSI");
125 STATISTIC(NumFastIselFailUIToFP,"Fast isel fails on UIToFP");
126 STATISTIC(NumFastIselFailSIToFP,"Fast isel fails on SIToFP");
127 STATISTIC(NumFastIselFailIntToPtr,"Fast isel fails on IntToPtr");
128 STATISTIC(NumFastIselFailPtrToInt,"Fast isel fails on PtrToInt");
129 STATISTIC(NumFastIselFailBitCast,"Fast isel fails on BitCast");
130 
131  // Other instructions...
132 STATISTIC(NumFastIselFailICmp,"Fast isel fails on ICmp");
133 STATISTIC(NumFastIselFailFCmp,"Fast isel fails on FCmp");
134 STATISTIC(NumFastIselFailPHI,"Fast isel fails on PHI");
135 STATISTIC(NumFastIselFailSelect,"Fast isel fails on Select");
136 STATISTIC(NumFastIselFailCall,"Fast isel fails on Call");
137 STATISTIC(NumFastIselFailShl,"Fast isel fails on Shl");
138 STATISTIC(NumFastIselFailLShr,"Fast isel fails on LShr");
139 STATISTIC(NumFastIselFailAShr,"Fast isel fails on AShr");
140 STATISTIC(NumFastIselFailVAArg,"Fast isel fails on VAArg");
141 STATISTIC(NumFastIselFailExtractElement,"Fast isel fails on ExtractElement");
142 STATISTIC(NumFastIselFailInsertElement,"Fast isel fails on InsertElement");
143 STATISTIC(NumFastIselFailShuffleVector,"Fast isel fails on ShuffleVector");
144 STATISTIC(NumFastIselFailExtractValue,"Fast isel fails on ExtractValue");
145 STATISTIC(NumFastIselFailInsertValue,"Fast isel fails on InsertValue");
146 STATISTIC(NumFastIselFailLandingPad,"Fast isel fails on LandingPad");
147 
148 // Intrinsic instructions...
149 STATISTIC(NumFastIselFailIntrinsicCall, "Fast isel fails on Intrinsic call");
150 STATISTIC(NumFastIselFailSAddWithOverflow,
151  "Fast isel fails on sadd.with.overflow");
152 STATISTIC(NumFastIselFailUAddWithOverflow,
153  "Fast isel fails on uadd.with.overflow");
154 STATISTIC(NumFastIselFailSSubWithOverflow,
155  "Fast isel fails on ssub.with.overflow");
156 STATISTIC(NumFastIselFailUSubWithOverflow,
157  "Fast isel fails on usub.with.overflow");
158 STATISTIC(NumFastIselFailSMulWithOverflow,
159  "Fast isel fails on smul.with.overflow");
160 STATISTIC(NumFastIselFailUMulWithOverflow,
161  "Fast isel fails on umul.with.overflow");
162 STATISTIC(NumFastIselFailFrameaddress, "Fast isel fails on Frameaddress");
163 STATISTIC(NumFastIselFailSqrt, "Fast isel fails on sqrt call");
164 STATISTIC(NumFastIselFailStackMap, "Fast isel fails on StackMap call");
165 STATISTIC(NumFastIselFailPatchPoint, "Fast isel fails on PatchPoint call");
166 #endif
167 
168 static cl::opt<bool>
169 EnableFastISelVerbose("fast-isel-verbose", cl::Hidden,
170  cl::desc("Enable verbose messages in the \"fast\" "
171  "instruction selector"));
173  "fast-isel-abort", cl::Hidden,
174  cl::desc("Enable abort calls when \"fast\" instruction selection "
175  "fails to lower an instruction: 0 disable the abort, 1 will "
176  "abort but for args, calls and terminators, 2 will also "
177  "abort for argument lowering, and 3 will never fallback "
178  "to SelectionDAG."));
179 
180 static cl::opt<bool>
181 UseMBPI("use-mbpi",
182  cl::desc("use Machine Branch Probability Info"),
183  cl::init(true), cl::Hidden);
184 
185 #ifndef NDEBUG
187 FilterDAGBasicBlockName("filter-view-dags", cl::Hidden,
188  cl::desc("Only display the basic block whose name "
189  "matches this for all view-*-dags options"));
190 static cl::opt<bool>
191 ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden,
192  cl::desc("Pop up a window to show dags before the first "
193  "dag combine pass"));
194 static cl::opt<bool>
195 ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden,
196  cl::desc("Pop up a window to show dags before legalize types"));
197 static cl::opt<bool>
198 ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,
199  cl::desc("Pop up a window to show dags before legalize"));
200 static cl::opt<bool>
201 ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden,
202  cl::desc("Pop up a window to show dags before the second "
203  "dag combine pass"));
204 static cl::opt<bool>
205 ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden,
206  cl::desc("Pop up a window to show dags before the post legalize types"
207  " dag combine pass"));
208 static cl::opt<bool>
209 ViewISelDAGs("view-isel-dags", cl::Hidden,
210  cl::desc("Pop up a window to show isel dags as they are selected"));
211 static cl::opt<bool>
212 ViewSchedDAGs("view-sched-dags", cl::Hidden,
213  cl::desc("Pop up a window to show sched dags as they are processed"));
214 static cl::opt<bool>
215 ViewSUnitDAGs("view-sunit-dags", cl::Hidden,
216  cl::desc("Pop up a window to show SUnit dags after they are processed"));
217 #else
218 static const bool ViewDAGCombine1 = false,
219  ViewLegalizeTypesDAGs = false, ViewLegalizeDAGs = false,
220  ViewDAGCombine2 = false,
221  ViewDAGCombineLT = false,
222  ViewISelDAGs = false, ViewSchedDAGs = false,
223  ViewSUnitDAGs = false;
224 #endif
225 
226 //===---------------------------------------------------------------------===//
227 ///
228 /// RegisterScheduler class - Track the registration of instruction schedulers.
229 ///
230 //===---------------------------------------------------------------------===//
232 
233 //===---------------------------------------------------------------------===//
234 ///
235 /// ISHeuristic command line option for instruction schedulers.
236 ///
237 //===---------------------------------------------------------------------===//
240 ISHeuristic("pre-RA-sched",
242  cl::desc("Instruction schedulers available (before register"
243  " allocation):"));
244 
245 static RegisterScheduler
246 defaultListDAGScheduler("default", "Best scheduler for the target",
248 
249 namespace llvm {
250  //===--------------------------------------------------------------------===//
251  /// \brief This class is used by SelectionDAGISel to temporarily override
252  /// the optimization level on a per-function basis.
254  SelectionDAGISel &IS;
255  CodeGenOpt::Level SavedOptLevel;
256  bool SavedFastISel;
257 
258  public:
260  CodeGenOpt::Level NewOptLevel) : IS(ISel) {
261  SavedOptLevel = IS.OptLevel;
262  if (NewOptLevel == SavedOptLevel)
263  return;
264  IS.OptLevel = NewOptLevel;
265  IS.TM.setOptLevel(NewOptLevel);
266  SavedFastISel = IS.TM.Options.EnableFastISel;
267  if (NewOptLevel == CodeGenOpt::None)
268  IS.TM.setFastISel(true);
269  DEBUG(dbgs() << "\nChanging optimization level for Function "
270  << IS.MF->getFunction()->getName() << "\n");
271  DEBUG(dbgs() << "\tBefore: -O" << SavedOptLevel
272  << " ; After: -O" << NewOptLevel << "\n");
273  }
274 
276  if (IS.OptLevel == SavedOptLevel)
277  return;
278  DEBUG(dbgs() << "\nRestoring optimization level for Function "
279  << IS.MF->getFunction()->getName() << "\n");
280  DEBUG(dbgs() << "\tBefore: -O" << IS.OptLevel
281  << " ; After: -O" << SavedOptLevel << "\n");
282  IS.OptLevel = SavedOptLevel;
283  IS.TM.setOptLevel(SavedOptLevel);
284  IS.TM.setFastISel(SavedFastISel);
285  }
286  };
287 
288  //===--------------------------------------------------------------------===//
289  /// createDefaultScheduler - This creates an instruction scheduler appropriate
290  /// for the target.
292  CodeGenOpt::Level OptLevel) {
293  const TargetLowering *TLI = IS->TLI;
294  const TargetSubtargetInfo &ST = IS->MF->getSubtarget();
295 
296  if (OptLevel == CodeGenOpt::None ||
299  return createSourceListDAGScheduler(IS, OptLevel);
301  return createBURRListDAGScheduler(IS, OptLevel);
303  return createHybridListDAGScheduler(IS, OptLevel);
304  if (TLI->getSchedulingPreference() == Sched::VLIW)
305  return createVLIWDAGScheduler(IS, OptLevel);
306  assert(TLI->getSchedulingPreference() == Sched::ILP &&
307  "Unknown sched type!");
308  return createILPListDAGScheduler(IS, OptLevel);
309  }
310 }
311 
312 // EmitInstrWithCustomInserter - This method should be implemented by targets
313 // that mark instructions with the 'usesCustomInserter' flag. These
314 // instructions are special in various ways, which require special support to
315 // insert. The specified MachineInstr is created but not inserted into any
316 // basic blocks, and this method is called to expand it into a sequence of
317 // instructions, potentially also creating new basic blocks and control flow.
318 // When new basic blocks are inserted and the edges from MBB to its successors
319 // are modified, the method should insert pairs of <OldSucc, NewSucc> into the
320 // DenseMap.
323  MachineBasicBlock *MBB) const {
324 #ifndef NDEBUG
325  dbgs() << "If a target marks an instruction with "
326  "'usesCustomInserter', it must implement "
327  "TargetLowering::EmitInstrWithCustomInserter!";
328 #endif
329  llvm_unreachable(nullptr);
330 }
331 
333  SDNode *Node) const {
334  assert(!MI->hasPostISelHook() &&
335  "If a target marks an instruction with 'hasPostISelHook', "
336  "it must implement TargetLowering::AdjustInstrPostInstrSelection!");
337 }
338 
339 //===----------------------------------------------------------------------===//
340 // SelectionDAGISel code
341 //===----------------------------------------------------------------------===//
342 
344  CodeGenOpt::Level OL) :
345  MachineFunctionPass(ID), TM(tm),
346  FuncInfo(new FunctionLoweringInfo()),
347  CurDAG(new SelectionDAG(tm, OL)),
348  SDB(new SelectionDAGBuilder(*CurDAG, *FuncInfo, OL)),
349  GFI(),
350  OptLevel(OL),
351  DAGSize(0) {
357  }
358 
360  delete SDB;
361  delete CurDAG;
362  delete FuncInfo;
363 }
364 
374 }
375 
376 /// SplitCriticalSideEffectEdges - Look for critical edges with a PHI value that
377 /// may trap on it. In this case we have to split the edge so that the path
378 /// through the predecessor block that doesn't go to the phi block doesn't
379 /// execute the possibly trapping instruction.
380 ///
381 /// This is required for correctness, so it must be done at -O0.
382 ///
384  // Loop for blocks with phi nodes.
385  for (Function::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB) {
386  PHINode *PN = dyn_cast<PHINode>(BB->begin());
387  if (!PN) continue;
388 
389  ReprocessBlock:
390  // For each block with a PHI node, check to see if any of the input values
391  // are potentially trapping constant expressions. Constant expressions are
392  // the only potentially trapping value that can occur as the argument to a
393  // PHI.
394  for (BasicBlock::iterator I = BB->begin(); (PN = dyn_cast<PHINode>(I)); ++I)
395  for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
397  if (!CE || !CE->canTrap()) continue;
398 
399  // The only case we have to worry about is when the edge is critical.
400  // Since this block has a PHI Node, we assume it has multiple input
401  // edges: check to see if the pred has multiple successors.
402  BasicBlock *Pred = PN->getIncomingBlock(i);
403  if (Pred->getTerminator()->getNumSuccessors() == 1)
404  continue;
405 
406  // Okay, we have to split this edge.
408  Pred->getTerminator(), GetSuccessorNumber(Pred, BB),
410  goto ReprocessBlock;
411  }
412  }
413 }
414 
416  // Do some sanity-checking on the command-line options.
418  "-fast-isel-verbose requires -fast-isel");
420  "-fast-isel-abort > 0 requires -fast-isel");
421 
422  const Function &Fn = *mf.getFunction();
423  MF = &mf;
424 
425  // Reset the target options before resetting the optimization
426  // level below.
427  // FIXME: This is a horrible hack and should be processed via
428  // codegen looking at the optimization level explicitly when
429  // it wants to look at it.
431  // Reset OptLevel to None for optnone functions.
432  CodeGenOpt::Level NewOptLevel = OptLevel;
434  NewOptLevel = CodeGenOpt::None;
435  OptLevelChanger OLC(*this, NewOptLevel);
436 
439  RegInfo = &MF->getRegInfo();
440  AA = &getAnalysis<AliasAnalysis>();
441  LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
442  GFI = Fn.hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) : nullptr;
443 
444  DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n");
445 
446  SplitCriticalSideEffectEdges(const_cast<Function&>(Fn), AA);
447 
448  CurDAG->init(*MF);
449  FuncInfo->set(Fn, *MF, CurDAG);
450 
452  FuncInfo->BPI = &getAnalysis<BranchProbabilityInfo>();
453  else
454  FuncInfo->BPI = nullptr;
455 
456  SDB->init(GFI, *AA, LibInfo);
457 
458  MF->setHasInlineAsm(false);
459 
460  SelectAllBasicBlocks(Fn);
461 
462  // If the first basic block in the function has live ins that need to be
463  // copied into vregs, emit the copies into the top of the block before
464  // emitting the code for the block.
465  MachineBasicBlock *EntryMBB = MF->begin();
467  RegInfo->EmitLiveInCopies(EntryMBB, TRI, *TII);
468 
470  if (!FuncInfo->ArgDbgValues.empty())
472  E = RegInfo->livein_end(); LI != E; ++LI)
473  if (LI->second)
474  LiveInMap.insert(std::make_pair(LI->first, LI->second));
475 
476  // Insert DBG_VALUE instructions for function arguments to the entry block.
477  for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) {
479  bool hasFI = MI->getOperand(0).isFI();
480  unsigned Reg =
481  hasFI ? TRI.getFrameRegister(*MF) : MI->getOperand(0).getReg();
483  EntryMBB->insert(EntryMBB->begin(), MI);
484  else {
486  if (Def) {
487  MachineBasicBlock::iterator InsertPos = Def;
488  // FIXME: VR def may not be in entry block.
489  Def->getParent()->insert(std::next(InsertPos), MI);
490  } else
491  DEBUG(dbgs() << "Dropping debug info for dead vreg"
492  << TargetRegisterInfo::virtReg2Index(Reg) << "\n");
493  }
494 
495  // If Reg is live-in then update debug info to track its copy in a vreg.
496  DenseMap<unsigned, unsigned>::iterator LDI = LiveInMap.find(Reg);
497  if (LDI != LiveInMap.end()) {
498  assert(!hasFI && "There's no handling of frame pointer updating here yet "
499  "- add if needed");
500  MachineInstr *Def = RegInfo->getVRegDef(LDI->second);
501  MachineBasicBlock::iterator InsertPos = Def;
502  const MDNode *Variable = MI->getDebugVariable();
503  const MDNode *Expr = MI->getDebugExpression();
504  DebugLoc DL = MI->getDebugLoc();
505  bool IsIndirect = MI->isIndirectDebugValue();
506  unsigned Offset = IsIndirect ? MI->getOperand(1).getImm() : 0;
507  assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&
508  "Expected inlined-at fields to agree");
509  // Def is never a terminator here, so it is ok to increment InsertPos.
510  BuildMI(*EntryMBB, ++InsertPos, DL, TII->get(TargetOpcode::DBG_VALUE),
511  IsIndirect, LDI->second, Offset, Variable, Expr);
512 
513  // If this vreg is directly copied into an exported register then
514  // that COPY instructions also need DBG_VALUE, if it is the only
515  // user of LDI->second.
516  MachineInstr *CopyUseMI = nullptr;
518  UI = RegInfo->use_instr_begin(LDI->second),
519  E = RegInfo->use_instr_end(); UI != E; ) {
520  MachineInstr *UseMI = &*(UI++);
521  if (UseMI->isDebugValue()) continue;
522  if (UseMI->isCopy() && !CopyUseMI && UseMI->getParent() == EntryMBB) {
523  CopyUseMI = UseMI; continue;
524  }
525  // Otherwise this is another use or second copy use.
526  CopyUseMI = nullptr; break;
527  }
528  if (CopyUseMI) {
529  // Use MI's debug location, which describes where Variable was
530  // declared, rather than whatever is attached to CopyUseMI.
531  MachineInstr *NewMI =
532  BuildMI(*MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
533  CopyUseMI->getOperand(0).getReg(), Offset, Variable, Expr);
534  MachineBasicBlock::iterator Pos = CopyUseMI;
535  EntryMBB->insertAfter(Pos, NewMI);
536  }
537  }
538  }
539 
540  // Determine if there are any calls in this machine function.
541  MachineFrameInfo *MFI = MF->getFrameInfo();
542  for (const auto &MBB : *MF) {
543  if (MFI->hasCalls() && MF->hasInlineAsm())
544  break;
545 
546  for (const auto &MI : MBB) {
547  const MCInstrDesc &MCID = TII->get(MI.getOpcode());
548  if ((MCID.isCall() && !MCID.isReturn()) ||
549  MI.isStackAligningInlineAsm()) {
550  MFI->setHasCalls(true);
551  }
552  if (MI.isInlineAsm()) {
553  MF->setHasInlineAsm(true);
554  }
555  }
556  }
557 
558  // Determine if there is a call to setjmp in the machine function.
559  MF->setExposesReturnsTwice(Fn.callsFunctionThatReturnsTwice());
560 
561  // Replace forward-declared registers with the registers containing
562  // the desired value.
563  MachineRegisterInfo &MRI = MF->getRegInfo();
566  I != E; ++I) {
567  unsigned From = I->first;
568  unsigned To = I->second;
569  // If To is also scheduled to be replaced, find what its ultimate
570  // replacement is.
571  for (;;) {
573  if (J == E) break;
574  To = J->second;
575  }
576  // Make sure the new register has a sufficiently constrained register class.
579  MRI.constrainRegClass(To, MRI.getRegClass(From));
580  // Replace it.
581 
582 
583  // Replacing one register with another won't touch the kill flags.
584  // We need to conservatively clear the kill flags as a kill on the old
585  // register might dominate existing uses of the new register.
586  if (!MRI.use_empty(To))
587  MRI.clearKillFlags(From);
588  MRI.replaceRegWith(From, To);
589  }
590 
591  // Freeze the set of reserved registers now that MachineFrameInfo has been
592  // set up. All the information required by getReservedRegs() should be
593  // available now.
594  MRI.freezeReservedRegs(*MF);
595 
596  // Release function-specific state. SDB and CurDAG are already cleared
597  // at this point.
598  FuncInfo->clear();
599 
600  DEBUG(dbgs() << "*** MachineFunction at end of ISel ***\n");
601  DEBUG(MF->print(dbgs()));
602 
603  return true;
604 }
605 
606 void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin,
608  bool &HadTailCall) {
609  // Lower the instructions. If a call is emitted as a tail call, cease emitting
610  // nodes for this block.
611  for (BasicBlock::const_iterator I = Begin; I != End && !SDB->HasTailCall; ++I)
612  SDB->visit(*I);
613 
614  // Make sure the root of the DAG is up-to-date.
616  HadTailCall = SDB->HasTailCall;
617  SDB->clear();
618 
619  // Final step, emit the lowered DAG as machine code.
620  CodeGenAndEmitDAG();
621 }
622 
623 void SelectionDAGISel::ComputeLiveOutVRegInfo() {
624  SmallPtrSet<SDNode*, 128> VisitedNodes;
625  SmallVector<SDNode*, 128> Worklist;
626 
627  Worklist.push_back(CurDAG->getRoot().getNode());
628 
629  APInt KnownZero;
630  APInt KnownOne;
631 
632  do {
633  SDNode *N = Worklist.pop_back_val();
634 
635  // If we've already seen this node, ignore it.
636  if (!VisitedNodes.insert(N).second)
637  continue;
638 
639  // Otherwise, add all chain operands to the worklist.
640  for (const SDValue &Op : N->op_values())
641  if (Op.getValueType() == MVT::Other)
642  Worklist.push_back(Op.getNode());
643 
644  // If this is a CopyToReg with a vreg dest, process it.
645  if (N->getOpcode() != ISD::CopyToReg)
646  continue;
647 
648  unsigned DestReg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
650  continue;
651 
652  // Ignore non-scalar or non-integer values.
653  SDValue Src = N->getOperand(2);
654  EVT SrcVT = Src.getValueType();
655  if (!SrcVT.isInteger() || SrcVT.isVector())
656  continue;
657 
658  unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src);
659  CurDAG->computeKnownBits(Src, KnownZero, KnownOne);
660  FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, KnownZero, KnownOne);
661  } while (!Worklist.empty());
662 }
663 
664 void SelectionDAGISel::CodeGenAndEmitDAG() {
665  std::string GroupName;
667  GroupName = "Instruction Selection and Scheduling";
668  std::string BlockName;
669  int BlockNumber = -1;
670  (void)BlockNumber;
671  bool MatchFilterBB = false; (void)MatchFilterBB;
672 #ifndef NDEBUG
673  MatchFilterBB = (FilterDAGBasicBlockName.empty() ||
676 #endif
677 #ifdef NDEBUG
681 #endif
682  {
683  BlockNumber = FuncInfo->MBB->getNumber();
684  BlockName =
685  (MF->getName() + ":" + FuncInfo->MBB->getBasicBlock()->getName()).str();
686  }
687  DEBUG(dbgs() << "Initial selection DAG: BB#" << BlockNumber
688  << " '" << BlockName << "'\n"; CurDAG->dump());
689 
690  if (ViewDAGCombine1 && MatchFilterBB)
691  CurDAG->viewGraph("dag-combine1 input for " + BlockName);
692 
693  // Run the DAG combiner in pre-legalize mode.
694  {
695  NamedRegionTimer T("DAG Combining 1", GroupName, TimePassesIsEnabled);
697  }
698 
699  DEBUG(dbgs() << "Optimized lowered selection DAG: BB#" << BlockNumber
700  << " '" << BlockName << "'\n"; CurDAG->dump());
701 
702  // Second step, hack on the DAG until it only uses operations and types that
703  // the target supports.
704  if (ViewLegalizeTypesDAGs && MatchFilterBB)
705  CurDAG->viewGraph("legalize-types input for " + BlockName);
706 
707  bool Changed;
708  {
709  NamedRegionTimer T("Type Legalization", GroupName, TimePassesIsEnabled);
710  Changed = CurDAG->LegalizeTypes();
711  }
712 
713  DEBUG(dbgs() << "Type-legalized selection DAG: BB#" << BlockNumber
714  << " '" << BlockName << "'\n"; CurDAG->dump());
715 
717 
718  if (Changed) {
719  if (ViewDAGCombineLT && MatchFilterBB)
720  CurDAG->viewGraph("dag-combine-lt input for " + BlockName);
721 
722  // Run the DAG combiner in post-type-legalize mode.
723  {
724  NamedRegionTimer T("DAG Combining after legalize types", GroupName,
727  }
728 
729  DEBUG(dbgs() << "Optimized type-legalized selection DAG: BB#" << BlockNumber
730  << " '" << BlockName << "'\n"; CurDAG->dump());
731 
732  }
733 
734  {
735  NamedRegionTimer T("Vector Legalization", GroupName, TimePassesIsEnabled);
736  Changed = CurDAG->LegalizeVectors();
737  }
738 
739  if (Changed) {
740  {
741  NamedRegionTimer T("Type Legalization 2", GroupName, TimePassesIsEnabled);
743  }
744 
745  if (ViewDAGCombineLT && MatchFilterBB)
746  CurDAG->viewGraph("dag-combine-lv input for " + BlockName);
747 
748  // Run the DAG combiner in post-type-legalize mode.
749  {
750  NamedRegionTimer T("DAG Combining after legalize vectors", GroupName,
753  }
754 
755  DEBUG(dbgs() << "Optimized vector-legalized selection DAG: BB#"
756  << BlockNumber << " '" << BlockName << "'\n"; CurDAG->dump());
757  }
758 
759  if (ViewLegalizeDAGs && MatchFilterBB)
760  CurDAG->viewGraph("legalize input for " + BlockName);
761 
762  {
763  NamedRegionTimer T("DAG Legalization", GroupName, TimePassesIsEnabled);
764  CurDAG->Legalize();
765  }
766 
767  DEBUG(dbgs() << "Legalized selection DAG: BB#" << BlockNumber
768  << " '" << BlockName << "'\n"; CurDAG->dump());
769 
770  if (ViewDAGCombine2 && MatchFilterBB)
771  CurDAG->viewGraph("dag-combine2 input for " + BlockName);
772 
773  // Run the DAG combiner in post-legalize mode.
774  {
775  NamedRegionTimer T("DAG Combining 2", GroupName, TimePassesIsEnabled);
777  }
778 
779  DEBUG(dbgs() << "Optimized legalized selection DAG: BB#" << BlockNumber
780  << " '" << BlockName << "'\n"; CurDAG->dump());
781 
782  if (OptLevel != CodeGenOpt::None)
783  ComputeLiveOutVRegInfo();
784 
785  if (ViewISelDAGs && MatchFilterBB)
786  CurDAG->viewGraph("isel input for " + BlockName);
787 
788  // Third, instruction select all of the operations to machine code, adding the
789  // code to the MachineBasicBlock.
790  {
791  NamedRegionTimer T("Instruction Selection", GroupName, TimePassesIsEnabled);
792  DoInstructionSelection();
793  }
794 
795  DEBUG(dbgs() << "Selected selection DAG: BB#" << BlockNumber
796  << " '" << BlockName << "'\n"; CurDAG->dump());
797 
798  if (ViewSchedDAGs && MatchFilterBB)
799  CurDAG->viewGraph("scheduler input for " + BlockName);
800 
801  // Schedule machine code.
802  ScheduleDAGSDNodes *Scheduler = CreateScheduler();
803  {
804  NamedRegionTimer T("Instruction Scheduling", GroupName,
806  Scheduler->Run(CurDAG, FuncInfo->MBB);
807  }
808 
809  if (ViewSUnitDAGs && MatchFilterBB) Scheduler->viewGraph();
810 
811  // Emit machine code to BB. This can change 'BB' to the last block being
812  // inserted into.
813  MachineBasicBlock *FirstMBB = FuncInfo->MBB, *LastMBB;
814  {
815  NamedRegionTimer T("Instruction Creation", GroupName, TimePassesIsEnabled);
816 
817  // FuncInfo->InsertPt is passed by reference and set to the end of the
818  // scheduled instructions.
819  LastMBB = FuncInfo->MBB = Scheduler->EmitSchedule(FuncInfo->InsertPt);
820  }
821 
822  // If the block was split, make sure we update any references that are used to
823  // update PHI nodes later on.
824  if (FirstMBB != LastMBB)
825  SDB->UpdateSplitBlock(FirstMBB, LastMBB);
826 
827  // Free the scheduler state.
828  {
829  NamedRegionTimer T("Instruction Scheduling Cleanup", GroupName,
831  delete Scheduler;
832  }
833 
834  // Free the SelectionDAG state, now that we're finished with it.
835  CurDAG->clear();
836 }
837 
838 namespace {
839 /// ISelUpdater - helper class to handle updates of the instruction selection
840 /// graph.
841 class ISelUpdater : public SelectionDAG::DAGUpdateListener {
842  SelectionDAG::allnodes_iterator &ISelPosition;
843 public:
844  ISelUpdater(SelectionDAG &DAG, SelectionDAG::allnodes_iterator &isp)
845  : SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}
846 
847  /// NodeDeleted - Handle nodes deleted from the graph. If the node being
848  /// deleted is the current ISelPosition node, update ISelPosition.
849  ///
850  void NodeDeleted(SDNode *N, SDNode *E) override {
851  if (ISelPosition == SelectionDAG::allnodes_iterator(N))
852  ++ISelPosition;
853  }
854 };
855 } // end anonymous namespace
856 
857 void SelectionDAGISel::DoInstructionSelection() {
858  DEBUG(dbgs() << "===== Instruction selection begins: BB#"
859  << FuncInfo->MBB->getNumber()
860  << " '" << FuncInfo->MBB->getName() << "'\n");
861 
863 
864  // Select target instructions for the DAG.
865  {
866  // Number all nodes with a topological order and set DAGSize.
868 
869  // Create a dummy node (which is not added to allnodes), that adds
870  // a reference to the root node, preventing it from being deleted,
871  // and tracking any changes of the root.
874  ++ISelPosition;
875 
876  // Make sure that ISelPosition gets properly updated when nodes are deleted
877  // in calls made from this function.
878  ISelUpdater ISU(*CurDAG, ISelPosition);
879 
880  // The AllNodes list is now topological-sorted. Visit the
881  // nodes by starting at the end of the list (the root of the
882  // graph) and preceding back toward the beginning (the entry
883  // node).
884  while (ISelPosition != CurDAG->allnodes_begin()) {
885  SDNode *Node = --ISelPosition;
886  // Skip dead nodes. DAGCombiner is expected to eliminate all dead nodes,
887  // but there are currently some corner cases that it misses. Also, this
888  // makes it theoretically possible to disable the DAGCombiner.
889  if (Node->use_empty())
890  continue;
891 
892  SDNode *ResNode = Select(Node);
893 
894  // FIXME: This is pretty gross. 'Select' should be changed to not return
895  // anything at all and this code should be nuked with a tactical strike.
896 
897  // If node should not be replaced, continue with the next one.
898  if (ResNode == Node || Node->getOpcode() == ISD::DELETED_NODE)
899  continue;
900  // Replace node.
901  if (ResNode) {
902  ReplaceUses(Node, ResNode);
903  }
904 
905  // If after the replacement this node is not used any more,
906  // remove this dead node.
907  if (Node->use_empty()) // Don't delete EntryToken, etc.
908  CurDAG->RemoveDeadNode(Node);
909  }
910 
911  CurDAG->setRoot(Dummy.getValue());
912  }
913 
914  DEBUG(dbgs() << "===== Instruction selection ends:\n");
915 
917 }
918 
919 /// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and
920 /// do other setup for EH landing-pad blocks.
921 bool SelectionDAGISel::PrepareEHLandingPad() {
923 
924  const TargetRegisterClass *PtrRC =
926 
927  // Add a label to mark the beginning of the landing pad. Deletion of the
928  // landing pad can thus be detected via the MachineModuleInfo.
929  MCSymbol *Label = MF->getMMI().addLandingPad(MBB);
930 
931  // Assign the call site to the landing pad's begin label.
933 
935  BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II)
936  .addSym(Label);
937 
938  // If this is an MSVC-style personality function, we need to split the landing
939  // pad into several BBs.
940  const BasicBlock *LLVMBB = MBB->getBasicBlock();
941  const LandingPadInst *LPadInst = LLVMBB->getLandingPadInst();
942  MF->getMMI().addPersonality(MBB, cast<Function>(LPadInst->getParent()
943  ->getParent()
944  ->getPersonalityFn()
945  ->stripPointerCasts()));
946  EHPersonality Personality = MF->getMMI().getPersonalityType();
947 
948  if (isMSVCEHPersonality(Personality)) {
950  const IntrinsicInst *ActionsCall =
952  // Get all invoke BBs that unwind to this landingpad.
954  MBB->pred_end());
955  if (ActionsCall && ActionsCall->getIntrinsicID() == Intrinsic::eh_actions) {
956  // If this is a call to llvm.eh.actions followed by indirectbr, then we've
957  // run WinEHPrepare, and we should remove this block from the machine CFG.
958  // Mark the targets of the indirectbr as landingpads instead.
959  for (const BasicBlock *LLVMSucc : successors(LLVMBB)) {
960  MachineBasicBlock *ClauseBB = FuncInfo->MBBMap[LLVMSucc];
961  // Add the edge from the invoke to the clause.
962  for (MachineBasicBlock *InvokeBB : InvokeBBs)
963  InvokeBB->addSuccessor(ClauseBB);
964 
965  // Mark the clause as a landing pad or MI passes will delete it.
966  ClauseBB->setIsLandingPad();
967  }
968  }
969 
970  // Remove the edge from the invoke to the lpad.
971  for (MachineBasicBlock *InvokeBB : InvokeBBs)
972  InvokeBB->removeSuccessor(MBB);
973 
974  // Don't select instructions for the landingpad.
975  return false;
976  }
977 
978  // Mark exception register as live in.
979  if (unsigned Reg = TLI->getExceptionPointerRegister())
981 
982  // Mark exception selector register as live in.
983  if (unsigned Reg = TLI->getExceptionSelectorRegister())
985 
986  return true;
987 }
988 
989 /// isFoldedOrDeadInstruction - Return true if the specified instruction is
990 /// side-effect free and is either dead or folded into a generated instruction.
991 /// Return false if it needs to be emitted.
993  FunctionLoweringInfo *FuncInfo) {
994  return !I->mayWriteToMemory() && // Side-effecting instructions aren't folded.
995  !isa<TerminatorInst>(I) && // Terminators aren't folded.
996  !isa<DbgInfoIntrinsic>(I) && // Debug instructions aren't folded.
997  !isa<LandingPadInst>(I) && // Landingpad instructions aren't folded.
998  !FuncInfo->isExportedInst(I); // Exported instrs must be computed.
999 }
1000 
1001 #ifndef NDEBUG
1002 // Collect per Instruction statistics for fast-isel misses. Only those
1003 // instructions that cause the bail are accounted for. It does not account for
1004 // instructions higher in the block. Thus, summing the per instructions stats
1005 // will not add up to what is reported by NumFastIselFailures.
1006 static void collectFailStats(const Instruction *I) {
1007  switch (I->getOpcode()) {
1008  default: assert (0 && "<Invalid operator> ");
1009 
1010  // Terminators
1011  case Instruction::Ret: NumFastIselFailRet++; return;
1012  case Instruction::Br: NumFastIselFailBr++; return;
1013  case Instruction::Switch: NumFastIselFailSwitch++; return;
1014  case Instruction::IndirectBr: NumFastIselFailIndirectBr++; return;
1015  case Instruction::Invoke: NumFastIselFailInvoke++; return;
1016  case Instruction::Resume: NumFastIselFailResume++; return;
1017  case Instruction::Unreachable: NumFastIselFailUnreachable++; return;
1018 
1019  // Standard binary operators...
1020  case Instruction::Add: NumFastIselFailAdd++; return;
1021  case Instruction::FAdd: NumFastIselFailFAdd++; return;
1022  case Instruction::Sub: NumFastIselFailSub++; return;
1023  case Instruction::FSub: NumFastIselFailFSub++; return;
1024  case Instruction::Mul: NumFastIselFailMul++; return;
1025  case Instruction::FMul: NumFastIselFailFMul++; return;
1026  case Instruction::UDiv: NumFastIselFailUDiv++; return;
1027  case Instruction::SDiv: NumFastIselFailSDiv++; return;
1028  case Instruction::FDiv: NumFastIselFailFDiv++; return;
1029  case Instruction::URem: NumFastIselFailURem++; return;
1030  case Instruction::SRem: NumFastIselFailSRem++; return;
1031  case Instruction::FRem: NumFastIselFailFRem++; return;
1032 
1033  // Logical operators...
1034  case Instruction::And: NumFastIselFailAnd++; return;
1035  case Instruction::Or: NumFastIselFailOr++; return;
1036  case Instruction::Xor: NumFastIselFailXor++; return;
1037 
1038  // Memory instructions...
1039  case Instruction::Alloca: NumFastIselFailAlloca++; return;
1040  case Instruction::Load: NumFastIselFailLoad++; return;
1041  case Instruction::Store: NumFastIselFailStore++; return;
1042  case Instruction::AtomicCmpXchg: NumFastIselFailAtomicCmpXchg++; return;
1043  case Instruction::AtomicRMW: NumFastIselFailAtomicRMW++; return;
1044  case Instruction::Fence: NumFastIselFailFence++; return;
1045  case Instruction::GetElementPtr: NumFastIselFailGetElementPtr++; return;
1046 
1047  // Convert instructions...
1048  case Instruction::Trunc: NumFastIselFailTrunc++; return;
1049  case Instruction::ZExt: NumFastIselFailZExt++; return;
1050  case Instruction::SExt: NumFastIselFailSExt++; return;
1051  case Instruction::FPTrunc: NumFastIselFailFPTrunc++; return;
1052  case Instruction::FPExt: NumFastIselFailFPExt++; return;
1053  case Instruction::FPToUI: NumFastIselFailFPToUI++; return;
1054  case Instruction::FPToSI: NumFastIselFailFPToSI++; return;
1055  case Instruction::UIToFP: NumFastIselFailUIToFP++; return;
1056  case Instruction::SIToFP: NumFastIselFailSIToFP++; return;
1057  case Instruction::IntToPtr: NumFastIselFailIntToPtr++; return;
1058  case Instruction::PtrToInt: NumFastIselFailPtrToInt++; return;
1059  case Instruction::BitCast: NumFastIselFailBitCast++; return;
1060 
1061  // Other instructions...
1062  case Instruction::ICmp: NumFastIselFailICmp++; return;
1063  case Instruction::FCmp: NumFastIselFailFCmp++; return;
1064  case Instruction::PHI: NumFastIselFailPHI++; return;
1065  case Instruction::Select: NumFastIselFailSelect++; return;
1066  case Instruction::Call: {
1067  if (auto const *Intrinsic = dyn_cast<IntrinsicInst>(I)) {
1068  switch (Intrinsic->getIntrinsicID()) {
1069  default:
1070  NumFastIselFailIntrinsicCall++; return;
1071  case Intrinsic::sadd_with_overflow:
1072  NumFastIselFailSAddWithOverflow++; return;
1073  case Intrinsic::uadd_with_overflow:
1074  NumFastIselFailUAddWithOverflow++; return;
1075  case Intrinsic::ssub_with_overflow:
1076  NumFastIselFailSSubWithOverflow++; return;
1077  case Intrinsic::usub_with_overflow:
1078  NumFastIselFailUSubWithOverflow++; return;
1079  case Intrinsic::smul_with_overflow:
1080  NumFastIselFailSMulWithOverflow++; return;
1081  case Intrinsic::umul_with_overflow:
1082  NumFastIselFailUMulWithOverflow++; return;
1083  case Intrinsic::frameaddress:
1084  NumFastIselFailFrameaddress++; return;
1085  case Intrinsic::sqrt:
1086  NumFastIselFailSqrt++; return;
1087  case Intrinsic::experimental_stackmap:
1088  NumFastIselFailStackMap++; return;
1089  case Intrinsic::experimental_patchpoint_void: // fall-through
1090  case Intrinsic::experimental_patchpoint_i64:
1091  NumFastIselFailPatchPoint++; return;
1092  }
1093  }
1094  NumFastIselFailCall++;
1095  return;
1096  }
1097  case Instruction::Shl: NumFastIselFailShl++; return;
1098  case Instruction::LShr: NumFastIselFailLShr++; return;
1099  case Instruction::AShr: NumFastIselFailAShr++; return;
1100  case Instruction::VAArg: NumFastIselFailVAArg++; return;
1101  case Instruction::ExtractElement: NumFastIselFailExtractElement++; return;
1102  case Instruction::InsertElement: NumFastIselFailInsertElement++; return;
1103  case Instruction::ShuffleVector: NumFastIselFailShuffleVector++; return;
1104  case Instruction::ExtractValue: NumFastIselFailExtractValue++; return;
1105  case Instruction::InsertValue: NumFastIselFailInsertValue++; return;
1106  case Instruction::LandingPad: NumFastIselFailLandingPad++; return;
1107  }
1108 }
1109 #endif
1110 
1111 void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
1112  // Initialize the Fast-ISel state, if needed.
1113  FastISel *FastIS = nullptr;
1115  FastIS = TLI->createFastISel(*FuncInfo, LibInfo);
1116 
1117  // Iterate over all basic blocks in the function.
1120  I = RPOT.begin(), E = RPOT.end(); I != E; ++I) {
1121  const BasicBlock *LLVMBB = *I;
1122 
1123  if (OptLevel != CodeGenOpt::None) {
1124  bool AllPredsVisited = true;
1125  for (const_pred_iterator PI = pred_begin(LLVMBB), PE = pred_end(LLVMBB);
1126  PI != PE; ++PI) {
1127  if (!FuncInfo->VisitedBBs.count(*PI)) {
1128  AllPredsVisited = false;
1129  break;
1130  }
1131  }
1132 
1133  if (AllPredsVisited) {
1134  for (BasicBlock::const_iterator I = LLVMBB->begin();
1135  const PHINode *PN = dyn_cast<PHINode>(I); ++I)
1137  } else {
1138  for (BasicBlock::const_iterator I = LLVMBB->begin();
1139  const PHINode *PN = dyn_cast<PHINode>(I); ++I)
1141  }
1142 
1143  FuncInfo->VisitedBBs.insert(LLVMBB);
1144  }
1145 
1146  BasicBlock::const_iterator const Begin = LLVMBB->getFirstNonPHI();
1147  BasicBlock::const_iterator const End = LLVMBB->end();
1148  BasicBlock::const_iterator BI = End;
1149 
1150  FuncInfo->MBB = FuncInfo->MBBMap[LLVMBB];
1152 
1153  // Setup an EH landing-pad block.
1156  if (LLVMBB->isLandingPad())
1157  if (!PrepareEHLandingPad())
1158  continue;
1159 
1160  // Before doing SelectionDAG ISel, see if FastISel has been requested.
1161  if (FastIS) {
1162  FastIS->startNewBlock();
1163 
1164  // Emit code for any incoming arguments. This must happen before
1165  // beginning FastISel on the entry block.
1166  if (LLVMBB == &Fn.getEntryBlock()) {
1167  ++NumEntryBlocks;
1168 
1169  // Lower any arguments needed in this block if this is the entry block.
1170  if (!FastIS->lowerArguments()) {
1171  // Fast isel failed to lower these arguments
1172  ++NumFastIselFailLowerArguments;
1173  if (EnableFastISelAbort > 1)
1174  report_fatal_error("FastISel didn't lower all arguments");
1175 
1176  // Use SelectionDAG argument lowering
1177  LowerArguments(Fn);
1179  SDB->clear();
1180  CodeGenAndEmitDAG();
1181  }
1182 
1183  // If we inserted any instructions at the beginning, make a note of
1184  // where they are, so we can be sure to emit subsequent instructions
1185  // after them.
1186  if (FuncInfo->InsertPt != FuncInfo->MBB->begin())
1187  FastIS->setLastLocalValue(std::prev(FuncInfo->InsertPt));
1188  else
1189  FastIS->setLastLocalValue(nullptr);
1190  }
1191 
1192  unsigned NumFastIselRemaining = std::distance(Begin, End);
1193  // Do FastISel on as many instructions as possible.
1194  for (; BI != Begin; --BI) {
1195  const Instruction *Inst = std::prev(BI);
1196 
1197  // If we no longer require this instruction, skip it.
1198  if (isFoldedOrDeadInstruction(Inst, FuncInfo)) {
1199  --NumFastIselRemaining;
1200  continue;
1201  }
1202 
1203  // Bottom-up: reset the insert pos at the top, after any local-value
1204  // instructions.
1205  FastIS->recomputeInsertPt();
1206 
1207  // Try to select the instruction with FastISel.
1208  if (FastIS->selectInstruction(Inst)) {
1209  --NumFastIselRemaining;
1210  ++NumFastIselSuccess;
1211  // If fast isel succeeded, skip over all the folded instructions, and
1212  // then see if there is a load right before the selected instructions.
1213  // Try to fold the load if so.
1214  const Instruction *BeforeInst = Inst;
1215  while (BeforeInst != Begin) {
1216  BeforeInst = std::prev(BasicBlock::const_iterator(BeforeInst));
1217  if (!isFoldedOrDeadInstruction(BeforeInst, FuncInfo))
1218  break;
1219  }
1220  if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&
1221  BeforeInst->hasOneUse() &&
1222  FastIS->tryToFoldLoad(cast<LoadInst>(BeforeInst), Inst)) {
1223  // If we succeeded, don't re-select the load.
1224  BI = std::next(BasicBlock::const_iterator(BeforeInst));
1225  --NumFastIselRemaining;
1226  ++NumFastIselSuccess;
1227  }
1228  continue;
1229  }
1230 
1231 #ifndef NDEBUG
1233  collectFailStats(Inst);
1234 #endif
1235 
1236  // Then handle certain instructions as single-LLVM-Instruction blocks.
1237  if (isa<CallInst>(Inst)) {
1238 
1240  dbgs() << "FastISel missed call: ";
1241  Inst->dump();
1242  }
1243  if (EnableFastISelAbort > 2)
1244  // FastISel selector couldn't handle something and bailed.
1245  // For the purpose of debugging, just abort.
1246  report_fatal_error("FastISel didn't select the entire block");
1247 
1248  if (!Inst->getType()->isVoidTy() && !Inst->use_empty()) {
1249  unsigned &R = FuncInfo->ValueMap[Inst];
1250  if (!R)
1251  R = FuncInfo->CreateRegs(Inst->getType());
1252  }
1253 
1254  bool HadTailCall = false;
1256  SelectBasicBlock(Inst, BI, HadTailCall);
1257 
1258  // If the call was emitted as a tail call, we're done with the block.
1259  // We also need to delete any previously emitted instructions.
1260  if (HadTailCall) {
1261  FastIS->removeDeadCode(SavedInsertPt, FuncInfo->MBB->end());
1262  --BI;
1263  break;
1264  }
1265 
1266  // Recompute NumFastIselRemaining as Selection DAG instruction
1267  // selection may have handled the call, input args, etc.
1268  unsigned RemainingNow = std::distance(Begin, BI);
1269  NumFastIselFailures += NumFastIselRemaining - RemainingNow;
1270  NumFastIselRemaining = RemainingNow;
1271  continue;
1272  }
1273 
1274  bool ShouldAbort = EnableFastISelAbort;
1276  if (isa<TerminatorInst>(Inst)) {
1277  // Use a different message for terminator misses.
1278  dbgs() << "FastISel missed terminator: ";
1279  // Don't abort unless for terminator unless the level is really high
1280  ShouldAbort = (EnableFastISelAbort > 2);
1281  } else {
1282  dbgs() << "FastISel miss: ";
1283  }
1284  Inst->dump();
1285  }
1286  if (ShouldAbort)
1287  // FastISel selector couldn't handle something and bailed.
1288  // For the purpose of debugging, just abort.
1289  report_fatal_error("FastISel didn't select the entire block");
1290 
1291  NumFastIselFailures += NumFastIselRemaining;
1292  break;
1293  }
1294 
1295  FastIS->recomputeInsertPt();
1296  } else {
1297  // Lower any arguments needed in this block if this is the entry block.
1298  if (LLVMBB == &Fn.getEntryBlock()) {
1299  ++NumEntryBlocks;
1300  LowerArguments(Fn);
1301  }
1302  }
1303 
1304  if (Begin != BI)
1305  ++NumDAGBlocks;
1306  else
1307  ++NumFastIselBlocks;
1308 
1309  if (Begin != BI) {
1310  // Run SelectionDAG instruction selection on the remainder of the block
1311  // not handled by FastISel. If FastISel is not run, this is the entire
1312  // block.
1313  bool HadTailCall;
1314  SelectBasicBlock(Begin, BI, HadTailCall);
1315  }
1316 
1317  FinishBasicBlock();
1318  FuncInfo->PHINodesToUpdate.clear();
1319  }
1320 
1321  delete FastIS;
1323  SDB->SPDescriptor.resetPerFunctionState();
1324 }
1325 
1326 /// Given that the input MI is before a partial terminator sequence TSeq, return
1327 /// true if M + TSeq also a partial terminator sequence.
1328 ///
1329 /// A Terminator sequence is a sequence of MachineInstrs which at this point in
1330 /// lowering copy vregs into physical registers, which are then passed into
1331 /// terminator instructors so we can satisfy ABI constraints. A partial
1332 /// terminator sequence is an improper subset of a terminator sequence (i.e. it
1333 /// may be the whole terminator sequence).
1335  // If we do not have a copy or an implicit def, we return true if and only if
1336  // MI is a debug value.
1337  if (!MI->isCopy() && !MI->isImplicitDef())
1338  // Sometimes DBG_VALUE MI sneak in between the copies from the vregs to the
1339  // physical registers if there is debug info associated with the terminator
1340  // of our mbb. We want to include said debug info in our terminator
1341  // sequence, so we return true in that case.
1342  return MI->isDebugValue();
1343 
1344  // We have left the terminator sequence if we are not doing one of the
1345  // following:
1346  //
1347  // 1. Copying a vreg into a physical register.
1348  // 2. Copying a vreg into a vreg.
1349  // 3. Defining a register via an implicit def.
1350 
1351  // OPI should always be a register definition...
1353  if (!OPI->isReg() || !OPI->isDef())
1354  return false;
1355 
1356  // Defining any register via an implicit def is always ok.
1357  if (MI->isImplicitDef())
1358  return true;
1359 
1360  // Grab the copy source...
1362  ++OPI2;
1363  assert(OPI2 != MI->operands_end()
1364  && "Should have a copy implying we should have 2 arguments.");
1365 
1366  // Make sure that the copy dest is not a vreg when the copy source is a
1367  // physical register.
1368  if (!OPI2->isReg() ||
1371  return false;
1372 
1373  return true;
1374 }
1375 
1376 /// Find the split point at which to splice the end of BB into its success stack
1377 /// protector check machine basic block.
1378 ///
1379 /// On many platforms, due to ABI constraints, terminators, even before register
1380 /// allocation, use physical registers. This creates an issue for us since
1381 /// physical registers at this point can not travel across basic
1382 /// blocks. Luckily, selectiondag always moves physical registers into vregs
1383 /// when they enter functions and moves them through a sequence of copies back
1384 /// into the physical registers right before the terminator creating a
1385 /// ``Terminator Sequence''. This function is searching for the beginning of the
1386 /// terminator sequence so that we can ensure that we splice off not just the
1387 /// terminator, but additionally the copies that move the vregs into the
1388 /// physical registers.
1392  //
1393  if (SplitPoint == BB->begin())
1394  return SplitPoint;
1395 
1396  MachineBasicBlock::iterator Start = BB->begin();
1397  MachineBasicBlock::iterator Previous = SplitPoint;
1398  --Previous;
1399 
1400  while (MIIsInTerminatorSequence(Previous)) {
1401  SplitPoint = Previous;
1402  if (Previous == Start)
1403  break;
1404  --Previous;
1405  }
1406 
1407  return SplitPoint;
1408 }
1409 
1410 void
1411 SelectionDAGISel::FinishBasicBlock() {
1412 
1413  DEBUG(dbgs() << "Total amount of phi nodes to update: "
1414  << FuncInfo->PHINodesToUpdate.size() << "\n";
1415  for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i)
1416  dbgs() << "Node " << i << " : ("
1417  << FuncInfo->PHINodesToUpdate[i].first
1418  << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n");
1419 
1420  // Next, now that we know what the last MBB the LLVM BB expanded is, update
1421  // PHI nodes in successors.
1422  for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
1424  assert(PHI->isPHI() &&
1425  "This is not a machine PHI node that we are updating!");
1426  if (!FuncInfo->MBB->isSuccessor(PHI->getParent()))
1427  continue;
1428  PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB);
1429  }
1430 
1431  // Handle stack protector.
1432  if (SDB->SPDescriptor.shouldEmitStackProtector()) {
1433  MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1434  MachineBasicBlock *SuccessMBB = SDB->SPDescriptor.getSuccessMBB();
1435 
1436  // Find the split point to split the parent mbb. At the same time copy all
1437  // physical registers used in the tail of parent mbb into virtual registers
1438  // before the split point and back into physical registers after the split
1439  // point. This prevents us needing to deal with Live-ins and many other
1440  // register allocation issues caused by us splitting the parent mbb. The
1441  // register allocator will clean up said virtual copies later on.
1442  MachineBasicBlock::iterator SplitPoint =
1444 
1445  // Splice the terminator of ParentMBB into SuccessMBB.
1446  SuccessMBB->splice(SuccessMBB->end(), ParentMBB,
1447  SplitPoint,
1448  ParentMBB->end());
1449 
1450  // Add compare/jump on neq/jump to the parent BB.
1451  FuncInfo->MBB = ParentMBB;
1452  FuncInfo->InsertPt = ParentMBB->end();
1454  CurDAG->setRoot(SDB->getRoot());
1455  SDB->clear();
1456  CodeGenAndEmitDAG();
1457 
1458  // CodeGen Failure MBB if we have not codegened it yet.
1459  MachineBasicBlock *FailureMBB = SDB->SPDescriptor.getFailureMBB();
1460  if (!FailureMBB->size()) {
1461  FuncInfo->MBB = FailureMBB;
1462  FuncInfo->InsertPt = FailureMBB->end();
1464  CurDAG->setRoot(SDB->getRoot());
1465  SDB->clear();
1466  CodeGenAndEmitDAG();
1467  }
1468 
1469  // Clear the Per-BB State.
1470  SDB->SPDescriptor.resetPerBBState();
1471  }
1472 
1473  for (unsigned i = 0, e = SDB->BitTestCases.size(); i != e; ++i) {
1474  // Lower header first, if it wasn't already lowered
1475  if (!SDB->BitTestCases[i].Emitted) {
1476  // Set the current basic block to the mbb we wish to insert the code into
1477  FuncInfo->MBB = SDB->BitTestCases[i].Parent;
1478  FuncInfo->InsertPt = FuncInfo->MBB->end();
1479  // Emit the code
1481  CurDAG->setRoot(SDB->getRoot());
1482  SDB->clear();
1483  CodeGenAndEmitDAG();
1484  }
1485 
1486  uint32_t UnhandledWeight = 0;
1487  for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j)
1488  UnhandledWeight += SDB->BitTestCases[i].Cases[j].ExtraWeight;
1489 
1490  for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j) {
1491  UnhandledWeight -= SDB->BitTestCases[i].Cases[j].ExtraWeight;
1492  // Set the current basic block to the mbb we wish to insert the code into
1493  FuncInfo->MBB = SDB->BitTestCases[i].Cases[j].ThisBB;
1494  FuncInfo->InsertPt = FuncInfo->MBB->end();
1495  // Emit the code
1496  if (j+1 != ej)
1498  SDB->BitTestCases[i].Cases[j+1].ThisBB,
1499  UnhandledWeight,
1500  SDB->BitTestCases[i].Reg,
1501  SDB->BitTestCases[i].Cases[j],
1502  FuncInfo->MBB);
1503  else
1505  SDB->BitTestCases[i].Default,
1506  UnhandledWeight,
1507  SDB->BitTestCases[i].Reg,
1508  SDB->BitTestCases[i].Cases[j],
1509  FuncInfo->MBB);
1510 
1511 
1512  CurDAG->setRoot(SDB->getRoot());
1513  SDB->clear();
1514  CodeGenAndEmitDAG();
1515  }
1516 
1517  // Update PHI Nodes
1518  for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
1519  pi != pe; ++pi) {
1521  MachineBasicBlock *PHIBB = PHI->getParent();
1522  assert(PHI->isPHI() &&
1523  "This is not a machine PHI node that we are updating!");
1524  // This is "default" BB. We have two jumps to it. From "header" BB and
1525  // from last "case" BB.
1526  if (PHIBB == SDB->BitTestCases[i].Default)
1527  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
1528  .addMBB(SDB->BitTestCases[i].Parent)
1529  .addReg(FuncInfo->PHINodesToUpdate[pi].second)
1530  .addMBB(SDB->BitTestCases[i].Cases.back().ThisBB);
1531  // One of "cases" BB.
1532  for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size();
1533  j != ej; ++j) {
1534  MachineBasicBlock* cBB = SDB->BitTestCases[i].Cases[j].ThisBB;
1535  if (cBB->isSuccessor(PHIBB))
1536  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(cBB);
1537  }
1538  }
1539  }
1540  SDB->BitTestCases.clear();
1541 
1542  // If the JumpTable record is filled in, then we need to emit a jump table.
1543  // Updating the PHI nodes is tricky in this case, since we need to determine
1544  // whether the PHI is a successor of the range check MBB or the jump table MBB
1545  for (unsigned i = 0, e = SDB->JTCases.size(); i != e; ++i) {
1546  // Lower header first, if it wasn't already lowered
1547  if (!SDB->JTCases[i].first.Emitted) {
1548  // Set the current basic block to the mbb we wish to insert the code into
1549  FuncInfo->MBB = SDB->JTCases[i].first.HeaderBB;
1550  FuncInfo->InsertPt = FuncInfo->MBB->end();
1551  // Emit the code
1552  SDB->visitJumpTableHeader(SDB->JTCases[i].second, SDB->JTCases[i].first,
1553  FuncInfo->MBB);
1554  CurDAG->setRoot(SDB->getRoot());
1555  SDB->clear();
1556  CodeGenAndEmitDAG();
1557  }
1558 
1559  // Set the current basic block to the mbb we wish to insert the code into
1560  FuncInfo->MBB = SDB->JTCases[i].second.MBB;
1561  FuncInfo->InsertPt = FuncInfo->MBB->end();
1562  // Emit the code
1563  SDB->visitJumpTable(SDB->JTCases[i].second);
1564  CurDAG->setRoot(SDB->getRoot());
1565  SDB->clear();
1566  CodeGenAndEmitDAG();
1567 
1568  // Update PHI Nodes
1569  for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
1570  pi != pe; ++pi) {
1572  MachineBasicBlock *PHIBB = PHI->getParent();
1573  assert(PHI->isPHI() &&
1574  "This is not a machine PHI node that we are updating!");
1575  // "default" BB. We can go there only from header BB.
1576  if (PHIBB == SDB->JTCases[i].second.Default)
1577  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
1578  .addMBB(SDB->JTCases[i].first.HeaderBB);
1579  // JT BB. Just iterate over successors here
1580  if (FuncInfo->MBB->isSuccessor(PHIBB))
1581  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(FuncInfo->MBB);
1582  }
1583  }
1584  SDB->JTCases.clear();
1585 
1586  // If we generated any switch lowering information, build and codegen any
1587  // additional DAGs necessary.
1588  for (unsigned i = 0, e = SDB->SwitchCases.size(); i != e; ++i) {
1589  // Set the current basic block to the mbb we wish to insert the code into
1590  FuncInfo->MBB = SDB->SwitchCases[i].ThisBB;
1591  FuncInfo->InsertPt = FuncInfo->MBB->end();
1592 
1593  // Determine the unique successors.
1595  Succs.push_back(SDB->SwitchCases[i].TrueBB);
1596  if (SDB->SwitchCases[i].TrueBB != SDB->SwitchCases[i].FalseBB)
1597  Succs.push_back(SDB->SwitchCases[i].FalseBB);
1598 
1599  // Emit the code. Note that this could result in FuncInfo->MBB being split.
1601  CurDAG->setRoot(SDB->getRoot());
1602  SDB->clear();
1603  CodeGenAndEmitDAG();
1604 
1605  // Remember the last block, now that any splitting is done, for use in
1606  // populating PHI nodes in successors.
1607  MachineBasicBlock *ThisBB = FuncInfo->MBB;
1608 
1609  // Handle any PHI nodes in successors of this chunk, as if we were coming
1610  // from the original BB before switch expansion. Note that PHI nodes can
1611  // occur multiple times in PHINodesToUpdate. We have to be very careful to
1612  // handle them the right number of times.
1613  for (unsigned i = 0, e = Succs.size(); i != e; ++i) {
1614  FuncInfo->MBB = Succs[i];
1615  FuncInfo->InsertPt = FuncInfo->MBB->end();
1616  // FuncInfo->MBB may have been removed from the CFG if a branch was
1617  // constant folded.
1618  if (ThisBB->isSuccessor(FuncInfo->MBB)) {
1620  MBBI = FuncInfo->MBB->begin(), MBBE = FuncInfo->MBB->end();
1621  MBBI != MBBE && MBBI->isPHI(); ++MBBI) {
1622  MachineInstrBuilder PHI(*MF, MBBI);
1623  // This value for this PHI node is recorded in PHINodesToUpdate.
1624  for (unsigned pn = 0; ; ++pn) {
1625  assert(pn != FuncInfo->PHINodesToUpdate.size() &&
1626  "Didn't find PHI entry!");
1627  if (FuncInfo->PHINodesToUpdate[pn].first == PHI) {
1628  PHI.addReg(FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB);
1629  break;
1630  }
1631  }
1632  }
1633  }
1634  }
1635  }
1636  SDB->SwitchCases.clear();
1637 }
1638 
1639 
1640 /// Create the scheduler. If a specific scheduler was specified
1641 /// via the SchedulerRegistry, use it, otherwise select the
1642 /// one preferred by the target.
1643 ///
1644 ScheduleDAGSDNodes *SelectionDAGISel::CreateScheduler() {
1646 
1647  if (!Ctor) {
1648  Ctor = ISHeuristic;
1650  }
1651 
1652  return Ctor(this, OptLevel);
1653 }
1654 
1655 //===----------------------------------------------------------------------===//
1656 // Helper functions used by the generated instruction selector.
1657 //===----------------------------------------------------------------------===//
1658 // Calls to these methods are generated by tblgen.
1659 
1660 /// CheckAndMask - The isel is trying to match something like (and X, 255). If
1661 /// the dag combiner simplified the 255, we still want to match. RHS is the
1662 /// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value
1663 /// specified in the .td file (e.g. 255).
1665  int64_t DesiredMaskS) const {
1666  const APInt &ActualMask = RHS->getAPIntValue();
1667  const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
1668 
1669  // If the actual mask exactly matches, success!
1670  if (ActualMask == DesiredMask)
1671  return true;
1672 
1673  // If the actual AND mask is allowing unallowed bits, this doesn't match.
1674  if (ActualMask.intersects(~DesiredMask))
1675  return false;
1676 
1677  // Otherwise, the DAG Combiner may have proven that the value coming in is
1678  // either already zero or is not demanded. Check for known zero input bits.
1679  APInt NeededMask = DesiredMask & ~ActualMask;
1680  if (CurDAG->MaskedValueIsZero(LHS, NeededMask))
1681  return true;
1682 
1683  // TODO: check to see if missing bits are just not demanded.
1684 
1685  // Otherwise, this pattern doesn't match.
1686  return false;
1687 }
1688 
1689 /// CheckOrMask - The isel is trying to match something like (or X, 255). If
1690 /// the dag combiner simplified the 255, we still want to match. RHS is the
1691 /// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value
1692 /// specified in the .td file (e.g. 255).
1694  int64_t DesiredMaskS) const {
1695  const APInt &ActualMask = RHS->getAPIntValue();
1696  const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
1697 
1698  // If the actual mask exactly matches, success!
1699  if (ActualMask == DesiredMask)
1700  return true;
1701 
1702  // If the actual AND mask is allowing unallowed bits, this doesn't match.
1703  if (ActualMask.intersects(~DesiredMask))
1704  return false;
1705 
1706  // Otherwise, the DAG Combiner may have proven that the value coming in is
1707  // either already zero or is not demanded. Check for known zero input bits.
1708  APInt NeededMask = DesiredMask & ~ActualMask;
1709 
1710  APInt KnownZero, KnownOne;
1711  CurDAG->computeKnownBits(LHS, KnownZero, KnownOne);
1712 
1713  // If all the missing bits in the or are already known to be set, match!
1714  if ((NeededMask & KnownOne) == NeededMask)
1715  return true;
1716 
1717  // TODO: check to see if missing bits are just not demanded.
1718 
1719  // Otherwise, this pattern doesn't match.
1720  return false;
1721 }
1722 
1723 /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
1724 /// by tblgen. Others should not call it.
1725 void SelectionDAGISel::
1726 SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops, SDLoc DL) {
1727  std::vector<SDValue> InOps;
1728  std::swap(InOps, Ops);
1729 
1730  Ops.push_back(InOps[InlineAsm::Op_InputChain]); // 0
1731  Ops.push_back(InOps[InlineAsm::Op_AsmString]); // 1
1732  Ops.push_back(InOps[InlineAsm::Op_MDNode]); // 2, !srcloc
1733  Ops.push_back(InOps[InlineAsm::Op_ExtraInfo]); // 3 (SideEffect, AlignStack)
1734 
1735  unsigned i = InlineAsm::Op_FirstOperand, e = InOps.size();
1736  if (InOps[e-1].getValueType() == MVT::Glue)
1737  --e; // Don't process a glue operand if it is here.
1738 
1739  while (i != e) {
1740  unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue();
1741  if (!InlineAsm::isMemKind(Flags)) {
1742  // Just skip over this operand, copying the operands verbatim.
1743  Ops.insert(Ops.end(), InOps.begin()+i,
1744  InOps.begin()+i+InlineAsm::getNumOperandRegisters(Flags) + 1);
1745  i += InlineAsm::getNumOperandRegisters(Flags) + 1;
1746  } else {
1747  assert(InlineAsm::getNumOperandRegisters(Flags) == 1 &&
1748  "Memory operand with multiple values?");
1749 
1750  unsigned TiedToOperand;
1751  if (InlineAsm::isUseOperandTiedToDef(Flags, TiedToOperand)) {
1752  // We need the constraint ID from the operand this is tied to.
1753  unsigned CurOp = InlineAsm::Op_FirstOperand;
1754  Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
1755  for (; TiedToOperand; --TiedToOperand) {
1756  CurOp += InlineAsm::getNumOperandRegisters(Flags)+1;
1757  Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
1758  }
1759  }
1760 
1761  // Otherwise, this is a memory operand. Ask the target to select it.
1762  std::vector<SDValue> SelOps;
1763  if (SelectInlineAsmMemoryOperand(InOps[i+1],
1765  SelOps))
1766  report_fatal_error("Could not match memory address. Inline asm"
1767  " failure!");
1768 
1769  // Add this to the output node.
1770  unsigned NewFlags =
1772  Ops.push_back(CurDAG->getTargetConstant(NewFlags, DL, MVT::i32));
1773  Ops.insert(Ops.end(), SelOps.begin(), SelOps.end());
1774  i += 2;
1775  }
1776  }
1777 
1778  // Add the glue input back if present.
1779  if (e != InOps.size())
1780  Ops.push_back(InOps.back());
1781 }
1782 
1783 /// findGlueUse - Return use of MVT::Glue value produced by the specified
1784 /// SDNode.
1785 ///
1787  unsigned FlagResNo = N->getNumValues()-1;
1788  for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
1789  SDUse &Use = I.getUse();
1790  if (Use.getResNo() == FlagResNo)
1791  return Use.getUser();
1792  }
1793  return nullptr;
1794 }
1795 
1796 /// findNonImmUse - Return true if "Use" is a non-immediate use of "Def".
1797 /// This function recursively traverses up the operand chain, ignoring
1798 /// certain nodes.
1799 static bool findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse,
1800  SDNode *Root, SmallPtrSetImpl<SDNode*> &Visited,
1801  bool IgnoreChains) {
1802  // The NodeID's are given uniques ID's where a node ID is guaranteed to be
1803  // greater than all of its (recursive) operands. If we scan to a point where
1804  // 'use' is smaller than the node we're scanning for, then we know we will
1805  // never find it.
1806  //
1807  // The Use may be -1 (unassigned) if it is a newly allocated node. This can
1808  // happen because we scan down to newly selected nodes in the case of glue
1809  // uses.
1810  if ((Use->getNodeId() < Def->getNodeId() && Use->getNodeId() != -1))
1811  return false;
1812 
1813  // Don't revisit nodes if we already scanned it and didn't fail, we know we
1814  // won't fail if we scan it again.
1815  if (!Visited.insert(Use).second)
1816  return false;
1817 
1818  for (const SDValue &Op : Use->op_values()) {
1819  // Ignore chain uses, they are validated by HandleMergeInputChains.
1820  if (Op.getValueType() == MVT::Other && IgnoreChains)
1821  continue;
1822 
1823  SDNode *N = Op.getNode();
1824  if (N == Def) {
1825  if (Use == ImmedUse || Use == Root)
1826  continue; // We are not looking for immediate use.
1827  assert(N != Root);
1828  return true;
1829  }
1830 
1831  // Traverse up the operand chain.
1832  if (findNonImmUse(N, Def, ImmedUse, Root, Visited, IgnoreChains))
1833  return true;
1834  }
1835  return false;
1836 }
1837 
1838 /// IsProfitableToFold - Returns true if it's profitable to fold the specific
1839 /// operand node N of U during instruction selection that starts at Root.
1841  SDNode *Root) const {
1842  if (OptLevel == CodeGenOpt::None) return false;
1843  return N.hasOneUse();
1844 }
1845 
1846 /// IsLegalToFold - Returns true if the specific operand node N of
1847 /// U can be folded during instruction selection that starts at Root.
1849  CodeGenOpt::Level OptLevel,
1850  bool IgnoreChains) {
1851  if (OptLevel == CodeGenOpt::None) return false;
1852 
1853  // If Root use can somehow reach N through a path that that doesn't contain
1854  // U then folding N would create a cycle. e.g. In the following
1855  // diagram, Root can reach N through X. If N is folded into into Root, then
1856  // X is both a predecessor and a successor of U.
1857  //
1858  // [N*] //
1859  // ^ ^ //
1860  // / \ //
1861  // [U*] [X]? //
1862  // ^ ^ //
1863  // \ / //
1864  // \ / //
1865  // [Root*] //
1866  //
1867  // * indicates nodes to be folded together.
1868  //
1869  // If Root produces glue, then it gets (even more) interesting. Since it
1870  // will be "glued" together with its glue use in the scheduler, we need to
1871  // check if it might reach N.
1872  //
1873  // [N*] //
1874  // ^ ^ //
1875  // / \ //
1876  // [U*] [X]? //
1877  // ^ ^ //
1878  // \ \ //
1879  // \ | //
1880  // [Root*] | //
1881  // ^ | //
1882  // f | //
1883  // | / //
1884  // [Y] / //
1885  // ^ / //
1886  // f / //
1887  // | / //
1888  // [GU] //
1889  //
1890  // If GU (glue use) indirectly reaches N (the load), and Root folds N
1891  // (call it Fold), then X is a predecessor of GU and a successor of
1892  // Fold. But since Fold and GU are glued together, this will create
1893  // a cycle in the scheduling graph.
1894 
1895  // If the node has glue, walk down the graph to the "lowest" node in the
1896  // glueged set.
1897  EVT VT = Root->getValueType(Root->getNumValues()-1);
1898  while (VT == MVT::Glue) {
1899  SDNode *GU = findGlueUse(Root);
1900  if (!GU)
1901  break;
1902  Root = GU;
1903  VT = Root->getValueType(Root->getNumValues()-1);
1904 
1905  // If our query node has a glue result with a use, we've walked up it. If
1906  // the user (which has already been selected) has a chain or indirectly uses
1907  // the chain, our WalkChainUsers predicate will not consider it. Because of
1908  // this, we cannot ignore chains in this predicate.
1909  IgnoreChains = false;
1910  }
1911 
1912 
1913  SmallPtrSet<SDNode*, 16> Visited;
1914  return !findNonImmUse(Root, N.getNode(), U, Root, Visited, IgnoreChains);
1915 }
1916 
1917 SDNode *SelectionDAGISel::Select_INLINEASM(SDNode *N) {
1918  SDLoc DL(N);
1919 
1920  std::vector<SDValue> Ops(N->op_begin(), N->op_end());
1922 
1923  const EVT VTs[] = {MVT::Other, MVT::Glue};
1924  SDValue New = CurDAG->getNode(ISD::INLINEASM, DL, VTs, Ops);
1925  New->setNodeId(-1);
1926  return New.getNode();
1927 }
1928 
1929 SDNode
1930 *SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) {
1931  SDLoc dl(Op);
1933  const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));
1934  unsigned Reg =
1935  TLI->getRegisterByName(RegStr->getString().data(), Op->getValueType(0),
1936  *CurDAG);
1937  SDValue New = CurDAG->getCopyFromReg(
1938  Op->getOperand(0), dl, Reg, Op->getValueType(0));
1939  New->setNodeId(-1);
1940  return New.getNode();
1941 }
1942 
1943 SDNode
1944 *SelectionDAGISel::Select_WRITE_REGISTER(SDNode *Op) {
1945  SDLoc dl(Op);
1947  const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));
1948  unsigned Reg = TLI->getRegisterByName(RegStr->getString().data(),
1949  Op->getOperand(2).getValueType(),
1950  *CurDAG);
1951  SDValue New = CurDAG->getCopyToReg(
1952  Op->getOperand(0), dl, Reg, Op->getOperand(2));
1953  New->setNodeId(-1);
1954  return New.getNode();
1955 }
1956 
1957 
1958 
1959 SDNode *SelectionDAGISel::Select_UNDEF(SDNode *N) {
1961 }
1962 
1963 /// GetVBR - decode a vbr encoding whose top bit is set.
1964 LLVM_ATTRIBUTE_ALWAYS_INLINE static uint64_t
1965 GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) {
1966  assert(Val >= 128 && "Not a VBR");
1967  Val &= 127; // Remove first vbr bit.
1968 
1969  unsigned Shift = 7;
1970  uint64_t NextBits;
1971  do {
1972  NextBits = MatcherTable[Idx++];
1973  Val |= (NextBits&127) << Shift;
1974  Shift += 7;
1975  } while (NextBits & 128);
1976 
1977  return Val;
1978 }
1979 
1980 
1981 /// UpdateChainsAndGlue - When a match is complete, this method updates uses of
1982 /// interior glue and chain results to use the new glue and chain results.
1983 void SelectionDAGISel::
1984 UpdateChainsAndGlue(SDNode *NodeToMatch, SDValue InputChain,
1985  const SmallVectorImpl<SDNode*> &ChainNodesMatched,
1986  SDValue InputGlue,
1987  const SmallVectorImpl<SDNode*> &GlueResultNodesMatched,
1988  bool isMorphNodeTo) {
1989  SmallVector<SDNode*, 4> NowDeadNodes;
1990 
1991  // Now that all the normal results are replaced, we replace the chain and
1992  // glue results if present.
1993  if (!ChainNodesMatched.empty()) {
1994  assert(InputChain.getNode() &&
1995  "Matched input chains but didn't produce a chain");
1996  // Loop over all of the nodes we matched that produced a chain result.
1997  // Replace all the chain results with the final chain we ended up with.
1998  for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
1999  SDNode *ChainNode = ChainNodesMatched[i];
2000 
2001  // If this node was already deleted, don't look at it.
2002  if (ChainNode->getOpcode() == ISD::DELETED_NODE)
2003  continue;
2004 
2005  // Don't replace the results of the root node if we're doing a
2006  // MorphNodeTo.
2007  if (ChainNode == NodeToMatch && isMorphNodeTo)
2008  continue;
2009 
2010  SDValue ChainVal = SDValue(ChainNode, ChainNode->getNumValues()-1);
2011  if (ChainVal.getValueType() == MVT::Glue)
2012  ChainVal = ChainVal.getValue(ChainVal->getNumValues()-2);
2013  assert(ChainVal.getValueType() == MVT::Other && "Not a chain?");
2014  CurDAG->ReplaceAllUsesOfValueWith(ChainVal, InputChain);
2015 
2016  // If the node became dead and we haven't already seen it, delete it.
2017  if (ChainNode->use_empty() &&
2018  !std::count(NowDeadNodes.begin(), NowDeadNodes.end(), ChainNode))
2019  NowDeadNodes.push_back(ChainNode);
2020  }
2021  }
2022 
2023  // If the result produces glue, update any glue results in the matched
2024  // pattern with the glue result.
2025  if (InputGlue.getNode()) {
2026  // Handle any interior nodes explicitly marked.
2027  for (unsigned i = 0, e = GlueResultNodesMatched.size(); i != e; ++i) {
2028  SDNode *FRN = GlueResultNodesMatched[i];
2029 
2030  // If this node was already deleted, don't look at it.
2031  if (FRN->getOpcode() == ISD::DELETED_NODE)
2032  continue;
2033 
2034  assert(FRN->getValueType(FRN->getNumValues()-1) == MVT::Glue &&
2035  "Doesn't have a glue result");
2037  InputGlue);
2038 
2039  // If the node became dead and we haven't already seen it, delete it.
2040  if (FRN->use_empty() &&
2041  !std::count(NowDeadNodes.begin(), NowDeadNodes.end(), FRN))
2042  NowDeadNodes.push_back(FRN);
2043  }
2044  }
2045 
2046  if (!NowDeadNodes.empty())
2047  CurDAG->RemoveDeadNodes(NowDeadNodes);
2048 
2049  DEBUG(dbgs() << "ISEL: Match complete!\n");
2050 }
2051 
2056 };
2057 
2058 /// WalkChainUsers - Walk down the users of the specified chained node that is
2059 /// part of the pattern we're matching, looking at all of the users we find.
2060 /// This determines whether something is an interior node, whether we have a
2061 /// non-pattern node in between two pattern nodes (which prevent folding because
2062 /// it would induce a cycle) and whether we have a TokenFactor node sandwiched
2063 /// between pattern nodes (in which case the TF becomes part of the pattern).
2064 ///
2065 /// The walk we do here is guaranteed to be small because we quickly get down to
2066 /// already selected nodes "below" us.
2067 static ChainResult
2068 WalkChainUsers(const SDNode *ChainedNode,
2069  SmallVectorImpl<SDNode*> &ChainedNodesInPattern,
2070  SmallVectorImpl<SDNode*> &InteriorChainedNodes) {
2071  ChainResult Result = CR_Simple;
2072 
2073  for (SDNode::use_iterator UI = ChainedNode->use_begin(),
2074  E = ChainedNode->use_end(); UI != E; ++UI) {
2075  // Make sure the use is of the chain, not some other value we produce.
2076  if (UI.getUse().getValueType() != MVT::Other) continue;
2077 
2078  SDNode *User = *UI;
2079 
2080  if (User->getOpcode() == ISD::HANDLENODE) // Root of the graph.
2081  continue;
2082 
2083  // If we see an already-selected machine node, then we've gone beyond the
2084  // pattern that we're selecting down into the already selected chunk of the
2085  // DAG.
2086  unsigned UserOpcode = User->getOpcode();
2087  if (User->isMachineOpcode() ||
2088  UserOpcode == ISD::CopyToReg ||
2089  UserOpcode == ISD::CopyFromReg ||
2090  UserOpcode == ISD::INLINEASM ||
2091  UserOpcode == ISD::EH_LABEL ||
2092  UserOpcode == ISD::LIFETIME_START ||
2093  UserOpcode == ISD::LIFETIME_END) {
2094  // If their node ID got reset to -1 then they've already been selected.
2095  // Treat them like a MachineOpcode.
2096  if (User->getNodeId() == -1)
2097  continue;
2098  }
2099 
2100  // If we have a TokenFactor, we handle it specially.
2101  if (User->getOpcode() != ISD::TokenFactor) {
2102  // If the node isn't a token factor and isn't part of our pattern, then it
2103  // must be a random chained node in between two nodes we're selecting.
2104  // This happens when we have something like:
2105  // x = load ptr
2106  // call
2107  // y = x+4
2108  // store y -> ptr
2109  // Because we structurally match the load/store as a read/modify/write,
2110  // but the call is chained between them. We cannot fold in this case
2111  // because it would induce a cycle in the graph.
2112  if (!std::count(ChainedNodesInPattern.begin(),
2113  ChainedNodesInPattern.end(), User))
2114  return CR_InducesCycle;
2115 
2116  // Otherwise we found a node that is part of our pattern. For example in:
2117  // x = load ptr
2118  // y = x+4
2119  // store y -> ptr
2120  // This would happen when we're scanning down from the load and see the
2121  // store as a user. Record that there is a use of ChainedNode that is
2122  // part of the pattern and keep scanning uses.
2123  Result = CR_LeadsToInteriorNode;
2124  InteriorChainedNodes.push_back(User);
2125  continue;
2126  }
2127 
2128  // If we found a TokenFactor, there are two cases to consider: first if the
2129  // TokenFactor is just hanging "below" the pattern we're matching (i.e. no
2130  // uses of the TF are in our pattern) we just want to ignore it. Second,
2131  // the TokenFactor can be sandwiched in between two chained nodes, like so:
2132  // [Load chain]
2133  // ^
2134  // |
2135  // [Load]
2136  // ^ ^
2137  // | \ DAG's like cheese
2138  // / \ do you?
2139  // / |
2140  // [TokenFactor] [Op]
2141  // ^ ^
2142  // | |
2143  // \ /
2144  // \ /
2145  // [Store]
2146  //
2147  // In this case, the TokenFactor becomes part of our match and we rewrite it
2148  // as a new TokenFactor.
2149  //
2150  // To distinguish these two cases, do a recursive walk down the uses.
2151  switch (WalkChainUsers(User, ChainedNodesInPattern, InteriorChainedNodes)) {
2152  case CR_Simple:
2153  // If the uses of the TokenFactor are just already-selected nodes, ignore
2154  // it, it is "below" our pattern.
2155  continue;
2156  case CR_InducesCycle:
2157  // If the uses of the TokenFactor lead to nodes that are not part of our
2158  // pattern that are not selected, folding would turn this into a cycle,
2159  // bail out now.
2160  return CR_InducesCycle;
2162  break; // Otherwise, keep processing.
2163  }
2164 
2165  // Okay, we know we're in the interesting interior case. The TokenFactor
2166  // is now going to be considered part of the pattern so that we rewrite its
2167  // uses (it may have uses that are not part of the pattern) with the
2168  // ultimate chain result of the generated code. We will also add its chain
2169  // inputs as inputs to the ultimate TokenFactor we create.
2170  Result = CR_LeadsToInteriorNode;
2171  ChainedNodesInPattern.push_back(User);
2172  InteriorChainedNodes.push_back(User);
2173  continue;
2174  }
2175 
2176  return Result;
2177 }
2178 
2179 /// HandleMergeInputChains - This implements the OPC_EmitMergeInputChains
2180 /// operation for when the pattern matched at least one node with a chains. The
2181 /// input vector contains a list of all of the chained nodes that we match. We
2182 /// must determine if this is a valid thing to cover (i.e. matching it won't
2183 /// induce cycles in the DAG) and if so, creating a TokenFactor node. that will
2184 /// be used as the input node chain for the generated nodes.
2185 static SDValue
2187  SelectionDAG *CurDAG) {
2188  // Walk all of the chained nodes we've matched, recursively scanning down the
2189  // users of the chain result. This adds any TokenFactor nodes that are caught
2190  // in between chained nodes to the chained and interior nodes list.
2191  SmallVector<SDNode*, 3> InteriorChainedNodes;
2192  for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
2193  if (WalkChainUsers(ChainNodesMatched[i], ChainNodesMatched,
2194  InteriorChainedNodes) == CR_InducesCycle)
2195  return SDValue(); // Would induce a cycle.
2196  }
2197 
2198  // Okay, we have walked all the matched nodes and collected TokenFactor nodes
2199  // that we are interested in. Form our input TokenFactor node.
2200  SmallVector<SDValue, 3> InputChains;
2201  for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
2202  // Add the input chain of this node to the InputChains list (which will be
2203  // the operands of the generated TokenFactor) if it's not an interior node.
2204  SDNode *N = ChainNodesMatched[i];
2205  if (N->getOpcode() != ISD::TokenFactor) {
2206  if (std::count(InteriorChainedNodes.begin(),InteriorChainedNodes.end(),N))
2207  continue;
2208 
2209  // Otherwise, add the input chain.
2210  SDValue InChain = ChainNodesMatched[i]->getOperand(0);
2211  assert(InChain.getValueType() == MVT::Other && "Not a chain");
2212  InputChains.push_back(InChain);
2213  continue;
2214  }
2215 
2216  // If we have a token factor, we want to add all inputs of the token factor
2217  // that are not part of the pattern we're matching.
2218  for (const SDValue &Op : N->op_values()) {
2219  if (!std::count(ChainNodesMatched.begin(), ChainNodesMatched.end(),
2220  Op.getNode()))
2221  InputChains.push_back(Op);
2222  }
2223  }
2224 
2225  if (InputChains.size() == 1)
2226  return InputChains[0];
2227  return CurDAG->getNode(ISD::TokenFactor, SDLoc(ChainNodesMatched[0]),
2228  MVT::Other, InputChains);
2229 }
2230 
2231 /// MorphNode - Handle morphing a node in place for the selector.
2232 SDNode *SelectionDAGISel::
2233 MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
2234  ArrayRef<SDValue> Ops, unsigned EmitNodeInfo) {
2235  // It is possible we're using MorphNodeTo to replace a node with no
2236  // normal results with one that has a normal result (or we could be
2237  // adding a chain) and the input could have glue and chains as well.
2238  // In this case we need to shift the operands down.
2239  // FIXME: This is a horrible hack and broken in obscure cases, no worse
2240  // than the old isel though.
2241  int OldGlueResultNo = -1, OldChainResultNo = -1;
2242 
2243  unsigned NTMNumResults = Node->getNumValues();
2244  if (Node->getValueType(NTMNumResults-1) == MVT::Glue) {
2245  OldGlueResultNo = NTMNumResults-1;
2246  if (NTMNumResults != 1 &&
2247  Node->getValueType(NTMNumResults-2) == MVT::Other)
2248  OldChainResultNo = NTMNumResults-2;
2249  } else if (Node->getValueType(NTMNumResults-1) == MVT::Other)
2250  OldChainResultNo = NTMNumResults-1;
2251 
2252  // Call the underlying SelectionDAG routine to do the transmogrification. Note
2253  // that this deletes operands of the old node that become dead.
2254  SDNode *Res = CurDAG->MorphNodeTo(Node, ~TargetOpc, VTList, Ops);
2255 
2256  // MorphNodeTo can operate in two ways: if an existing node with the
2257  // specified operands exists, it can just return it. Otherwise, it
2258  // updates the node in place to have the requested operands.
2259  if (Res == Node) {
2260  // If we updated the node in place, reset the node ID. To the isel,
2261  // this should be just like a newly allocated machine node.
2262  Res->setNodeId(-1);
2263  }
2264 
2265  unsigned ResNumResults = Res->getNumValues();
2266  // Move the glue if needed.
2267  if ((EmitNodeInfo & OPFL_GlueOutput) && OldGlueResultNo != -1 &&
2268  (unsigned)OldGlueResultNo != ResNumResults-1)
2269  CurDAG->ReplaceAllUsesOfValueWith(SDValue(Node, OldGlueResultNo),
2270  SDValue(Res, ResNumResults-1));
2271 
2272  if ((EmitNodeInfo & OPFL_GlueOutput) != 0)
2273  --ResNumResults;
2274 
2275  // Move the chain reference if needed.
2276  if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 &&
2277  (unsigned)OldChainResultNo != ResNumResults-1)
2278  CurDAG->ReplaceAllUsesOfValueWith(SDValue(Node, OldChainResultNo),
2279  SDValue(Res, ResNumResults-1));
2280 
2281  // Otherwise, no replacement happened because the node already exists. Replace
2282  // Uses of the old node with the new one.
2283  if (Res != Node)
2284  CurDAG->ReplaceAllUsesWith(Node, Res);
2285 
2286  return Res;
2287 }
2288 
2289 /// CheckSame - Implements OP_CheckSame.
2290 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2291 CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2292  SDValue N,
2293  const SmallVectorImpl<std::pair<SDValue, SDNode*> > &RecordedNodes) {
2294  // Accept if it is exactly the same as a previously recorded node.
2295  unsigned RecNo = MatcherTable[MatcherIndex++];
2296  assert(RecNo < RecordedNodes.size() && "Invalid CheckSame");
2297  return N == RecordedNodes[RecNo].first;
2298 }
2299 
2300 /// CheckChildSame - Implements OP_CheckChildXSame.
2301 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2302 CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2303  SDValue N,
2304  const SmallVectorImpl<std::pair<SDValue, SDNode*> > &RecordedNodes,
2305  unsigned ChildNo) {
2306  if (ChildNo >= N.getNumOperands())
2307  return false; // Match fails if out of range child #.
2308  return ::CheckSame(MatcherTable, MatcherIndex, N.getOperand(ChildNo),
2309  RecordedNodes);
2310 }
2311 
2312 /// CheckPatternPredicate - Implements OP_CheckPatternPredicate.
2313 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2314 CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2315  const SelectionDAGISel &SDISel) {
2316  return SDISel.CheckPatternPredicate(MatcherTable[MatcherIndex++]);
2317 }
2318 
2319 /// CheckNodePredicate - Implements OP_CheckNodePredicate.
2320 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2321 CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2322  const SelectionDAGISel &SDISel, SDNode *N) {
2323  return SDISel.CheckNodePredicate(N, MatcherTable[MatcherIndex++]);
2324 }
2325 
2326 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2327 CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2328  SDNode *N) {
2329  uint16_t Opc = MatcherTable[MatcherIndex++];
2330  Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
2331  return N->getOpcode() == Opc;
2332 }
2333 
2334 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2335 CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2336  const TargetLowering *TLI, const DataLayout &DL) {
2337  MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2338  if (N.getValueType() == VT) return true;
2339 
2340  // Handle the case when VT is iPTR.
2341  return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(DL);
2342 }
2343 
2344 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2345 CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2346  SDValue N, const TargetLowering *TLI, const DataLayout &DL,
2347  unsigned ChildNo) {
2348  if (ChildNo >= N.getNumOperands())
2349  return false; // Match fails if out of range child #.
2350  return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI,
2351  DL);
2352 }
2353 
2354 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2355 CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2356  SDValue N) {
2357  return cast<CondCodeSDNode>(N)->get() ==
2358  (ISD::CondCode)MatcherTable[MatcherIndex++];
2359 }
2360 
2361 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2362 CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2363  SDValue N, const TargetLowering *TLI, const DataLayout &DL) {
2364  MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2365  if (cast<VTSDNode>(N)->getVT() == VT)
2366  return true;
2367 
2368  // Handle the case when VT is iPTR.
2369  return VT == MVT::iPTR && cast<VTSDNode>(N)->getVT() == TLI->getPointerTy(DL);
2370 }
2371 
2372 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2373 CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2374  SDValue N) {
2375  int64_t Val = MatcherTable[MatcherIndex++];
2376  if (Val & 128)
2377  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2378 
2380  return C && C->getSExtValue() == Val;
2381 }
2382 
2383 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2384 CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2385  SDValue N, unsigned ChildNo) {
2386  if (ChildNo >= N.getNumOperands())
2387  return false; // Match fails if out of range child #.
2388  return ::CheckInteger(MatcherTable, MatcherIndex, N.getOperand(ChildNo));
2389 }
2390 
2391 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2392 CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2393  SDValue N, const SelectionDAGISel &SDISel) {
2394  int64_t Val = MatcherTable[MatcherIndex++];
2395  if (Val & 128)
2396  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2397 
2398  if (N->getOpcode() != ISD::AND) return false;
2399 
2401  return C && SDISel.CheckAndMask(N.getOperand(0), C, Val);
2402 }
2403 
2404 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2405 CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2406  SDValue N, const SelectionDAGISel &SDISel) {
2407  int64_t Val = MatcherTable[MatcherIndex++];
2408  if (Val & 128)
2409  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2410 
2411  if (N->getOpcode() != ISD::OR) return false;
2412 
2414  return C && SDISel.CheckOrMask(N.getOperand(0), C, Val);
2415 }
2416 
2417 /// IsPredicateKnownToFail - If we know how and can do so without pushing a
2418 /// scope, evaluate the current node. If the current predicate is known to
2419 /// fail, set Result=true and return anything. If the current predicate is
2420 /// known to pass, set Result=false and return the MatcherIndex to continue
2421 /// with. If the current predicate is unknown, set Result=false and return the
2422 /// MatcherIndex to continue with.
2423 static unsigned IsPredicateKnownToFail(const unsigned char *Table,
2424  unsigned Index, SDValue N,
2425  bool &Result,
2426  const SelectionDAGISel &SDISel,
2427  SmallVectorImpl<std::pair<SDValue, SDNode*> > &RecordedNodes) {
2428  switch (Table[Index++]) {
2429  default:
2430  Result = false;
2431  return Index-1; // Could not evaluate this predicate.
2433  Result = !::CheckSame(Table, Index, N, RecordedNodes);
2434  return Index;
2439  Result = !::CheckChildSame(Table, Index, N, RecordedNodes,
2440  Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Same);
2441  return Index;
2443  Result = !::CheckPatternPredicate(Table, Index, SDISel);
2444  return Index;
2446  Result = !::CheckNodePredicate(Table, Index, SDISel, N.getNode());
2447  return Index;
2449  Result = !::CheckOpcode(Table, Index, N.getNode());
2450  return Index;
2452  Result = !::CheckType(Table, Index, N, SDISel.TLI,
2453  SDISel.CurDAG->getDataLayout());
2454  return Index;
2463  Result = !::CheckChildType(
2464  Table, Index, N, SDISel.TLI, SDISel.CurDAG->getDataLayout(),
2465  Table[Index - 1] - SelectionDAGISel::OPC_CheckChild0Type);
2466  return Index;
2468  Result = !::CheckCondCode(Table, Index, N);
2469  return Index;
2471  Result = !::CheckValueType(Table, Index, N, SDISel.TLI,
2472  SDISel.CurDAG->getDataLayout());
2473  return Index;
2475  Result = !::CheckInteger(Table, Index, N);
2476  return Index;
2482  Result = !::CheckChildInteger(Table, Index, N,
2483  Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Integer);
2484  return Index;
2486  Result = !::CheckAndImm(Table, Index, N, SDISel);
2487  return Index;
2489  Result = !::CheckOrImm(Table, Index, N, SDISel);
2490  return Index;
2491  }
2492 }
2493 
2494 namespace {
2495 
2496 struct MatchScope {
2497  /// FailIndex - If this match fails, this is the index to continue with.
2498  unsigned FailIndex;
2499 
2500  /// NodeStack - The node stack when the scope was formed.
2501  SmallVector<SDValue, 4> NodeStack;
2502 
2503  /// NumRecordedNodes - The number of recorded nodes when the scope was formed.
2504  unsigned NumRecordedNodes;
2505 
2506  /// NumMatchedMemRefs - The number of matched memref entries.
2507  unsigned NumMatchedMemRefs;
2508 
2509  /// InputChain/InputGlue - The current chain/glue
2510  SDValue InputChain, InputGlue;
2511 
2512  /// HasChainNodesMatched - True if the ChainNodesMatched list is non-empty.
2513  bool HasChainNodesMatched, HasGlueResultNodesMatched;
2514 };
2515 
2516 /// \\brief A DAG update listener to keep the matching state
2517 /// (i.e. RecordedNodes and MatchScope) uptodate if the target is allowed to
2518 /// change the DAG while matching. X86 addressing mode matcher is an example
2519 /// for this.
2520 class MatchStateUpdater : public SelectionDAG::DAGUpdateListener
2521 {
2523  SmallVectorImpl<MatchScope> &MatchScopes;
2524 public:
2525  MatchStateUpdater(SelectionDAG &DAG,
2526  SmallVectorImpl<std::pair<SDValue, SDNode*> > &RN,
2528  SelectionDAG::DAGUpdateListener(DAG),
2529  RecordedNodes(RN), MatchScopes(MS) { }
2530 
2531  void NodeDeleted(SDNode *N, SDNode *E) override {
2532  // Some early-returns here to avoid the search if we deleted the node or
2533  // if the update comes from MorphNodeTo (MorphNodeTo is the last thing we
2534  // do, so it's unnecessary to update matching state at that point).
2535  // Neither of these can occur currently because we only install this
2536  // update listener during matching a complex patterns.
2537  if (!E || E->isMachineOpcode())
2538  return;
2539  // Performing linear search here does not matter because we almost never
2540  // run this code. You'd have to have a CSE during complex pattern
2541  // matching.
2542  for (auto &I : RecordedNodes)
2543  if (I.first.getNode() == N)
2544  I.first.setNode(E);
2545 
2546  for (auto &I : MatchScopes)
2547  for (auto &J : I.NodeStack)
2548  if (J.getNode() == N)
2549  J.setNode(E);
2550  }
2551 };
2552 }
2553 
2555 SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
2556  unsigned TableSize) {
2557  // FIXME: Should these even be selected? Handle these cases in the caller?
2558  switch (NodeToMatch->getOpcode()) {
2559  default:
2560  break;
2561  case ISD::EntryToken: // These nodes remain the same.
2562  case ISD::BasicBlock:
2563  case ISD::Register:
2564  case ISD::RegisterMask:
2565  case ISD::HANDLENODE:
2566  case ISD::MDNODE_SDNODE:
2567  case ISD::TargetConstant:
2568  case ISD::TargetConstantFP:
2570  case ISD::TargetFrameIndex:
2572  case ISD::MCSymbol:
2574  case ISD::TargetJumpTable:
2577  case ISD::TokenFactor:
2578  case ISD::CopyFromReg:
2579  case ISD::CopyToReg:
2580  case ISD::EH_LABEL:
2581  case ISD::LIFETIME_START:
2582  case ISD::LIFETIME_END:
2583  NodeToMatch->setNodeId(-1); // Mark selected.
2584  return nullptr;
2585  case ISD::AssertSext:
2586  case ISD::AssertZext:
2587  CurDAG->ReplaceAllUsesOfValueWith(SDValue(NodeToMatch, 0),
2588  NodeToMatch->getOperand(0));
2589  return nullptr;
2590  case ISD::INLINEASM: return Select_INLINEASM(NodeToMatch);
2591  case ISD::READ_REGISTER: return Select_READ_REGISTER(NodeToMatch);
2592  case ISD::WRITE_REGISTER: return Select_WRITE_REGISTER(NodeToMatch);
2593  case ISD::UNDEF: return Select_UNDEF(NodeToMatch);
2594  }
2595 
2596  assert(!NodeToMatch->isMachineOpcode() && "Node already selected!");
2597 
2598  // Set up the node stack with NodeToMatch as the only node on the stack.
2599  SmallVector<SDValue, 8> NodeStack;
2600  SDValue N = SDValue(NodeToMatch, 0);
2601  NodeStack.push_back(N);
2602 
2603  // MatchScopes - Scopes used when matching, if a match failure happens, this
2604  // indicates where to continue checking.
2605  SmallVector<MatchScope, 8> MatchScopes;
2606 
2607  // RecordedNodes - This is the set of nodes that have been recorded by the
2608  // state machine. The second value is the parent of the node, or null if the
2609  // root is recorded.
2610  SmallVector<std::pair<SDValue, SDNode*>, 8> RecordedNodes;
2611 
2612  // MatchedMemRefs - This is the set of MemRef's we've seen in the input
2613  // pattern.
2614  SmallVector<MachineMemOperand*, 2> MatchedMemRefs;
2615 
2616  // These are the current input chain and glue for use when generating nodes.
2617  // Various Emit operations change these. For example, emitting a copytoreg
2618  // uses and updates these.
2619  SDValue InputChain, InputGlue;
2620 
2621  // ChainNodesMatched - If a pattern matches nodes that have input/output
2622  // chains, the OPC_EmitMergeInputChains operation is emitted which indicates
2623  // which ones they are. The result is captured into this list so that we can
2624  // update the chain results when the pattern is complete.
2625  SmallVector<SDNode*, 3> ChainNodesMatched;
2626  SmallVector<SDNode*, 3> GlueResultNodesMatched;
2627 
2628  DEBUG(dbgs() << "ISEL: Starting pattern match on root node: ";
2629  NodeToMatch->dump(CurDAG);
2630  dbgs() << '\n');
2631 
2632  // Determine where to start the interpreter. Normally we start at opcode #0,
2633  // but if the state machine starts with an OPC_SwitchOpcode, then we
2634  // accelerate the first lookup (which is guaranteed to be hot) with the
2635  // OpcodeOffset table.
2636  unsigned MatcherIndex = 0;
2637 
2638  if (!OpcodeOffset.empty()) {
2639  // Already computed the OpcodeOffset table, just index into it.
2640  if (N.getOpcode() < OpcodeOffset.size())
2641  MatcherIndex = OpcodeOffset[N.getOpcode()];
2642  DEBUG(dbgs() << " Initial Opcode index to " << MatcherIndex << "\n");
2643 
2644  } else if (MatcherTable[0] == OPC_SwitchOpcode) {
2645  // Otherwise, the table isn't computed, but the state machine does start
2646  // with an OPC_SwitchOpcode instruction. Populate the table now, since this
2647  // is the first time we're selecting an instruction.
2648  unsigned Idx = 1;
2649  while (1) {
2650  // Get the size of this case.
2651  unsigned CaseSize = MatcherTable[Idx++];
2652  if (CaseSize & 128)
2653  CaseSize = GetVBR(CaseSize, MatcherTable, Idx);
2654  if (CaseSize == 0) break;
2655 
2656  // Get the opcode, add the index to the table.
2657  uint16_t Opc = MatcherTable[Idx++];
2658  Opc |= (unsigned short)MatcherTable[Idx++] << 8;
2659  if (Opc >= OpcodeOffset.size())
2660  OpcodeOffset.resize((Opc+1)*2);
2661  OpcodeOffset[Opc] = Idx;
2662  Idx += CaseSize;
2663  }
2664 
2665  // Okay, do the lookup for the first opcode.
2666  if (N.getOpcode() < OpcodeOffset.size())
2667  MatcherIndex = OpcodeOffset[N.getOpcode()];
2668  }
2669 
2670  while (1) {
2671  assert(MatcherIndex < TableSize && "Invalid index");
2672 #ifndef NDEBUG
2673  unsigned CurrentOpcodeIndex = MatcherIndex;
2674 #endif
2675  BuiltinOpcodes Opcode = (BuiltinOpcodes)MatcherTable[MatcherIndex++];
2676  switch (Opcode) {
2677  case OPC_Scope: {
2678  // Okay, the semantics of this operation are that we should push a scope
2679  // then evaluate the first child. However, pushing a scope only to have
2680  // the first check fail (which then pops it) is inefficient. If we can
2681  // determine immediately that the first check (or first several) will
2682  // immediately fail, don't even bother pushing a scope for them.
2683  unsigned FailIndex;
2684 
2685  while (1) {
2686  unsigned NumToSkip = MatcherTable[MatcherIndex++];
2687  if (NumToSkip & 128)
2688  NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
2689  // Found the end of the scope with no match.
2690  if (NumToSkip == 0) {
2691  FailIndex = 0;
2692  break;
2693  }
2694 
2695  FailIndex = MatcherIndex+NumToSkip;
2696 
2697  unsigned MatcherIndexOfPredicate = MatcherIndex;
2698  (void)MatcherIndexOfPredicate; // silence warning.
2699 
2700  // If we can't evaluate this predicate without pushing a scope (e.g. if
2701  // it is a 'MoveParent') or if the predicate succeeds on this node, we
2702  // push the scope and evaluate the full predicate chain.
2703  bool Result;
2704  MatcherIndex = IsPredicateKnownToFail(MatcherTable, MatcherIndex, N,
2705  Result, *this, RecordedNodes);
2706  if (!Result)
2707  break;
2708 
2709  DEBUG(dbgs() << " Skipped scope entry (due to false predicate) at "
2710  << "index " << MatcherIndexOfPredicate
2711  << ", continuing at " << FailIndex << "\n");
2712  ++NumDAGIselRetries;
2713 
2714  // Otherwise, we know that this case of the Scope is guaranteed to fail,
2715  // move to the next case.
2716  MatcherIndex = FailIndex;
2717  }
2718 
2719  // If the whole scope failed to match, bail.
2720  if (FailIndex == 0) break;
2721 
2722  // Push a MatchScope which indicates where to go if the first child fails
2723  // to match.
2724  MatchScope NewEntry;
2725  NewEntry.FailIndex = FailIndex;
2726  NewEntry.NodeStack.append(NodeStack.begin(), NodeStack.end());
2727  NewEntry.NumRecordedNodes = RecordedNodes.size();
2728  NewEntry.NumMatchedMemRefs = MatchedMemRefs.size();
2729  NewEntry.InputChain = InputChain;
2730  NewEntry.InputGlue = InputGlue;
2731  NewEntry.HasChainNodesMatched = !ChainNodesMatched.empty();
2732  NewEntry.HasGlueResultNodesMatched = !GlueResultNodesMatched.empty();
2733  MatchScopes.push_back(NewEntry);
2734  continue;
2735  }
2736  case OPC_RecordNode: {
2737  // Remember this node, it may end up being an operand in the pattern.
2738  SDNode *Parent = nullptr;
2739  if (NodeStack.size() > 1)
2740  Parent = NodeStack[NodeStack.size()-2].getNode();
2741  RecordedNodes.push_back(std::make_pair(N, Parent));
2742  continue;
2743  }
2744 
2748  case OPC_RecordChild6: case OPC_RecordChild7: {
2749  unsigned ChildNo = Opcode-OPC_RecordChild0;
2750  if (ChildNo >= N.getNumOperands())
2751  break; // Match fails if out of range child #.
2752 
2753  RecordedNodes.push_back(std::make_pair(N->getOperand(ChildNo),
2754  N.getNode()));
2755  continue;
2756  }
2757  case OPC_RecordMemRef:
2758  MatchedMemRefs.push_back(cast<MemSDNode>(N)->getMemOperand());
2759  continue;
2760 
2761  case OPC_CaptureGlueInput:
2762  // If the current node has an input glue, capture it in InputGlue.
2763  if (N->getNumOperands() != 0 &&
2764  N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue)
2765  InputGlue = N->getOperand(N->getNumOperands()-1);
2766  continue;
2767 
2768  case OPC_MoveChild: {
2769  unsigned ChildNo = MatcherTable[MatcherIndex++];
2770  if (ChildNo >= N.getNumOperands())
2771  break; // Match fails if out of range child #.
2772  N = N.getOperand(ChildNo);
2773  NodeStack.push_back(N);
2774  continue;
2775  }
2776 
2777  case OPC_MoveParent:
2778  // Pop the current node off the NodeStack.
2779  NodeStack.pop_back();
2780  assert(!NodeStack.empty() && "Node stack imbalance!");
2781  N = NodeStack.back();
2782  continue;
2783 
2784  case OPC_CheckSame:
2785  if (!::CheckSame(MatcherTable, MatcherIndex, N, RecordedNodes)) break;
2786  continue;
2787 
2790  if (!::CheckChildSame(MatcherTable, MatcherIndex, N, RecordedNodes,
2791  Opcode-OPC_CheckChild0Same))
2792  break;
2793  continue;
2794 
2796  if (!::CheckPatternPredicate(MatcherTable, MatcherIndex, *this)) break;
2797  continue;
2798  case OPC_CheckPredicate:
2799  if (!::CheckNodePredicate(MatcherTable, MatcherIndex, *this,
2800  N.getNode()))
2801  break;
2802  continue;
2803  case OPC_CheckComplexPat: {
2804  unsigned CPNum = MatcherTable[MatcherIndex++];
2805  unsigned RecNo = MatcherTable[MatcherIndex++];
2806  assert(RecNo < RecordedNodes.size() && "Invalid CheckComplexPat");
2807 
2808  // If target can modify DAG during matching, keep the matching state
2809  // consistent.
2810  std::unique_ptr<MatchStateUpdater> MSU;
2812  MSU.reset(new MatchStateUpdater(*CurDAG, RecordedNodes,
2813  MatchScopes));
2814 
2815  if (!CheckComplexPattern(NodeToMatch, RecordedNodes[RecNo].second,
2816  RecordedNodes[RecNo].first, CPNum,
2817  RecordedNodes))
2818  break;
2819  continue;
2820  }
2821  case OPC_CheckOpcode:
2822  if (!::CheckOpcode(MatcherTable, MatcherIndex, N.getNode())) break;
2823  continue;
2824 
2825  case OPC_CheckType:
2826  if (!::CheckType(MatcherTable, MatcherIndex, N, TLI,
2827  CurDAG->getDataLayout()))
2828  break;
2829  continue;
2830 
2831  case OPC_SwitchOpcode: {
2832  unsigned CurNodeOpcode = N.getOpcode();
2833  unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
2834  unsigned CaseSize;
2835  while (1) {
2836  // Get the size of this case.
2837  CaseSize = MatcherTable[MatcherIndex++];
2838  if (CaseSize & 128)
2839  CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
2840  if (CaseSize == 0) break;
2841 
2842  uint16_t Opc = MatcherTable[MatcherIndex++];
2843  Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
2844 
2845  // If the opcode matches, then we will execute this case.
2846  if (CurNodeOpcode == Opc)
2847  break;
2848 
2849  // Otherwise, skip over this case.
2850  MatcherIndex += CaseSize;
2851  }
2852 
2853  // If no cases matched, bail out.
2854  if (CaseSize == 0) break;
2855 
2856  // Otherwise, execute the case we found.
2857  DEBUG(dbgs() << " OpcodeSwitch from " << SwitchStart
2858  << " to " << MatcherIndex << "\n");
2859  continue;
2860  }
2861 
2862  case OPC_SwitchType: {
2863  MVT CurNodeVT = N.getSimpleValueType();
2864  unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
2865  unsigned CaseSize;
2866  while (1) {
2867  // Get the size of this case.
2868  CaseSize = MatcherTable[MatcherIndex++];
2869  if (CaseSize & 128)
2870  CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
2871  if (CaseSize == 0) break;
2872 
2873  MVT CaseVT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2874  if (CaseVT == MVT::iPTR)
2875  CaseVT = TLI->getPointerTy(CurDAG->getDataLayout());
2876 
2877  // If the VT matches, then we will execute this case.
2878  if (CurNodeVT == CaseVT)
2879  break;
2880 
2881  // Otherwise, skip over this case.
2882  MatcherIndex += CaseSize;
2883  }
2884 
2885  // If no cases matched, bail out.
2886  if (CaseSize == 0) break;
2887 
2888  // Otherwise, execute the case we found.
2889  DEBUG(dbgs() << " TypeSwitch[" << EVT(CurNodeVT).getEVTString()
2890  << "] from " << SwitchStart << " to " << MatcherIndex<<'\n');
2891  continue;
2892  }
2897  if (!::CheckChildType(MatcherTable, MatcherIndex, N, TLI,
2898  CurDAG->getDataLayout(),
2899  Opcode - OPC_CheckChild0Type))
2900  break;
2901  continue;
2902  case OPC_CheckCondCode:
2903  if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break;
2904  continue;
2905  case OPC_CheckValueType:
2906  if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI,
2907  CurDAG->getDataLayout()))
2908  break;
2909  continue;
2910  case OPC_CheckInteger:
2911  if (!::CheckInteger(MatcherTable, MatcherIndex, N)) break;
2912  continue;
2916  if (!::CheckChildInteger(MatcherTable, MatcherIndex, N,
2917  Opcode-OPC_CheckChild0Integer)) break;
2918  continue;
2919  case OPC_CheckAndImm:
2920  if (!::CheckAndImm(MatcherTable, MatcherIndex, N, *this)) break;
2921  continue;
2922  case OPC_CheckOrImm:
2923  if (!::CheckOrImm(MatcherTable, MatcherIndex, N, *this)) break;
2924  continue;
2925 
2927  assert(NodeStack.size() != 1 && "No parent node");
2928  // Verify that all intermediate nodes between the root and this one have
2929  // a single use.
2930  bool HasMultipleUses = false;
2931  for (unsigned i = 1, e = NodeStack.size()-1; i != e; ++i)
2932  if (!NodeStack[i].hasOneUse()) {
2933  HasMultipleUses = true;
2934  break;
2935  }
2936  if (HasMultipleUses) break;
2937 
2938  // Check to see that the target thinks this is profitable to fold and that
2939  // we can fold it without inducing cycles in the graph.
2940  if (!IsProfitableToFold(N, NodeStack[NodeStack.size()-2].getNode(),
2941  NodeToMatch) ||
2942  !IsLegalToFold(N, NodeStack[NodeStack.size()-2].getNode(),
2943  NodeToMatch, OptLevel,
2944  true/*We validate our own chains*/))
2945  break;
2946 
2947  continue;
2948  }
2949  case OPC_EmitInteger: {
2951  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2952  int64_t Val = MatcherTable[MatcherIndex++];
2953  if (Val & 128)
2954  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2955  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
2956  CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch),
2957  VT), nullptr));
2958  continue;
2959  }
2960  case OPC_EmitRegister: {
2962  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2963  unsigned RegNo = MatcherTable[MatcherIndex++];
2964  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
2965  CurDAG->getRegister(RegNo, VT), nullptr));
2966  continue;
2967  }
2968  case OPC_EmitRegister2: {
2969  // For targets w/ more than 256 register names, the register enum
2970  // values are stored in two bytes in the matcher table (just like
2971  // opcodes).
2973  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2974  unsigned RegNo = MatcherTable[MatcherIndex++];
2975  RegNo |= MatcherTable[MatcherIndex++] << 8;
2976  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
2977  CurDAG->getRegister(RegNo, VT), nullptr));
2978  continue;
2979  }
2980 
2981  case OPC_EmitConvertToTarget: {
2982  // Convert from IMM/FPIMM to target version.
2983  unsigned RecNo = MatcherTable[MatcherIndex++];
2984  assert(RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget");
2985  SDValue Imm = RecordedNodes[RecNo].first;
2986 
2987  if (Imm->getOpcode() == ISD::Constant) {
2988  const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();
2989  Imm = CurDAG->getConstant(*Val, SDLoc(NodeToMatch), Imm.getValueType(),
2990  true);
2991  } else if (Imm->getOpcode() == ISD::ConstantFP) {
2992  const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();
2993  Imm = CurDAG->getConstantFP(*Val, SDLoc(NodeToMatch),
2994  Imm.getValueType(), true);
2995  }
2996 
2997  RecordedNodes.push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
2998  continue;
2999  }
3000 
3001  case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0
3002  case OPC_EmitMergeInputChains1_1: { // OPC_EmitMergeInputChains, 1, 1
3003  // These are space-optimized forms of OPC_EmitMergeInputChains.
3004  assert(!InputChain.getNode() &&
3005  "EmitMergeInputChains should be the first chain producing node");
3006  assert(ChainNodesMatched.empty() &&
3007  "Should only have one EmitMergeInputChains per match");
3008 
3009  // Read all of the chained nodes.
3010  unsigned RecNo = Opcode == OPC_EmitMergeInputChains1_1;
3011  assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains");
3012  ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3013 
3014  // FIXME: What if other value results of the node have uses not matched
3015  // by this pattern?
3016  if (ChainNodesMatched.back() != NodeToMatch &&
3017  !RecordedNodes[RecNo].first.hasOneUse()) {
3018  ChainNodesMatched.clear();
3019  break;
3020  }
3021 
3022  // Merge the input chains if they are not intra-pattern references.
3023  InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3024 
3025  if (!InputChain.getNode())
3026  break; // Failed to merge.
3027  continue;
3028  }
3029 
3030  case OPC_EmitMergeInputChains: {
3031  assert(!InputChain.getNode() &&
3032  "EmitMergeInputChains should be the first chain producing node");
3033  // This node gets a list of nodes we matched in the input that have
3034  // chains. We want to token factor all of the input chains to these nodes
3035  // together. However, if any of the input chains is actually one of the
3036  // nodes matched in this pattern, then we have an intra-match reference.
3037  // Ignore these because the newly token factored chain should not refer to
3038  // the old nodes.
3039  unsigned NumChains = MatcherTable[MatcherIndex++];
3040  assert(NumChains != 0 && "Can't TF zero chains");
3041 
3042  assert(ChainNodesMatched.empty() &&
3043  "Should only have one EmitMergeInputChains per match");
3044 
3045  // Read all of the chained nodes.
3046  for (unsigned i = 0; i != NumChains; ++i) {
3047  unsigned RecNo = MatcherTable[MatcherIndex++];
3048  assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains");
3049  ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3050 
3051  // FIXME: What if other value results of the node have uses not matched
3052  // by this pattern?
3053  if (ChainNodesMatched.back() != NodeToMatch &&
3054  !RecordedNodes[RecNo].first.hasOneUse()) {
3055  ChainNodesMatched.clear();
3056  break;
3057  }
3058  }
3059 
3060  // If the inner loop broke out, the match fails.
3061  if (ChainNodesMatched.empty())
3062  break;
3063 
3064  // Merge the input chains if they are not intra-pattern references.
3065  InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3066 
3067  if (!InputChain.getNode())
3068  break; // Failed to merge.
3069 
3070  continue;
3071  }
3072 
3073  case OPC_EmitCopyToReg: {
3074  unsigned RecNo = MatcherTable[MatcherIndex++];
3075  assert(RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg");
3076  unsigned DestPhysReg = MatcherTable[MatcherIndex++];
3077 
3078  if (!InputChain.getNode())
3079  InputChain = CurDAG->getEntryNode();
3080 
3081  InputChain = CurDAG->getCopyToReg(InputChain, SDLoc(NodeToMatch),
3082  DestPhysReg, RecordedNodes[RecNo].first,
3083  InputGlue);
3084 
3085  InputGlue = InputChain.getValue(1);
3086  continue;
3087  }
3088 
3089  case OPC_EmitNodeXForm: {
3090  unsigned XFormNo = MatcherTable[MatcherIndex++];
3091  unsigned RecNo = MatcherTable[MatcherIndex++];
3092  assert(RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm");
3093  SDValue Res = RunSDNodeXForm(RecordedNodes[RecNo].first, XFormNo);
3094  RecordedNodes.push_back(std::pair<SDValue,SDNode*>(Res, nullptr));
3095  continue;
3096  }
3097 
3098  case OPC_EmitNode:
3099  case OPC_MorphNodeTo: {
3100  uint16_t TargetOpc = MatcherTable[MatcherIndex++];
3101  TargetOpc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3102  unsigned EmitNodeInfo = MatcherTable[MatcherIndex++];
3103  // Get the result VT list.
3104  unsigned NumVTs = MatcherTable[MatcherIndex++];
3105  SmallVector<EVT, 4> VTs;
3106  for (unsigned i = 0; i != NumVTs; ++i) {
3108  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3109  if (VT == MVT::iPTR)
3110  VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy;
3111  VTs.push_back(VT);
3112  }
3113 
3114  if (EmitNodeInfo & OPFL_Chain)
3115  VTs.push_back(MVT::Other);
3116  if (EmitNodeInfo & OPFL_GlueOutput)
3117  VTs.push_back(MVT::Glue);
3118 
3119  // This is hot code, so optimize the two most common cases of 1 and 2
3120  // results.
3121  SDVTList VTList;
3122  if (VTs.size() == 1)
3123  VTList = CurDAG->getVTList(VTs[0]);
3124  else if (VTs.size() == 2)
3125  VTList = CurDAG->getVTList(VTs[0], VTs[1]);
3126  else
3127  VTList = CurDAG->getVTList(VTs);
3128 
3129  // Get the operand list.
3130  unsigned NumOps = MatcherTable[MatcherIndex++];
3132  for (unsigned i = 0; i != NumOps; ++i) {
3133  unsigned RecNo = MatcherTable[MatcherIndex++];
3134  if (RecNo & 128)
3135  RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex);
3136 
3137  assert(RecNo < RecordedNodes.size() && "Invalid EmitNode");
3138  Ops.push_back(RecordedNodes[RecNo].first);
3139  }
3140 
3141  // If there are variadic operands to add, handle them now.
3142  if (EmitNodeInfo & OPFL_VariadicInfo) {
3143  // Determine the start index to copy from.
3144  unsigned FirstOpToCopy = getNumFixedFromVariadicInfo(EmitNodeInfo);
3145  FirstOpToCopy += (EmitNodeInfo & OPFL_Chain) ? 1 : 0;
3146  assert(NodeToMatch->getNumOperands() >= FirstOpToCopy &&
3147  "Invalid variadic node");
3148  // Copy all of the variadic operands, not including a potential glue
3149  // input.
3150  for (unsigned i = FirstOpToCopy, e = NodeToMatch->getNumOperands();
3151  i != e; ++i) {
3152  SDValue V = NodeToMatch->getOperand(i);
3153  if (V.getValueType() == MVT::Glue) break;
3154  Ops.push_back(V);
3155  }
3156  }
3157 
3158  // If this has chain/glue inputs, add them.
3159  if (EmitNodeInfo & OPFL_Chain)
3160  Ops.push_back(InputChain);
3161  if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr)
3162  Ops.push_back(InputGlue);
3163 
3164  // Create the node.
3165  SDNode *Res = nullptr;
3166  if (Opcode != OPC_MorphNodeTo) {
3167  // If this is a normal EmitNode command, just create the new node and
3168  // add the results to the RecordedNodes list.
3169  Res = CurDAG->getMachineNode(TargetOpc, SDLoc(NodeToMatch),
3170  VTList, Ops);
3171 
3172  // Add all the non-glue/non-chain results to the RecordedNodes list.
3173  for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
3174  if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue) break;
3175  RecordedNodes.push_back(std::pair<SDValue,SDNode*>(SDValue(Res, i),
3176  nullptr));
3177  }
3178 
3179  } else if (NodeToMatch->getOpcode() != ISD::DELETED_NODE) {
3180  Res = MorphNode(NodeToMatch, TargetOpc, VTList, Ops, EmitNodeInfo);
3181  } else {
3182  // NodeToMatch was eliminated by CSE when the target changed the DAG.
3183  // We will visit the equivalent node later.
3184  DEBUG(dbgs() << "Node was eliminated by CSE\n");
3185  return nullptr;
3186  }
3187 
3188  // If the node had chain/glue results, update our notion of the current
3189  // chain and glue.
3190  if (EmitNodeInfo & OPFL_GlueOutput) {
3191  InputGlue = SDValue(Res, VTs.size()-1);
3192  if (EmitNodeInfo & OPFL_Chain)
3193  InputChain = SDValue(Res, VTs.size()-2);
3194  } else if (EmitNodeInfo & OPFL_Chain)
3195  InputChain = SDValue(Res, VTs.size()-1);
3196 
3197  // If the OPFL_MemRefs glue is set on this node, slap all of the
3198  // accumulated memrefs onto it.
3199  //
3200  // FIXME: This is vastly incorrect for patterns with multiple outputs
3201  // instructions that access memory and for ComplexPatterns that match
3202  // loads.
3203  if (EmitNodeInfo & OPFL_MemRefs) {
3204  // Only attach load or store memory operands if the generated
3205  // instruction may load or store.
3206  const MCInstrDesc &MCID = TII->get(TargetOpc);
3207  bool mayLoad = MCID.mayLoad();
3208  bool mayStore = MCID.mayStore();
3209 
3210  unsigned NumMemRefs = 0;
3212  MatchedMemRefs.begin(), E = MatchedMemRefs.end(); I != E; ++I) {
3213  if ((*I)->isLoad()) {
3214  if (mayLoad)
3215  ++NumMemRefs;
3216  } else if ((*I)->isStore()) {
3217  if (mayStore)
3218  ++NumMemRefs;
3219  } else {
3220  ++NumMemRefs;
3221  }
3222  }
3223 
3224  MachineSDNode::mmo_iterator MemRefs =
3225  MF->allocateMemRefsArray(NumMemRefs);
3226 
3227  MachineSDNode::mmo_iterator MemRefsPos = MemRefs;
3229  MatchedMemRefs.begin(), E = MatchedMemRefs.end(); I != E; ++I) {
3230  if ((*I)->isLoad()) {
3231  if (mayLoad)
3232  *MemRefsPos++ = *I;
3233  } else if ((*I)->isStore()) {
3234  if (mayStore)
3235  *MemRefsPos++ = *I;
3236  } else {
3237  *MemRefsPos++ = *I;
3238  }
3239  }
3240 
3241  cast<MachineSDNode>(Res)
3242  ->setMemRefs(MemRefs, MemRefs + NumMemRefs);
3243  }
3244 
3245  DEBUG(dbgs() << " "
3246  << (Opcode == OPC_MorphNodeTo ? "Morphed" : "Created")
3247  << " node: "; Res->dump(CurDAG); dbgs() << "\n");
3248 
3249  // If this was a MorphNodeTo then we're completely done!
3250  if (Opcode == OPC_MorphNodeTo) {
3251  // Update chain and glue uses.
3252  UpdateChainsAndGlue(NodeToMatch, InputChain, ChainNodesMatched,
3253  InputGlue, GlueResultNodesMatched, true);
3254  return Res;
3255  }
3256 
3257  continue;
3258  }
3259 
3260  case OPC_MarkGlueResults: {
3261  unsigned NumNodes = MatcherTable[MatcherIndex++];
3262 
3263  // Read and remember all the glue-result nodes.
3264  for (unsigned i = 0; i != NumNodes; ++i) {
3265  unsigned RecNo = MatcherTable[MatcherIndex++];
3266  if (RecNo & 128)
3267  RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex);
3268 
3269  assert(RecNo < RecordedNodes.size() && "Invalid MarkGlueResults");
3270  GlueResultNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3271  }
3272  continue;
3273  }
3274 
3275  case OPC_CompleteMatch: {
3276  // The match has been completed, and any new nodes (if any) have been
3277  // created. Patch up references to the matched dag to use the newly
3278  // created nodes.
3279  unsigned NumResults = MatcherTable[MatcherIndex++];
3280 
3281  for (unsigned i = 0; i != NumResults; ++i) {
3282  unsigned ResSlot = MatcherTable[MatcherIndex++];
3283  if (ResSlot & 128)
3284  ResSlot = GetVBR(ResSlot, MatcherTable, MatcherIndex);
3285 
3286  assert(ResSlot < RecordedNodes.size() && "Invalid CompleteMatch");
3287  SDValue Res = RecordedNodes[ResSlot].first;
3288 
3289  assert(i < NodeToMatch->getNumValues() &&
3290  NodeToMatch->getValueType(i) != MVT::Other &&
3291  NodeToMatch->getValueType(i) != MVT::Glue &&
3292  "Invalid number of results to complete!");
3293  assert((NodeToMatch->getValueType(i) == Res.getValueType() ||
3294  NodeToMatch->getValueType(i) == MVT::iPTR ||
3295  Res.getValueType() == MVT::iPTR ||
3296  NodeToMatch->getValueType(i).getSizeInBits() ==
3297  Res.getValueType().getSizeInBits()) &&
3298  "invalid replacement");
3299  CurDAG->ReplaceAllUsesOfValueWith(SDValue(NodeToMatch, i), Res);
3300  }
3301 
3302  // If the root node defines glue, add it to the glue nodes to update list.
3303  if (NodeToMatch->getValueType(NodeToMatch->getNumValues()-1) == MVT::Glue)
3304  GlueResultNodesMatched.push_back(NodeToMatch);
3305 
3306  // Update chain and glue uses.
3307  UpdateChainsAndGlue(NodeToMatch, InputChain, ChainNodesMatched,
3308  InputGlue, GlueResultNodesMatched, false);
3309 
3310  assert(NodeToMatch->use_empty() &&
3311  "Didn't replace all uses of the node?");
3312 
3313  // FIXME: We just return here, which interacts correctly with SelectRoot
3314  // above. We should fix this to not return an SDNode* anymore.
3315  return nullptr;
3316  }
3317  }
3318 
3319  // If the code reached this point, then the match failed. See if there is
3320  // another child to try in the current 'Scope', otherwise pop it until we
3321  // find a case to check.
3322  DEBUG(dbgs() << " Match failed at index " << CurrentOpcodeIndex << "\n");
3323  ++NumDAGIselRetries;
3324  while (1) {
3325  if (MatchScopes.empty()) {
3326  CannotYetSelect(NodeToMatch);
3327  return nullptr;
3328  }
3329 
3330  // Restore the interpreter state back to the point where the scope was
3331  // formed.
3332  MatchScope &LastScope = MatchScopes.back();
3333  RecordedNodes.resize(LastScope.NumRecordedNodes);
3334  NodeStack.clear();
3335  NodeStack.append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());
3336  N = NodeStack.back();
3337 
3338  if (LastScope.NumMatchedMemRefs != MatchedMemRefs.size())
3339  MatchedMemRefs.resize(LastScope.NumMatchedMemRefs);
3340  MatcherIndex = LastScope.FailIndex;
3341 
3342  DEBUG(dbgs() << " Continuing at " << MatcherIndex << "\n");
3343 
3344  InputChain = LastScope.InputChain;
3345  InputGlue = LastScope.InputGlue;
3346  if (!LastScope.HasChainNodesMatched)
3347  ChainNodesMatched.clear();
3348  if (!LastScope.HasGlueResultNodesMatched)
3349  GlueResultNodesMatched.clear();
3350 
3351  // Check to see what the offset is at the new MatcherIndex. If it is zero
3352  // we have reached the end of this scope, otherwise we have another child
3353  // in the current scope to try.
3354  unsigned NumToSkip = MatcherTable[MatcherIndex++];
3355  if (NumToSkip & 128)
3356  NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3357 
3358  // If we have another child in this scope to match, update FailIndex and
3359  // try it.
3360  if (NumToSkip != 0) {
3361  LastScope.FailIndex = MatcherIndex+NumToSkip;
3362  break;
3363  }
3364 
3365  // End of this scope, pop it and try the next child in the containing
3366  // scope.
3367  MatchScopes.pop_back();
3368  }
3369  }
3370 }
3371 
3372 
3373 
3374 void SelectionDAGISel::CannotYetSelect(SDNode *N) {
3375  std::string msg;
3376  raw_string_ostream Msg(msg);
3377  Msg << "Cannot select: ";
3378 
3379  if (N->getOpcode() != ISD::INTRINSIC_W_CHAIN &&
3381  N->getOpcode() != ISD::INTRINSIC_VOID) {
3382  N->printrFull(Msg, CurDAG);
3383  Msg << "\nIn function: " << MF->getName();
3384  } else {
3385  bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other;
3386  unsigned iid =
3387  cast<ConstantSDNode>(N->getOperand(HasInputChain))->getZExtValue();
3388  if (iid < Intrinsic::num_intrinsics)
3389  Msg << "intrinsic %" << Intrinsic::getName((Intrinsic::ID)iid);
3390  else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo())
3391  Msg << "target intrinsic %" << TII->getName(iid);
3392  else
3393  Msg << "unknown intrinsic #" << iid;
3394  }
3395  report_fatal_error(Msg.str());
3396 }
3397 
3398 char SelectionDAGISel::ID = 0;
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
bool use_empty() const
Return true if there are no uses of this node.
std::vector< BitTestBlock > BitTestCases
BitTestCases - Vector of BitTestBlock structures used to communicate SwitchInst code generation infor...
SelectionDAGBuilder * SDB
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:104
mop_iterator operands_end()
Definition: MachineInstr.h:290
SDValue getValue(unsigned R) const
bool hasPostISelHook(QueryType Type=IgnoreBundle) const
Return true if this instruction requires adjustment after instruction selection by calling a target h...
Definition: MachineInstr.h:645
static SDNode * findGlueUse(SDNode *N)
findGlueUse - Return use of MVT::Glue value produced by the specified SDNode.
void EmitLiveInCopies(MachineBasicBlock *EntryMBB, const TargetRegisterInfo &TRI, const TargetInstrInfo &TII)
EmitLiveInCopies - Emit copies to initialize livein virtual registers into the given entry block...
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
virtual bool enableMachineScheduler() const
True if the subtarget should run MachineScheduler after aggressive coalescing.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
static int getNumFixedFromVariadicInfo(unsigned Flags)
getNumFixedFromVariadicInfo - Transform an EmitNode flags word into the number of fixed arity values ...
bool LegalizeTypes()
This transforms the SelectionDAG into a SelectionDAG that only uses types natively supported by the t...
SDValue getCopyToReg(SDValue Chain, SDLoc dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:522
GCFunctionInfo * GFI
void dump() const
Dump this node, for debugging.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:42
STATISTIC(NumFunctions,"Total number of functions")
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR...
Definition: ISDOpcodes.h:615
static unsigned virtReg2Index(unsigned Reg)
virtReg2Index - Convert a virtual register number to a 0-based index.
livein_iterator livein_end() const
Various leaf nodes.
Definition: ISDOpcodes.h:60
static cl::opt< bool > ViewISelDAGs("view-isel-dags", cl::Hidden, cl::desc("Pop up a window to show isel dags as they are selected"))
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:39
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
int getNumber() const
getNumber - MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a M...
void setIsLandingPad(bool V=true)
setIsLandingPad - Indicates the block is a landing pad.
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
static MachineBasicBlock::iterator FindSplitPointForStackProtector(MachineBasicBlock *BB, DebugLoc DL)
Find the split point at which to splice the end of BB into its success stack protector check machine ...
iterator end()
Definition: Function.h:459
Intrinsic::ID getIntrinsicID() const
getIntrinsicID - Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:44
void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef< unsigned > Sites)
setCallSiteLandingPad - Map the landing pad's EH symbol to the call site indexes. ...
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
const DIExpression * getDebugExpression() const
Return the complex address expression referenced by this DBG_VALUE instruction.
Definition: MachineInstr.h:249
ScheduleDAGSDNodes *(* FunctionPassCtor)(SelectionDAGISel *, CodeGenOpt::Level)
bool isReturn() const
Return true if the instruction is a return.
Definition: MCInstrDesc.h:211
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
bool mayStore() const
Return true if this instruction could possibly modify memory.
Definition: MCInstrDesc.h:356
iterator getFirstTerminator()
getFirstTerminator - returns an iterator to the first terminator instruction of this basic block...
static unsigned getFlagWord(unsigned Kind, unsigned NumOps)
Definition: InlineAsm.h:268
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:138
void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &)
Clients of various APIs that cause global effects on the DAG can optionally implement this interface...
Definition: SelectionDAG.h:225
static bool isVirtualRegister(unsigned Reg)
isVirtualRegister - Return true if the specified register number is in the virtual register namespace...
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in the KnownZero/KnownO...
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
void addLiveIn(unsigned Reg)
Adds the specified register as a live in.
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel)
CheckPatternPredicate - Implements OP_CheckPatternPredicate.
void visitJumpTableHeader(JumpTable &JT, JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
Definition: SelectionDAG.h:253
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:122
std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:188
unsigned getNumOperands() const
Return the number of values used by this operation.
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:111
unsigned getNumOperands() const
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
A debug info location.
Definition: DebugLoc.h:34
Metadata node.
Definition: Metadata.h:740
const SDValue & getOperand(unsigned Num) const
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
unsigned GetSuccessorNumber(BasicBlock *BB, BasicBlock *Succ)
Search for the specified successor of basic block BB and return its position in the terminator instru...
Definition: CFG.cpp:72
RegisterPassParser class - Handle the addition of new machine passes.
void setNodeId(int Id)
Set unique node id.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:344
void viewGraph(const std::string &Title)
Pop up a GraphViz/gv window with the DAG rendered using 'dot'.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL, unsigned ChildNo)
DebugLoc getCurDebugLoc() const
virtual void AdjustInstrPostInstrSelection(MachineInstr *MI, SDNode *Node) const
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag...
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:45
static bool isUseOperandTiedToDef(unsigned Flag, unsigned &Idx)
isUseOperandTiedToDef - Return true if the flag of the inline asm operand indicates it is an use oper...
Definition: InlineAsm.h:338
MachineFunction * MF
const TargetLibraryInfo * LibInfo
void set(const Function &Fn, MachineFunction &MF, SelectionDAG *DAG)
set - Initialize this FunctionLoweringInfo with the given Function and its associated MachineFunction...
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const char *reason, bool gen_crash_diag=true)
Reports a serious error, calling any installed error handler.
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:150
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:188
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:242
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:231
void initializeBranchProbabilityInfoPass(PassRegistry &)
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1329
static bool isFoldedOrDeadInstruction(const Instruction *I, FunctionLoweringInfo *FuncInfo)
isFoldedOrDeadInstruction - Return true if the specified instruction is side-effect free and is eithe...
StackProtectorDescriptor SPDescriptor
A StackProtectorDescriptor structure used to communicate stack protector information in between Selec...
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:169
bool isVector() const
isVector - Return true if this is a vector value type.
Definition: ValueTypes.h:115
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
SDValue getRoot()
getRoot - Return the current virtual root of the Selection DAG, flushing any PendingLoad items...
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block...
AnalysisUsage & addRequired()
void setLastLocalValue(MachineInstr *I)
Update the position of the last instruction emitted for materializing constants for use in the curren...
Definition: FastISel.h:225
MachineMemOperand - A description of a memory reference used in the backend.
void setOptLevel(CodeGenOpt::Level Level) const
Overrides the optimization level.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
static bool findNonImmUse(SDNode *Use, SDNode *Def, SDNode *ImmedUse, SDNode *Root, SmallPtrSetImpl< SDNode * > &Visited, bool IgnoreChains)
findNonImmUse - Return true if "Use" is a non-immediate use of "Def".
bool CheckAndMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckAndMask - The isel is trying to match something like (and X, 255).
void visitSwitchCase(CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
Option class for critical edge splitting.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOpt::Level OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val()
Definition: SmallVector.h:406
NamedRegionTimer - This class is basically a combination of TimeRegion and Timer. ...
Definition: Timer.h:147
void AddLiveOutRegInfo(unsigned Reg, unsigned NumSignBits, const APInt &KnownZero, const APInt &KnownOne)
AddLiveOutRegInfo - Adds LiveOutInfo for a register.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:69
static SDValue HandleMergeInputChains(SmallVectorImpl< SDNode * > &ChainNodesMatched, SelectionDAG *CurDAG)
HandleMergeInputChains - This implements the OPC_EmitMergeInputChains operation for when the pattern ...
void visitJumpTable(JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
use_instr_iterator use_instr_begin(unsigned RegNo) const
const TargetLowering * TLI
#define LLVM_ATTRIBUTE_ALWAYS_INLINE
LLVM_ATTRIBUTE_ALWAYS_INLINE - On compilers where we have a directive to do so, mark a method "always...
Definition: Compiler.h:196
bool isCall() const
Return true if the instruction is a call.
Definition: MCInstrDesc.h:214
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
bool isReg() const
isReg - Tests if this is a MO_Register operand.
CopyToReg - This node has three operands: a chain, a register number to set to this value...
Definition: ISDOpcodes.h:161
Instruction * getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:165
static MachinePassRegistry Registry
RegisterScheduler class - Track the registration of instruction schedulers.
const TargetRegisterClass * getRegClass(unsigned Reg) const
getRegClass - Return the register class of the specified virtual register.
void freezeReservedRegs(const MachineFunction &)
freezeReservedRegs - Called by the register allocator to freeze the set of reserved registers before ...
Reg
All possible values of the reg field in the ModR/M byte.
SDNode * SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, unsigned TableSize)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
An analysis pass which caches information about the entire Module.
Definition: GCMetadata.h:154
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDNode *N)
static ConstantInt * ExtractElement(Constant *V, Constant *Idx)
virtual unsigned getFrameRegister(const MachineFunction &MF) const =0
Debug information queries.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
Definition: ValueTypes.h:110
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:571
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
unsigned DAGSize
DAGSize - Size of DAG being instruction selected.
static cl::opt< bool > EnableFastISelVerbose("fast-isel-verbose", cl::Hidden, cl::desc("Enable verbose messages in the \"fast\" ""instruction selector"))
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
defusechain_iterator - This class provides iterator support for machine operands in the function that...
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:30
ConstantExpr - a constant value that is initialized with an expression using other constant values...
Definition: Constants.h:852
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
ScheduleDAGSDNodes * createHybridListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createHybridListDAGScheduler - This creates a bottom up register pressure aware list scheduler that m...
#define T
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: ArrayRef.h:31
void initializeAliasAnalysisAnalysisGroup(PassRegistry &)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:804
const APInt & getAPIntValue() const
MachinePassRegistry - Track the registration of machine passes.
void addPersonality(MachineBasicBlock *LandingPad, const Function *Personality)
addPersonality - Provide the personality function for the exception information.
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification, or lowering of the constant.
Definition: ISDOpcodes.h:116
int64_t getImm() const
static LLVM_ATTRIBUTE_ALWAYS_INLINE uint64_t GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx)
GetVBR - decode a vbr encoding whose top bit is set.
ScheduleDAGSDNodes * createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createBURRListDAGScheduler - This creates a bottom up list scheduler that schedules nodes in source c...
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
iterator begin()
Definition: Function.h:457
void init(MachineFunction &mf)
Prepare this SelectionDAG to process code in the given MachineFunction.
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:142
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:284
const TargetRegisterClass * constrainRegClass(unsigned Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
const BasicBlock * getBasicBlock() const
getBasicBlock - Return the LLVM basic block that this instance corresponded to originally.
UNDEF - An undefined node.
Definition: ISDOpcodes.h:169
unsigned getNumIncomingValues() const
getNumIncomingValues - Return the number of incoming edges
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:120
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:711
MachineRegisterInfo * RegInfo
void clear()
clear - Clear out all the function-specific state.
bool isDebugValue() const
Definition: MachineInstr.h:748
bool isImplicitDef() const
Definition: MachineInstr.h:759
DBG_VALUE - a mapping of the llvm.dbg.value intrinsic.
Definition: TargetOpcodes.h:69
unsigned getNumSuccessors() const
Return the number of successors that this terminator has.
Definition: InstrTypes.h:57
SDNode * getNode() const
get the SDNode which holds the desired result
IMPLICIT_DEF - This is the MachineInstr-level equivalent of undef.
Definition: TargetOpcodes.h:52
static cl::opt< RegisterScheduler::FunctionPassCtor, false, RegisterPassParser< RegisterScheduler > > ISHeuristic("pre-RA-sched", cl::init(&createDefaultScheduler), cl::Hidden, cl::desc("Instruction schedulers available (before register"" allocation):"))
ISHeuristic command line option for instruction schedulers.
bundle_iterator< MachineInstr, instr_iterator > iterator
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
BasicBlock * SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions())
SplitCriticalEdge - If this edge is a critical edge, insert a new node to split the critical edge...
CriticalEdgeSplittingOptions & setMergeIdenticalEdges()
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:325
virtual FastISel * createFastISel(FunctionLoweringInfo &, const TargetLibraryInfo *) const
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:157
CodeGenOpt::Level OptLevel
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
LandingPadInst - The landingpad instruction holds all of the information necessary to generate correc...
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:85
static cl::opt< int > EnableFastISelAbort("fast-isel-abort", cl::Hidden, cl::desc("Enable abort calls when \"fast\" instruction selection ""fails to lower an instruction: 0 disable the abort, 1 will ""abort but for args, calls and terminators, 2 will also ""abort for argument lowering, and 3 will never fallback ""to SelectionDAG."))
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
virtual bool CheckNodePredicate(SDNode *N, unsigned PredNo) const
CheckNodePredicate - This function is generated by tblgen in the target.
Constant * stripPointerCasts()
Definition: Constant.h:170
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1165
ScheduleDAGSDNodes * createVLIWDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createVLIWDAGScheduler - Scheduler for VLIW targets.
bool HasTailCall
HasTailCall - This is set to true if a call in the current block has been translated as a tail call...
static void setDefault(FunctionPassCtor C)
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
ScheduleDAGSDNodes * createDefaultScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createDefaultScheduler - This creates an instruction scheduler appropriate for the target...
MVT - Machine Value Type.
LLVM Basic Block Representation.
Definition: BasicBlock.h:65
const SDValue & getOperand(unsigned i) const
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
bool canTrap() const
canTrap - Return true if evaluation of this constant could trap.
Definition: Constants.cpp:354
ScheduleDAGSDNodes * createBURRListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createBURRListDAGScheduler - This creates a bottom up register usage reduction list scheduler...
void Combine(CombineLevel Level, AliasAnalysis &AA, CodeGenOpt::Level OptLevel)
This iterates over the nodes in the SelectionDAG, folding certain types of nodes together, or eliminating superfluous nodes.
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
Definition: FastISel.cpp:352
const char * getName(unsigned Opcode) const
Returns the name for the instructions with the given opcode.
Definition: MCInstrInfo.h:51
bool isIndirectDebugValue() const
A DBG_VALUE is indirect iff the first operand is a register and the second operand is an immediate...
Definition: MachineInstr.h:751
This file contains the declarations for the subclasses of Constant, which represent the different fla...
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:273
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
Definition: APInt.h:1895
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:233
LandingPadInst * getLandingPadInst()
Return the landingpad instruction associated with the landing pad.
Definition: BasicBlock.cpp:418
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:264
APInt Xor(const APInt &LHS, const APInt &RHS)
Bitwise XOR function for APInt.
Definition: APInt.h:1900
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:114
EHPersonality getPersonalityType()
Classify the personality function amongst known EH styles.
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag...
Definition: InlineAsm.h:332
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
bool isCopy() const
Definition: MachineInstr.h:778
Represent the analysis usage information of a pass.
This class provides iterator support for SDUse operands that use a specific SDNode.
BasicBlock * getIncomingBlock(unsigned i) const
getIncomingBlock - Return incoming basic block number i.
SDValue getCopyFromReg(SDValue Chain, SDLoc dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:547
void visitBitTestCase(BitTestBlock &BB, MachineBasicBlock *NextMBB, uint32_t BranchWeightToNext, unsigned Reg, BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We're checking to see if we can fold LI into FoldInst.
Definition: FastISel.cpp:2070
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, unsigned ChildNo)
static unsigned getMemoryConstraintID(unsigned Flag)
Definition: InlineAsm.h:325
void printrFull(raw_ostream &O, const SelectionDAG *G=nullptr) const
Print a SelectionDAG node and all children down to the leaves.
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:108
SDValue getTargetConstant(uint64_t Val, SDLoc DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:436
virtual unsigned getRegisterByName(const char *RegName, EVT VT, SelectionDAG &DAG) const
Return the register ID of the name passed in.
static void collectFailStats(const Instruction *I)
unsigned getOpcode() const
static cl::opt< bool > UseMBPI("use-mbpi", cl::desc("use Machine Branch Probability Info"), cl::init(true), cl::Hidden)
static void SplitCriticalSideEffectEdges(Function &Fn, AliasAnalysis *AA)
SplitCriticalSideEffectEdges - Look for critical edges with a PHI value that may trap on it...
for(unsigned i=0, e=MI->getNumOperands();i!=e;++i)
Interval::pred_iterator pred_end(Interval *I)
Definition: Interval.h:117
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:57
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
addLandingPad - Add a new panding pad.
void setHasInlineAsm(bool B)
Set a flag that indicates that the function contains inline assembly.
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection...
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
static cl::opt< bool > ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the second ""dag combine pass"))
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, const MCInstrDesc &MCID)
BuildMI - Builder interface.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
unsigned ExceptionPointerVirtReg
If the current MBB is a landing pad, the exception pointer and exception selector registers are copie...
SmallPtrSet< const BasicBlock *, 4 > VisitedBBs
VisitedBBs - The set of basic blocks visited thus far by instruction selection.
static bool isMemKind(unsigned Flag)
Definition: InlineAsm.h:317
EVT - Extended Value Type.
Definition: ValueTypes.h:31
unsigned getExceptionPointerRegister() const
If a physical register, this returns the register that receives the exception address on entry to a l...
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
void ComputePHILiveOutRegInfo(const PHINode *)
ComputePHILiveOutRegInfo - Compute LiveOutInfo for a PHI's destination register based on the LiveOutI...
HANDLENODE node - Used as a handle for various purposes.
Definition: ISDOpcodes.h:627
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:335
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:45
MachineBasicBlock * MBB
MBB - The current block.
bool mayWriteToMemory() const
mayWriteToMemory - Return true if this instruction may modify memory.
std::vector< NodeType * >::reverse_iterator rpo_iterator
virtual bool enableMachineSchedDefaultSched() const
True if the machine scheduler should disable the TLI preference for preRA scheduling with the source ...
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:576
void dump() const
Support for debugging, callable in GDB: V->dump()
Definition: AsmWriter.cpp:3353
TargetIntrinsicInfo - Interface to description of machine instruction set.
static cl::opt< bool > ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the first ""dag combine pass"))
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
Definition: FastISel.cpp:338
virtual SDNode * Select(SDNode *N)=0
Select - Main hook targets implement to select a node.
bool hasCalls() const
Return true if the current function has any function calls.
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:936
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
const DILocalVariable * getDebugVariable() const
Return the debug variable referenced by this DBG_VALUE instruction.
Definition: MachineInstr.h:242
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:299
This is the shared class of boolean and integer constants.
Definition: Constants.h:47
ScheduleDAGSDNodes * createILPListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createILPListDAGScheduler - This creates a bottom up register pressure aware list scheduler that trie...
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:317
Value * getIncomingValue(unsigned i) const
getIncomingValue - Return incoming value number x
iterator end()
Definition: BasicBlock.h:233
static FunctionPassCtor getDefault()
DenseMap< unsigned, unsigned > RegFixups
RegFixups - Registers which need to be replaced after isel is done.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:179
SDNode * SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type...
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
SmallVector< MachineInstr *, 8 > ArgDbgValues
ArgDbgValues - A list of DBG_VALUE instructions created during isel for function arguments that are i...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:222
void clear()
clear - Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuil...
virtual const TargetLowering * getTargetLowering() const
void setFastISel(bool Enable)
SelectionDAGISel(TargetMachine &tm, CodeGenOpt::Level OL=CodeGenOpt::Default)
bool isSuccessor(const MachineBasicBlock *MBB) const
isSuccessor - Return true if the specified MBB is a successor of this block.
void visit(const Instruction &I)
bool mayLoad() const
Return true if this instruction could possibly read memory.
Definition: MCInstrDesc.h:350
livein_iterator livein_begin() const
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
MachineFrameInfo * getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel, SDNode *N)
CheckNodePredicate - Implements OP_CheckNodePredicate.
Represents one node in the SelectionDAG.
const BasicBlock & getEntryBlock() const
Definition: Function.h:442
virtual bool CheckComplexPattern(SDNode *Root, SDNode *Parent, SDValue N, unsigned PatternNo, SmallVectorImpl< std::pair< SDValue, SDNode * > > &Result)
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
UpdateSplitBlock - When an MBB was split during scheduling, update the references that need to refer ...
static cl::opt< bool > ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize types"))
unsigned CreateRegs(Type *Ty)
CreateRegs - Allocate the appropriate number of virtual registers of the correctly promoted or expand...
void startNewBlock()
Set the current block to which generated machine instructions will be appended, and clear the local C...
Definition: FastISel.cpp:96
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:576
static cl::opt< bool > ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize"))
StringRef getName() const
getName - Return the name of the corresponding LLVM basic block, or "(null)".
Class for arbitrary precision integers.
Definition: APInt.h:73
static bool MIIsInTerminatorSequence(const MachineInstr *MI)
Given that the input MI is before a partial terminator sequence TSeq, return true if M + TSeq also a ...
static unsigned IsPredicateKnownToFail(const unsigned char *Table, unsigned Index, SDValue N, bool &Result, const SelectionDAGISel &SDISel, SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
IsPredicateKnownToFail - If we know how and can do so without pushing a scope, evaluate the current n...
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
BranchProbabilityInfo * BPI
machine Machine Instruction Scheduler
Function must not be optimized.
Definition: Attributes.h:98
int64_t getSExtValue() const
op_iterator op_begin() const
This file defines the FastISel class.
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
static use_iterator use_end()
std::vector< JumpTableBlock > JTCases
JTCases - Vector of JumpTable structures used to communicate SwitchInst code generation information...
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:285
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.cpp:379
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:238
void replaceRegWith(unsigned FromReg, unsigned ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes, unsigned ChildNo)
CheckChildSame - Implements OP_CheckChildXSame.
iterator_range< value_op_iterator > op_values() const
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
Definition: APInt.h:1890
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
TargetSubtargetInfo - Generic base class for all target subtargets.
Representation of each machine instruction.
Definition: MachineInstr.h:51
static ChainResult WalkChainUsers(const SDNode *ChainedNode, SmallVectorImpl< SDNode * > &ChainedNodesInPattern, SmallVectorImpl< SDNode * > &InteriorChainedNodes)
WalkChainUsers - Walk down the users of the specified chained node that is part of the pattern we're ...
static bool isPhysicalRegister(unsigned Reg)
isPhysicalRegister - Return true if the specified register number is in the physical register namespa...
Represents a use of a SDNode.
Analysis pass providing branch probability information.
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:321
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:217
std::string getName(ID id, ArrayRef< Type * > Tys=None)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:500
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Constant * getPersonalityFn() const
Definition: Function.h:133
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
iterator begin()
Definition: DenseMap.h:64
void SelectInlineAsmMemoryOperands(std::vector< SDValue > &Ops, SDLoc DL)
SelectInlineAsmMemoryOperands - Calls to this are automatically generated by tblgen.
virtual void viewGraph(const Twine &Name, const Twine &Title)
viewGraph - Pop up a GraphViz/gv window with the ScheduleDAG rendered using 'dot'.
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:233
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
void initializeGCModuleInfoPass(PassRegistry &)
#define I(x, y, z)
Definition: MD5.cpp:54
static cl::opt< bool > ViewSUnitDAGs("view-sunit-dags", cl::Hidden, cl::desc("Pop up a window to show SUnit dags after they are processed"))
#define N
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:124
bool isLandingPad() const
Return true if this basic block is a landing pad.
Definition: BasicBlock.cpp:413
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:311
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
bool callsFunctionThatReturnsTwice() const
callsFunctionThatReturnsTwice - Return true if the function has a call to setjmp or other function th...
Definition: Function.cpp:916
void resetTargetOptions(const Function &F) const
Reset the target options based on the function's attributes.
iterator end()
Definition: DenseMap.h:68
void clearKillFlags(unsigned Reg) const
clearKillFlags - Iterate over all the uses of the given register and clear the kill flag from the Mac...
iterator find(const KeyT &Val)
Definition: DenseMap.h:124
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
op_iterator op_end() const
void Run(SelectionDAG *dag, MachineBasicBlock *bb)
Run - perform scheduling.
MachineSDNode * getMachineNode(unsigned Opcode, SDLoc dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s), MachineInstr opcode, and operands.
OptLevelChanger(SelectionDAGISel &ISel, CodeGenOpt::Level NewOptLevel)
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
bool CheckOrMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckOrMask - The isel is trying to match something like (or X, 255).
int getNodeId() const
Return the unique node id.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:166
EVT getValueType() const
Return the ValueType of the referenced return value.
MachineInstr * getVRegDef(unsigned Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode...
SDValue getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isTarget=false, bool isOpaque=false)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
std::vector< std::pair< unsigned, unsigned > >::const_iterator livein_iterator
unsigned getReg() const
getReg - Returns the register number.
const MDNode * getMD() const
bool use_empty() const
Definition: Value.h:275
virtual bool CheckPatternPredicate(unsigned PredNo) const
CheckPatternPredicate - This function is generated by tblgen in the target.
This class is used to form a handle around another node that is persistent and is updated across invo...
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:465
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void InvalidatePHILiveOutRegInfo(const PHINode *PN)
InvalidatePHILiveOutRegInfo - Invalidates a PHI's LiveOutInfo, to be called when a block is visited b...
virtual const TargetInstrInfo * getInstrInfo() const
SDValue getRegister(unsigned Reg, EVT VT)
succ_range successors(BasicBlock *BB)
Definition: IR/CFG.h:271
mop_iterator operands_begin()
Definition: MachineInstr.h:289
void init(GCFunctionInfo *gfi, AliasAnalysis &aa, const TargetLibraryInfo *li)
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:112
virtual MachineBasicBlock * EmitSchedule(MachineBasicBlock::iterator &InsertPos)
EmitSchedule - Insert MachineInstrs into the MachineBasicBlock according to the order specified in Se...
static use_instr_iterator use_instr_end()
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
CheckSame - Implements OP_CheckSame.
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
static cl::opt< bool > ViewSchedDAGs("view-sched-dags", cl::Hidden, cl::desc("Pop up a window to show sched dags as they are processed"))
std::vector< CaseBlock > SwitchCases
SwitchCases - Vector of CaseBlock structures used to communicate SwitchInst code generation informati...
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.Val alone...
#define DEBUG(X)
Definition: Debug.h:92
Primary interface to the complete machine description for the target machine.
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
SDValue getControlRoot()
getControlRoot - Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports items.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
MachineModuleInfo & getMMI() const
A single uniqued string.
Definition: Metadata.h:508
void clearDanglingDebugInfo()
clearDanglingDebugInfo - Clear the dangling debug information map.
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
SDValue getConstantFP(double Val, SDLoc DL, EVT VT, bool isTarget=false)
static cl::opt< bool > EnableFastISelVerbose2("fast-isel-verbose2", cl::Hidden, cl::desc("Enable extra verbose messages in the \"fast\" ""instruction selector"))
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
LPadToCallSiteMap - Map a landing pad to the call site indexes.
unsigned getExceptionSelectorRegister() const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:194
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:338
bool TimePassesIsEnabled
If the user specifies the -time-passes argument on an LLVM tool command line then the value of this b...
Definition: IRReader.cpp:26
SDNode * getUser()
This returns the SDNode that contains this Use.
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
static cl::opt< std::string > FilterDAGBasicBlockName("filter-view-dags", cl::Hidden, cl::desc("Only display the basic block whose name ""matches this for all view-*-dags options"))
const TargetInstrInfo * TII
DenseMap< const Value *, unsigned > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num)
allocateMemRefsArray - Allocate an array to hold MachineMemOperand pointers.
void visitBitTestHeader(BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests" ...
const BasicBlock * getParent() const
Definition: Instruction.h:72
bool isExportedInst(const Value *V)
isExportedInst - Return true if the specified value is an instruction exported from its block...
unsigned getResNo() const
Convenience function for get().getResNo().
bool isMSVCEHPersonality(EHPersonality Pers)
Returns true if this is an MSVC personality function.
FunctionLoweringInfo * FuncInfo
static cl::opt< bool > ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the post legalize types"" dag combine pass"))
static RegisterScheduler defaultListDAGScheduler("default","Best scheduler for the target", createDefaultScheduler)
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
IntrinsicInst - A useful wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:37
This file describes how to lower LLVM code to machine code.
virtual SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo)
bool isVoidTy() const
isVoidTy - Return true if this is 'void'.
Definition: Type.h:137
bool use_empty(unsigned RegNo) const
use_empty - Return true if there are no instructions using the specified register.
This class is used by SelectionDAGISel to temporarily override the optimization level on a per-functi...
void resize(size_type N)
Definition: SmallVector.h:376