LLVM  16.0.0git
SelectionDAGISel.cpp
Go to the documentation of this file.
1 //===- SelectionDAGISel.cpp - Implement the SelectionDAGISel class --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the SelectionDAGISel class.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "ScheduleDAGSDNodes.h"
15 #include "SelectionDAGBuilder.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/DenseMap.h"
19 #include "llvm/ADT/STLExtras.h"
20 #include "llvm/ADT/SmallPtrSet.h"
21 #include "llvm/ADT/SmallVector.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/ADT/StringRef.h"
27 #include "llvm/Analysis/CFG.h"
36 #include "llvm/CodeGen/FastISel.h"
54 #include "llvm/CodeGen/StackMaps.h"
62 #include "llvm/IR/BasicBlock.h"
63 #include "llvm/IR/Constants.h"
64 #include "llvm/IR/DataLayout.h"
66 #include "llvm/IR/DebugLoc.h"
67 #include "llvm/IR/DiagnosticInfo.h"
68 #include "llvm/IR/Function.h"
69 #include "llvm/IR/InlineAsm.h"
70 #include "llvm/IR/InstIterator.h"
71 #include "llvm/IR/Instruction.h"
72 #include "llvm/IR/Instructions.h"
73 #include "llvm/IR/IntrinsicInst.h"
74 #include "llvm/IR/Intrinsics.h"
75 #include "llvm/IR/IntrinsicsWebAssembly.h"
76 #include "llvm/IR/Metadata.h"
77 #include "llvm/IR/Statepoint.h"
78 #include "llvm/IR/Type.h"
79 #include "llvm/IR/User.h"
80 #include "llvm/IR/Value.h"
81 #include "llvm/InitializePasses.h"
82 #include "llvm/MC/MCInstrDesc.h"
83 #include "llvm/Pass.h"
85 #include "llvm/Support/Casting.h"
86 #include "llvm/Support/CodeGen.h"
88 #include "llvm/Support/Compiler.h"
89 #include "llvm/Support/Debug.h"
91 #include "llvm/Support/KnownBits.h"
93 #include "llvm/Support/Timer.h"
99 #include <algorithm>
100 #include <cassert>
101 #include <cstdint>
102 #include <iterator>
103 #include <limits>
104 #include <memory>
105 #include <string>
106 #include <utility>
107 #include <vector>
108 
109 using namespace llvm;
110 
111 #define DEBUG_TYPE "isel"
112 
113 STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on");
114 STATISTIC(NumFastIselSuccess, "Number of instructions fast isel selected");
115 STATISTIC(NumFastIselBlocks, "Number of blocks selected entirely by fast isel");
116 STATISTIC(NumDAGBlocks, "Number of blocks selected using DAG");
117 STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path");
118 STATISTIC(NumEntryBlocks, "Number of entry blocks encountered");
119 STATISTIC(NumFastIselFailLowerArguments,
120  "Number of entry blocks where fast isel failed to lower arguments");
121 
123  "fast-isel-abort", cl::Hidden,
124  cl::desc("Enable abort calls when \"fast\" instruction selection "
125  "fails to lower an instruction: 0 disable the abort, 1 will "
126  "abort but for args, calls and terminators, 2 will also "
127  "abort for argument lowering, and 3 will never fallback "
128  "to SelectionDAG."));
129 
131  "fast-isel-report-on-fallback", cl::Hidden,
132  cl::desc("Emit a diagnostic when \"fast\" instruction selection "
133  "falls back to SelectionDAG."));
134 
135 static cl::opt<bool>
136 UseMBPI("use-mbpi",
137  cl::desc("use Machine Branch Probability Info"),
138  cl::init(true), cl::Hidden);
139 
140 #ifndef NDEBUG
142 FilterDAGBasicBlockName("filter-view-dags", cl::Hidden,
143  cl::desc("Only display the basic block whose name "
144  "matches this for all view-*-dags options"));
145 static cl::opt<bool>
146 ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden,
147  cl::desc("Pop up a window to show dags before the first "
148  "dag combine pass"));
149 static cl::opt<bool>
150 ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden,
151  cl::desc("Pop up a window to show dags before legalize types"));
152 static cl::opt<bool>
153  ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden,
154  cl::desc("Pop up a window to show dags before the post "
155  "legalize types dag combine pass"));
156 static cl::opt<bool>
157  ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,
158  cl::desc("Pop up a window to show dags before legalize"));
159 static cl::opt<bool>
160 ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden,
161  cl::desc("Pop up a window to show dags before the second "
162  "dag combine pass"));
163 static cl::opt<bool>
164 ViewISelDAGs("view-isel-dags", cl::Hidden,
165  cl::desc("Pop up a window to show isel dags as they are selected"));
166 static cl::opt<bool>
167 ViewSchedDAGs("view-sched-dags", cl::Hidden,
168  cl::desc("Pop up a window to show sched dags as they are processed"));
169 static cl::opt<bool>
170 ViewSUnitDAGs("view-sunit-dags", cl::Hidden,
171  cl::desc("Pop up a window to show SUnit dags after they are processed"));
172 #else
173 static const bool ViewDAGCombine1 = false, ViewLegalizeTypesDAGs = false,
174  ViewDAGCombineLT = false, ViewLegalizeDAGs = false,
175  ViewDAGCombine2 = false, ViewISelDAGs = false,
176  ViewSchedDAGs = false, ViewSUnitDAGs = false;
177 #endif
178 
179 //===---------------------------------------------------------------------===//
180 ///
181 /// RegisterScheduler class - Track the registration of instruction schedulers.
182 ///
183 //===---------------------------------------------------------------------===//
186 
187 //===---------------------------------------------------------------------===//
188 ///
189 /// ISHeuristic command line option for instruction schedulers.
190 ///
191 //===---------------------------------------------------------------------===//
194 ISHeuristic("pre-RA-sched",
196  cl::desc("Instruction schedulers available (before register"
197  " allocation):"));
198 
199 static RegisterScheduler
200 defaultListDAGScheduler("default", "Best scheduler for the target",
202 
203 namespace llvm {
204 
205  //===--------------------------------------------------------------------===//
206  /// This class is used by SelectionDAGISel to temporarily override
207  /// the optimization level on a per-function basis.
209  SelectionDAGISel &IS;
210  CodeGenOpt::Level SavedOptLevel;
211  bool SavedFastISel;
212 
213  public:
215  CodeGenOpt::Level NewOptLevel) : IS(ISel) {
216  SavedOptLevel = IS.OptLevel;
217  SavedFastISel = IS.TM.Options.EnableFastISel;
218  if (NewOptLevel == SavedOptLevel)
219  return;
220  IS.OptLevel = NewOptLevel;
221  IS.TM.setOptLevel(NewOptLevel);
222  LLVM_DEBUG(dbgs() << "\nChanging optimization level for Function "
223  << IS.MF->getFunction().getName() << "\n");
224  LLVM_DEBUG(dbgs() << "\tBefore: -O" << SavedOptLevel << " ; After: -O"
225  << NewOptLevel << "\n");
226  if (NewOptLevel == CodeGenOpt::None) {
228  LLVM_DEBUG(
229  dbgs() << "\tFastISel is "
230  << (IS.TM.Options.EnableFastISel ? "enabled" : "disabled")
231  << "\n");
232  }
233  }
234 
236  if (IS.OptLevel == SavedOptLevel)
237  return;
238  LLVM_DEBUG(dbgs() << "\nRestoring optimization level for Function "
239  << IS.MF->getFunction().getName() << "\n");
240  LLVM_DEBUG(dbgs() << "\tBefore: -O" << IS.OptLevel << " ; After: -O"
241  << SavedOptLevel << "\n");
242  IS.OptLevel = SavedOptLevel;
243  IS.TM.setOptLevel(SavedOptLevel);
244  IS.TM.setFastISel(SavedFastISel);
245  }
246  };
247 
248  //===--------------------------------------------------------------------===//
249  /// createDefaultScheduler - This creates an instruction scheduler appropriate
250  /// for the target.
252  CodeGenOpt::Level OptLevel) {
253  const TargetLowering *TLI = IS->TLI;
254  const TargetSubtargetInfo &ST = IS->MF->getSubtarget();
255 
256  // Try first to see if the Target has its own way of selecting a scheduler
257  if (auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) {
258  return SchedulerCtor(IS, OptLevel);
259  }
260 
261  if (OptLevel == CodeGenOpt::None ||
262  (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) ||
264  return createSourceListDAGScheduler(IS, OptLevel);
266  return createBURRListDAGScheduler(IS, OptLevel);
268  return createHybridListDAGScheduler(IS, OptLevel);
269  if (TLI->getSchedulingPreference() == Sched::VLIW)
270  return createVLIWDAGScheduler(IS, OptLevel);
271  if (TLI->getSchedulingPreference() == Sched::Fast)
272  return createFastDAGScheduler(IS, OptLevel);
274  return createDAGLinearizer(IS, OptLevel);
276  "Unknown sched type!");
277  return createILPListDAGScheduler(IS, OptLevel);
278  }
279 
280 } // end namespace llvm
281 
282 // EmitInstrWithCustomInserter - This method should be implemented by targets
283 // that mark instructions with the 'usesCustomInserter' flag. These
284 // instructions are special in various ways, which require special support to
285 // insert. The specified MachineInstr is created but not inserted into any
286 // basic blocks, and this method is called to expand it into a sequence of
287 // instructions, potentially also creating new basic blocks and control flow.
288 // When new basic blocks are inserted and the edges from MBB to its successors
289 // are modified, the method should insert pairs of <OldSucc, NewSucc> into the
290 // DenseMap.
293  MachineBasicBlock *MBB) const {
294 #ifndef NDEBUG
295  dbgs() << "If a target marks an instruction with "
296  "'usesCustomInserter', it must implement "
297  "TargetLowering::EmitInstrWithCustomInserter!\n";
298 #endif
299  llvm_unreachable(nullptr);
300 }
301 
303  SDNode *Node) const {
304  assert(!MI.hasPostISelHook() &&
305  "If a target marks an instruction with 'hasPostISelHook', "
306  "it must implement TargetLowering::AdjustInstrPostInstrSelection!");
307 }
308 
309 //===----------------------------------------------------------------------===//
310 // SelectionDAGISel code
311 //===----------------------------------------------------------------------===//
312 
314  : MachineFunctionPass(ID), TM(tm), FuncInfo(new FunctionLoweringInfo()),
315  SwiftError(new SwiftErrorValueTracking()),
316  CurDAG(new SelectionDAG(tm, OL)),
317  SDB(std::make_unique<SelectionDAGBuilder>(*CurDAG, *FuncInfo, *SwiftError,
318  OL)),
319  OptLevel(OL) {
325 }
326 
328  delete CurDAG;
329  delete SwiftError;
330 }
331 
333  if (OptLevel != CodeGenOpt::None)
344  if (OptLevel != CodeGenOpt::None)
347 }
348 
349 static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F,
350  MachineModuleInfo &MMI) {
351  // Only needed for MSVC
352  if (!TT.isWindowsMSVCEnvironment())
353  return;
354 
355  // If it's already set, nothing to do.
356  if (MMI.usesMSVCFloatingPoint())
357  return;
358 
359  for (const Instruction &I : instructions(F)) {
360  if (I.getType()->isFPOrFPVectorTy()) {
361  MMI.setUsesMSVCFloatingPoint(true);
362  return;
363  }
364  for (const auto &Op : I.operands()) {
365  if (Op->getType()->isFPOrFPVectorTy()) {
366  MMI.setUsesMSVCFloatingPoint(true);
367  return;
368  }
369  }
370  }
371 }
372 
374  // If we already selected that function, we do not need to run SDISel.
375  if (mf.getProperties().hasProperty(
377  return false;
378  // Do some sanity-checking on the command-line options.
380  "-fast-isel-abort > 0 requires -fast-isel");
381 
382  const Function &Fn = mf.getFunction();
383  MF = &mf;
384 
385  // Decide what flavour of variable location debug-info will be used, before
386  // we change the optimisation level.
389 
390  // Reset the target options before resetting the optimization
391  // level below.
392  // FIXME: This is a horrible hack and should be processed via
393  // codegen looking at the optimization level explicitly when
394  // it wants to look at it.
396  // Reset OptLevel to None for optnone functions.
397  CodeGenOpt::Level NewOptLevel = OptLevel;
398  if (OptLevel != CodeGenOpt::None && skipFunction(Fn))
399  NewOptLevel = CodeGenOpt::None;
400  OptLevelChanger OLC(*this, NewOptLevel);
401 
404  RegInfo = &MF->getRegInfo();
405  LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(Fn);
406  GFI = Fn.hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) : nullptr;
407  ORE = std::make_unique<OptimizationRemarkEmitter>(&Fn);
408  AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(mf.getFunction());
409  auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
410  BlockFrequencyInfo *BFI = nullptr;
411  if (PSI && PSI->hasProfileSummary() && OptLevel != CodeGenOpt::None)
412  BFI = &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI();
413 
414  LLVM_DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n");
415 
416  CurDAG->init(*MF, *ORE, this, LibInfo,
417  getAnalysisIfAvailable<LegacyDivergenceAnalysis>(), PSI, BFI);
418  FuncInfo->set(Fn, *MF, CurDAG);
420 
421  // Now get the optional analyzes if we want to.
422  // This is based on the possibly changed OptLevel (after optnone is taken
423  // into account). That's unfortunate but OK because it just means we won't
424  // ask for passes that have been required anyway.
425 
427  FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
428  else
429  FuncInfo->BPI = nullptr;
430 
431  if (OptLevel != CodeGenOpt::None)
432  AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
433  else
434  AA = nullptr;
435 
436  SDB->init(GFI, AA, AC, LibInfo);
437 
438  MF->setHasInlineAsm(false);
439 
440  FuncInfo->SplitCSR = false;
441 
442  // We split CSR if the target supports it for the given function
443  // and the function has only return exits.
445  FuncInfo->SplitCSR = true;
446 
447  // Collect all the return blocks.
448  for (const BasicBlock &BB : Fn) {
449  if (!succ_empty(&BB))
450  continue;
451 
452  const Instruction *Term = BB.getTerminator();
453  if (isa<UnreachableInst>(Term) || isa<ReturnInst>(Term))
454  continue;
455 
456  // Bail out if the exit block is not Return nor Unreachable.
457  FuncInfo->SplitCSR = false;
458  break;
459  }
460  }
461 
462  MachineBasicBlock *EntryMBB = &MF->front();
463  if (FuncInfo->SplitCSR)
464  // This performs initialization so lowering for SplitCSR will be correct.
465  TLI->initializeSplitCSR(EntryMBB);
466 
467  SelectAllBasicBlocks(Fn);
469  DiagnosticInfoISelFallback DiagFallback(Fn);
470  Fn.getContext().diagnose(DiagFallback);
471  }
472 
473  // Replace forward-declared registers with the registers containing
474  // the desired value.
475  // Note: it is important that this happens **before** the call to
476  // EmitLiveInCopies, since implementations can skip copies of unused
477  // registers. If we don't apply the reg fixups before, some registers may
478  // appear as unused and will be skipped, resulting in bad MI.
480  for (DenseMap<Register, Register>::iterator I = FuncInfo->RegFixups.begin(),
481  E = FuncInfo->RegFixups.end();
482  I != E; ++I) {
483  Register From = I->first;
484  Register To = I->second;
485  // If To is also scheduled to be replaced, find what its ultimate
486  // replacement is.
487  while (true) {
488  DenseMap<Register, Register>::iterator J = FuncInfo->RegFixups.find(To);
489  if (J == E)
490  break;
491  To = J->second;
492  }
493  // Make sure the new register has a sufficiently constrained register class.
496  // Replace it.
497 
498  // Replacing one register with another won't touch the kill flags.
499  // We need to conservatively clear the kill flags as a kill on the old
500  // register might dominate existing uses of the new register.
501  if (!MRI.use_empty(To))
503  MRI.replaceRegWith(From, To);
504  }
505 
506  // If the first basic block in the function has live ins that need to be
507  // copied into vregs, emit the copies into the top of the block before
508  // emitting the code for the block.
510  RegInfo->EmitLiveInCopies(EntryMBB, TRI, *TII);
511 
512  // Insert copies in the entry block and the return blocks.
513  if (FuncInfo->SplitCSR) {
515  // Collect all the return blocks.
516  for (MachineBasicBlock &MBB : mf) {
517  if (!MBB.succ_empty())
518  continue;
519 
521  if (Term != MBB.end() && Term->isReturn()) {
522  Returns.push_back(&MBB);
523  continue;
524  }
525  }
526  TLI->insertCopiesSplitCSR(EntryMBB, Returns);
527  }
528 
530  if (!FuncInfo->ArgDbgValues.empty())
531  for (std::pair<unsigned, unsigned> LI : RegInfo->liveins())
532  if (LI.second)
533  LiveInMap.insert(LI);
534 
535  // Insert DBG_VALUE instructions for function arguments to the entry block.
536  bool InstrRef = MF->useDebugInstrRef();
537  for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) {
538  MachineInstr *MI = FuncInfo->ArgDbgValues[e - i - 1];
539  assert(MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
540  "Function parameters should not be described by DBG_VALUE_LIST.");
541  bool hasFI = MI->getOperand(0).isFI();
542  Register Reg =
543  hasFI ? TRI.getFrameRegister(*MF) : MI->getOperand(0).getReg();
545  EntryMBB->insert(EntryMBB->begin(), MI);
546  else {
548  if (Def) {
549  MachineBasicBlock::iterator InsertPos = Def;
550  // FIXME: VR def may not be in entry block.
551  Def->getParent()->insert(std::next(InsertPos), MI);
552  } else
553  LLVM_DEBUG(dbgs() << "Dropping debug info for dead vreg"
554  << Register::virtReg2Index(Reg) << "\n");
555  }
556 
557  // Don't try and extend through copies in instruction referencing mode.
558  if (InstrRef)
559  continue;
560 
561  // If Reg is live-in then update debug info to track its copy in a vreg.
563  if (LDI != LiveInMap.end()) {
564  assert(!hasFI && "There's no handling of frame pointer updating here yet "
565  "- add if needed");
566  MachineInstr *Def = RegInfo->getVRegDef(LDI->second);
567  MachineBasicBlock::iterator InsertPos = Def;
568  const MDNode *Variable = MI->getDebugVariable();
569  const MDNode *Expr = MI->getDebugExpression();
570  DebugLoc DL = MI->getDebugLoc();
571  bool IsIndirect = MI->isIndirectDebugValue();
572  if (IsIndirect)
573  assert(MI->getOperand(1).getImm() == 0 &&
574  "DBG_VALUE with nonzero offset");
575  assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&
576  "Expected inlined-at fields to agree");
577  assert(MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
578  "Didn't expect to see a DBG_VALUE_LIST here");
579  // Def is never a terminator here, so it is ok to increment InsertPos.
580  BuildMI(*EntryMBB, ++InsertPos, DL, TII->get(TargetOpcode::DBG_VALUE),
581  IsIndirect, LDI->second, Variable, Expr);
582 
583  // If this vreg is directly copied into an exported register then
584  // that COPY instructions also need DBG_VALUE, if it is the only
585  // user of LDI->second.
586  MachineInstr *CopyUseMI = nullptr;
588  UI = RegInfo->use_instr_begin(LDI->second),
589  E = RegInfo->use_instr_end(); UI != E; ) {
590  MachineInstr *UseMI = &*(UI++);
591  if (UseMI->isDebugValue()) continue;
592  if (UseMI->isCopy() && !CopyUseMI && UseMI->getParent() == EntryMBB) {
593  CopyUseMI = UseMI; continue;
594  }
595  // Otherwise this is another use or second copy use.
596  CopyUseMI = nullptr; break;
597  }
598  if (CopyUseMI &&
599  TRI.getRegSizeInBits(LDI->second, MRI) ==
600  TRI.getRegSizeInBits(CopyUseMI->getOperand(0).getReg(), MRI)) {
601  // Use MI's debug location, which describes where Variable was
602  // declared, rather than whatever is attached to CopyUseMI.
603  MachineInstr *NewMI =
604  BuildMI(*MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
605  CopyUseMI->getOperand(0).getReg(), Variable, Expr);
606  MachineBasicBlock::iterator Pos = CopyUseMI;
607  EntryMBB->insertAfter(Pos, NewMI);
608  }
609  }
610  }
611 
612  // For debug-info, in instruction referencing mode, we need to perform some
613  // post-isel maintenence.
616 
617  // Determine if there are any calls in this machine function.
618  MachineFrameInfo &MFI = MF->getFrameInfo();
619  for (const auto &MBB : *MF) {
620  if (MFI.hasCalls() && MF->hasInlineAsm())
621  break;
622 
623  for (const auto &MI : MBB) {
624  const MCInstrDesc &MCID = TII->get(MI.getOpcode());
625  if ((MCID.isCall() && !MCID.isReturn()) ||
626  MI.isStackAligningInlineAsm()) {
627  MFI.setHasCalls(true);
628  }
629  if (MI.isInlineAsm()) {
630  MF->setHasInlineAsm(true);
631  }
632  }
633  }
634 
635  // Determine if there is a call to setjmp in the machine function.
637 
638  // Determine if floating point is used for msvc
640 
641  // Release function-specific state. SDB and CurDAG are already cleared
642  // at this point.
643  FuncInfo->clear();
644 
645  LLVM_DEBUG(dbgs() << "*** MachineFunction at end of ISel ***\n");
646  LLVM_DEBUG(MF->print(dbgs()));
647 
648  return true;
649 }
650 
654  bool ShouldAbort) {
655  // Print the function name explicitly if we don't have a debug location (which
656  // makes the diagnostic less useful) or if we're going to emit a raw error.
657  if (!R.getLocation().isValid() || ShouldAbort)
658  R << (" (in function: " + MF.getName() + ")").str();
659 
660  if (ShouldAbort)
661  report_fatal_error(Twine(R.getMsg()));
662 
663  ORE.emit(R);
664  LLVM_DEBUG(dbgs() << R.getMsg() << "\n");
665 }
666 
667 void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin,
669  bool &HadTailCall) {
670  // Allow creating illegal types during DAG building for the basic block.
672 
673  // Lower the instructions. If a call is emitted as a tail call, cease emitting
674  // nodes for this block.
675  for (BasicBlock::const_iterator I = Begin; I != End && !SDB->HasTailCall; ++I) {
676  if (!ElidedArgCopyInstrs.count(&*I))
677  SDB->visit(*I);
678  }
679 
680  // Make sure the root of the DAG is up-to-date.
681  CurDAG->setRoot(SDB->getControlRoot());
682  HadTailCall = SDB->HasTailCall;
683  SDB->resolveOrClearDbgInfo();
684  SDB->clear();
685 
686  // Final step, emit the lowered DAG as machine code.
687  CodeGenAndEmitDAG();
688 }
689 
690 void SelectionDAGISel::ComputeLiveOutVRegInfo() {
692  SmallVector<SDNode*, 128> Worklist;
693 
694  Worklist.push_back(CurDAG->getRoot().getNode());
695  Added.insert(CurDAG->getRoot().getNode());
696 
697  KnownBits Known;
698 
699  do {
700  SDNode *N = Worklist.pop_back_val();
701 
702  // Otherwise, add all chain operands to the worklist.
703  for (const SDValue &Op : N->op_values())
704  if (Op.getValueType() == MVT::Other && Added.insert(Op.getNode()).second)
705  Worklist.push_back(Op.getNode());
706 
707  // If this is a CopyToReg with a vreg dest, process it.
708  if (N->getOpcode() != ISD::CopyToReg)
709  continue;
710 
711  unsigned DestReg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
712  if (!Register::isVirtualRegister(DestReg))
713  continue;
714 
715  // Ignore non-integer values.
716  SDValue Src = N->getOperand(2);
717  EVT SrcVT = Src.getValueType();
718  if (!SrcVT.isInteger())
719  continue;
720 
721  unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src);
722  Known = CurDAG->computeKnownBits(Src);
723  FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, Known);
724  } while (!Worklist.empty());
725 }
726 
727 void SelectionDAGISel::CodeGenAndEmitDAG() {
728  StringRef GroupName = "sdag";
729  StringRef GroupDescription = "Instruction Selection and Scheduling";
730  std::string BlockName;
731  bool MatchFilterBB = false; (void)MatchFilterBB;
732 #ifndef NDEBUG
734  getAnalysis<TargetTransformInfoWrapperPass>().getTTI(*FuncInfo->Fn);
735 #endif
736 
737  // Pre-type legalization allow creation of any node types.
739 
740 #ifndef NDEBUG
741  MatchFilterBB = (FilterDAGBasicBlockName.empty() ||
743  FuncInfo->MBB->getBasicBlock()->getName());
744 #endif
745 #ifdef NDEBUG
749 #endif
750  {
751  BlockName =
752  (MF->getName() + ":" + FuncInfo->MBB->getBasicBlock()->getName()).str();
753  }
754  LLVM_DEBUG(dbgs() << "Initial selection DAG: "
755  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
756  << "'\n";
757  CurDAG->dump());
758 
759 #ifndef NDEBUG
760  if (TTI.hasBranchDivergence())
762 #endif
763 
764  if (ViewDAGCombine1 && MatchFilterBB)
765  CurDAG->viewGraph("dag-combine1 input for " + BlockName);
766 
767  // Run the DAG combiner in pre-legalize mode.
768  {
769  NamedRegionTimer T("combine1", "DAG Combining 1", GroupName,
770  GroupDescription, TimePassesIsEnabled);
772  }
773 
774  LLVM_DEBUG(dbgs() << "Optimized lowered selection DAG: "
775  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
776  << "'\n";
777  CurDAG->dump());
778 
779 #ifndef NDEBUG
780  if (TTI.hasBranchDivergence())
782 #endif
783 
784  // Second step, hack on the DAG until it only uses operations and types that
785  // the target supports.
786  if (ViewLegalizeTypesDAGs && MatchFilterBB)
787  CurDAG->viewGraph("legalize-types input for " + BlockName);
788 
789  bool Changed;
790  {
791  NamedRegionTimer T("legalize_types", "Type Legalization", GroupName,
792  GroupDescription, TimePassesIsEnabled);
793  Changed = CurDAG->LegalizeTypes();
794  }
795 
796  LLVM_DEBUG(dbgs() << "Type-legalized selection DAG: "
797  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
798  << "'\n";
799  CurDAG->dump());
800 
801 #ifndef NDEBUG
802  if (TTI.hasBranchDivergence())
804 #endif
805 
806  // Only allow creation of legal node types.
808 
809  if (Changed) {
810  if (ViewDAGCombineLT && MatchFilterBB)
811  CurDAG->viewGraph("dag-combine-lt input for " + BlockName);
812 
813  // Run the DAG combiner in post-type-legalize mode.
814  {
815  NamedRegionTimer T("combine_lt", "DAG Combining after legalize types",
816  GroupName, GroupDescription, TimePassesIsEnabled);
818  }
819 
820  LLVM_DEBUG(dbgs() << "Optimized type-legalized selection DAG: "
821  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
822  << "'\n";
823  CurDAG->dump());
824 
825 #ifndef NDEBUG
826  if (TTI.hasBranchDivergence())
828 #endif
829  }
830 
831  {
832  NamedRegionTimer T("legalize_vec", "Vector Legalization", GroupName,
833  GroupDescription, TimePassesIsEnabled);
834  Changed = CurDAG->LegalizeVectors();
835  }
836 
837  if (Changed) {
838  LLVM_DEBUG(dbgs() << "Vector-legalized selection DAG: "
839  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
840  << "'\n";
841  CurDAG->dump());
842 
843 #ifndef NDEBUG
844  if (TTI.hasBranchDivergence())
846 #endif
847 
848  {
849  NamedRegionTimer T("legalize_types2", "Type Legalization 2", GroupName,
850  GroupDescription, TimePassesIsEnabled);
852  }
853 
854  LLVM_DEBUG(dbgs() << "Vector/type-legalized selection DAG: "
855  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
856  << "'\n";
857  CurDAG->dump());
858 
859 #ifndef NDEBUG
860  if (TTI.hasBranchDivergence())
862 #endif
863 
864  if (ViewDAGCombineLT && MatchFilterBB)
865  CurDAG->viewGraph("dag-combine-lv input for " + BlockName);
866 
867  // Run the DAG combiner in post-type-legalize mode.
868  {
869  NamedRegionTimer T("combine_lv", "DAG Combining after legalize vectors",
870  GroupName, GroupDescription, TimePassesIsEnabled);
872  }
873 
874  LLVM_DEBUG(dbgs() << "Optimized vector-legalized selection DAG: "
875  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
876  << "'\n";
877  CurDAG->dump());
878 
879 #ifndef NDEBUG
880  if (TTI.hasBranchDivergence())
882 #endif
883  }
884 
885  if (ViewLegalizeDAGs && MatchFilterBB)
886  CurDAG->viewGraph("legalize input for " + BlockName);
887 
888  {
889  NamedRegionTimer T("legalize", "DAG Legalization", GroupName,
890  GroupDescription, TimePassesIsEnabled);
891  CurDAG->Legalize();
892  }
893 
894  LLVM_DEBUG(dbgs() << "Legalized selection DAG: "
895  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
896  << "'\n";
897  CurDAG->dump());
898 
899 #ifndef NDEBUG
900  if (TTI.hasBranchDivergence())
902 #endif
903 
904  if (ViewDAGCombine2 && MatchFilterBB)
905  CurDAG->viewGraph("dag-combine2 input for " + BlockName);
906 
907  // Run the DAG combiner in post-legalize mode.
908  {
909  NamedRegionTimer T("combine2", "DAG Combining 2", GroupName,
910  GroupDescription, TimePassesIsEnabled);
912  }
913 
914  LLVM_DEBUG(dbgs() << "Optimized legalized selection DAG: "
915  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
916  << "'\n";
917  CurDAG->dump());
918 
919 #ifndef NDEBUG
920  if (TTI.hasBranchDivergence())
922 #endif
923 
924  if (OptLevel != CodeGenOpt::None)
925  ComputeLiveOutVRegInfo();
926 
927  if (ViewISelDAGs && MatchFilterBB)
928  CurDAG->viewGraph("isel input for " + BlockName);
929 
930  // Third, instruction select all of the operations to machine code, adding the
931  // code to the MachineBasicBlock.
932  {
933  NamedRegionTimer T("isel", "Instruction Selection", GroupName,
934  GroupDescription, TimePassesIsEnabled);
935  DoInstructionSelection();
936  }
937 
938  LLVM_DEBUG(dbgs() << "Selected selection DAG: "
939  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
940  << "'\n";
941  CurDAG->dump());
942 
943  if (ViewSchedDAGs && MatchFilterBB)
944  CurDAG->viewGraph("scheduler input for " + BlockName);
945 
946  // Schedule machine code.
947  ScheduleDAGSDNodes *Scheduler = CreateScheduler();
948  {
949  NamedRegionTimer T("sched", "Instruction Scheduling", GroupName,
950  GroupDescription, TimePassesIsEnabled);
951  Scheduler->Run(CurDAG, FuncInfo->MBB);
952  }
953 
954  if (ViewSUnitDAGs && MatchFilterBB)
955  Scheduler->viewGraph();
956 
957  // Emit machine code to BB. This can change 'BB' to the last block being
958  // inserted into.
959  MachineBasicBlock *FirstMBB = FuncInfo->MBB, *LastMBB;
960  {
961  NamedRegionTimer T("emit", "Instruction Creation", GroupName,
962  GroupDescription, TimePassesIsEnabled);
963 
964  // FuncInfo->InsertPt is passed by reference and set to the end of the
965  // scheduled instructions.
966  LastMBB = FuncInfo->MBB = Scheduler->EmitSchedule(FuncInfo->InsertPt);
967  }
968 
969  // If the block was split, make sure we update any references that are used to
970  // update PHI nodes later on.
971  if (FirstMBB != LastMBB)
972  SDB->UpdateSplitBlock(FirstMBB, LastMBB);
973 
974  // Free the scheduler state.
975  {
976  NamedRegionTimer T("cleanup", "Instruction Scheduling Cleanup", GroupName,
977  GroupDescription, TimePassesIsEnabled);
978  delete Scheduler;
979  }
980 
981  // Free the SelectionDAG state, now that we're finished with it.
982  CurDAG->clear();
983 }
984 
985 namespace {
986 
987 /// ISelUpdater - helper class to handle updates of the instruction selection
988 /// graph.
989 class ISelUpdater : public SelectionDAG::DAGUpdateListener {
990  SelectionDAG::allnodes_iterator &ISelPosition;
991 
992 public:
993  ISelUpdater(SelectionDAG &DAG, SelectionDAG::allnodes_iterator &isp)
994  : SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}
995 
996  /// NodeDeleted - Handle nodes deleted from the graph. If the node being
997  /// deleted is the current ISelPosition node, update ISelPosition.
998  ///
999  void NodeDeleted(SDNode *N, SDNode *E) override {
1000  if (ISelPosition == SelectionDAG::allnodes_iterator(N))
1001  ++ISelPosition;
1002  }
1003 
1004  /// NodeInserted - Handle new nodes inserted into the graph: propagate
1005  /// metadata from root nodes that also applies to new nodes, in case the root
1006  /// is later deleted.
1007  void NodeInserted(SDNode *N) override {
1008  SDNode *CurNode = &*ISelPosition;
1009  if (MDNode *MD = DAG.getPCSections(CurNode))
1010  DAG.addPCSections(N, MD);
1011  }
1012 };
1013 
1014 } // end anonymous namespace
1015 
1016 // This function is used to enforce the topological node id property
1017 // leveraged during instruction selection. Before the selection process all
1018 // nodes are given a non-negative id such that all nodes have a greater id than
1019 // their operands. As this holds transitively we can prune checks that a node N
1020 // is a predecessor of M another by not recursively checking through M's
1021 // operands if N's ID is larger than M's ID. This significantly improves
1022 // performance of various legality checks (e.g. IsLegalToFold / UpdateChains).
1023 
1024 // However, when we fuse multiple nodes into a single node during the
1025 // selection we may induce a predecessor relationship between inputs and
1026 // outputs of distinct nodes being merged, violating the topological property.
1027 // Should a fused node have a successor which has yet to be selected,
1028 // our legality checks would be incorrect. To avoid this we mark all unselected
1029 // successor nodes, i.e. id != -1, as invalid for pruning by bit-negating (x =>
1030 // (-(x+1))) the ids and modify our pruning check to ignore negative Ids of M.
1031 // We use bit-negation to more clearly enforce that node id -1 can only be
1032 // achieved by selected nodes. As the conversion is reversable to the original
1033 // Id, topological pruning can still be leveraged when looking for unselected
1034 // nodes. This method is called internally in all ISel replacement related
1035 // functions.
1038  Nodes.push_back(Node);
1039 
1040  while (!Nodes.empty()) {
1041  SDNode *N = Nodes.pop_back_val();
1042  for (auto *U : N->uses()) {
1043  auto UId = U->getNodeId();
1044  if (UId > 0) {
1045  InvalidateNodeId(U);
1046  Nodes.push_back(U);
1047  }
1048  }
1049  }
1050 }
1051 
1052 // InvalidateNodeId - As explained in EnforceNodeIdInvariant, mark a
1053 // NodeId with the equivalent node id which is invalid for topological
1054 // pruning.
1056  int InvalidId = -(N->getNodeId() + 1);
1057  N->setNodeId(InvalidId);
1058 }
1059 
1060 // getUninvalidatedNodeId - get original uninvalidated node id.
1062  int Id = N->getNodeId();
1063  if (Id < -1)
1064  return -(Id + 1);
1065  return Id;
1066 }
1067 
1068 void SelectionDAGISel::DoInstructionSelection() {
1069  LLVM_DEBUG(dbgs() << "===== Instruction selection begins: "
1070  << printMBBReference(*FuncInfo->MBB) << " '"
1071  << FuncInfo->MBB->getName() << "'\n");
1072 
1074 
1075  // Select target instructions for the DAG.
1076  {
1077  // Number all nodes with a topological order and set DAGSize.
1079 
1080  // Create a dummy node (which is not added to allnodes), that adds
1081  // a reference to the root node, preventing it from being deleted,
1082  // and tracking any changes of the root.
1085  ++ISelPosition;
1086 
1087  // Make sure that ISelPosition gets properly updated when nodes are deleted
1088  // in calls made from this function. New nodes inherit relevant metadata.
1089  ISelUpdater ISU(*CurDAG, ISelPosition);
1090 
1091  // The AllNodes list is now topological-sorted. Visit the
1092  // nodes by starting at the end of the list (the root of the
1093  // graph) and preceding back toward the beginning (the entry
1094  // node).
1095  while (ISelPosition != CurDAG->allnodes_begin()) {
1096  SDNode *Node = &*--ISelPosition;
1097  // Skip dead nodes. DAGCombiner is expected to eliminate all dead nodes,
1098  // but there are currently some corner cases that it misses. Also, this
1099  // makes it theoretically possible to disable the DAGCombiner.
1100  if (Node->use_empty())
1101  continue;
1102 
1103 #ifndef NDEBUG
1105  Nodes.push_back(Node);
1106 
1107  while (!Nodes.empty()) {
1108  auto N = Nodes.pop_back_val();
1109  if (N->getOpcode() == ISD::TokenFactor || N->getNodeId() < 0)
1110  continue;
1111  for (const SDValue &Op : N->op_values()) {
1112  if (Op->getOpcode() == ISD::TokenFactor)
1113  Nodes.push_back(Op.getNode());
1114  else {
1115  // We rely on topological ordering of node ids for checking for
1116  // cycles when fusing nodes during selection. All unselected nodes
1117  // successors of an already selected node should have a negative id.
1118  // This assertion will catch such cases. If this assertion triggers
1119  // it is likely you using DAG-level Value/Node replacement functions
1120  // (versus equivalent ISEL replacement) in backend-specific
1121  // selections. See comment in EnforceNodeIdInvariant for more
1122  // details.
1123  assert(Op->getNodeId() != -1 &&
1124  "Node has already selected predecessor node");
1125  }
1126  }
1127  }
1128 #endif
1129 
1130  // When we are using non-default rounding modes or FP exception behavior
1131  // FP operations are represented by StrictFP pseudo-operations. For
1132  // targets that do not (yet) understand strict FP operations directly,
1133  // we convert them to normal FP opcodes instead at this point. This
1134  // will allow them to be handled by existing target-specific instruction
1135  // selectors.
1136  if (!TLI->isStrictFPEnabled() && Node->isStrictFPOpcode()) {
1137  // For some opcodes, we need to call TLI->getOperationAction using
1138  // the first operand type instead of the result type. Note that this
1139  // must match what SelectionDAGLegalize::LegalizeOp is doing.
1140  EVT ActionVT;
1141  switch (Node->getOpcode()) {
1144  case ISD::STRICT_LRINT:
1145  case ISD::STRICT_LLRINT:
1146  case ISD::STRICT_LROUND:
1147  case ISD::STRICT_LLROUND:
1148  case ISD::STRICT_FSETCC:
1149  case ISD::STRICT_FSETCCS:
1150  ActionVT = Node->getOperand(1).getValueType();
1151  break;
1152  default:
1153  ActionVT = Node->getValueType(0);
1154  break;
1155  }
1156  if (TLI->getOperationAction(Node->getOpcode(), ActionVT)
1158  Node = CurDAG->mutateStrictFPToFP(Node);
1159  }
1160 
1161  LLVM_DEBUG(dbgs() << "\nISEL: Starting selection on root node: ";
1162  Node->dump(CurDAG));
1163 
1164  Select(Node);
1165  }
1166 
1167  CurDAG->setRoot(Dummy.getValue());
1168  }
1169 
1170  LLVM_DEBUG(dbgs() << "\n===== Instruction selection ends:\n");
1171 
1173 }
1174 
1176  for (const User *U : CPI->users()) {
1177  if (const IntrinsicInst *EHPtrCall = dyn_cast<IntrinsicInst>(U)) {
1178  Intrinsic::ID IID = EHPtrCall->getIntrinsicID();
1179  if (IID == Intrinsic::eh_exceptionpointer ||
1180  IID == Intrinsic::eh_exceptioncode)
1181  return true;
1182  }
1183  }
1184  return false;
1185 }
1186 
1187 // wasm.landingpad.index intrinsic is for associating a landing pad index number
1188 // with a catchpad instruction. Retrieve the landing pad index in the intrinsic
1189 // and store the mapping in the function.
1191  const CatchPadInst *CPI) {
1192  MachineFunction *MF = MBB->getParent();
1193  // In case of single catch (...), we don't emit LSDA, so we don't need
1194  // this information.
1195  bool IsSingleCatchAllClause =
1196  CPI->arg_size() == 1 &&
1197  cast<Constant>(CPI->getArgOperand(0))->isNullValue();
1198  // cathchpads for longjmp use an empty type list, e.g. catchpad within %0 []
1199  // and they don't need LSDA info
1200  bool IsCatchLongjmp = CPI->arg_size() == 0;
1201  if (!IsSingleCatchAllClause && !IsCatchLongjmp) {
1202  // Create a mapping from landing pad label to landing pad index.
1203  bool IntrFound = false;
1204  for (const User *U : CPI->users()) {
1205  if (const auto *Call = dyn_cast<IntrinsicInst>(U)) {
1206  Intrinsic::ID IID = Call->getIntrinsicID();
1207  if (IID == Intrinsic::wasm_landingpad_index) {
1208  Value *IndexArg = Call->getArgOperand(1);
1209  int Index = cast<ConstantInt>(IndexArg)->getZExtValue();
1211  IntrFound = true;
1212  break;
1213  }
1214  }
1215  }
1216  assert(IntrFound && "wasm.landingpad.index intrinsic not found!");
1217  (void)IntrFound;
1218  }
1219 }
1220 
1221 /// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and
1222 /// do other setup for EH landing-pad blocks.
1223 bool SelectionDAGISel::PrepareEHLandingPad() {
1224  MachineBasicBlock *MBB = FuncInfo->MBB;
1225  const Constant *PersonalityFn = FuncInfo->Fn->getPersonalityFn();
1226  const BasicBlock *LLVMBB = MBB->getBasicBlock();
1227  const TargetRegisterClass *PtrRC =
1229 
1230  auto Pers = classifyEHPersonality(PersonalityFn);
1231 
1232  // Catchpads have one live-in register, which typically holds the exception
1233  // pointer or code.
1234  if (isFuncletEHPersonality(Pers)) {
1235  if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI())) {
1236  if (hasExceptionPointerOrCodeUser(CPI)) {
1237  // Get or create the virtual register to hold the pointer or code. Mark
1238  // the live in physreg and copy into the vreg.
1239  MCPhysReg EHPhysReg = TLI->getExceptionPointerRegister(PersonalityFn);
1240  assert(EHPhysReg && "target lacks exception pointer register");
1241  MBB->addLiveIn(EHPhysReg);
1242  unsigned VReg = FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC);
1243  BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(),
1244  TII->get(TargetOpcode::COPY), VReg)
1245  .addReg(EHPhysReg, RegState::Kill);
1246  }
1247  }
1248  return true;
1249  }
1250 
1251  // Add a label to mark the beginning of the landing pad. Deletion of the
1252  // landing pad can thus be detected via the MachineModuleInfo.
1254 
1255  const MCInstrDesc &II = TII->get(TargetOpcode::EH_LABEL);
1256  BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II)
1257  .addSym(Label);
1258 
1259  // If the unwinder does not preserve all registers, ensure that the
1260  // function marks the clobbered registers as used.
1262  if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
1264 
1265  if (Pers == EHPersonality::Wasm_CXX) {
1266  if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI()))
1268  } else {
1269  // Assign the call site to the landing pad's begin label.
1270  MF->setCallSiteLandingPad(Label, SDB->LPadToCallSiteMap[MBB]);
1271  // Mark exception register as live in.
1272  if (unsigned Reg = TLI->getExceptionPointerRegister(PersonalityFn))
1273  FuncInfo->ExceptionPointerVirtReg = MBB->addLiveIn(Reg, PtrRC);
1274  // Mark exception selector register as live in.
1275  if (unsigned Reg = TLI->getExceptionSelectorRegister(PersonalityFn))
1276  FuncInfo->ExceptionSelectorVirtReg = MBB->addLiveIn(Reg, PtrRC);
1277  }
1278 
1279  return true;
1280 }
1281 
1282 /// isFoldedOrDeadInstruction - Return true if the specified instruction is
1283 /// side-effect free and is either dead or folded into a generated instruction.
1284 /// Return false if it needs to be emitted.
1286  const FunctionLoweringInfo &FuncInfo) {
1287  return !I->mayWriteToMemory() && // Side-effecting instructions aren't folded.
1288  !I->isTerminator() && // Terminators aren't folded.
1289  !isa<DbgInfoIntrinsic>(I) && // Debug instructions aren't folded.
1290  !I->isEHPad() && // EH pad instructions aren't folded.
1291  !FuncInfo.isExportedInst(I); // Exported instrs must be computed.
1292 }
1293 
1294 /// Collect llvm.dbg.declare information. This is done after argument lowering
1295 /// in case the declarations refer to arguments.
1297  MachineFunction *MF = FuncInfo.MF;
1298  const DataLayout &DL = MF->getDataLayout();
1299  for (const BasicBlock &BB : *FuncInfo.Fn) {
1300  for (const Instruction &I : BB) {
1301  const DbgDeclareInst *DI = dyn_cast<DbgDeclareInst>(&I);
1302  if (!DI)
1303  continue;
1304 
1305  assert(DI->getVariable() && "Missing variable");
1306  assert(DI->getDebugLoc() && "Missing location");
1307  const Value *Address = DI->getAddress();
1308  if (!Address) {
1309  LLVM_DEBUG(dbgs() << "processDbgDeclares skipping " << *DI
1310  << " (bad address)\n");
1311  continue;
1312  }
1313 
1314  // Look through casts and constant offset GEPs. These mostly come from
1315  // inalloca.
1316  APInt Offset(DL.getTypeSizeInBits(Address->getType()), 0);
1317  Address = Address->stripAndAccumulateInBoundsConstantOffsets(DL, Offset);
1318 
1319  // Check if the variable is a static alloca or a byval or inalloca
1320  // argument passed in memory. If it is not, then we will ignore this
1321  // intrinsic and handle this during isel like dbg.value.
1322  int FI = std::numeric_limits<int>::max();
1323  if (const auto *AI = dyn_cast<AllocaInst>(Address)) {
1324  auto SI = FuncInfo.StaticAllocaMap.find(AI);
1325  if (SI != FuncInfo.StaticAllocaMap.end())
1326  FI = SI->second;
1327  } else if (const auto *Arg = dyn_cast<Argument>(Address))
1328  FI = FuncInfo.getArgumentFrameIndex(Arg);
1329 
1330  if (FI == std::numeric_limits<int>::max())
1331  continue;
1332 
1333  DIExpression *Expr = DI->getExpression();
1334  if (Offset.getBoolValue())
1336  Offset.getZExtValue());
1337  LLVM_DEBUG(dbgs() << "processDbgDeclares: setVariableDbgInfo FI=" << FI
1338  << ", " << *DI << "\n");
1339  MF->setVariableDbgInfo(DI->getVariable(), Expr, FI, DI->getDebugLoc());
1340  }
1341  }
1342 }
1343 
1344 void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
1345  FastISelFailed = false;
1346  // Initialize the Fast-ISel state, if needed.
1347  FastISel *FastIS = nullptr;
1348  if (TM.Options.EnableFastISel) {
1349  LLVM_DEBUG(dbgs() << "Enabling fast-isel\n");
1350  FastIS = TLI->createFastISel(*FuncInfo, LibInfo);
1351  if (FastIS)
1353  }
1354 
1356 
1357  // Lower arguments up front. An RPO iteration always visits the entry block
1358  // first.
1359  assert(*RPOT.begin() == &Fn.getEntryBlock());
1360  ++NumEntryBlocks;
1361 
1362  // Set up FuncInfo for ISel. Entry blocks never have PHIs.
1363  FuncInfo->MBB = FuncInfo->MBBMap[&Fn.getEntryBlock()];
1364  FuncInfo->InsertPt = FuncInfo->MBB->begin();
1365 
1367 
1368  if (!FastIS) {
1369  LowerArguments(Fn);
1370  } else {
1371  // See if fast isel can lower the arguments.
1372  FastIS->startNewBlock();
1373  if (!FastIS->lowerArguments()) {
1374  FastISelFailed = true;
1375  // Fast isel failed to lower these arguments
1376  ++NumFastIselFailLowerArguments;
1377 
1378  OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1379  Fn.getSubprogram(),
1380  &Fn.getEntryBlock());
1381  R << "FastISel didn't lower all arguments: "
1382  << ore::NV("Prototype", Fn.getType());
1384 
1385  // Use SelectionDAG argument lowering
1386  LowerArguments(Fn);
1387  CurDAG->setRoot(SDB->getControlRoot());
1388  SDB->clear();
1389  CodeGenAndEmitDAG();
1390  }
1391 
1392  // If we inserted any instructions at the beginning, make a note of
1393  // where they are, so we can be sure to emit subsequent instructions
1394  // after them.
1395  if (FuncInfo->InsertPt != FuncInfo->MBB->begin())
1396  FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));
1397  else
1398  FastIS->setLastLocalValue(nullptr);
1399  }
1400 
1401  bool Inserted = SwiftError->createEntriesInEntryBlock(SDB->getCurDebugLoc());
1402 
1403  if (FastIS && Inserted)
1404  FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));
1405 
1407 
1408  // Iterate over all basic blocks in the function.
1409  StackProtector &SP = getAnalysis<StackProtector>();
1410  for (const BasicBlock *LLVMBB : RPOT) {
1411  if (OptLevel != CodeGenOpt::None) {
1412  bool AllPredsVisited = true;
1413  for (const BasicBlock *Pred : predecessors(LLVMBB)) {
1414  if (!FuncInfo->VisitedBBs.count(Pred)) {
1415  AllPredsVisited = false;
1416  break;
1417  }
1418  }
1419 
1420  if (AllPredsVisited) {
1421  for (const PHINode &PN : LLVMBB->phis())
1422  FuncInfo->ComputePHILiveOutRegInfo(&PN);
1423  } else {
1424  for (const PHINode &PN : LLVMBB->phis())
1425  FuncInfo->InvalidatePHILiveOutRegInfo(&PN);
1426  }
1427 
1428  FuncInfo->VisitedBBs.insert(LLVMBB);
1429  }
1430 
1431  BasicBlock::const_iterator const Begin =
1432  LLVMBB->getFirstNonPHI()->getIterator();
1433  BasicBlock::const_iterator const End = LLVMBB->end();
1434  BasicBlock::const_iterator BI = End;
1435 
1436  FuncInfo->MBB = FuncInfo->MBBMap[LLVMBB];
1437  if (!FuncInfo->MBB)
1438  continue; // Some blocks like catchpads have no code or MBB.
1439 
1440  // Insert new instructions after any phi or argument setup code.
1441  FuncInfo->InsertPt = FuncInfo->MBB->end();
1442 
1443  // Setup an EH landing-pad block.
1444  FuncInfo->ExceptionPointerVirtReg = 0;
1445  FuncInfo->ExceptionSelectorVirtReg = 0;
1446  if (LLVMBB->isEHPad())
1447  if (!PrepareEHLandingPad())
1448  continue;
1449 
1450  // Before doing SelectionDAG ISel, see if FastISel has been requested.
1451  if (FastIS) {
1452  if (LLVMBB != &Fn.getEntryBlock())
1453  FastIS->startNewBlock();
1454 
1455  unsigned NumFastIselRemaining = std::distance(Begin, End);
1456 
1457  // Pre-assign swifterror vregs.
1458  SwiftError->preassignVRegs(FuncInfo->MBB, Begin, End);
1459 
1460  // Do FastISel on as many instructions as possible.
1461  for (; BI != Begin; --BI) {
1462  const Instruction *Inst = &*std::prev(BI);
1463 
1464  // If we no longer require this instruction, skip it.
1465  if (isFoldedOrDeadInstruction(Inst, *FuncInfo) ||
1466  ElidedArgCopyInstrs.count(Inst)) {
1467  --NumFastIselRemaining;
1468  continue;
1469  }
1470 
1471  // Bottom-up: reset the insert pos at the top, after any local-value
1472  // instructions.
1473  FastIS->recomputeInsertPt();
1474 
1475  // Try to select the instruction with FastISel.
1476  if (FastIS->selectInstruction(Inst)) {
1477  --NumFastIselRemaining;
1478  ++NumFastIselSuccess;
1479  // If fast isel succeeded, skip over all the folded instructions, and
1480  // then see if there is a load right before the selected instructions.
1481  // Try to fold the load if so.
1482  const Instruction *BeforeInst = Inst;
1483  while (BeforeInst != &*Begin) {
1484  BeforeInst = &*std::prev(BasicBlock::const_iterator(BeforeInst));
1485  if (!isFoldedOrDeadInstruction(BeforeInst, *FuncInfo))
1486  break;
1487  }
1488  if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&
1489  BeforeInst->hasOneUse() &&
1490  FastIS->tryToFoldLoad(cast<LoadInst>(BeforeInst), Inst)) {
1491  // If we succeeded, don't re-select the load.
1492  LLVM_DEBUG(dbgs()
1493  << "FastISel folded load: " << *BeforeInst << "\n");
1494  BI = std::next(BasicBlock::const_iterator(BeforeInst));
1495  --NumFastIselRemaining;
1496  ++NumFastIselSuccess;
1497  }
1498  continue;
1499  }
1500 
1501  FastISelFailed = true;
1502 
1503  // Then handle certain instructions as single-LLVM-Instruction blocks.
1504  // We cannot separate out GCrelocates to their own blocks since we need
1505  // to keep track of gc-relocates for a particular gc-statepoint. This is
1506  // done by SelectionDAGBuilder::LowerAsSTATEPOINT, called before
1507  // visitGCRelocate.
1508  if (isa<CallInst>(Inst) && !isa<GCStatepointInst>(Inst) &&
1509  !isa<GCRelocateInst>(Inst) && !isa<GCResultInst>(Inst)) {
1510  OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1511  Inst->getDebugLoc(), LLVMBB);
1512 
1513  R << "FastISel missed call";
1514 
1515  if (R.isEnabled() || EnableFastISelAbort) {
1516  std::string InstStrStorage;
1517  raw_string_ostream InstStr(InstStrStorage);
1518  InstStr << *Inst;
1519 
1520  R << ": " << InstStr.str();
1521  }
1522 
1524 
1525  if (!Inst->getType()->isVoidTy() && !Inst->getType()->isTokenTy() &&
1526  !Inst->use_empty()) {
1527  Register &R = FuncInfo->ValueMap[Inst];
1528  if (!R)
1529  R = FuncInfo->CreateRegs(Inst);
1530  }
1531 
1532  bool HadTailCall = false;
1533  MachineBasicBlock::iterator SavedInsertPt = FuncInfo->InsertPt;
1534  SelectBasicBlock(Inst->getIterator(), BI, HadTailCall);
1535 
1536  // If the call was emitted as a tail call, we're done with the block.
1537  // We also need to delete any previously emitted instructions.
1538  if (HadTailCall) {
1539  FastIS->removeDeadCode(SavedInsertPt, FuncInfo->MBB->end());
1540  --BI;
1541  break;
1542  }
1543 
1544  // Recompute NumFastIselRemaining as Selection DAG instruction
1545  // selection may have handled the call, input args, etc.
1546  unsigned RemainingNow = std::distance(Begin, BI);
1547  NumFastIselFailures += NumFastIselRemaining - RemainingNow;
1548  NumFastIselRemaining = RemainingNow;
1549  continue;
1550  }
1551 
1552  OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1553  Inst->getDebugLoc(), LLVMBB);
1554 
1555  bool ShouldAbort = EnableFastISelAbort;
1556  if (Inst->isTerminator()) {
1557  // Use a different message for terminator misses.
1558  R << "FastISel missed terminator";
1559  // Don't abort for terminator unless the level is really high
1560  ShouldAbort = (EnableFastISelAbort > 2);
1561  } else {
1562  R << "FastISel missed";
1563  }
1564 
1565  if (R.isEnabled() || EnableFastISelAbort) {
1566  std::string InstStrStorage;
1567  raw_string_ostream InstStr(InstStrStorage);
1568  InstStr << *Inst;
1569  R << ": " << InstStr.str();
1570  }
1571 
1572  reportFastISelFailure(*MF, *ORE, R, ShouldAbort);
1573 
1574  NumFastIselFailures += NumFastIselRemaining;
1575  break;
1576  }
1577 
1578  FastIS->recomputeInsertPt();
1579  }
1580 
1581  if (SP.shouldEmitSDCheck(*LLVMBB)) {
1582  bool FunctionBasedInstrumentation =
1584  SDB->SPDescriptor.initialize(LLVMBB, FuncInfo->MBBMap[LLVMBB],
1585  FunctionBasedInstrumentation);
1586  }
1587 
1588  if (Begin != BI)
1589  ++NumDAGBlocks;
1590  else
1591  ++NumFastIselBlocks;
1592 
1593  if (Begin != BI) {
1594  // Run SelectionDAG instruction selection on the remainder of the block
1595  // not handled by FastISel. If FastISel is not run, this is the entire
1596  // block.
1597  bool HadTailCall;
1598  SelectBasicBlock(Begin, BI, HadTailCall);
1599 
1600  // But if FastISel was run, we already selected some of the block.
1601  // If we emitted a tail-call, we need to delete any previously emitted
1602  // instruction that follows it.
1603  if (FastIS && HadTailCall && FuncInfo->InsertPt != FuncInfo->MBB->end())
1604  FastIS->removeDeadCode(FuncInfo->InsertPt, FuncInfo->MBB->end());
1605  }
1606 
1607  if (FastIS)
1608  FastIS->finishBasicBlock();
1609  FinishBasicBlock();
1610  FuncInfo->PHINodesToUpdate.clear();
1611  ElidedArgCopyInstrs.clear();
1612  }
1613 
1615 
1617 
1618  delete FastIS;
1619  SDB->clearDanglingDebugInfo();
1620  SDB->SPDescriptor.resetPerFunctionState();
1621 }
1622 
1623 void
1624 SelectionDAGISel::FinishBasicBlock() {
1625  LLVM_DEBUG(dbgs() << "Total amount of phi nodes to update: "
1626  << FuncInfo->PHINodesToUpdate.size() << "\n";
1627  for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e;
1628  ++i) dbgs()
1629  << "Node " << i << " : (" << FuncInfo->PHINodesToUpdate[i].first
1630  << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n");
1631 
1632  // Next, now that we know what the last MBB the LLVM BB expanded is, update
1633  // PHI nodes in successors.
1634  for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
1635  MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first);
1636  assert(PHI->isPHI() &&
1637  "This is not a machine PHI node that we are updating!");
1638  if (!FuncInfo->MBB->isSuccessor(PHI->getParent()))
1639  continue;
1640  PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB);
1641  }
1642 
1643  // Handle stack protector.
1644  if (SDB->SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
1645  // The target provides a guard check function. There is no need to
1646  // generate error handling code or to split current basic block.
1647  MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1648 
1649  // Add load and check to the basicblock.
1650  FuncInfo->MBB = ParentMBB;
1651  FuncInfo->InsertPt =
1652  findSplitPointForStackProtector(ParentMBB, *TII);
1653  SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
1654  CurDAG->setRoot(SDB->getRoot());
1655  SDB->clear();
1656  CodeGenAndEmitDAG();
1657 
1658  // Clear the Per-BB State.
1659  SDB->SPDescriptor.resetPerBBState();
1660  } else if (SDB->SPDescriptor.shouldEmitStackProtector()) {
1661  MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1662  MachineBasicBlock *SuccessMBB = SDB->SPDescriptor.getSuccessMBB();
1663 
1664  // Find the split point to split the parent mbb. At the same time copy all
1665  // physical registers used in the tail of parent mbb into virtual registers
1666  // before the split point and back into physical registers after the split
1667  // point. This prevents us needing to deal with Live-ins and many other
1668  // register allocation issues caused by us splitting the parent mbb. The
1669  // register allocator will clean up said virtual copies later on.
1670  MachineBasicBlock::iterator SplitPoint =
1671  findSplitPointForStackProtector(ParentMBB, *TII);
1672 
1673  // Splice the terminator of ParentMBB into SuccessMBB.
1674  SuccessMBB->splice(SuccessMBB->end(), ParentMBB,
1675  SplitPoint,
1676  ParentMBB->end());
1677 
1678  // Add compare/jump on neq/jump to the parent BB.
1679  FuncInfo->MBB = ParentMBB;
1680  FuncInfo->InsertPt = ParentMBB->end();
1681  SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
1682  CurDAG->setRoot(SDB->getRoot());
1683  SDB->clear();
1684  CodeGenAndEmitDAG();
1685 
1686  // CodeGen Failure MBB if we have not codegened it yet.
1687  MachineBasicBlock *FailureMBB = SDB->SPDescriptor.getFailureMBB();
1688  if (FailureMBB->empty()) {
1689  FuncInfo->MBB = FailureMBB;
1690  FuncInfo->InsertPt = FailureMBB->end();
1691  SDB->visitSPDescriptorFailure(SDB->SPDescriptor);
1692  CurDAG->setRoot(SDB->getRoot());
1693  SDB->clear();
1694  CodeGenAndEmitDAG();
1695  }
1696 
1697  // Clear the Per-BB State.
1698  SDB->SPDescriptor.resetPerBBState();
1699  }
1700 
1701  // Lower each BitTestBlock.
1702  for (auto &BTB : SDB->SL->BitTestCases) {
1703  // Lower header first, if it wasn't already lowered
1704  if (!BTB.Emitted) {
1705  // Set the current basic block to the mbb we wish to insert the code into
1706  FuncInfo->MBB = BTB.Parent;
1707  FuncInfo->InsertPt = FuncInfo->MBB->end();
1708  // Emit the code
1709  SDB->visitBitTestHeader(BTB, FuncInfo->MBB);
1710  CurDAG->setRoot(SDB->getRoot());
1711  SDB->clear();
1712  CodeGenAndEmitDAG();
1713  }
1714 
1715  BranchProbability UnhandledProb = BTB.Prob;
1716  for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
1717  UnhandledProb -= BTB.Cases[j].ExtraProb;
1718  // Set the current basic block to the mbb we wish to insert the code into
1719  FuncInfo->MBB = BTB.Cases[j].ThisBB;
1720  FuncInfo->InsertPt = FuncInfo->MBB->end();
1721  // Emit the code
1722 
1723  // If all cases cover a contiguous range, it is not necessary to jump to
1724  // the default block after the last bit test fails. This is because the
1725  // range check during bit test header creation has guaranteed that every
1726  // case here doesn't go outside the range. In this case, there is no need
1727  // to perform the last bit test, as it will always be true. Instead, make
1728  // the second-to-last bit-test fall through to the target of the last bit
1729  // test, and delete the last bit test.
1730 
1731  MachineBasicBlock *NextMBB;
1732  if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1733  // Second-to-last bit-test with contiguous range or omitted range
1734  // check: fall through to the target of the final bit test.
1735  NextMBB = BTB.Cases[j + 1].TargetBB;
1736  } else if (j + 1 == ej) {
1737  // For the last bit test, fall through to Default.
1738  NextMBB = BTB.Default;
1739  } else {
1740  // Otherwise, fall through to the next bit test.
1741  NextMBB = BTB.Cases[j + 1].ThisBB;
1742  }
1743 
1744  SDB->visitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j],
1745  FuncInfo->MBB);
1746 
1747  CurDAG->setRoot(SDB->getRoot());
1748  SDB->clear();
1749  CodeGenAndEmitDAG();
1750 
1751  if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1752  // Since we're not going to use the final bit test, remove it.
1753  BTB.Cases.pop_back();
1754  break;
1755  }
1756  }
1757 
1758  // Update PHI Nodes
1759  for (const std::pair<MachineInstr *, unsigned> &P :
1760  FuncInfo->PHINodesToUpdate) {
1761  MachineInstrBuilder PHI(*MF, P.first);
1762  MachineBasicBlock *PHIBB = PHI->getParent();
1763  assert(PHI->isPHI() &&
1764  "This is not a machine PHI node that we are updating!");
1765  // This is "default" BB. We have two jumps to it. From "header" BB and
1766  // from last "case" BB, unless the latter was skipped.
1767  if (PHIBB == BTB.Default) {
1768  PHI.addReg(P.second).addMBB(BTB.Parent);
1769  if (!BTB.ContiguousRange) {
1770  PHI.addReg(P.second).addMBB(BTB.Cases.back().ThisBB);
1771  }
1772  }
1773  // One of "cases" BB.
1774  for (const SwitchCG::BitTestCase &BT : BTB.Cases) {
1775  MachineBasicBlock* cBB = BT.ThisBB;
1776  if (cBB->isSuccessor(PHIBB))
1777  PHI.addReg(P.second).addMBB(cBB);
1778  }
1779  }
1780  }
1781  SDB->SL->BitTestCases.clear();
1782 
1783  // If the JumpTable record is filled in, then we need to emit a jump table.
1784  // Updating the PHI nodes is tricky in this case, since we need to determine
1785  // whether the PHI is a successor of the range check MBB or the jump table MBB
1786  for (unsigned i = 0, e = SDB->SL->JTCases.size(); i != e; ++i) {
1787  // Lower header first, if it wasn't already lowered
1788  if (!SDB->SL->JTCases[i].first.Emitted) {
1789  // Set the current basic block to the mbb we wish to insert the code into
1790  FuncInfo->MBB = SDB->SL->JTCases[i].first.HeaderBB;
1791  FuncInfo->InsertPt = FuncInfo->MBB->end();
1792  // Emit the code
1793  SDB->visitJumpTableHeader(SDB->SL->JTCases[i].second,
1794  SDB->SL->JTCases[i].first, FuncInfo->MBB);
1795  CurDAG->setRoot(SDB->getRoot());
1796  SDB->clear();
1797  CodeGenAndEmitDAG();
1798  }
1799 
1800  // Set the current basic block to the mbb we wish to insert the code into
1801  FuncInfo->MBB = SDB->SL->JTCases[i].second.MBB;
1802  FuncInfo->InsertPt = FuncInfo->MBB->end();
1803  // Emit the code
1804  SDB->visitJumpTable(SDB->SL->JTCases[i].second);
1805  CurDAG->setRoot(SDB->getRoot());
1806  SDB->clear();
1807  CodeGenAndEmitDAG();
1808 
1809  // Update PHI Nodes
1810  for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
1811  pi != pe; ++pi) {
1812  MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);
1813  MachineBasicBlock *PHIBB = PHI->getParent();
1814  assert(PHI->isPHI() &&
1815  "This is not a machine PHI node that we are updating!");
1816  // "default" BB. We can go there only from header BB.
1817  if (PHIBB == SDB->SL->JTCases[i].second.Default)
1818  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
1819  .addMBB(SDB->SL->JTCases[i].first.HeaderBB);
1820  // JT BB. Just iterate over successors here
1821  if (FuncInfo->MBB->isSuccessor(PHIBB))
1822  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(FuncInfo->MBB);
1823  }
1824  }
1825  SDB->SL->JTCases.clear();
1826 
1827  // If we generated any switch lowering information, build and codegen any
1828  // additional DAGs necessary.
1829  for (unsigned i = 0, e = SDB->SL->SwitchCases.size(); i != e; ++i) {
1830  // Set the current basic block to the mbb we wish to insert the code into
1831  FuncInfo->MBB = SDB->SL->SwitchCases[i].ThisBB;
1832  FuncInfo->InsertPt = FuncInfo->MBB->end();
1833 
1834  // Determine the unique successors.
1836  Succs.push_back(SDB->SL->SwitchCases[i].TrueBB);
1837  if (SDB->SL->SwitchCases[i].TrueBB != SDB->SL->SwitchCases[i].FalseBB)
1838  Succs.push_back(SDB->SL->SwitchCases[i].FalseBB);
1839 
1840  // Emit the code. Note that this could result in FuncInfo->MBB being split.
1841  SDB->visitSwitchCase(SDB->SL->SwitchCases[i], FuncInfo->MBB);
1842  CurDAG->setRoot(SDB->getRoot());
1843  SDB->clear();
1844  CodeGenAndEmitDAG();
1845 
1846  // Remember the last block, now that any splitting is done, for use in
1847  // populating PHI nodes in successors.
1848  MachineBasicBlock *ThisBB = FuncInfo->MBB;
1849 
1850  // Handle any PHI nodes in successors of this chunk, as if we were coming
1851  // from the original BB before switch expansion. Note that PHI nodes can
1852  // occur multiple times in PHINodesToUpdate. We have to be very careful to
1853  // handle them the right number of times.
1854  for (unsigned i = 0, e = Succs.size(); i != e; ++i) {
1855  FuncInfo->MBB = Succs[i];
1856  FuncInfo->InsertPt = FuncInfo->MBB->end();
1857  // FuncInfo->MBB may have been removed from the CFG if a branch was
1858  // constant folded.
1859  if (ThisBB->isSuccessor(FuncInfo->MBB)) {
1861  MBBI = FuncInfo->MBB->begin(), MBBE = FuncInfo->MBB->end();
1862  MBBI != MBBE && MBBI->isPHI(); ++MBBI) {
1864  // This value for this PHI node is recorded in PHINodesToUpdate.
1865  for (unsigned pn = 0; ; ++pn) {
1866  assert(pn != FuncInfo->PHINodesToUpdate.size() &&
1867  "Didn't find PHI entry!");
1868  if (FuncInfo->PHINodesToUpdate[pn].first == PHI) {
1869  PHI.addReg(FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB);
1870  break;
1871  }
1872  }
1873  }
1874  }
1875  }
1876  }
1877  SDB->SL->SwitchCases.clear();
1878 }
1879 
1880 /// Create the scheduler. If a specific scheduler was specified
1881 /// via the SchedulerRegistry, use it, otherwise select the
1882 /// one preferred by the target.
1883 ///
1884 ScheduleDAGSDNodes *SelectionDAGISel::CreateScheduler() {
1885  return ISHeuristic(this, OptLevel);
1886 }
1887 
1888 //===----------------------------------------------------------------------===//
1889 // Helper functions used by the generated instruction selector.
1890 //===----------------------------------------------------------------------===//
1891 // Calls to these methods are generated by tblgen.
1892 
1893 /// CheckAndMask - The isel is trying to match something like (and X, 255). If
1894 /// the dag combiner simplified the 255, we still want to match. RHS is the
1895 /// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value
1896 /// specified in the .td file (e.g. 255).
1898  int64_t DesiredMaskS) const {
1899  const APInt &ActualMask = RHS->getAPIntValue();
1900  const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
1901 
1902  // If the actual mask exactly matches, success!
1903  if (ActualMask == DesiredMask)
1904  return true;
1905 
1906  // If the actual AND mask is allowing unallowed bits, this doesn't match.
1907  if (!ActualMask.isSubsetOf(DesiredMask))
1908  return false;
1909 
1910  // Otherwise, the DAG Combiner may have proven that the value coming in is
1911  // either already zero or is not demanded. Check for known zero input bits.
1912  APInt NeededMask = DesiredMask & ~ActualMask;
1913  if (CurDAG->MaskedValueIsZero(LHS, NeededMask))
1914  return true;
1915 
1916  // TODO: check to see if missing bits are just not demanded.
1917 
1918  // Otherwise, this pattern doesn't match.
1919  return false;
1920 }
1921 
1922 /// CheckOrMask - The isel is trying to match something like (or X, 255). If
1923 /// the dag combiner simplified the 255, we still want to match. RHS is the
1924 /// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value
1925 /// specified in the .td file (e.g. 255).
1927  int64_t DesiredMaskS) const {
1928  const APInt &ActualMask = RHS->getAPIntValue();
1929  const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
1930 
1931  // If the actual mask exactly matches, success!
1932  if (ActualMask == DesiredMask)
1933  return true;
1934 
1935  // If the actual AND mask is allowing unallowed bits, this doesn't match.
1936  if (!ActualMask.isSubsetOf(DesiredMask))
1937  return false;
1938 
1939  // Otherwise, the DAG Combiner may have proven that the value coming in is
1940  // either already zero or is not demanded. Check for known zero input bits.
1941  APInt NeededMask = DesiredMask & ~ActualMask;
1943 
1944  // If all the missing bits in the or are already known to be set, match!
1945  if (NeededMask.isSubsetOf(Known.One))
1946  return true;
1947 
1948  // TODO: check to see if missing bits are just not demanded.
1949 
1950  // Otherwise, this pattern doesn't match.
1951  return false;
1952 }
1953 
1954 /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
1955 /// by tblgen. Others should not call it.
1957  const SDLoc &DL) {
1958  std::vector<SDValue> InOps;
1959  std::swap(InOps, Ops);
1960 
1961  Ops.push_back(InOps[InlineAsm::Op_InputChain]); // 0
1962  Ops.push_back(InOps[InlineAsm::Op_AsmString]); // 1
1963  Ops.push_back(InOps[InlineAsm::Op_MDNode]); // 2, !srcloc
1964  Ops.push_back(InOps[InlineAsm::Op_ExtraInfo]); // 3 (SideEffect, AlignStack)
1965 
1966  unsigned i = InlineAsm::Op_FirstOperand, e = InOps.size();
1967  if (InOps[e-1].getValueType() == MVT::Glue)
1968  --e; // Don't process a glue operand if it is here.
1969 
1970  while (i != e) {
1971  unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue();
1972  if (!InlineAsm::isMemKind(Flags) && !InlineAsm::isFuncKind(Flags)) {
1973  // Just skip over this operand, copying the operands verbatim.
1974  Ops.insert(Ops.end(), InOps.begin()+i,
1975  InOps.begin()+i+InlineAsm::getNumOperandRegisters(Flags) + 1);
1976  i += InlineAsm::getNumOperandRegisters(Flags) + 1;
1977  } else {
1979  "Memory operand with multiple values?");
1980 
1981  unsigned TiedToOperand;
1982  if (InlineAsm::isUseOperandTiedToDef(Flags, TiedToOperand)) {
1983  // We need the constraint ID from the operand this is tied to.
1984  unsigned CurOp = InlineAsm::Op_FirstOperand;
1985  Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
1986  for (; TiedToOperand; --TiedToOperand) {
1987  CurOp += InlineAsm::getNumOperandRegisters(Flags)+1;
1988  Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
1989  }
1990  }
1991 
1992  // Otherwise, this is a memory operand. Ask the target to select it.
1993  std::vector<SDValue> SelOps;
1994  unsigned ConstraintID = InlineAsm::getMemoryConstraintID(Flags);
1995  if (SelectInlineAsmMemoryOperand(InOps[i+1], ConstraintID, SelOps))
1996  report_fatal_error("Could not match memory address. Inline asm"
1997  " failure!");
1998 
1999  // Add this to the output node.
2000  unsigned NewFlags =
2001  InlineAsm::isMemKind(Flags)
2003  : InlineAsm::getFlagWord(InlineAsm::Kind_Func, SelOps.size());
2004  NewFlags = InlineAsm::getFlagWordForMem(NewFlags, ConstraintID);
2005  Ops.push_back(CurDAG->getTargetConstant(NewFlags, DL, MVT::i32));
2006  llvm::append_range(Ops, SelOps);
2007  i += 2;
2008  }
2009  }
2010 
2011  // Add the glue input back if present.
2012  if (e != InOps.size())
2013  Ops.push_back(InOps.back());
2014 }
2015 
2016 /// findGlueUse - Return use of MVT::Glue value produced by the specified
2017 /// SDNode.
2018 ///
2020  unsigned FlagResNo = N->getNumValues()-1;
2021  for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
2022  SDUse &Use = I.getUse();
2023  if (Use.getResNo() == FlagResNo)
2024  return Use.getUser();
2025  }
2026  return nullptr;
2027 }
2028 
2029 /// findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path
2030 /// beyond "ImmedUse". We may ignore chains as they are checked separately.
2031 static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse,
2032  bool IgnoreChains) {
2035  // Only check if we have non-immediate uses of Def.
2036  if (ImmedUse->isOnlyUserOf(Def))
2037  return false;
2038 
2039  // We don't care about paths to Def that go through ImmedUse so mark it
2040  // visited and mark non-def operands as used.
2041  Visited.insert(ImmedUse);
2042  for (const SDValue &Op : ImmedUse->op_values()) {
2043  SDNode *N = Op.getNode();
2044  // Ignore chain deps (they are validated by
2045  // HandleMergeInputChains) and immediate uses
2046  if ((Op.getValueType() == MVT::Other && IgnoreChains) || N == Def)
2047  continue;
2048  if (!Visited.insert(N).second)
2049  continue;
2050  WorkList.push_back(N);
2051  }
2052 
2053  // Initialize worklist to operands of Root.
2054  if (Root != ImmedUse) {
2055  for (const SDValue &Op : Root->op_values()) {
2056  SDNode *N = Op.getNode();
2057  // Ignore chains (they are validated by HandleMergeInputChains)
2058  if ((Op.getValueType() == MVT::Other && IgnoreChains) || N == Def)
2059  continue;
2060  if (!Visited.insert(N).second)
2061  continue;
2062  WorkList.push_back(N);
2063  }
2064  }
2065 
2066  return SDNode::hasPredecessorHelper(Def, Visited, WorkList, 0, true);
2067 }
2068 
2069 /// IsProfitableToFold - Returns true if it's profitable to fold the specific
2070 /// operand node N of U during instruction selection that starts at Root.
2072  SDNode *Root) const {
2073  if (OptLevel == CodeGenOpt::None) return false;
2074  return N.hasOneUse();
2075 }
2076 
2077 /// IsLegalToFold - Returns true if the specific operand node N of
2078 /// U can be folded during instruction selection that starts at Root.
2080  CodeGenOpt::Level OptLevel,
2081  bool IgnoreChains) {
2082  if (OptLevel == CodeGenOpt::None) return false;
2083 
2084  // If Root use can somehow reach N through a path that that doesn't contain
2085  // U then folding N would create a cycle. e.g. In the following
2086  // diagram, Root can reach N through X. If N is folded into Root, then
2087  // X is both a predecessor and a successor of U.
2088  //
2089  // [N*] //
2090  // ^ ^ //
2091  // / \ //
2092  // [U*] [X]? //
2093  // ^ ^ //
2094  // \ / //
2095  // \ / //
2096  // [Root*] //
2097  //
2098  // * indicates nodes to be folded together.
2099  //
2100  // If Root produces glue, then it gets (even more) interesting. Since it
2101  // will be "glued" together with its glue use in the scheduler, we need to
2102  // check if it might reach N.
2103  //
2104  // [N*] //
2105  // ^ ^ //
2106  // / \ //
2107  // [U*] [X]? //
2108  // ^ ^ //
2109  // \ \ //
2110  // \ | //
2111  // [Root*] | //
2112  // ^ | //
2113  // f | //
2114  // | / //
2115  // [Y] / //
2116  // ^ / //
2117  // f / //
2118  // | / //
2119  // [GU] //
2120  //
2121  // If GU (glue use) indirectly reaches N (the load), and Root folds N
2122  // (call it Fold), then X is a predecessor of GU and a successor of
2123  // Fold. But since Fold and GU are glued together, this will create
2124  // a cycle in the scheduling graph.
2125 
2126  // If the node has glue, walk down the graph to the "lowest" node in the
2127  // glueged set.
2128  EVT VT = Root->getValueType(Root->getNumValues()-1);
2129  while (VT == MVT::Glue) {
2130  SDNode *GU = findGlueUse(Root);
2131  if (!GU)
2132  break;
2133  Root = GU;
2134  VT = Root->getValueType(Root->getNumValues()-1);
2135 
2136  // If our query node has a glue result with a use, we've walked up it. If
2137  // the user (which has already been selected) has a chain or indirectly uses
2138  // the chain, HandleMergeInputChains will not consider it. Because of
2139  // this, we cannot ignore chains in this predicate.
2140  IgnoreChains = false;
2141  }
2142 
2143  return !findNonImmUse(Root, N.getNode(), U, IgnoreChains);
2144 }
2145 
2146 void SelectionDAGISel::Select_INLINEASM(SDNode *N) {
2147  SDLoc DL(N);
2148 
2149  std::vector<SDValue> Ops(N->op_begin(), N->op_end());
2151 
2152  const EVT VTs[] = {MVT::Other, MVT::Glue};
2153  SDValue New = CurDAG->getNode(N->getOpcode(), DL, VTs, Ops);
2154  New->setNodeId(-1);
2155  ReplaceUses(N, New.getNode());
2157 }
2158 
2159 void SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) {
2160  SDLoc dl(Op);
2161  MDNodeSDNode *MD = cast<MDNodeSDNode>(Op->getOperand(1));
2162  const MDString *RegStr = cast<MDString>(MD->getMD()->getOperand(0));
2163 
2164  EVT VT = Op->getValueType(0);
2165  LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT();
2166  Register Reg =
2167  TLI->getRegisterByName(RegStr->getString().data(), Ty,
2170  Op->getOperand(0), dl, Reg, Op->getValueType(0));
2171  New->setNodeId(-1);
2172  ReplaceUses(Op, New.getNode());
2174 }
2175 
2176 void SelectionDAGISel::Select_WRITE_REGISTER(SDNode *Op) {
2177  SDLoc dl(Op);
2178  MDNodeSDNode *MD = cast<MDNodeSDNode>(Op->getOperand(1));
2179  const MDString *RegStr = cast<MDString>(MD->getMD()->getOperand(0));
2180 
2181  EVT VT = Op->getOperand(2).getValueType();
2182  LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT();
2183 
2184  Register Reg = TLI->getRegisterByName(RegStr->getString().data(), Ty,
2187  Op->getOperand(0), dl, Reg, Op->getOperand(2));
2188  New->setNodeId(-1);
2189  ReplaceUses(Op, New.getNode());
2191 }
2192 
2193 void SelectionDAGISel::Select_UNDEF(SDNode *N) {
2194  CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0));
2195 }
2196 
2197 void SelectionDAGISel::Select_FREEZE(SDNode *N) {
2198  // TODO: We don't have FREEZE pseudo-instruction in MachineInstr-level now.
2199  // If FREEZE instruction is added later, the code below must be changed as
2200  // well.
2201  CurDAG->SelectNodeTo(N, TargetOpcode::COPY, N->getValueType(0),
2202  N->getOperand(0));
2203 }
2204 
2205 void SelectionDAGISel::Select_ARITH_FENCE(SDNode *N) {
2206  CurDAG->SelectNodeTo(N, TargetOpcode::ARITH_FENCE, N->getValueType(0),
2207  N->getOperand(0));
2208 }
2209 
2210 void SelectionDAGISel::pushStackMapLiveVariable(SmallVectorImpl<SDValue> &Ops,
2211  SDValue OpVal, SDLoc DL) {
2212  SDNode *OpNode = OpVal.getNode();
2213 
2214  // FrameIndex nodes should have been directly emitted to TargetFrameIndex
2215  // nodes at DAG-construction time.
2216  assert(OpNode->getOpcode() != ISD::FrameIndex);
2217 
2218  if (OpNode->getOpcode() == ISD::Constant) {
2219  Ops.push_back(
2220  CurDAG->getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
2221  Ops.push_back(
2222  CurDAG->getTargetConstant(cast<ConstantSDNode>(OpNode)->getZExtValue(),
2223  DL, OpVal.getValueType()));
2224  } else {
2225  Ops.push_back(OpVal);
2226  }
2227 }
2228 
2229 void SelectionDAGISel::Select_STACKMAP(SDNode *N) {
2231  auto *It = N->op_begin();
2232  SDLoc DL(N);
2233 
2234  // Stash the chain and glue operands so we can move them to the end.
2235  SDValue Chain = *It++;
2236  SDValue InFlag = *It++;
2237 
2238  // <id> operand.
2239  SDValue ID = *It++;
2240  assert(ID.getValueType() == MVT::i64);
2241  Ops.push_back(ID);
2242 
2243  // <numShadowBytes> operand.
2244  SDValue Shad = *It++;
2245  assert(Shad.getValueType() == MVT::i32);
2246  Ops.push_back(Shad);
2247 
2248  // Live variable operands.
2249  for (; It != N->op_end(); It++)
2250  pushStackMapLiveVariable(Ops, *It, DL);
2251 
2252  Ops.push_back(Chain);
2253  Ops.push_back(InFlag);
2254 
2256  CurDAG->SelectNodeTo(N, TargetOpcode::STACKMAP, NodeTys, Ops);
2257 }
2258 
2259 void SelectionDAGISel::Select_PATCHPOINT(SDNode *N) {
2261  auto *It = N->op_begin();
2262  SDLoc DL(N);
2263 
2264  // Cache arguments that will be moved to the end in the target node.
2265  SDValue Chain = *It++;
2266  Optional<SDValue> Glue;
2267  if (It->getValueType() == MVT::Glue)
2268  Glue = *It++;
2269  SDValue RegMask = *It++;
2270 
2271  // <id> operand.
2272  SDValue ID = *It++;
2273  assert(ID.getValueType() == MVT::i64);
2274  Ops.push_back(ID);
2275 
2276  // <numShadowBytes> operand.
2277  SDValue Shad = *It++;
2278  assert(Shad.getValueType() == MVT::i32);
2279  Ops.push_back(Shad);
2280 
2281  // Add the callee.
2282  Ops.push_back(*It++);
2283 
2284  // Add <numArgs>.
2285  SDValue NumArgs = *It++;
2286  assert(NumArgs.getValueType() == MVT::i32);
2287  Ops.push_back(NumArgs);
2288 
2289  // Calling convention.
2290  Ops.push_back(*It++);
2291 
2292  // Push the args for the call.
2293  for (uint64_t I = cast<ConstantSDNode>(NumArgs)->getZExtValue(); I != 0; I--)
2294  Ops.push_back(*It++);
2295 
2296  // Now push the live variables.
2297  for (; It != N->op_end(); It++)
2298  pushStackMapLiveVariable(Ops, *It, DL);
2299 
2300  // Finally, the regmask, chain and (if present) glue are moved to the end.
2301  Ops.push_back(RegMask);
2302  Ops.push_back(Chain);
2303  if (Glue.has_value())
2304  Ops.push_back(Glue.value());
2305 
2306  SDVTList NodeTys = N->getVTList();
2307  CurDAG->SelectNodeTo(N, TargetOpcode::PATCHPOINT, NodeTys, Ops);
2308 }
2309 
2310 /// GetVBR - decode a vbr encoding whose top bit is set.
2312 GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) {
2313  assert(Val >= 128 && "Not a VBR");
2314  Val &= 127; // Remove first vbr bit.
2315 
2316  unsigned Shift = 7;
2317  uint64_t NextBits;
2318  do {
2319  NextBits = MatcherTable[Idx++];
2320  Val |= (NextBits&127) << Shift;
2321  Shift += 7;
2322  } while (NextBits & 128);
2323 
2324  return Val;
2325 }
2326 
2327 /// When a match is complete, this method updates uses of interior chain results
2328 /// to use the new results.
2329 void SelectionDAGISel::UpdateChains(
2330  SDNode *NodeToMatch, SDValue InputChain,
2331  SmallVectorImpl<SDNode *> &ChainNodesMatched, bool isMorphNodeTo) {
2332  SmallVector<SDNode*, 4> NowDeadNodes;
2333 
2334  // Now that all the normal results are replaced, we replace the chain and
2335  // glue results if present.
2336  if (!ChainNodesMatched.empty()) {
2337  assert(InputChain.getNode() &&
2338  "Matched input chains but didn't produce a chain");
2339  // Loop over all of the nodes we matched that produced a chain result.
2340  // Replace all the chain results with the final chain we ended up with.
2341  for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
2342  SDNode *ChainNode = ChainNodesMatched[i];
2343  // If ChainNode is null, it's because we replaced it on a previous
2344  // iteration and we cleared it out of the map. Just skip it.
2345  if (!ChainNode)
2346  continue;
2347 
2348  assert(ChainNode->getOpcode() != ISD::DELETED_NODE &&
2349  "Deleted node left in chain");
2350 
2351  // Don't replace the results of the root node if we're doing a
2352  // MorphNodeTo.
2353  if (ChainNode == NodeToMatch && isMorphNodeTo)
2354  continue;
2355 
2356  SDValue ChainVal = SDValue(ChainNode, ChainNode->getNumValues()-1);
2357  if (ChainVal.getValueType() == MVT::Glue)
2358  ChainVal = ChainVal.getValue(ChainVal->getNumValues()-2);
2359  assert(ChainVal.getValueType() == MVT::Other && "Not a chain?");
2361  *CurDAG, [&](SDNode *N, SDNode *E) {
2362  std::replace(ChainNodesMatched.begin(), ChainNodesMatched.end(), N,
2363  static_cast<SDNode *>(nullptr));
2364  });
2365  if (ChainNode->getOpcode() != ISD::TokenFactor)
2366  ReplaceUses(ChainVal, InputChain);
2367 
2368  // If the node became dead and we haven't already seen it, delete it.
2369  if (ChainNode != NodeToMatch && ChainNode->use_empty() &&
2370  !llvm::is_contained(NowDeadNodes, ChainNode))
2371  NowDeadNodes.push_back(ChainNode);
2372  }
2373  }
2374 
2375  if (!NowDeadNodes.empty())
2376  CurDAG->RemoveDeadNodes(NowDeadNodes);
2377 
2378  LLVM_DEBUG(dbgs() << "ISEL: Match complete!\n");
2379 }
2380 
2381 /// HandleMergeInputChains - This implements the OPC_EmitMergeInputChains
2382 /// operation for when the pattern matched at least one node with a chains. The
2383 /// input vector contains a list of all of the chained nodes that we match. We
2384 /// must determine if this is a valid thing to cover (i.e. matching it won't
2385 /// induce cycles in the DAG) and if so, creating a TokenFactor node. that will
2386 /// be used as the input node chain for the generated nodes.
2387 static SDValue
2389  SelectionDAG *CurDAG) {
2390 
2393  SmallVector<SDValue, 3> InputChains;
2394  unsigned int Max = 8192;
2395 
2396  // Quick exit on trivial merge.
2397  if (ChainNodesMatched.size() == 1)
2398  return ChainNodesMatched[0]->getOperand(0);
2399 
2400  // Add chains that aren't already added (internal). Peek through
2401  // token factors.
2402  std::function<void(const SDValue)> AddChains = [&](const SDValue V) {
2403  if (V.getValueType() != MVT::Other)
2404  return;
2405  if (V->getOpcode() == ISD::EntryToken)
2406  return;
2407  if (!Visited.insert(V.getNode()).second)
2408  return;
2409  if (V->getOpcode() == ISD::TokenFactor) {
2410  for (const SDValue &Op : V->op_values())
2411  AddChains(Op);
2412  } else
2413  InputChains.push_back(V);
2414  };
2415 
2416  for (auto *N : ChainNodesMatched) {
2417  Worklist.push_back(N);
2418  Visited.insert(N);
2419  }
2420 
2421  while (!Worklist.empty())
2422  AddChains(Worklist.pop_back_val()->getOperand(0));
2423 
2424  // Skip the search if there are no chain dependencies.
2425  if (InputChains.size() == 0)
2426  return CurDAG->getEntryNode();
2427 
2428  // If one of these chains is a successor of input, we must have a
2429  // node that is both the predecessor and successor of the
2430  // to-be-merged nodes. Fail.
2431  Visited.clear();
2432  for (SDValue V : InputChains)
2433  Worklist.push_back(V.getNode());
2434 
2435  for (auto *N : ChainNodesMatched)
2436  if (SDNode::hasPredecessorHelper(N, Visited, Worklist, Max, true))
2437  return SDValue();
2438 
2439  // Return merged chain.
2440  if (InputChains.size() == 1)
2441  return InputChains[0];
2442  return CurDAG->getNode(ISD::TokenFactor, SDLoc(ChainNodesMatched[0]),
2443  MVT::Other, InputChains);
2444 }
2445 
2446 /// MorphNode - Handle morphing a node in place for the selector.
2447 SDNode *SelectionDAGISel::
2448 MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
2449  ArrayRef<SDValue> Ops, unsigned EmitNodeInfo) {
2450  // It is possible we're using MorphNodeTo to replace a node with no
2451  // normal results with one that has a normal result (or we could be
2452  // adding a chain) and the input could have glue and chains as well.
2453  // In this case we need to shift the operands down.
2454  // FIXME: This is a horrible hack and broken in obscure cases, no worse
2455  // than the old isel though.
2456  int OldGlueResultNo = -1, OldChainResultNo = -1;
2457 
2458  unsigned NTMNumResults = Node->getNumValues();
2459  if (Node->getValueType(NTMNumResults-1) == MVT::Glue) {
2460  OldGlueResultNo = NTMNumResults-1;
2461  if (NTMNumResults != 1 &&
2462  Node->getValueType(NTMNumResults-2) == MVT::Other)
2463  OldChainResultNo = NTMNumResults-2;
2464  } else if (Node->getValueType(NTMNumResults-1) == MVT::Other)
2465  OldChainResultNo = NTMNumResults-1;
2466 
2467  // Call the underlying SelectionDAG routine to do the transmogrification. Note
2468  // that this deletes operands of the old node that become dead.
2469  SDNode *Res = CurDAG->MorphNodeTo(Node, ~TargetOpc, VTList, Ops);
2470 
2471  // MorphNodeTo can operate in two ways: if an existing node with the
2472  // specified operands exists, it can just return it. Otherwise, it
2473  // updates the node in place to have the requested operands.
2474  if (Res == Node) {
2475  // If we updated the node in place, reset the node ID. To the isel,
2476  // this should be just like a newly allocated machine node.
2477  Res->setNodeId(-1);
2478  }
2479 
2480  unsigned ResNumResults = Res->getNumValues();
2481  // Move the glue if needed.
2482  if ((EmitNodeInfo & OPFL_GlueOutput) && OldGlueResultNo != -1 &&
2483  (unsigned)OldGlueResultNo != ResNumResults-1)
2484  ReplaceUses(SDValue(Node, OldGlueResultNo),
2485  SDValue(Res, ResNumResults - 1));
2486 
2487  if ((EmitNodeInfo & OPFL_GlueOutput) != 0)
2488  --ResNumResults;
2489 
2490  // Move the chain reference if needed.
2491  if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 &&
2492  (unsigned)OldChainResultNo != ResNumResults-1)
2493  ReplaceUses(SDValue(Node, OldChainResultNo),
2494  SDValue(Res, ResNumResults - 1));
2495 
2496  // Otherwise, no replacement happened because the node already exists. Replace
2497  // Uses of the old node with the new one.
2498  if (Res != Node) {
2499  ReplaceNode(Node, Res);
2500  } else {
2502  }
2503 
2504  return Res;
2505 }
2506 
2507 /// CheckSame - Implements OP_CheckSame.
2508 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2509 CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2510  const SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes) {
2511  // Accept if it is exactly the same as a previously recorded node.
2512  unsigned RecNo = MatcherTable[MatcherIndex++];
2513  assert(RecNo < RecordedNodes.size() && "Invalid CheckSame");
2514  return N == RecordedNodes[RecNo].first;
2515 }
2516 
2517 /// CheckChildSame - Implements OP_CheckChildXSame.
2519  const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2520  const SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes,
2521  unsigned ChildNo) {
2522  if (ChildNo >= N.getNumOperands())
2523  return false; // Match fails if out of range child #.
2524  return ::CheckSame(MatcherTable, MatcherIndex, N.getOperand(ChildNo),
2525  RecordedNodes);
2526 }
2527 
2528 /// CheckPatternPredicate - Implements OP_CheckPatternPredicate.
2529 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2530 CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2531  const SelectionDAGISel &SDISel) {
2532  return SDISel.CheckPatternPredicate(MatcherTable[MatcherIndex++]);
2533 }
2534 
2535 /// CheckNodePredicate - Implements OP_CheckNodePredicate.
2536 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2537 CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2538  const SelectionDAGISel &SDISel, SDNode *N) {
2539  return SDISel.CheckNodePredicate(N, MatcherTable[MatcherIndex++]);
2540 }
2541 
2542 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2543 CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2544  SDNode *N) {
2545  uint16_t Opc = MatcherTable[MatcherIndex++];
2546  Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
2547  return N->getOpcode() == Opc;
2548 }
2549 
2550 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2551 CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2552  const TargetLowering *TLI, const DataLayout &DL) {
2553  MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2554  if (N.getValueType() == VT) return true;
2555 
2556  // Handle the case when VT is iPTR.
2557  return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(DL);
2558 }
2559 
2560 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2561 CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2562  SDValue N, const TargetLowering *TLI, const DataLayout &DL,
2563  unsigned ChildNo) {
2564  if (ChildNo >= N.getNumOperands())
2565  return false; // Match fails if out of range child #.
2566  return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI,
2567  DL);
2568 }
2569 
2570 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2571 CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2572  SDValue N) {
2573  return cast<CondCodeSDNode>(N)->get() ==
2574  (ISD::CondCode)MatcherTable[MatcherIndex++];
2575 }
2576 
2577 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2578 CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2579  SDValue N) {
2580  if (2 >= N.getNumOperands())
2581  return false;
2582  return ::CheckCondCode(MatcherTable, MatcherIndex, N.getOperand(2));
2583 }
2584 
2585 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2586 CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2587  SDValue N, const TargetLowering *TLI, const DataLayout &DL) {
2588  MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2589  if (cast<VTSDNode>(N)->getVT() == VT)
2590  return true;
2591 
2592  // Handle the case when VT is iPTR.
2593  return VT == MVT::iPTR && cast<VTSDNode>(N)->getVT() == TLI->getPointerTy(DL);
2594 }
2595 
2596 // Bit 0 stores the sign of the immediate. The upper bits contain the magnitude
2597 // shifted left by 1.
2599  if ((V & 1) == 0)
2600  return V >> 1;
2601  if (V != 1)
2602  return -(V >> 1);
2603  // There is no such thing as -0 with integers. "-0" really means MININT.
2604  return 1ULL << 63;
2605 }
2606 
2607 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2608 CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2609  SDValue N) {
2610  int64_t Val = MatcherTable[MatcherIndex++];
2611  if (Val & 128)
2612  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2613 
2614  Val = decodeSignRotatedValue(Val);
2615 
2616  ConstantSDNode *C = dyn_cast<ConstantSDNode>(N);
2617  return C && C->getSExtValue() == Val;
2618 }
2619 
2620 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2621 CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2622  SDValue N, unsigned ChildNo) {
2623  if (ChildNo >= N.getNumOperands())
2624  return false; // Match fails if out of range child #.
2625  return ::CheckInteger(MatcherTable, MatcherIndex, N.getOperand(ChildNo));
2626 }
2627 
2628 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2629 CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2630  SDValue N, const SelectionDAGISel &SDISel) {
2631  int64_t Val = MatcherTable[MatcherIndex++];
2632  if (Val & 128)
2633  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2634 
2635  if (N->getOpcode() != ISD::AND) return false;
2636 
2637  ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2638  return C && SDISel.CheckAndMask(N.getOperand(0), C, Val);
2639 }
2640 
2641 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2642 CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2643  const SelectionDAGISel &SDISel) {
2644  int64_t Val = MatcherTable[MatcherIndex++];
2645  if (Val & 128)
2646  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2647 
2648  if (N->getOpcode() != ISD::OR) return false;
2649 
2650  ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2651  return C && SDISel.CheckOrMask(N.getOperand(0), C, Val);
2652 }
2653 
2654 /// IsPredicateKnownToFail - If we know how and can do so without pushing a
2655 /// scope, evaluate the current node. If the current predicate is known to
2656 /// fail, set Result=true and return anything. If the current predicate is
2657 /// known to pass, set Result=false and return the MatcherIndex to continue
2658 /// with. If the current predicate is unknown, set Result=false and return the
2659 /// MatcherIndex to continue with.
2660 static unsigned IsPredicateKnownToFail(const unsigned char *Table,
2661  unsigned Index, SDValue N,
2662  bool &Result,
2663  const SelectionDAGISel &SDISel,
2664  SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) {
2665  switch (Table[Index++]) {
2666  default:
2667  Result = false;
2668  return Index-1; // Could not evaluate this predicate.
2670  Result = !::CheckSame(Table, Index, N, RecordedNodes);
2671  return Index;
2676  Result = !::CheckChildSame(Table, Index, N, RecordedNodes,
2678  return Index;
2680  Result = !::CheckPatternPredicate(Table, Index, SDISel);
2681  return Index;
2683  Result = !::CheckNodePredicate(Table, Index, SDISel, N.getNode());
2684  return Index;
2686  Result = !::CheckOpcode(Table, Index, N.getNode());
2687  return Index;
2689  Result = !::CheckType(Table, Index, N, SDISel.TLI,
2690  SDISel.CurDAG->getDataLayout());
2691  return Index;
2693  unsigned Res = Table[Index++];
2694  Result = !::CheckType(Table, Index, N.getValue(Res), SDISel.TLI,
2695  SDISel.CurDAG->getDataLayout());
2696  return Index;
2697  }
2706  Result = !::CheckChildType(
2707  Table, Index, N, SDISel.TLI, SDISel.CurDAG->getDataLayout(),
2709  return Index;
2711  Result = !::CheckCondCode(Table, Index, N);
2712  return Index;
2714  Result = !::CheckChild2CondCode(Table, Index, N);
2715  return Index;
2717  Result = !::CheckValueType(Table, Index, N, SDISel.TLI,
2718  SDISel.CurDAG->getDataLayout());
2719  return Index;
2721  Result = !::CheckInteger(Table, Index, N);
2722  return Index;
2728  Result = !::CheckChildInteger(Table, Index, N,
2730  return Index;
2732  Result = !::CheckAndImm(Table, Index, N, SDISel);
2733  return Index;
2735  Result = !::CheckOrImm(Table, Index, N, SDISel);
2736  return Index;
2737  }
2738 }
2739 
2740 namespace {
2741 
2742 struct MatchScope {
2743  /// FailIndex - If this match fails, this is the index to continue with.
2744  unsigned FailIndex;
2745 
2746  /// NodeStack - The node stack when the scope was formed.
2747  SmallVector<SDValue, 4> NodeStack;
2748 
2749  /// NumRecordedNodes - The number of recorded nodes when the scope was formed.
2750  unsigned NumRecordedNodes;
2751 
2752  /// NumMatchedMemRefs - The number of matched memref entries.
2753  unsigned NumMatchedMemRefs;
2754 
2755  /// InputChain/InputGlue - The current chain/glue
2756  SDValue InputChain, InputGlue;
2757 
2758  /// HasChainNodesMatched - True if the ChainNodesMatched list is non-empty.
2759  bool HasChainNodesMatched;
2760 };
2761 
2762 /// \A DAG update listener to keep the matching state
2763 /// (i.e. RecordedNodes and MatchScope) uptodate if the target is allowed to
2764 /// change the DAG while matching. X86 addressing mode matcher is an example
2765 /// for this.
2766 class MatchStateUpdater : public SelectionDAG::DAGUpdateListener
2767 {
2768  SDNode **NodeToMatch;
2770  SmallVectorImpl<MatchScope> &MatchScopes;
2771 
2772 public:
2773  MatchStateUpdater(SelectionDAG &DAG, SDNode **NodeToMatch,
2774  SmallVectorImpl<std::pair<SDValue, SDNode *>> &RN,
2776  : SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch),
2777  RecordedNodes(RN), MatchScopes(MS) {}
2778 
2779  void NodeDeleted(SDNode *N, SDNode *E) override {
2780  // Some early-returns here to avoid the search if we deleted the node or
2781  // if the update comes from MorphNodeTo (MorphNodeTo is the last thing we
2782  // do, so it's unnecessary to update matching state at that point).
2783  // Neither of these can occur currently because we only install this
2784  // update listener during matching a complex patterns.
2785  if (!E || E->isMachineOpcode())
2786  return;
2787  // Check if NodeToMatch was updated.
2788  if (N == *NodeToMatch)
2789  *NodeToMatch = E;
2790  // Performing linear search here does not matter because we almost never
2791  // run this code. You'd have to have a CSE during complex pattern
2792  // matching.
2793  for (auto &I : RecordedNodes)
2794  if (I.first.getNode() == N)
2795  I.first.setNode(E);
2796 
2797  for (auto &I : MatchScopes)
2798  for (auto &J : I.NodeStack)
2799  if (J.getNode() == N)
2800  J.setNode(E);
2801  }
2802 };
2803 
2804 } // end anonymous namespace
2805 
2807  const unsigned char *MatcherTable,
2808  unsigned TableSize) {
2809  // FIXME: Should these even be selected? Handle these cases in the caller?
2810  switch (NodeToMatch->getOpcode()) {
2811  default:
2812  break;
2813  case ISD::EntryToken: // These nodes remain the same.
2814  case ISD::BasicBlock:
2815  case ISD::Register:
2816  case ISD::RegisterMask:
2817  case ISD::HANDLENODE:
2818  case ISD::MDNODE_SDNODE:
2819  case ISD::TargetConstant:
2820  case ISD::TargetConstantFP:
2822  case ISD::TargetFrameIndex:
2824  case ISD::MCSymbol:
2826  case ISD::TargetJumpTable:
2829  case ISD::TokenFactor:
2830  case ISD::CopyFromReg:
2831  case ISD::CopyToReg:
2832  case ISD::EH_LABEL:
2833  case ISD::ANNOTATION_LABEL:
2834  case ISD::LIFETIME_START:
2835  case ISD::LIFETIME_END:
2836  case ISD::PSEUDO_PROBE:
2837  NodeToMatch->setNodeId(-1); // Mark selected.
2838  return;
2839  case ISD::AssertSext:
2840  case ISD::AssertZext:
2841  case ISD::AssertAlign:
2842  ReplaceUses(SDValue(NodeToMatch, 0), NodeToMatch->getOperand(0));
2843  CurDAG->RemoveDeadNode(NodeToMatch);
2844  return;
2845  case ISD::INLINEASM:
2846  case ISD::INLINEASM_BR:
2847  Select_INLINEASM(NodeToMatch);
2848  return;
2849  case ISD::READ_REGISTER:
2850  Select_READ_REGISTER(NodeToMatch);
2851  return;
2852  case ISD::WRITE_REGISTER:
2853  Select_WRITE_REGISTER(NodeToMatch);
2854  return;
2855  case ISD::UNDEF:
2856  Select_UNDEF(NodeToMatch);
2857  return;
2858  case ISD::FREEZE:
2859  Select_FREEZE(NodeToMatch);
2860  return;
2861  case ISD::ARITH_FENCE:
2862  Select_ARITH_FENCE(NodeToMatch);
2863  return;
2864  case ISD::STACKMAP:
2865  Select_STACKMAP(NodeToMatch);
2866  return;
2867  case ISD::PATCHPOINT:
2868  Select_PATCHPOINT(NodeToMatch);
2869  return;
2870  }
2871 
2872  assert(!NodeToMatch->isMachineOpcode() && "Node already selected!");
2873 
2874  // Set up the node stack with NodeToMatch as the only node on the stack.
2875  SmallVector<SDValue, 8> NodeStack;
2876  SDValue N = SDValue(NodeToMatch, 0);
2877  NodeStack.push_back(N);
2878 
2879  // MatchScopes - Scopes used when matching, if a match failure happens, this
2880  // indicates where to continue checking.
2881  SmallVector<MatchScope, 8> MatchScopes;
2882 
2883  // RecordedNodes - This is the set of nodes that have been recorded by the
2884  // state machine. The second value is the parent of the node, or null if the
2885  // root is recorded.
2886  SmallVector<std::pair<SDValue, SDNode*>, 8> RecordedNodes;
2887 
2888  // MatchedMemRefs - This is the set of MemRef's we've seen in the input
2889  // pattern.
2890  SmallVector<MachineMemOperand*, 2> MatchedMemRefs;
2891 
2892  // These are the current input chain and glue for use when generating nodes.
2893  // Various Emit operations change these. For example, emitting a copytoreg
2894  // uses and updates these.
2895  SDValue InputChain, InputGlue;
2896 
2897  // ChainNodesMatched - If a pattern matches nodes that have input/output
2898  // chains, the OPC_EmitMergeInputChains operation is emitted which indicates
2899  // which ones they are. The result is captured into this list so that we can
2900  // update the chain results when the pattern is complete.
2901  SmallVector<SDNode*, 3> ChainNodesMatched;
2902 
2903  LLVM_DEBUG(dbgs() << "ISEL: Starting pattern match\n");
2904 
2905  // Determine where to start the interpreter. Normally we start at opcode #0,
2906  // but if the state machine starts with an OPC_SwitchOpcode, then we
2907  // accelerate the first lookup (which is guaranteed to be hot) with the
2908  // OpcodeOffset table.
2909  unsigned MatcherIndex = 0;
2910 
2911  if (!OpcodeOffset.empty()) {
2912  // Already computed the OpcodeOffset table, just index into it.
2913  if (N.getOpcode() < OpcodeOffset.size())
2914  MatcherIndex = OpcodeOffset[N.getOpcode()];
2915  LLVM_DEBUG(dbgs() << " Initial Opcode index to " << MatcherIndex << "\n");
2916 
2917  } else if (MatcherTable[0] == OPC_SwitchOpcode) {
2918  // Otherwise, the table isn't computed, but the state machine does start
2919  // with an OPC_SwitchOpcode instruction. Populate the table now, since this
2920  // is the first time we're selecting an instruction.
2921  unsigned Idx = 1;
2922  while (true) {
2923  // Get the size of this case.
2924  unsigned CaseSize = MatcherTable[Idx++];
2925  if (CaseSize & 128)
2926  CaseSize = GetVBR(CaseSize, MatcherTable, Idx);
2927  if (CaseSize == 0) break;
2928 
2929  // Get the opcode, add the index to the table.
2930  uint16_t Opc = MatcherTable[Idx++];
2931  Opc |= (unsigned short)MatcherTable[Idx++] << 8;
2932  if (Opc >= OpcodeOffset.size())
2933  OpcodeOffset.resize((Opc+1)*2);
2934  OpcodeOffset[Opc] = Idx;
2935  Idx += CaseSize;
2936  }
2937 
2938  // Okay, do the lookup for the first opcode.
2939  if (N.getOpcode() < OpcodeOffset.size())
2940  MatcherIndex = OpcodeOffset[N.getOpcode()];
2941  }
2942 
2943  while (true) {
2944  assert(MatcherIndex < TableSize && "Invalid index");
2945 #ifndef NDEBUG
2946  unsigned CurrentOpcodeIndex = MatcherIndex;
2947 #endif
2948  BuiltinOpcodes Opcode = (BuiltinOpcodes)MatcherTable[MatcherIndex++];
2949  switch (Opcode) {
2950  case OPC_Scope: {
2951  // Okay, the semantics of this operation are that we should push a scope
2952  // then evaluate the first child. However, pushing a scope only to have
2953  // the first check fail (which then pops it) is inefficient. If we can
2954  // determine immediately that the first check (or first several) will
2955  // immediately fail, don't even bother pushing a scope for them.
2956  unsigned FailIndex;
2957 
2958  while (true) {
2959  unsigned NumToSkip = MatcherTable[MatcherIndex++];
2960  if (NumToSkip & 128)
2961  NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
2962  // Found the end of the scope with no match.
2963  if (NumToSkip == 0) {
2964  FailIndex = 0;
2965  break;
2966  }
2967 
2968  FailIndex = MatcherIndex+NumToSkip;
2969 
2970  unsigned MatcherIndexOfPredicate = MatcherIndex;
2971  (void)MatcherIndexOfPredicate; // silence warning.
2972 
2973  // If we can't evaluate this predicate without pushing a scope (e.g. if
2974  // it is a 'MoveParent') or if the predicate succeeds on this node, we
2975  // push the scope and evaluate the full predicate chain.
2976  bool Result;
2977  MatcherIndex = IsPredicateKnownToFail(MatcherTable, MatcherIndex, N,
2978  Result, *this, RecordedNodes);
2979  if (!Result)
2980  break;
2981 
2982  LLVM_DEBUG(
2983  dbgs() << " Skipped scope entry (due to false predicate) at "
2984  << "index " << MatcherIndexOfPredicate << ", continuing at "
2985  << FailIndex << "\n");
2986  ++NumDAGIselRetries;
2987 
2988  // Otherwise, we know that this case of the Scope is guaranteed to fail,
2989  // move to the next case.
2990  MatcherIndex = FailIndex;
2991  }
2992 
2993  // If the whole scope failed to match, bail.
2994  if (FailIndex == 0) break;
2995 
2996  // Push a MatchScope which indicates where to go if the first child fails
2997  // to match.
2998  MatchScope NewEntry;
2999  NewEntry.FailIndex = FailIndex;
3000  NewEntry.NodeStack.append(NodeStack.begin(), NodeStack.end());
3001  NewEntry.NumRecordedNodes = RecordedNodes.size();
3002  NewEntry.NumMatchedMemRefs = MatchedMemRefs.size();
3003  NewEntry.InputChain = InputChain;
3004  NewEntry.InputGlue = InputGlue;
3005  NewEntry.HasChainNodesMatched = !ChainNodesMatched.empty();
3006  MatchScopes.push_back(NewEntry);
3007  continue;
3008  }
3009  case OPC_RecordNode: {
3010  // Remember this node, it may end up being an operand in the pattern.
3011  SDNode *Parent = nullptr;
3012  if (NodeStack.size() > 1)
3013  Parent = NodeStack[NodeStack.size()-2].getNode();
3014  RecordedNodes.push_back(std::make_pair(N, Parent));
3015  continue;
3016  }
3017 
3021  case OPC_RecordChild6: case OPC_RecordChild7: {
3022  unsigned ChildNo = Opcode-OPC_RecordChild0;
3023  if (ChildNo >= N.getNumOperands())
3024  break; // Match fails if out of range child #.
3025 
3026  RecordedNodes.push_back(std::make_pair(N->getOperand(ChildNo),
3027  N.getNode()));
3028  continue;
3029  }
3030  case OPC_RecordMemRef:
3031  if (auto *MN = dyn_cast<MemSDNode>(N))
3032  MatchedMemRefs.push_back(MN->getMemOperand());
3033  else {
3034  LLVM_DEBUG(dbgs() << "Expected MemSDNode "; N->dump(CurDAG);
3035  dbgs() << '\n');
3036  }
3037 
3038  continue;
3039 
3040  case OPC_CaptureGlueInput:
3041  // If the current node has an input glue, capture it in InputGlue.
3042  if (N->getNumOperands() != 0 &&
3043  N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue)
3044  InputGlue = N->getOperand(N->getNumOperands()-1);
3045  continue;
3046 
3047  case OPC_MoveChild: {
3048  unsigned ChildNo = MatcherTable[MatcherIndex++];
3049  if (ChildNo >= N.getNumOperands())
3050  break; // Match fails if out of range child #.
3051  N = N.getOperand(ChildNo);
3052  NodeStack.push_back(N);
3053  continue;
3054  }
3055 
3056  case OPC_MoveChild0: case OPC_MoveChild1:
3057  case OPC_MoveChild2: case OPC_MoveChild3:
3058  case OPC_MoveChild4: case OPC_MoveChild5:
3059  case OPC_MoveChild6: case OPC_MoveChild7: {
3060  unsigned ChildNo = Opcode-OPC_MoveChild0;
3061  if (ChildNo >= N.getNumOperands())
3062  break; // Match fails if out of range child #.
3063  N = N.getOperand(ChildNo);
3064  NodeStack.push_back(N);
3065  continue;
3066  }
3067 
3068  case OPC_MoveParent:
3069  // Pop the current node off the NodeStack.
3070  NodeStack.pop_back();
3071  assert(!NodeStack.empty() && "Node stack imbalance!");
3072  N = NodeStack.back();
3073  continue;
3074 
3075  case OPC_CheckSame:
3076  if (!::CheckSame(MatcherTable, MatcherIndex, N, RecordedNodes)) break;
3077  continue;
3078 
3081  if (!::CheckChildSame(MatcherTable, MatcherIndex, N, RecordedNodes,
3082  Opcode-OPC_CheckChild0Same))
3083  break;
3084  continue;
3085 
3087  if (!::CheckPatternPredicate(MatcherTable, MatcherIndex, *this)) break;
3088  continue;
3089  case OPC_CheckPredicate:
3090  if (!::CheckNodePredicate(MatcherTable, MatcherIndex, *this,
3091  N.getNode()))
3092  break;
3093  continue;
3095  unsigned OpNum = MatcherTable[MatcherIndex++];
3097 
3098  for (unsigned i = 0; i < OpNum; ++i)
3099  Operands.push_back(RecordedNodes[MatcherTable[MatcherIndex++]].first);
3100 
3101  unsigned PredNo = MatcherTable[MatcherIndex++];
3102  if (!CheckNodePredicateWithOperands(N.getNode(), PredNo, Operands))
3103  break;
3104  continue;
3105  }
3106  case OPC_CheckComplexPat: {
3107  unsigned CPNum = MatcherTable[MatcherIndex++];
3108  unsigned RecNo = MatcherTable[MatcherIndex++];
3109  assert(RecNo < RecordedNodes.size() && "Invalid CheckComplexPat");
3110 
3111  // If target can modify DAG during matching, keep the matching state
3112  // consistent.
3113  std::unique_ptr<MatchStateUpdater> MSU;
3115  MSU.reset(new MatchStateUpdater(*CurDAG, &NodeToMatch, RecordedNodes,
3116  MatchScopes));
3117 
3118  if (!CheckComplexPattern(NodeToMatch, RecordedNodes[RecNo].second,
3119  RecordedNodes[RecNo].first, CPNum,
3120  RecordedNodes))
3121  break;
3122  continue;
3123  }
3124  case OPC_CheckOpcode:
3125  if (!::CheckOpcode(MatcherTable, MatcherIndex, N.getNode())) break;
3126  continue;
3127 
3128  case OPC_CheckType:
3129  if (!::CheckType(MatcherTable, MatcherIndex, N, TLI,
3130  CurDAG->getDataLayout()))
3131  break;
3132  continue;
3133 
3134  case OPC_CheckTypeRes: {
3135  unsigned Res = MatcherTable[MatcherIndex++];
3136  if (!::CheckType(MatcherTable, MatcherIndex, N.getValue(Res), TLI,
3137  CurDAG->getDataLayout()))
3138  break;
3139  continue;
3140  }
3141 
3142  case OPC_SwitchOpcode: {
3143  unsigned CurNodeOpcode = N.getOpcode();
3144  unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3145  unsigned CaseSize;
3146  while (true) {
3147  // Get the size of this case.
3148  CaseSize = MatcherTable[MatcherIndex++];
3149  if (CaseSize & 128)
3150  CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3151  if (CaseSize == 0) break;
3152 
3153  uint16_t Opc = MatcherTable[MatcherIndex++];
3154  Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3155 
3156  // If the opcode matches, then we will execute this case.
3157  if (CurNodeOpcode == Opc)
3158  break;
3159 
3160  // Otherwise, skip over this case.
3161  MatcherIndex += CaseSize;
3162  }
3163 
3164  // If no cases matched, bail out.
3165  if (CaseSize == 0) break;
3166 
3167  // Otherwise, execute the case we found.
3168  LLVM_DEBUG(dbgs() << " OpcodeSwitch from " << SwitchStart << " to "
3169  << MatcherIndex << "\n");
3170  continue;
3171  }
3172 
3173  case OPC_SwitchType: {
3174  MVT CurNodeVT = N.getSimpleValueType();
3175  unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3176  unsigned CaseSize;
3177  while (true) {
3178  // Get the size of this case.
3179  CaseSize = MatcherTable[MatcherIndex++];
3180  if (CaseSize & 128)
3181  CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3182  if (CaseSize == 0) break;
3183 
3184  MVT CaseVT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3185  if (CaseVT == MVT::iPTR)
3186  CaseVT = TLI->getPointerTy(CurDAG->getDataLayout());
3187 
3188  // If the VT matches, then we will execute this case.
3189  if (CurNodeVT == CaseVT)
3190  break;
3191 
3192  // Otherwise, skip over this case.
3193  MatcherIndex += CaseSize;
3194  }
3195 
3196  // If no cases matched, bail out.
3197  if (CaseSize == 0) break;
3198 
3199  // Otherwise, execute the case we found.
3200  LLVM_DEBUG(dbgs() << " TypeSwitch[" << EVT(CurNodeVT).getEVTString()
3201  << "] from " << SwitchStart << " to " << MatcherIndex
3202  << '\n');
3203  continue;
3204  }
3209  if (!::CheckChildType(MatcherTable, MatcherIndex, N, TLI,
3210  CurDAG->getDataLayout(),
3211  Opcode - OPC_CheckChild0Type))
3212  break;
3213  continue;
3214  case OPC_CheckCondCode:
3215  if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break;
3216  continue;
3218  if (!::CheckChild2CondCode(MatcherTable, MatcherIndex, N)) break;
3219  continue;
3220  case OPC_CheckValueType:
3221  if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI,
3222  CurDAG->getDataLayout()))
3223  break;
3224  continue;
3225  case OPC_CheckInteger:
3226  if (!::CheckInteger(MatcherTable, MatcherIndex, N)) break;
3227  continue;
3231  if (!::CheckChildInteger(MatcherTable, MatcherIndex, N,
3232  Opcode-OPC_CheckChild0Integer)) break;
3233  continue;
3234  case OPC_CheckAndImm:
3235  if (!::CheckAndImm(MatcherTable, MatcherIndex, N, *this)) break;
3236  continue;
3237  case OPC_CheckOrImm:
3238  if (!::CheckOrImm(MatcherTable, MatcherIndex, N, *this)) break;
3239  continue;
3240  case OPC_CheckImmAllOnesV:
3241  if (!ISD::isConstantSplatVectorAllOnes(N.getNode()))
3242  break;
3243  continue;
3244  case OPC_CheckImmAllZerosV:
3245  if (!ISD::isConstantSplatVectorAllZeros(N.getNode()))
3246  break;
3247  continue;
3248 
3250  assert(NodeStack.size() != 1 && "No parent node");
3251  // Verify that all intermediate nodes between the root and this one have
3252  // a single use (ignoring chains, which are handled in UpdateChains).
3253  bool HasMultipleUses = false;
3254  for (unsigned i = 1, e = NodeStack.size()-1; i != e; ++i) {
3255  unsigned NNonChainUses = 0;
3256  SDNode *NS = NodeStack[i].getNode();
3257  for (auto UI = NS->use_begin(), UE = NS->use_end(); UI != UE; ++UI)
3258  if (UI.getUse().getValueType() != MVT::Other)
3259  if (++NNonChainUses > 1) {
3260  HasMultipleUses = true;
3261  break;
3262  }
3263  if (HasMultipleUses) break;
3264  }
3265  if (HasMultipleUses) break;
3266 
3267  // Check to see that the target thinks this is profitable to fold and that
3268  // we can fold it without inducing cycles in the graph.
3269  if (!IsProfitableToFold(N, NodeStack[NodeStack.size()-2].getNode(),
3270  NodeToMatch) ||
3271  !IsLegalToFold(N, NodeStack[NodeStack.size()-2].getNode(),
3272  NodeToMatch, OptLevel,
3273  true/*We validate our own chains*/))
3274  break;
3275 
3276  continue;
3277  }
3278  case OPC_EmitInteger:
3279  case OPC_EmitStringInteger: {
3281  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3282  int64_t Val = MatcherTable[MatcherIndex++];
3283  if (Val & 128)
3284  Val = GetVBR(Val, MatcherTable, MatcherIndex);
3285  if (Opcode == OPC_EmitInteger)
3286  Val = decodeSignRotatedValue(Val);
3287  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3288  CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch),
3289  VT), nullptr));
3290  continue;
3291  }
3292  case OPC_EmitRegister: {
3294  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3295  unsigned RegNo = MatcherTable[MatcherIndex++];
3296  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3297  CurDAG->getRegister(RegNo, VT), nullptr));
3298  continue;
3299  }
3300  case OPC_EmitRegister2: {
3301  // For targets w/ more than 256 register names, the register enum
3302  // values are stored in two bytes in the matcher table (just like
3303  // opcodes).
3305  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3306  unsigned RegNo = MatcherTable[MatcherIndex++];
3307  RegNo |= MatcherTable[MatcherIndex++] << 8;
3308  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3309  CurDAG->getRegister(RegNo, VT), nullptr));
3310  continue;
3311  }
3312 
3313  case OPC_EmitConvertToTarget: {
3314  // Convert from IMM/FPIMM to target version.
3315  unsigned RecNo = MatcherTable[MatcherIndex++];
3316  assert(RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget");
3317  SDValue Imm = RecordedNodes[RecNo].first;
3318 
3319  if (Imm->getOpcode() == ISD::Constant) {
3320  const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();
3321  Imm = CurDAG->getTargetConstant(*Val, SDLoc(NodeToMatch),
3322  Imm.getValueType());
3323  } else if (Imm->getOpcode() == ISD::ConstantFP) {
3324  const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();
3325  Imm = CurDAG->getTargetConstantFP(*Val, SDLoc(NodeToMatch),
3326  Imm.getValueType());
3327  }
3328 
3329  RecordedNodes.push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
3330  continue;
3331  }
3332 
3333  case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0
3334  case OPC_EmitMergeInputChains1_1: // OPC_EmitMergeInputChains, 1, 1
3335  case OPC_EmitMergeInputChains1_2: { // OPC_EmitMergeInputChains, 1, 2
3336  // These are space-optimized forms of OPC_EmitMergeInputChains.
3337  assert(!InputChain.getNode() &&
3338  "EmitMergeInputChains should be the first chain producing node");
3339  assert(ChainNodesMatched.empty() &&
3340  "Should only have one EmitMergeInputChains per match");
3341 
3342  // Read all of the chained nodes.
3343  unsigned RecNo = Opcode - OPC_EmitMergeInputChains1_0;
3344  assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains");
3345  ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3346 
3347  // If the chained node is not the root, we can't fold it if it has
3348  // multiple uses.
3349  // FIXME: What if other value results of the node have uses not matched
3350  // by this pattern?
3351  if (ChainNodesMatched.back() != NodeToMatch &&
3352  !RecordedNodes[RecNo].first.hasOneUse()) {
3353  ChainNodesMatched.clear();
3354  break;
3355  }
3356 
3357  // Merge the input chains if they are not intra-pattern references.
3358  InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3359 
3360  if (!InputChain.getNode())
3361  break; // Failed to merge.
3362  continue;
3363  }
3364 
3365  case OPC_EmitMergeInputChains: {
3366  assert(!InputChain.getNode() &&
3367  "EmitMergeInputChains should be the first chain producing node");
3368  // This node gets a list of nodes we matched in the input that have
3369  // chains. We want to token factor all of the input chains to these nodes
3370  // together. However, if any of the input chains is actually one of the
3371  // nodes matched in this pattern, then we have an intra-match reference.
3372  // Ignore these because the newly token factored chain should not refer to
3373  // the old nodes.
3374  unsigned NumChains = MatcherTable[MatcherIndex++];
3375  assert(NumChains != 0 && "Can't TF zero chains");
3376 
3377  assert(ChainNodesMatched.empty() &&
3378  "Should only have one EmitMergeInputChains per match");
3379 
3380  // Read all of the chained nodes.
3381  for (unsigned i = 0; i != NumChains; ++i) {
3382  unsigned RecNo = MatcherTable[MatcherIndex++];
3383  assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains");
3384  ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3385 
3386  // If the chained node is not the root, we can't fold it if it has
3387  // multiple uses.
3388  // FIXME: What if other value results of the node have uses not matched
3389  // by this pattern?
3390  if (ChainNodesMatched.back() != NodeToMatch &&
3391  !RecordedNodes[RecNo].first.hasOneUse()) {
3392  ChainNodesMatched.clear();
3393  break;
3394  }
3395  }
3396 
3397  // If the inner loop broke out, the match fails.
3398  if (ChainNodesMatched.empty())
3399  break;
3400 
3401  // Merge the input chains if they are not intra-pattern references.
3402  InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3403 
3404  if (!InputChain.getNode())
3405  break; // Failed to merge.
3406 
3407  continue;
3408  }
3409 
3410  case OPC_EmitCopyToReg:
3411  case OPC_EmitCopyToReg2: {
3412  unsigned RecNo = MatcherTable[MatcherIndex++];
3413  assert(RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg");
3414  unsigned DestPhysReg = MatcherTable[MatcherIndex++];
3415  if (Opcode == OPC_EmitCopyToReg2)
3416  DestPhysReg |= MatcherTable[MatcherIndex++] << 8;
3417 
3418  if (!InputChain.getNode())
3419  InputChain = CurDAG->getEntryNode();
3420 
3421  InputChain = CurDAG->getCopyToReg(InputChain, SDLoc(NodeToMatch),
3422  DestPhysReg, RecordedNodes[RecNo].first,
3423  InputGlue);
3424 
3425  InputGlue = InputChain.getValue(1);
3426  continue;
3427  }
3428 
3429  case OPC_EmitNodeXForm: {
3430  unsigned XFormNo = MatcherTable[MatcherIndex++];
3431  unsigned RecNo = MatcherTable[MatcherIndex++];
3432  assert(RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm");
3433  SDValue Res = RunSDNodeXForm(RecordedNodes[RecNo].first, XFormNo);
3434  RecordedNodes.push_back(std::pair<SDValue,SDNode*>(Res, nullptr));
3435  continue;
3436  }
3437  case OPC_Coverage: {
3438  // This is emitted right before MorphNode/EmitNode.
3439  // So it should be safe to assume that this node has been selected
3440  unsigned index = MatcherTable[MatcherIndex++];
3441  index |= (MatcherTable[MatcherIndex++] << 8);
3442  dbgs() << "COVERED: " << getPatternForIndex(index) << "\n";
3443  dbgs() << "INCLUDED: " << getIncludePathForIndex(index) << "\n";
3444  continue;
3445  }
3446 
3447  case OPC_EmitNode: case OPC_MorphNodeTo:
3448  case OPC_EmitNode0: case OPC_EmitNode1: case OPC_EmitNode2:
3450  uint16_t TargetOpc = MatcherTable[MatcherIndex++];
3451  TargetOpc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3452  unsigned EmitNodeInfo = MatcherTable[MatcherIndex++];
3453  // Get the result VT list.
3454  unsigned NumVTs;
3455  // If this is one of the compressed forms, get the number of VTs based
3456  // on the Opcode. Otherwise read the next byte from the table.
3457  if (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2)
3458  NumVTs = Opcode - OPC_MorphNodeTo0;
3459  else if (Opcode >= OPC_EmitNode0 && Opcode <= OPC_EmitNode2)
3460  NumVTs = Opcode - OPC_EmitNode0;
3461  else
3462  NumVTs = MatcherTable[MatcherIndex++];
3463  SmallVector<EVT, 4> VTs;
3464  for (unsigned i = 0; i != NumVTs; ++i) {
3466  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3467  if (VT == MVT::iPTR)
3468  VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy;
3469  VTs.push_back(VT);
3470  }
3471 
3472  if (EmitNodeInfo & OPFL_Chain)
3473  VTs.push_back(MVT::Other);
3474  if (EmitNodeInfo & OPFL_GlueOutput)
3475  VTs.push_back(MVT::Glue);
3476 
3477  // This is hot code, so optimize the two most common cases of 1 and 2
3478  // results.
3479  SDVTList VTList;
3480  if (VTs.size() == 1)
3481  VTList = CurDAG->getVTList(VTs[0]);
3482  else if (VTs.size() == 2)
3483  VTList = CurDAG->getVTList(VTs[0], VTs[1]);
3484  else
3485  VTList = CurDAG->getVTList(VTs);
3486 
3487  // Get the operand list.
3488  unsigned NumOps = MatcherTable[MatcherIndex++];
3490  for (unsigned i = 0; i != NumOps; ++i) {
3491  unsigned RecNo = MatcherTable[MatcherIndex++];
3492  if (RecNo & 128)
3493  RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex);
3494 
3495  assert(RecNo < RecordedNodes.size() && "Invalid EmitNode");
3496  Ops.push_back(RecordedNodes[RecNo].first);
3497  }
3498 
3499  // If there are variadic operands to add, handle them now.
3500  if (EmitNodeInfo & OPFL_VariadicInfo) {
3501  // Determine the start index to copy from.
3502  unsigned FirstOpToCopy = getNumFixedFromVariadicInfo(EmitNodeInfo);
3503  FirstOpToCopy += (EmitNodeInfo & OPFL_Chain) ? 1 : 0;
3504  assert(NodeToMatch->getNumOperands() >= FirstOpToCopy &&
3505  "Invalid variadic node");
3506  // Copy all of the variadic operands, not including a potential glue
3507  // input.
3508  for (unsigned i = FirstOpToCopy, e = NodeToMatch->getNumOperands();
3509  i != e; ++i) {
3510  SDValue V = NodeToMatch->getOperand(i);
3511  if (V.getValueType() == MVT::Glue) break;
3512  Ops.push_back(V);
3513  }
3514  }
3515 
3516  // If this has chain/glue inputs, add them.
3517  if (EmitNodeInfo & OPFL_Chain)
3518  Ops.push_back(InputChain);
3519  if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr)
3520  Ops.push_back(InputGlue);
3521 
3522  // Check whether any matched node could raise an FP exception. Since all
3523  // such nodes must have a chain, it suffices to check ChainNodesMatched.
3524  // We need to perform this check before potentially modifying one of the
3525  // nodes via MorphNode.
3526  bool MayRaiseFPException =
3527  llvm::any_of(ChainNodesMatched, [this](SDNode *N) {
3528  return mayRaiseFPException(N) && !N->getFlags().hasNoFPExcept();
3529  });
3530 
3531  // Create the node.
3532  MachineSDNode *Res = nullptr;
3533  bool IsMorphNodeTo = Opcode == OPC_MorphNodeTo ||
3534  (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2);
3535  if (!IsMorphNodeTo) {
3536  // If this is a normal EmitNode command, just create the new node and
3537  // add the results to the RecordedNodes list.
3538  Res = CurDAG->getMachineNode(TargetOpc, SDLoc(NodeToMatch),
3539  VTList, Ops);
3540 
3541  // Add all the non-glue/non-chain results to the RecordedNodes list.
3542  for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
3543  if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue) break;
3544  RecordedNodes.push_back(std::pair<SDValue,SDNode*>(SDValue(Res, i),
3545  nullptr));
3546  }
3547  } else {
3548  assert(NodeToMatch->getOpcode() != ISD::DELETED_NODE &&
3549  "NodeToMatch was removed partway through selection");
3551  SDNode *E) {
3553  auto &Chain = ChainNodesMatched;
3554  assert((!E || !is_contained(Chain, N)) &&
3555  "Chain node replaced during MorphNode");
3556  llvm::erase_value(Chain, N);
3557  });
3558  Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList,
3559  Ops, EmitNodeInfo));
3560  }
3561 
3562  // Set the NoFPExcept flag when no original matched node could
3563  // raise an FP exception, but the new node potentially might.
3565  SDNodeFlags Flags = Res->getFlags();
3566  Flags.setNoFPExcept(true);
3567  Res->setFlags(Flags);
3568  }
3569 
3570  // If the node had chain/glue results, update our notion of the current
3571  // chain and glue.
3572  if (EmitNodeInfo & OPFL_GlueOutput) {
3573  InputGlue = SDValue(Res, VTs.size()-1);
3574  if (EmitNodeInfo & OPFL_Chain)
3575  InputChain = SDValue(Res, VTs.size()-2);
3576  } else if (EmitNodeInfo & OPFL_Chain)
3577  InputChain = SDValue(Res, VTs.size()-1);
3578 
3579  // If the OPFL_MemRefs glue is set on this node, slap all of the
3580  // accumulated memrefs onto it.
3581  //
3582  // FIXME: This is vastly incorrect for patterns with multiple outputs
3583  // instructions that access memory and for ComplexPatterns that match
3584  // loads.
3585  if (EmitNodeInfo & OPFL_MemRefs) {
3586  // Only attach load or store memory operands if the generated
3587  // instruction may load or store.
3588  const MCInstrDesc &MCID = TII->get(TargetOpc);
3589  bool mayLoad = MCID.mayLoad();
3590  bool mayStore = MCID.mayStore();
3591 
3592  // We expect to have relatively few of these so just filter them into a
3593  // temporary buffer so that we can easily add them to the instruction.
3594  SmallVector<MachineMemOperand *, 4> FilteredMemRefs;
3595  for (MachineMemOperand *MMO : MatchedMemRefs) {
3596  if (MMO->isLoad()) {
3597  if (mayLoad)
3598  FilteredMemRefs.push_back(MMO);
3599  } else if (MMO->isStore()) {
3600  if (mayStore)
3601  FilteredMemRefs.push_back(MMO);
3602  } else {
3603  FilteredMemRefs.push_back(MMO);
3604  }
3605  }
3606 
3607  CurDAG->setNodeMemRefs(Res, FilteredMemRefs);
3608  }
3609 
3610  LLVM_DEBUG(if (!MatchedMemRefs.empty() && Res->memoperands_empty()) dbgs()
3611  << " Dropping mem operands\n";
3612  dbgs() << " " << (IsMorphNodeTo ? "Morphed" : "Created")
3613  << " node: ";
3614  Res->dump(CurDAG););
3615 
3616  // If this was a MorphNodeTo then we're completely done!
3617  if (IsMorphNodeTo) {
3618  // Update chain uses.
3619  UpdateChains(Res, InputChain, ChainNodesMatched, true);
3620  return;
3621  }
3622  continue;
3623  }
3624 
3625  case OPC_CompleteMatch: {
3626  // The match has been completed, and any new nodes (if any) have been
3627  // created. Patch up references to the matched dag to use the newly
3628  // created nodes.
3629  unsigned NumResults = MatcherTable[MatcherIndex++];
3630 
3631  for (unsigned i = 0; i != NumResults; ++i) {
3632  unsigned ResSlot = MatcherTable[MatcherIndex++];
3633  if (ResSlot & 128)
3634  ResSlot = GetVBR(ResSlot, MatcherTable, MatcherIndex);
3635 
3636  assert(ResSlot < RecordedNodes.size() && "Invalid CompleteMatch");
3637  SDValue Res = RecordedNodes[ResSlot].first;
3638 
3639  assert(i < NodeToMatch->getNumValues() &&
3640  NodeToMatch->getValueType(i) != MVT::Other &&
3641  NodeToMatch->getValueType(i) != MVT::Glue &&
3642  "Invalid number of results to complete!");
3643  assert((NodeToMatch->getValueType(i) == Res.getValueType() ||
3644  NodeToMatch->getValueType(i) == MVT::iPTR ||
3645  Res.getValueType() == MVT::iPTR ||
3646  NodeToMatch->getValueType(i).getSizeInBits() ==
3647  Res.getValueSizeInBits()) &&
3648  "invalid replacement");
3649  ReplaceUses(SDValue(NodeToMatch, i), Res);
3650  }
3651 
3652  // Update chain uses.
3653  UpdateChains(NodeToMatch, InputChain, ChainNodesMatched, false);
3654 
3655  // If the root node defines glue, we need to update it to the glue result.
3656  // TODO: This never happens in our tests and I think it can be removed /
3657  // replaced with an assert, but if we do it this the way the change is
3658  // NFC.
3659  if (NodeToMatch->getValueType(NodeToMatch->getNumValues() - 1) ==
3660  MVT::Glue &&
3661  InputGlue.getNode())
3662  ReplaceUses(SDValue(NodeToMatch, NodeToMatch->getNumValues() - 1),
3663  InputGlue);
3664 
3665  assert(NodeToMatch->use_empty() &&
3666  "Didn't replace all uses of the node?");
3667  CurDAG->RemoveDeadNode(NodeToMatch);
3668 
3669  return;
3670  }
3671  }
3672 
3673  // If the code reached this point, then the match failed. See if there is
3674  // another child to try in the current 'Scope', otherwise pop it until we
3675  // find a case to check.
3676  LLVM_DEBUG(dbgs() << " Match failed at index " << CurrentOpcodeIndex
3677  << "\n");
3678  ++NumDAGIselRetries;
3679  while (true) {
3680  if (MatchScopes.empty()) {
3681  CannotYetSelect(NodeToMatch);
3682  return;
3683  }
3684 
3685  // Restore the interpreter state back to the point where the scope was
3686  // formed.
3687  MatchScope &LastScope = MatchScopes.back();
3688  RecordedNodes.resize(LastScope.NumRecordedNodes);
3689  NodeStack.clear();
3690  NodeStack.append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());
3691  N = NodeStack.back();
3692 
3693  if (LastScope.NumMatchedMemRefs != MatchedMemRefs.size())
3694  MatchedMemRefs.resize(LastScope.NumMatchedMemRefs);
3695  MatcherIndex = LastScope.FailIndex;
3696 
3697  LLVM_DEBUG(dbgs() << " Continuing at " << MatcherIndex << "\n");
3698 
3699  InputChain = LastScope.InputChain;
3700  InputGlue = LastScope.InputGlue;
3701  if (!LastScope.HasChainNodesMatched)
3702  ChainNodesMatched.clear();
3703 
3704  // Check to see what the offset is at the new MatcherIndex. If it is zero
3705  // we have reached the end of this scope, otherwise we have another child
3706  // in the current scope to try.
3707  unsigned NumToSkip = MatcherTable[MatcherIndex++];
3708  if (NumToSkip & 128)
3709  NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3710 
3711  // If we have another child in this scope to match, update FailIndex and
3712  // try it.
3713  if (NumToSkip != 0) {
3714  LastScope.FailIndex = MatcherIndex+NumToSkip;
3715  break;
3716  }
3717 
3718  // End of this scope, pop it and try the next child in the containing
3719  // scope.
3720  MatchScopes.pop_back();
3721  }
3722  }
3723 }
3724 
3725 /// Return whether the node may raise an FP exception.
3727  // For machine opcodes, consult the MCID flag.
3728  if (N->isMachineOpcode()) {
3729  const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
3730  return MCID.mayRaiseFPException();
3731  }
3732 
3733  // For ISD opcodes, only StrictFP opcodes may raise an FP
3734  // exception.
3735  if (N->isTargetOpcode())
3736  return N->isTargetStrictFPOpcode();
3737  return N->isStrictFPOpcode();
3738 }
3739 
3741  assert(N->getOpcode() == ISD::OR && "Unexpected opcode");
3742  auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
3743  if (!C)
3744  return false;
3745 
3746  // Detect when "or" is used to add an offset to a stack object.
3747  if (auto *FN = dyn_cast<FrameIndexSDNode>(N->getOperand(0))) {
3748  MachineFrameInfo &MFI = MF->getFrameInfo();
3749  Align A = MFI.getObjectAlign(FN->getIndex());
3750  int32_t Off = C->getSExtValue();
3751  // If the alleged offset fits in the zero bits guaranteed by
3752  // the alignment, then this or is really an add.
3753  return (Off >= 0) && (((A.value() - 1) & Off) == unsigned(Off));
3754  }
3755  return false;
3756 }
3757 
3758 void SelectionDAGISel::CannotYetSelect(SDNode *N) {
3759  std::string msg;
3760  raw_string_ostream Msg(msg);
3761  Msg << "Cannot select: ";
3762 
3763  if (N->getOpcode() != ISD::INTRINSIC_W_CHAIN &&
3764  N->getOpcode() != ISD::INTRINSIC_WO_CHAIN &&
3765  N->getOpcode() != ISD::INTRINSIC_VOID) {
3766  N->printrFull(Msg, CurDAG);
3767  Msg << "\nIn function: " << MF->getName();
3768  } else {
3769  bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other;
3770  unsigned iid =
3771  cast<ConstantSDNode>(N->getOperand(HasInputChain))->getZExtValue();
3772  if (iid < Intrinsic::num_intrinsics)
3773  Msg << "intrinsic %" << Intrinsic::getBaseName((Intrinsic::ID)iid);
3774  else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo())
3775  Msg << "target intrinsic %" << TII->getName(iid);
3776  else
3777  Msg << "unknown intrinsic #" << iid;
3778  }
3779  report_fatal_error(Twine(Msg.str()));
3780 }
3781 
3782 char SelectionDAGISel::ID = 0;
llvm::DIExpression::prepend
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
Definition: DebugInfoMetadata.cpp:1492
llvm::FunctionLoweringInfo::Fn
const Function * Fn
Definition: FunctionLoweringInfo.h:54
llvm::Intrinsic::getBaseName
StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
Definition: Function.cpp:937
llvm::MachineInstr::isDebugValue
bool isDebugValue() const
Definition: MachineInstr.h:1261
llvm::SelectionDAGISel::OPC_MoveChild2
@ OPC_MoveChild2
Definition: SelectionDAGISel.h:127
i
i
Definition: README.txt:29
llvm::MachineFunction::hasInlineAsm
bool hasInlineAsm() const
Returns true if the function contains any inline assembly.
Definition: MachineFunction.h:730
llvm::M68kBeads::Term
@ Term
Definition: M68kBaseInfo.h:71
ValueTypes.h
CheckSame
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * >> &RecordedNodes)
CheckSame - Implements OP_CheckSame.
Definition: SelectionDAGISel.cpp:2509
llvm::SelectionDAGISel::FastISelFailed
bool FastISelFailed
Definition: SelectionDAGISel.h:57
llvm::DbgVariableIntrinsic::getExpression
DIExpression * getExpression() const
Definition: IntrinsicInst.h:291
AssumptionCache.h
llvm::ConstantSDNode
Definition: SelectionDAGNodes.h:1581
llvm::SelectionDAGISel::OPC_CheckComplexPat
@ OPC_CheckComplexPat
Definition: SelectionDAGISel.h:149
llvm::BasicBlock::end
iterator end()
Definition: BasicBlock.h:308
llvm::ISD::STRICT_FSETCC
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:475
llvm::FunctionLoweringInfo::StaticAllocaMap
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
Definition: FunctionLoweringInfo.h:128
StackProtector.h
llvm::SelectionDAGISel::OPC_EmitInteger
@ OPC_EmitInteger
Definition: SelectionDAGISel.h:155
llvm::OptimizationRemarkMissed
Diagnostic information for missed-optimization remarks.
Definition: DiagnosticInfo.h:734
llvm::createDAGLinearizer
ScheduleDAGSDNodes * createDAGLinearizer(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createDAGLinearizer - This creates a "no-scheduling" scheduler which linearize the DAG using topologi...
Definition: ScheduleDAGFast.cpp:820
isFoldedOrDeadInstruction
static bool isFoldedOrDeadInstruction(const Instruction *I, const FunctionLoweringInfo &FuncInfo)
isFoldedOrDeadInstruction - Return true if the specified instruction is side-effect free and is eithe...
Definition: SelectionDAGISel.cpp:1285
llvm::MachineFunctionProperties::hasProperty
bool hasProperty(Property P) const
Definition: MachineFunction.h:192
llvm::ISD::INTRINSIC_VOID
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
llvm::InlineAsm::Op_ExtraInfo
@ Op_ExtraInfo
Definition: InlineAsm.h:221
llvm::SDUse
Represents a use of a SDNode.
Definition: SelectionDAGNodes.h:284
llvm::SelectionDAGISel::OPC_CheckChild2Integer
@ OPC_CheckChild2Integer
Definition: SelectionDAGISel.h:145
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:108
MachineInstr.h
llvm::SelectionDAGISel::UseInstrRefDebugInfo
bool UseInstrRefDebugInfo
Definition: SelectionDAGISel.h:59
llvm::SelectionDAGISel::TLI
const TargetLowering * TLI
Definition: SelectionDAGISel.h:56
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::SelectionDAGISel::getIncludePathForIndex
virtual StringRef getIncludePathForIndex(unsigned index)
getIncludePathForIndex - get the td source location of pattern instantiation
Definition: SelectionDAGISel.h:248
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::DbgDeclareInst::getAddress
Value * getAddress() const
Definition: IntrinsicInst.h:350
llvm::DiagnosticInfoISelFallback
Diagnostic information for ISel fallback path.
Definition: DiagnosticInfo.h:954
llvm::SelectionDAG::AssignTopologicalOrder
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
Definition: SelectionDAG.cpp:10611
llvm::FuncletPadInst::arg_size
unsigned arg_size() const
arg_size - Return the number of funcletpad arguments.
Definition: InstrTypes.h:2311
GCMetadata.h
llvm::SelectionDAGISel::TM
TargetMachine & TM
Definition: SelectionDAGISel.h:43
llvm::MCSymbol
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
llvm::ISD::PATCHPOINT
@ PATCHPOINT
Definition: ISDOpcodes.h:1299
llvm::SDNode::getValueType
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Definition: SelectionDAGNodes.h:985
UseMI
MachineInstrBuilder & UseMI
Definition: AArch64ExpandPseudoInsts.cpp:107
llvm::SelectionDAGISel::isOrEquivalentToAdd
bool isOrEquivalentToAdd(const SDNode *N) const
Definition: SelectionDAGISel.cpp:3740
llvm::ScheduleDAGSDNodes
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
Definition: ScheduleDAGSDNodes.h:46
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1105
llvm::SelectionDAGISel::OPC_EmitMergeInputChains1_2
@ OPC_EmitMergeInputChains1_2
Definition: SelectionDAGISel.h:163
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::ISD::OR
@ OR
Definition: ISDOpcodes.h:667
llvm::SelectionDAGISel::EnforceNodeIdInvariant
static void EnforceNodeIdInvariant(SDNode *N)
Definition: SelectionDAGISel.cpp:1036
llvm::TargetLowering::EmitInstrWithCustomInserter
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Definition: SelectionDAGISel.cpp:292
CheckChildSame
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * >> &RecordedNodes, unsigned ChildNo)
CheckChildSame - Implements OP_CheckChildXSame.
Definition: SelectionDAGISel.cpp:2518
llvm::Value::hasOneUse
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
llvm::MachineBasicBlock::getBasicBlock
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Definition: MachineBasicBlock.h:209
llvm::SelectionDAGISel::CheckComplexPattern
virtual bool CheckComplexPattern(SDNode *Root, SDNode *Parent, SDValue N, unsigned PatternNo, SmallVectorImpl< std::pair< SDValue, SDNode * > > &Result)
Definition: SelectionDAGISel.h:291
Metadata.h
llvm::ISD::LIFETIME_END
@ LIFETIME_END
Definition: ISDOpcodes.h:1225
llvm::SelectionDAGISel::OPC_RecordChild7
@ OPC_RecordChild7
Definition: SelectionDAGISel.h:123
llvm::SelectionDAGISel::PostprocessISelDAG
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
Definition: SelectionDAGISel.h:85
llvm::TargetLoweringBase::isStrictFPEnabled
bool isStrictFPEnabled() const
Return true if the target support strict float operation.
Definition: TargetLowering.h:340
PHI
Rewrite undef for PHI
Definition: AMDGPURewriteUndefForPHI.cpp:101
IntrinsicInst.h
computeUsesMSVCFloatingPoint
static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F, MachineModuleInfo &MMI)
Definition: SelectionDAGISel.cpp:349
hasExceptionPointerOrCodeUser
static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI)
Definition: SelectionDAGISel.cpp:1175
DebugInfoMetadata.h
llvm::ISD::AssertSext
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
llvm::SelectionDAG::getCopyToReg
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:767
MCInstrDesc.h
llvm::SelectionDAGISel::DAGSize
unsigned DAGSize
DAGSize - Size of DAG being instruction selected.
Definition: SelectionDAGISel.h:206
EnableFastISelAbort
static cl::opt< int > EnableFastISelAbort("fast-isel-abort", cl::Hidden, cl::desc("Enable abort calls when \"fast\" instruction selection " "fails to lower an instruction: 0 disable the abort, 1 will " "abort but for args, calls and terminators, 2 will also " "abort for argument lowering, and 3 will never fallback " "to SelectionDAG."))
InstIterator.h
MachinePassRegistry.h
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:50
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:159
llvm::SelectionDAG::VerifyDAGDivergence
void VerifyDAGDivergence()
Definition: SelectionDAG.cpp:10530
llvm::SelectionDAGISel::OPC_EmitNode
@ OPC_EmitNode
Definition: SelectionDAGISel.h:167
llvm::Function
Definition: Function.h:60
llvm::ISD::ConstantFP
@ ConstantFP
Definition: ISDOpcodes.h:77
StringRef.h
llvm::ISD::LIFETIME_START
@ LIFETIME_START
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:1224
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
Pass.h
SchedulerRegistry.h
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:95
llvm::ISD::STRICT_UINT_TO_FP
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:449
llvm::SelectionDAGISel::SelectInlineAsmMemoryOperands
void SelectInlineAsmMemoryOperands(std::vector< SDValue > &Ops, const SDLoc &DL)
SelectInlineAsmMemoryOperands - Calls to this are automatically generated by tblgen.
Definition: SelectionDAGISel.cpp:1956
llvm::StackProtector::copyToMachineFrameInfo
void copyToMachineFrameInfo(MachineFrameInfo &MFI) const
Definition: StackProtector.cpp:599
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:628
llvm::SDNode::op_values
iterator_range< value_op_iterator > op_values() const
Definition: SelectionDAGNodes.h:943
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::TargetMachine::getO0WantsFastISel
bool getO0WantsFastISel()
Definition: TargetMachine.h:247
Statistic.h
llvm::SelectionDAG::init
void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, LegacyDivergenceAnalysis *Divergence, ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin)
Prepare this SelectionDAG to process code in the given MachineFunction.
Definition: SelectionDAG.cpp:1285
llvm::MachineFunction::useDebugInstrRef
bool useDebugInstrRef() const
Returns true if the function's variable locations should be tracked with instruction referencing.
Definition: MachineFunction.cpp:1243
InlineAsm.h
CheckType
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
Definition: SelectionDAGISel.cpp:2551
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:9353
llvm::FastISel::finishBasicBlock
void finishBasicBlock()
Flush the local value map.
Definition: FastISel.cpp:136
llvm::printMBBReference
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Definition: MachineBasicBlock.cpp:117
llvm::FunctionPass::skipFunction
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:174
llvm::Function::getSubprogram
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1626
llvm::MachineSDNode
An SDNode that represents everything that will be needed to construct a MachineInstr.
Definition: SelectionDAGNodes.h:2896
llvm::createDefaultScheduler
ScheduleDAGSDNodes * createDefaultScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createDefaultScheduler - This creates an instruction scheduler appropriate for the target.
Definition: SelectionDAGISel.cpp:251
ErrorHandling.h
llvm::FastISel::lowerArguments
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:138
llvm::TargetTransformInfo
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Definition: TargetTransformInfo.h:172
LazyBlockFrequencyInfo.h
llvm::SelectionDAGISel::OPC_CheckOrImm
@ OPC_CheckOrImm
Definition: SelectionDAGISel.h:150
llvm::SelectionDAGISel::OPC_RecordChild3
@ OPC_RecordChild3
Definition: SelectionDAGISel.h:122
llvm::Function::getEntryBlock
const BasicBlock & getEntryBlock() const
Definition: Function.h:691
llvm::Sched::ILP
@ ILP
Definition: TargetLowering.h:102
llvm::SelectionDAG::allnodes_begin
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:525
llvm::SelectionDAG::getRoot
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:545
llvm::SelectionDAGISel::AC
AssumptionCache * AC
Definition: SelectionDAGISel.h:52
llvm::SelectionDAGISel::OPC_EmitNode1
@ OPC_EmitNode1
Definition: SelectionDAGISel.h:169
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:462
llvm::SwiftErrorValueTracking::createEntriesInEntryBlock
bool createEntriesInEntryBlock(DebugLoc DbgLoc)
Create initial definitions of swifterror values in the entry block of the current function.
Definition: SwiftErrorValueTracking.cpp:115
llvm::SwiftErrorValueTracking::propagateVRegs
void propagateVRegs()
Propagate assigned swifterror vregs through a function, synthesizing PHI nodes when needed to maintai...
Definition: SwiftErrorValueTracking.cpp:147
llvm::ISD::EH_LABEL
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:1033
llvm::SelectionDAGISel::OPC_CheckChild1Integer
@ OPC_CheckChild1Integer
Definition: SelectionDAGISel.h:145
llvm::SelectionDAGISel::OPC_CheckFoldableChainNode
@ OPC_CheckFoldableChainNode
Definition: SelectionDAGISel.h:153
llvm::MachineRegisterInfo::defusechain_instr_iterator
defusechain_iterator - This class provides iterator support for machine operands in the function that...
Definition: MachineRegisterInfo.h:277
llvm::HandleSDNode
This class is used to form a handle around another node that is persistent and is updated across invo...
Definition: SelectionDAGNodes.h:1232
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::createFastDAGScheduler
ScheduleDAGSDNodes * createFastDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createFastDAGScheduler - This creates a "fast" scheduler.
Definition: ScheduleDAGFast.cpp:815
llvm::SelectionDAGISel::OPC_CheckChild1Same
@ OPC_CheckChild1Same
Definition: SelectionDAGISel.h:131
OptimizationRemarkEmitter.h
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:463
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
MachineBasicBlock.h
llvm::TargetSubtargetInfo::getRegisterInfo
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Definition: TargetSubtargetInfo.h:127
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:140
llvm::MVT::Glue
@ Glue
Definition: MachineValueType.h:273
llvm::SelectionDAGISel::OPC_MorphNodeTo2
@ OPC_MorphNodeTo2
Definition: SelectionDAGISel.h:172
llvm::SelectionDAGISel::OPC_MoveChild7
@ OPC_MoveChild7
Definition: SelectionDAGISel.h:128
llvm::SDNode::use_iterator
This class provides iterator support for SDUse operands that use a specific SDNode.
Definition: SelectionDAGNodes.h:745
APInt.h
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:237
llvm::SelectionDAGISel::CheckPatternPredicate
virtual bool CheckPatternPredicate(unsigned PredNo) const
CheckPatternPredicate - This function is generated by tblgen in the target.
Definition: SelectionDAGISel.h:268
llvm::LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage
static void getLazyBFIAnalysisUsage(AnalysisUsage &AU)
Helper for client passes to set up the analysis usage on behalf of this pass.
Definition: LazyBlockFrequencyInfo.cpp:62
llvm::Function::getContext
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:321
llvm::DenseMapIterator
Definition: DenseMap.h:57
Shift
bool Shift
Definition: README.txt:468
llvm::InlineAsm::isUseOperandTiedToDef
static bool isUseOperandTiedToDef(unsigned Flag, unsigned &Idx)
isUseOperandTiedToDef - Return true if the flag of the inline asm operand indicates it is an use oper...
Definition: InlineAsm.h:369
ViewSchedDAGs
static cl::opt< bool > ViewSchedDAGs("view-sched-dags", cl::Hidden, cl::desc("Pop up a window to show sched dags as they are processed"))
DenseMap.h
llvm::TargetLowering::initializeSplitCSR
virtual void initializeSplitCSR(MachineBasicBlock *Entry) const
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
Definition: TargetLowering.h:4033
llvm::TimePassesIsEnabled
bool TimePassesIsEnabled
If the user specifies the -time-passes argument on an LLVM tool command line then the value of this b...
Definition: PassTimingInfo.cpp:37
CheckChildInteger
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, unsigned ChildNo)
Definition: SelectionDAGISel.cpp:2621
llvm::SelectionDAGISel::OPC_CaptureGlueInput
@ OPC_CaptureGlueInput
Definition: SelectionDAGISel.h:125
llvm::SelectionDAGISel::OPC_EmitNodeXForm
@ OPC_EmitNodeXForm
Definition: SelectionDAGISel.h:166
TargetInstrInfo.h
llvm::MachineModuleInfo::setUsesMSVCFloatingPoint
void setUsesMSVCFloatingPoint(bool b)
Definition: MachineModuleInfo.h:186
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition: MachineMemOperand.h:127
llvm::FunctionLoweringInfo::isExportedInst
bool isExportedInst(const Value *V) const
isExportedInst - Return true if the specified value is an instruction exported from its block.
Definition: FunctionLoweringInfo.h:198
EHPersonalities.h
llvm::TargetLoweringBase::getExceptionPointerRegister
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
Definition: TargetLowering.h:1830
llvm::RegisterScheduler::Registry
static MachinePassRegistry< FunctionPassCtor > Registry
RegisterScheduler class - Track the registration of instruction schedulers.
Definition: SchedulerRegistry.h:38
ViewDAGCombine1
static cl::opt< bool > ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the first " "dag combine pass"))
llvm::MachineFunction::setCallSiteLandingPad
void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef< unsigned > Sites)
Map the landing pad's EH symbol to the call site indexes.
Definition: MachineFunction.cpp:861
llvm::SelectionDAGISel::OPC_CheckChild0Same
@ OPC_CheckChild0Same
Definition: SelectionDAGISel.h:131
llvm::DbgVariableIntrinsic::getVariable
DILocalVariable * getVariable() const
Definition: IntrinsicInst.h:287
llvm::Optional
Definition: APInt.h:33
LLVM_ATTRIBUTE_ALWAYS_INLINE
#define LLVM_ATTRIBUTE_ALWAYS_INLINE
LLVM_ATTRIBUTE_ALWAYS_INLINE - On compilers where we have a directive to do so, mark a method "always...
Definition: Compiler.h:230
T
#define T
Definition: Mips16ISelLowering.cpp:341
llvm::MachineInstr::isCopy
bool isCopy() const
Definition: MachineInstr.h:1336
llvm::SelectionDAGISel::OPC_CheckPredicate
@ OPC_CheckPredicate
Definition: SelectionDAGISel.h:134
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:136
llvm::SelectionDAGISel::OPC_Coverage
@ OPC_Coverage
Definition: SelectionDAGISel.h:175
llvm::ISD::isConstantSplatVectorAllZeros
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
Definition: SelectionDAG.cpp:220
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
llvm::TargetLowering::supportSplitCSR
virtual bool supportSplitCSR(MachineFunction *MF) const
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
Definition: TargetLowering.h:4023
llvm::SelectionDAG::RemoveDeadNodes
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
Definition: SelectionDAG.cpp:943
llvm::SelectionDAGISel::OPFL_GlueOutput
@ OPFL_GlueOutput
Definition: SelectionDAGISel.h:182
STLExtras.h
llvm::DIExpression
DWARF expression.
Definition: DebugInfoMetadata.h:2596
llvm::AfterLegalizeTypes
@ AfterLegalizeTypes
Definition: DAGCombine.h:17
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
llvm::SelectionDAGISel::OPC_CheckAndImm
@ OPC_CheckAndImm
Definition: SelectionDAGISel.h:150
llvm::SelectionDAGISel::OPC_EmitCopyToReg
@ OPC_EmitCopyToReg
Definition: SelectionDAGISel.h:164
llvm::SelectionDAGISel::OPC_SwitchType
@ OPC_SwitchType
Definition: SelectionDAGISel.h:140
llvm::Sched::Fast
@ Fast
Definition: TargetLowering.h:104
llvm::succ_empty
bool succ_empty(const Instruction *I)
Definition: CFG.h:255
llvm::SelectionDAGISel::OPC_CheckChild0Type
@ OPC_CheckChild0Type
Definition: SelectionDAGISel.h:141
replace
static void replace(Module &M, GlobalVariable *Old, GlobalVariable *New)
Definition: ConstantMerge.cpp:116
llvm::Sched::Linearize
@ Linearize
Definition: TargetLowering.h:105
llvm::SelectionDAGISel::OPC_CheckCondCode
@ OPC_CheckCondCode
Definition: SelectionDAGISel.h:147
SelectionDAG.h
llvm::SelectionDAGISel::OPC_CheckChild7Type
@ OPC_CheckChild7Type
Definition: SelectionDAGISel.h:143
llvm::initializeGCModuleInfoPass
void initializeGCModuleInfoPass(PassRegistry &)
processDbgDeclares
static void processDbgDeclares(FunctionLoweringInfo &FuncInfo)
Collect llvm.dbg.declare information.
Definition: SelectionDAGISel.cpp:1296
llvm::ISD::FREEZE
@ FREEZE
Definition: ISDOpcodes.h:216
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1628
llvm::SelectionDAG::clear
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block.
Definition: SelectionDAG.cpp:1365
FilterDAGBasicBlockName
static cl::opt< std::string > FilterDAGBasicBlockName("filter-view-dags", cl::Hidden, cl::desc("Only display the basic block whose name " "matches this for all view-*-dags options"))
llvm::TargetLoweringBase::getOperationAction
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
Definition: TargetLowering.h:1112
llvm::initializeTargetLibraryInfoWrapperPassPass
void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &)
llvm::SelectionDAGISel::OPC_CheckChild1Type
@ OPC_CheckChild1Type
Definition: SelectionDAGISel.h:141
llvm::SelectionDAGISel::OPC_EmitStringInteger
@ OPC_EmitStringInteger
Definition: SelectionDAGISel.h:156
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: MachineFunctionPass.cpp:167
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
new
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM ID Predecessors according to mbb< bb27, 0x8b0a7c0 > Note ADDri is not a two address instruction its result reg1037 is an operand of the PHI node in bb76 and its operand reg1039 is the result of the PHI node We should treat it as a two address code and make sure the ADDri is scheduled after any node that reads reg1039 Use info(i.e. register scavenger) to assign it a free register to allow reuse the collector could move the objects and invalidate the derived pointer This is bad enough in the first but safe points can crop up unpredictably **array_addr i32 n y store obj * new
Definition: README.txt:125
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::RISCVFenceField::R
@ R
Definition: RISCVBaseInfo.h:265
MachineRegisterInfo.h
KnownBits.h
llvm::SelectionDAGISel::PreprocessISelDAG
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
Definition: SelectionDAGISel.h:81
llvm::ISD::ARITH_FENCE
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1150
llvm::SelectionDAG::getRegister
SDValue getRegister(unsigned Reg, EVT VT)
Definition: SelectionDAG.cpp:2121
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::ISD::INLINEASM
@ INLINEASM
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:1025
llvm::InlineAsm::Kind_Mem
@ Kind_Mem
Definition: InlineAsm.h:245
llvm::EVT::isSimple
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:129
llvm::ISD::PSEUDO_PROBE
@ PSEUDO_PROBE
Pseudo probe for AutoFDO, as a place holder in a basic block to improve the sample counts quality.
Definition: ISDOpcodes.h:1244
llvm::TargetIntrinsicInfo
TargetIntrinsicInfo - Interface to description of machine instruction set.
Definition: TargetIntrinsicInfo.h:29
mapWasmLandingPadIndex
static void mapWasmLandingPadIndex(MachineBasicBlock *MBB, const CatchPadInst *CPI)
Definition: SelectionDAGISel.cpp:1190
CheckAndImm
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
Definition: SelectionDAGISel.cpp:2629
AliasAnalysis.h
UseMBPI
static cl::opt< bool > UseMBPI("use-mbpi", cl::desc("use Machine Branch Probability Info"), cl::init(true), cl::Hidden)
MachineValueType.h
llvm::MVT::SimpleValueType
SimpleValueType
Definition: MachineValueType.h:33
llvm::MachineFunction::setExposesReturnsTwice
void setExposesReturnsTwice(bool B)
setCallsSetJmp - Set a flag that indicates if there's a call to a "returns twice" function.
Definition: MachineFunction.h:725
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::classifyEHPersonality
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Definition: EHPersonalities.cpp:22
llvm::MachineFunction::setWasmLandingPadIndex
void setWasmLandingPadIndex(const MachineBasicBlock *LPad, unsigned Index)
Map the landing pad to its index. Used for Wasm exception handling.
Definition: MachineFunction.h:1151
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:187
llvm::TargetLowering::AdjustInstrPostInstrSelection
virtual void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
Definition: SelectionDAGISel.cpp:302
llvm::SelectionDAGISel::OPC_EmitMergeInputChains1_0
@ OPC_EmitMergeInputChains1_0
Definition: SelectionDAGISel.h:161
llvm::SelectionDAGISel::ElidedArgCopyInstrs
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
Definition: SelectionDAGISel.h:58
llvm::SelectionDAGISel::OPC_MoveChild5
@ OPC_MoveChild5
Definition: SelectionDAGISel.h:128
Instruction.h
CommandLine.h
llvm::BeforeLegalizeTypes
@ BeforeLegalizeTypes
Definition: DAGCombine.h:16
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
TargetLowering.h
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
llvm::FuncletPadInst::getArgOperand
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th funcletpad argument.
Definition: InstrTypes.h:2327
llvm::SDNode::getOpcode
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
Definition: SelectionDAGNodes.h:644
llvm::MachineFunction::front
const MachineBasicBlock & front() const
Definition: MachineFunction.h:866
llvm::BlockFrequencyInfo
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Definition: BlockFrequencyInfo.h:37
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:667
reportFastISelFailure
static void reportFastISelFailure(MachineFunction &MF, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R, bool ShouldAbort)
Definition: SelectionDAGISel.cpp:651
llvm::SelectionDAGISel::OPC_RecordChild6
@ OPC_RecordChild6
Definition: SelectionDAGISel.h:123
llvm::MachineBasicBlock::insertAfter
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
Definition: MachineBasicBlock.h:930
llvm::SelectionDAGISel::OPC_RecordChild1
@ OPC_RecordChild1
Definition: SelectionDAGISel.h:122
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1140
ViewDAGCombineLT
static cl::opt< bool > ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the post " "legalize types dag combine pass"))
CheckValueType
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
Definition: SelectionDAGISel.cpp:2586
llvm::MachineFunctionProperties::Property::Selected
@ Selected
TargetMachine.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:24
CheckOrImm
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
Definition: SelectionDAGISel.cpp:2642
llvm::SelectionDAGISel::OPC_EmitNode0
@ OPC_EmitNode0
Definition: SelectionDAGISel.h:169
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:220
SelectionDAGNodes.h
Constants.h
llvm::SelectionDAGISel::RunSDNodeXForm
virtual SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo)
Definition: SelectionDAGISel.h:297
llvm::ISD::Constant
@ Constant
Definition: ISDOpcodes.h:76
llvm::SelectionDAGISel::SDB
std::unique_ptr< SelectionDAGBuilder > SDB
Definition: SelectionDAGISel.h:50
llvm::SelectionDAGISel::OPC_CheckOpcode
@ OPC_CheckOpcode
Definition: SelectionDAGISel.h:136
llvm::SelectionDAGISel::OPC_CheckPatternPredicate
@ OPC_CheckPatternPredicate
Definition: SelectionDAGISel.h:133
llvm::BranchProbabilityInfoWrapperPass
Legacy analysis pass which computes BranchProbabilityInfo.
Definition: BranchProbabilityInfo.h:438
FunctionLoweringInfo.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::SelectionDAG::LegalizeTypes
bool LegalizeTypes()
This transforms the SelectionDAG into a SelectionDAG that only uses types natively supported by the t...
Definition: LegalizeTypes.cpp:1063
llvm::DIExpression::ApplyOffset
@ ApplyOffset
Definition: DebugInfoMetadata.h:2801
llvm::User
Definition: User.h:44
llvm::SelectionDAGISel::OPC_MoveChild
@ OPC_MoveChild
Definition: SelectionDAGISel.h:126
llvm::ISD::CopyToReg
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::SelectionDAGISel::RegInfo
MachineRegisterInfo * RegInfo
Definition: SelectionDAGISel.h:48
Intrinsics.h
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:526
llvm::SelectionDAGISel::OptLevel
CodeGenOpt::Level OptLevel
Definition: SelectionDAGISel.h:54
llvm::SelectionDAGISel::CheckNodePredicate
virtual bool CheckNodePredicate(SDNode *N, unsigned PredNo) const
CheckNodePredicate - This function is generated by tblgen in the target.
Definition: SelectionDAGISel.h:276
llvm::getLLTForMVT
LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
Definition: LowLevelType.cpp:64
llvm::TargetMachine::getIntrinsicInfo
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
Definition: TargetMachine.h:212
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3486
ViewLegalizeTypesDAGs
static cl::opt< bool > ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize types"))
llvm::KnownBits::One
APInt One
Definition: KnownBits.h:25
llvm::SelectionDAGISel::ComplexPatternFuncMutatesDAG
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
Definition: SelectionDAGISel.h:306
llvm::TargetTransformInfo::hasBranchDivergence
bool hasBranchDivergence() const
Return true if branch divergence exists.
Definition: TargetTransformInfo.cpp:237
llvm::MachineBasicBlock::isSuccessor
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
Definition: MachineBasicBlock.cpp:924
llvm::StackProtector
Definition: StackProtector.h:36
llvm::SelectionDAG::MaskedValueIsZero
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
Definition: SelectionDAG.cpp:2532
llvm::createILPListDAGScheduler
ScheduleDAGSDNodes * createILPListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createILPListDAGScheduler - This creates a bottom up register pressure aware list scheduler that trie...
Definition: ScheduleDAGRRList.cpp:3191
llvm::Register::isPhysicalRegister
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:65
SI
@ SI
Definition: SIInstrInfo.cpp:7882
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:46
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::SelectionDAG::Legalize
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
Definition: LegalizeDAG.cpp:5068
ViewLegalizeDAGs
static cl::opt< bool > ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize"))
CheckInteger
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition: SelectionDAGISel.cpp:2608
llvm::MachineFunction::getProperties
const MachineFunctionProperties & getProperties() const
Get the function properties.
Definition: MachineFunction.h:748
TargetLibraryInfo.h
llvm::BitTracker
Definition: BitTracker.h:35
llvm::SelectionDAGISel::ReplaceNode
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
Definition: SelectionDAGISel.h:231
llvm::SelectionDAGISel::ORE
std::unique_ptr< OptimizationRemarkEmitter > ORE
Current optimization remark emitter.
Definition: SelectionDAGISel.h:63
llvm::SelectionDAGISel::OPC_MorphNodeTo1
@ OPC_MorphNodeTo1
Definition: SelectionDAGISel.h:172
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
llvm::ISD::DELETED_NODE
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:197
CheckCondCode
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition: SelectionDAGISel.cpp:2571
llvm::SelectionDAG::setRoot
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:554
llvm::EVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
llvm::Instruction
Definition: Instruction.h:42
llvm::MachineModuleInfo
This class contains meta information specific to a module.
Definition: MachineModuleInfo.h:74
llvm::ISD::MDNODE_SDNODE
@ MDNODE_SDNODE
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR.
Definition: ISDOpcodes.h:1102
llvm::InlineAsm::Op_FirstOperand
@ Op_FirstOperand
Definition: InlineAsm.h:222
llvm::createSourceListDAGScheduler
ScheduleDAGSDNodes * createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createBURRListDAGScheduler - This creates a bottom up list scheduler that schedules nodes in source c...
Definition: ScheduleDAGRRList.cpp:3161
llvm::SDNode::use_empty
bool use_empty() const
Return true if there are no uses of this node.
Definition: SelectionDAGNodes.h:715
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
llvm::BasicBlock::phis
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:364
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
decodeSignRotatedValue
static uint64_t decodeSignRotatedValue(uint64_t V)
Decode a signed value stored with the sign bit in the LSB for dense VBR encoding.
Definition: SelectionDAGISel.cpp:2598
CheckPatternPredicate
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel)
CheckPatternPredicate - Implements OP_CheckPatternPredicate.
Definition: SelectionDAGISel.cpp:2530
llvm::SelectionDAG::DAGNodeDeletedListener
Definition: SelectionDAG.h:330
llvm::predecessors
auto predecessors(MachineBasicBlock *BB)
Definition: MachineSSAContext.h:30
ScheduleDAGSDNodes.h
llvm::ConstantFP
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:257
llvm::SmallVectorImpl::resize
void resize(size_type N)
Definition: SmallVector.h:642
llvm::Use::getUser
User * getUser() const
Returns the User that contains this Use.
Definition: Use.h:72
llvm::StringRef::data
const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
llvm::SelectionDAG::dump
void dump() const
Definition: SelectionDAGDumper.cpp:925
llvm::SelectionDAGISel::ID
static char ID
Definition: SelectionDAGISel.h:65
llvm::ISD::AssertAlign
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
Definition: ISDOpcodes.h:68
DebugLoc.h
SmallPtrSet.h
llvm::SwiftErrorValueTracking::preassignVRegs
void preassignVRegs(MachineBasicBlock *MBB, BasicBlock::const_iterator Begin, BasicBlock::const_iterator End)
Definition: SwiftErrorValueTracking.cpp:258
llvm::RegisterScheduler::FunctionPassCtor
ScheduleDAGSDNodes *(*)(SelectionDAGISel *, CodeGenOpt::Level) FunctionPassCtor
Definition: SchedulerRegistry.h:36
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:666
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
llvm::InlineAsm::Op_AsmString
@ Op_AsmString
Definition: InlineAsm.h:219
llvm::ISD::TargetGlobalAddress
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:164
llvm::BasicBlock::getFirstNonPHI
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:209
llvm::ISD::EntryToken
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:47
CheckNodePredicate
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel, SDNode *N)
CheckNodePredicate - Implements OP_CheckNodePredicate.
Definition: SelectionDAGISel.cpp:2537
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
findNonImmUse
static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse, bool IgnoreChains)
findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path beyond "ImmedUse".
Definition: SelectionDAGISel.cpp:2031
llvm::SelectionDAGISel::OPC_CheckSame
@ OPC_CheckSame
Definition: SelectionDAGISel.h:130
llvm::MachineRegisterInfo::getVRegDef
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
Definition: MachineRegisterInfo.cpp:396
ViewISelDAGs
static cl::opt< bool > ViewISelDAGs("view-isel-dags", cl::Hidden, cl::desc("Pop up a window to show isel dags as they are selected"))
llvm::SDValue::getValueSizeInBits
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
Definition: SelectionDAGNodes.h:199
llvm::ISD::CopyFromReg
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:208
llvm::RegisterPassParser
RegisterPassParser class - Handle the addition of new machine passes.
Definition: MachinePassRegistry.h:135
llvm::EHPersonality::Wasm_CXX
@ Wasm_CXX
llvm::MachineRegisterInfo::use_empty
bool use_empty(Register RegNo) const
use_empty - Return true if there are no instructions using the specified register.
Definition: MachineRegisterInfo.h:514
Statepoint.h
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::erase_value
void erase_value(Container &C, ValueType V)
Wrapper function to remove a value from a container:
Definition: STLExtras.h:1980
Type.h
BranchProbability.h
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:94
llvm::TargetMachine::resetTargetOptions
void resetTargetOptions(const Function &F) const
Reset the target options based on the function's attributes.
Definition: TargetMachine.cpp:53
llvm::Sched::RegPressure
@ RegPressure
Definition: TargetLowering.h:100
ViewSUnitDAGs
static cl::opt< bool > ViewSUnitDAGs("view-sunit-dags", cl::Hidden, cl::desc("Pop up a window to show SUnit dags after they are processed"))
llvm::InlineAsm::isMemKind
static bool isMemKind(unsigned Flag)
Definition: InlineAsm.h:301
llvm::OptLevelChanger::~OptLevelChanger
~OptLevelChanger()
Definition: SelectionDAGISel.cpp:235
llvm::TargetLowering::createFastISel
virtual FastISel * createFastISel(FunctionLoweringInfo &, const TargetLibraryInfo *) const
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
Definition: TargetLowering.h:4530
Operands
mir Rename Register Operands
Definition: MIRNamerPass.cpp:74
llvm::MachineRegisterInfo::getRegClass
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
Definition: MachineRegisterInfo.h:647
llvm::MCID::MayRaiseFPException
@ MayRaiseFPException
Definition: MCInstrDesc.h:169
llvm::ISD::TargetConstantFP
@ TargetConstantFP
Definition: ISDOpcodes.h:159
llvm::SelectionDAGISel::InvalidateNodeId
static void InvalidateNodeId(SDNode *N)
Definition: SelectionDAGISel.cpp:1055
llvm::APInt::isSubsetOf
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1227
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1292
llvm::TargetLowering::insertCopiesSplitCSR
virtual void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const
Insert explicit copies in entry and exit blocks.
Definition: TargetLowering.h:4041
llvm::MachineFunction::getMMI
MachineModuleInfo & getMMI() const
Definition: MachineFunction.h:608
llvm::ISD::STRICT_FSETCCS
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:476
llvm::MachineRegisterInfo::clearKillFlags
void clearKillFlags(Register Reg) const
clearKillFlags - Iterate over all the uses of the given register and clear the kill flag from the Mac...
Definition: MachineRegisterInfo.cpp:433
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:657
llvm::MCInstrDesc::mayLoad
bool mayLoad() const
Return true if this instruction could possibly read memory.
Definition: MCInstrDesc.h:435
llvm::ISD::WRITE_REGISTER
@ WRITE_REGISTER
Definition: ISDOpcodes.h:119
Timer.h
BasicBlock.h
llvm::cl::opt
Definition: CommandLine.h:1412
llvm::MachineFunction::setHasInlineAsm
void setHasInlineAsm(bool B)
Set a flag that indicates that the function contains inline assembly.
Definition: MachineFunction.h:735
llvm::SDNode::use_begin
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
Definition: SelectionDAGNodes.h:799
llvm::ISD::Register
@ Register
Definition: ISDOpcodes.h:74
llvm::SelectionDAGISel::OPC_CheckChild4Integer
@ OPC_CheckChild4Integer
Definition: SelectionDAGISel.h:146
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
llvm::FastISel::selectInstruction
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1476
llvm::SelectionDAGISel::OPC_CheckValueType
@ OPC_CheckValueType
Definition: SelectionDAGISel.h:148
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::SelectionDAG::RemoveDeadNode
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
Definition: SelectionDAG.cpp:997
llvm::SelectionDAGISel::OPC_CheckImmAllZerosV
@ OPC_CheckImmAllZerosV
Definition: SelectionDAGISel.h:152
BranchProbabilityInfo.h
llvm::SelectionDAGISel::OPC_CheckInteger
@ OPC_CheckInteger
Definition: SelectionDAGISel.h:144
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:340
llvm::SelectionDAGISel::IsProfitableToFold
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
Definition: SelectionDAGISel.cpp:2071
llvm::AfterLegalizeDAG
@ AfterLegalizeDAG
Definition: DAGCombine.h:19
CheckChild2CondCode
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition: SelectionDAGISel.cpp:2578
Index
uint32_t Index
Definition: ELFObjHandler.cpp:82
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
index
splat index
Definition: README_ALTIVEC.txt:181
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
llvm::SelectionDAGISel::OPC_RecordMemRef
@ OPC_RecordMemRef
Definition: SelectionDAGISel.h:124
llvm::TargetLibraryInfoWrapperPass
Definition: TargetLibraryInfo.h:474
uint64_t
ProfileSummaryInfo.h
llvm::SelectionDAGISel::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: SelectionDAGISel.cpp:332
llvm::TargetTransformInfoWrapperPass
Wrapper pass for TargetTransformInfo.
Definition: TargetTransformInfo.h:2641
llvm::SelectionDAGISel::TII
const TargetInstrInfo * TII
Definition: SelectionDAGISel.h:55
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:650
llvm::MCInstrDesc::isCall
bool isCall() const
Return true if the instruction is a call.
Definition: MCInstrDesc.h:285
llvm::DbgDeclareInst
This represents the llvm.dbg.declare instruction.
Definition: IntrinsicInst.h:348
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::MachineRegisterInfo::use_instr_begin
use_instr_iterator use_instr_begin(Register RegNo) const
Definition: MachineRegisterInfo.h:485
llvm::SelectionDAGISel::OPC_RecordChild0
@ OPC_RecordChild0
Definition: SelectionDAGISel.h:122
HandleMergeInputChains
static SDValue HandleMergeInputChains(SmallVectorImpl< SDNode * > &ChainNodesMatched, SelectionDAG *CurDAG)
HandleMergeInputChains - This implements the OPC_EmitMergeInputChains operation for when the pattern ...
Definition: SelectionDAGISel.cpp:2388
llvm::ISD::AssertZext
@ AssertZext
Definition: ISDOpcodes.h:62
llvm::Function::hasGC
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:306
llvm::ISD::STRICT_LRINT
@ STRICT_LRINT
Definition: ISDOpcodes.h:432
llvm::SelectionDAG::mutateStrictFPToFP
SDNode * mutateStrictFPToFP(SDNode *Node)
Mutate the specified strict FP node to its non-strict equivalent, unlinking the node from its chain a...
Definition: SelectionDAG.cpp:9743
llvm::SelectionDAGISel::FuncInfo
std::unique_ptr< FunctionLoweringInfo > FuncInfo
Definition: SelectionDAGISel.h:45
llvm::TargetRegisterInfo::getFrameRegister
virtual Register getFrameRegister(const MachineFunction &MF) const =0
Debug information queries.
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:53
llvm::SDNode::setNodeId
void setNodeId(int Id)
Set unique node id.
Definition: SelectionDAGNodes.h:728
llvm::SelectionDAG::getCopyFromReg
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:793
llvm::DenseMap< unsigned, unsigned >
llvm::SelectionDAGISel::IsLegalToFold
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOpt::Level OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
Definition: SelectionDAGISel.cpp:2079
llvm::SelectionDAGISel::mayRaiseFPException
bool mayRaiseFPException(SDNode *Node) const
Return whether the node may raise an FP exception.
Definition: SelectionDAGISel.cpp:3726
llvm::SelectionDAGISel::OPC_CheckChild2Same
@ OPC_CheckChild2Same
Definition: SelectionDAGISel.h:132
llvm::SDNode::getOperand
const SDValue & getOperand(unsigned Num) const
Definition: SelectionDAGNodes.h:920
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::SwiftErrorValueTracking
Definition: SwiftErrorValueTracking.h:34
llvm::MCPhysReg
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:21
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:9047
CheckOpcode
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDNode *N)
Definition: SelectionDAGISel.cpp:2543
llvm::MachineFrameInfo::getObjectAlign
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
Definition: MachineFrameInfo.h:484
llvm::SelectionDAGISel::OPC_CheckChild4Type
@ OPC_CheckChild4Type
Definition: SelectionDAGISel.h:142
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:447
llvm::ProfileSummaryInfoWrapperPass
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Definition: ProfileSummaryInfo.h:193
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1843
llvm::MachineRegisterInfo::liveins
ArrayRef< std::pair< MCRegister, Register > > liveins() const
Definition: MachineRegisterInfo.h:971
llvm::MachineFunction::finalizeDebugInstrRefs
void finalizeDebugInstrRefs()
Finalise any partially emitted debug instructions.
Definition: MachineFunction.cpp:1188
llvm::SelectionDAGISel::getPatternForIndex
virtual StringRef getPatternForIndex(unsigned index)
getPatternForIndex - Patterns selected by tablegen during ISEL
Definition: SelectionDAGISel.h:243
MachineFunctionPass.h
llvm::SelectionDAG::MorphNodeTo
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
Definition: SelectionDAG.cpp:9690
llvm::SelectionDAG::NewNodesMustHaveLegalTypes
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
Definition: SelectionDAG.h:384
llvm::Register::isVirtualRegister
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:71
llvm::SelectionDAGISel::OPC_CheckImmAllOnesV
@ OPC_CheckImmAllOnesV
Definition: SelectionDAGISel.h:151
llvm::SelectionDAGISel::OPC_SwitchOpcode
@ OPC_SwitchOpcode
Definition: SelectionDAGISel.h:137
llvm::DenseMapBase::find
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:150
llvm::MachineFunction::getName
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Definition: MachineFunction.cpp:567
llvm::SelectionDAGISel::OPC_CheckChild3Integer
@ OPC_CheckChild3Integer
Definition: SelectionDAGISel.h:146
llvm::ISD::STRICT_LROUND
@ STRICT_LROUND
Definition: ISDOpcodes.h:430
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:179
llvm::TargetMachine::Options
TargetOptions Options
Definition: TargetMachine.h:118
llvm::SelectionDAGISel::CheckOrMask
bool CheckOrMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckOrMask - The isel is trying to match something like (or X, 255).
Definition: SelectionDAGISel.cpp:1926
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
llvm::SelectionDAGISel::~SelectionDAGISel
~SelectionDAGISel() override
Definition: SelectionDAGISel.cpp:327
llvm::FunctionLoweringInfo
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Definition: FunctionLoweringInfo.h:52
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
llvm::SDNode::getNodeId
int getNodeId() const
Return the unique node id.
Definition: SelectionDAGNodes.h:725
llvm::createVLIWDAGScheduler
ScheduleDAGSDNodes * createVLIWDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createVLIWDAGScheduler - Scheduler for VLIW targets.
Definition: ScheduleDAGVLIW.cpp:269
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
llvm::MachineFunction::getFrameInfo
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Definition: MachineFunction.h:673
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:261
llvm::OptimizationRemarkEmitter::emit
void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Definition: OptimizationRemarkEmitter.cpp:77
llvm::ISD::STACKMAP
@ STACKMAP
Definition: ISDOpcodes.h:1293
llvm::SelectionDAGISel::OPC_EmitMergeInputChains
@ OPC_EmitMergeInputChains
Definition: SelectionDAGISel.h:160
llvm::InlineAsm::getMemoryConstraintID
static unsigned getMemoryConstraintID(unsigned Flag)
Definition: InlineAsm.h:355
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1424
llvm::ISD::BasicBlock
@ BasicBlock
Various leaf nodes.
Definition: ISDOpcodes.h:71
llvm::SelectionDAGISel::CurDAG
SelectionDAG * CurDAG
Definition: SelectionDAGISel.h:49
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:82
llvm::SelectionDAG::getMachineNode
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
Definition: SelectionDAG.cpp:9791
CodeGenCommonISel.h
MachineModuleInfo.h
llvm::ISD::TargetGlobalTLSAddress
@ TargetGlobalTLSAddress
Definition: ISDOpcodes.h:165
llvm::Sched::VLIW
@ VLIW
Definition: TargetLowering.h:103
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:31
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::SwiftErrorValueTracking::setFunction
void setFunction(MachineFunction &MF)
Initialize data structures for specified new function.
Definition: SwiftErrorValueTracking.cpp:79
llvm::FastISel
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:359
llvm::SelectionDAGISel::getUninvalidatedNodeId
static int getUninvalidatedNodeId(SDNode *N)
Definition: SelectionDAGISel.cpp:1061
llvm::SelectionDAGISel::SelectionDAGISel
SelectionDAGISel(TargetMachine &tm, CodeGenOpt::Level OL=CodeGenOpt::Default)
Definition: SelectionDAGISel.cpp:313
llvm::SelectionDAG::setNodeMemRefs
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
Definition: SelectionDAG.cpp:9559
SwiftErrorValueTracking.h
llvm::MDNode
Metadata node.
Definition: Metadata.h:944
llvm::ISD::STRICT_LLRINT
@ STRICT_LLRINT
Definition: ISDOpcodes.h:433
llvm::AfterLegalizeVectorOps
@ AfterLegalizeVectorOps
Definition: DAGCombine.h:18
llvm::MCInstrInfo::getName
StringRef getName(unsigned Opcode) const
Returns the name for the instructions with the given opcode.
Definition: MCInstrInfo.h:69
llvm::SelectionDAG::SelectNodeTo
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
Definition: SelectionDAG.cpp:9583
llvm::TargetOptions::EnableFastISel
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
Definition: TargetOptions.h:232
llvm::SDNode::isMachineOpcode
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
Definition: SelectionDAGNodes.h:699
llvm::ISD::TargetBlockAddress
@ TargetBlockAddress
Definition: ISDOpcodes.h:170
llvm::SelectionDAGISel::OPC_CheckPredicateWithOperands
@ OPC_CheckPredicateWithOperands
Definition: SelectionDAGISel.h:135
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:75
llvm::MachineFunction
Definition: MachineFunction.h:257
llvm::MCInstrDesc::mayStore
bool mayStore() const
Return true if this instruction could possibly modify memory.
Definition: MCInstrDesc.h:441
llvm::ISD::ANNOTATION_LABEL
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
Definition: ISDOpcodes.h:1039
llvm::SelectionDAGISel::CheckNodePredicateWithOperands
virtual bool CheckNodePredicateWithOperands(SDNode *N, unsigned PredNo, const SmallVectorImpl< SDValue > &Operands) const
CheckNodePredicateWithOperands - This function is generated by tblgen in the target.
Definition: SelectionDAGISel.h:285
llvm::MDNodeSDNode
Definition: SelectionDAGNodes.h:2143
llvm::createHybridListDAGScheduler
ScheduleDAGSDNodes * createHybridListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createHybridListDAGScheduler - This creates a bottom up register pressure aware list scheduler that m...
Definition: ScheduleDAGRRList.cpp:3175
llvm::CodeGenOpt::None
@ None
Definition: CodeGen.h:53
CFG.h
llvm::ISD::HANDLENODE
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
Definition: ISDOpcodes.h:1116
llvm::MachineBasicBlock::succ_empty
bool succ_empty() const
Definition: MachineBasicBlock.h:384
TargetOptions.h
SelectionDAGISel.h
ViewDAGCombine2
static cl::opt< bool > ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the second " "dag combine pass"))
llvm::ISD::TargetConstantPool
@ TargetConstantPool
Definition: ISDOpcodes.h:168
Scheduler
Machine Instruction Scheduler
Definition: MachineScheduler.cpp:222
llvm::NVPTXISD::Dummy
@ Dummy
Definition: NVPTXISelLowering.h:60
llvm::MachinePassRegistry
MachinePassRegistry - Track the registration of machine passes.
Definition: MachinePassRegistry.h:73
llvm::MachineBasicBlock::getFirstTerminator
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Definition: MachineBasicBlock.cpp:239
llvm::SmallPtrSetImplBase::clear
void clear()
Definition: SmallPtrSet.h:95
llvm::Sched::Source
@ Source
Definition: TargetLowering.h:99
llvm::AssumptionCacheTracker
An immutable pass that tracks lazily created AssumptionCache objects.
Definition: AssumptionCache.h:202
llvm::initializeAAResultsWrapperPassPass
void initializeAAResultsWrapperPassPass(PassRegistry &)
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::StackProtector::shouldEmitSDCheck
bool shouldEmitSDCheck(const BasicBlock &BB) const
Definition: StackProtector.cpp:595
llvm::OptimizationRemarkEmitter
The optimization diagnostic interface.
Definition: OptimizationRemarkEmitter.h:33
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1716
llvm::SDNode::use_end
static use_iterator use_end()
Definition: SelectionDAGNodes.h:803
llvm::MachineFrameInfo::setHasCalls
void setHasCalls(bool V)
Definition: MachineFrameInfo.h:614
DataLayout.h
llvm::SelectionDAGISel::OPC_CheckChild3Type
@ OPC_CheckChild3Type
Definition: SelectionDAGISel.h:142
llvm::MVT::i64
@ i64
Definition: MachineValueType.h:49
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:1009
llvm::SelectionDAGISel::SelectInlineAsmMemoryOperand
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
Definition: SelectionDAGISel.h:95
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition: AArch64SLSHardening.cpp:75
llvm::SelectionDAGBuilder
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
Definition: SelectionDAGBuilder.h:95
llvm::InlineAsm::getFlagWord
static unsigned getFlagWord(unsigned Kind, unsigned NumOps)
Definition: InlineAsm.h:293
llvm::Sched::Hybrid
@ Hybrid
Definition: TargetLowering.h:101
llvm::ISD::STRICT_SINT_TO_FP
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:448
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::AnalysisUsage::addPreserved
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Definition: PassAnalysisSupport.h:98
llvm::SelectionDAG::useInstrRefDebugInfo
void useInstrRefDebugInfo(bool Flag)
Signal whether instruction referencing variable locations are desired for this function's debug-info.
Definition: SelectionDAG.h:1868
llvm::ms_demangle::IntrinsicFunctionKind::New
@ New
llvm::BranchProbability
Definition: BranchProbability.h:30
Compiler.h
llvm::append_range
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:1988
llvm::TargetLowering::getRegisterByName
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
Definition: TargetLowering.h:4413
llvm::ilist_node_impl::getIterator
self_iterator getIterator()