Bug Summary

File:lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
Warning:line 1506, column 9
Called C++ object pointer is null

Annotated Source Code

1//===-- SelectionDAGISel.cpp - Implement the SelectionDAGISel class -------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This implements the SelectionDAGISel class.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ScheduleDAGSDNodes.h"
15#include "SelectionDAGBuilder.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/DenseMap.h"
18#include "llvm/ADT/None.h"
19#include "llvm/ADT/PostOrderIterator.h"
20#include "llvm/ADT/SmallPtrSet.h"
21#include "llvm/ADT/SmallSet.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/ADT/Statistic.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/Analysis/AliasAnalysis.h"
27#include "llvm/Analysis/BranchProbabilityInfo.h"
28#include "llvm/Analysis/CFG.h"
29#include "llvm/Analysis/OptimizationDiagnosticInfo.h"
30#include "llvm/Analysis/TargetLibraryInfo.h"
31#include "llvm/CodeGen/FastISel.h"
32#include "llvm/CodeGen/FunctionLoweringInfo.h"
33#include "llvm/CodeGen/GCMetadata.h"
34#include "llvm/CodeGen/MachineBasicBlock.h"
35#include "llvm/CodeGen/MachineFrameInfo.h"
36#include "llvm/CodeGen/MachineFunction.h"
37#include "llvm/CodeGen/MachineFunctionPass.h"
38#include "llvm/CodeGen/MachineInstr.h"
39#include "llvm/CodeGen/MachineInstrBuilder.h"
40#include "llvm/CodeGen/MachineMemOperand.h"
41#include "llvm/CodeGen/MachineOperand.h"
42#include "llvm/CodeGen/MachinePassRegistry.h"
43#include "llvm/CodeGen/MachineRegisterInfo.h"
44#include "llvm/CodeGen/MachineValueType.h"
45#include "llvm/CodeGen/SchedulerRegistry.h"
46#include "llvm/CodeGen/SelectionDAG.h"
47#include "llvm/CodeGen/SelectionDAGISel.h"
48#include "llvm/CodeGen/SelectionDAGNodes.h"
49#include "llvm/CodeGen/StackProtector.h"
50#include "llvm/CodeGen/ValueTypes.h"
51#include "llvm/IR/BasicBlock.h"
52#include "llvm/IR/Constants.h"
53#include "llvm/IR/DebugInfoMetadata.h"
54#include "llvm/IR/DebugLoc.h"
55#include "llvm/IR/DiagnosticInfo.h"
56#include "llvm/IR/Function.h"
57#include "llvm/IR/InlineAsm.h"
58#include "llvm/IR/InstrTypes.h"
59#include "llvm/IR/Instruction.h"
60#include "llvm/IR/Instructions.h"
61#include "llvm/IR/IntrinsicInst.h"
62#include "llvm/IR/Intrinsics.h"
63#include "llvm/IR/Metadata.h"
64#include "llvm/IR/Type.h"
65#include "llvm/IR/User.h"
66#include "llvm/MC/MCInstrDesc.h"
67#include "llvm/MC/MCRegisterInfo.h"
68#include "llvm/Pass.h"
69#include "llvm/Support/BranchProbability.h"
70#include "llvm/Support/Casting.h"
71#include "llvm/Support/CodeGen.h"
72#include "llvm/Support/CommandLine.h"
73#include "llvm/Support/Compiler.h"
74#include "llvm/Support/Debug.h"
75#include "llvm/Support/ErrorHandling.h"
76#include "llvm/Support/KnownBits.h"
77#include "llvm/Support/Timer.h"
78#include "llvm/Support/raw_ostream.h"
79#include "llvm/Target/TargetInstrInfo.h"
80#include "llvm/Target/TargetIntrinsicInfo.h"
81#include "llvm/Target/TargetLowering.h"
82#include "llvm/Target/TargetMachine.h"
83#include "llvm/Target/TargetOptions.h"
84#include "llvm/Target/TargetRegisterInfo.h"
85#include "llvm/Target/TargetSubtargetInfo.h"
86#include "llvm/Transforms/Utils/BasicBlockUtils.h"
87#include <algorithm>
88#include <cassert>
89#include <cstdint>
90#include <iterator>
91#include <memory>
92#include <string>
93#include <utility>
94#include <vector>
95
96using namespace llvm;
97
98#define DEBUG_TYPE"isel" "isel"
99
100STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on")static llvm::Statistic NumFastIselFailures = {"isel", "NumFastIselFailures"
, "Number of instructions fast isel failed on", {0}, false}
;
101STATISTIC(NumFastIselSuccess, "Number of instructions fast isel selected")static llvm::Statistic NumFastIselSuccess = {"isel", "NumFastIselSuccess"
, "Number of instructions fast isel selected", {0}, false}
;
102STATISTIC(NumFastIselBlocks, "Number of blocks selected entirely by fast isel")static llvm::Statistic NumFastIselBlocks = {"isel", "NumFastIselBlocks"
, "Number of blocks selected entirely by fast isel", {0}, false
}
;
103STATISTIC(NumDAGBlocks, "Number of blocks selected using DAG")static llvm::Statistic NumDAGBlocks = {"isel", "NumDAGBlocks"
, "Number of blocks selected using DAG", {0}, false}
;
104STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path")static llvm::Statistic NumDAGIselRetries = {"isel", "NumDAGIselRetries"
, "Number of times dag isel has to try another path", {0}, false
}
;
105STATISTIC(NumEntryBlocks, "Number of entry blocks encountered")static llvm::Statistic NumEntryBlocks = {"isel", "NumEntryBlocks"
, "Number of entry blocks encountered", {0}, false}
;
106STATISTIC(NumFastIselFailLowerArguments,static llvm::Statistic NumFastIselFailLowerArguments = {"isel"
, "NumFastIselFailLowerArguments", "Number of entry blocks where fast isel failed to lower arguments"
, {0}, false}
107 "Number of entry blocks where fast isel failed to lower arguments")static llvm::Statistic NumFastIselFailLowerArguments = {"isel"
, "NumFastIselFailLowerArguments", "Number of entry blocks where fast isel failed to lower arguments"
, {0}, false}
;
108
109static cl::opt<int> EnableFastISelAbort(
110 "fast-isel-abort", cl::Hidden,
111 cl::desc("Enable abort calls when \"fast\" instruction selection "
112 "fails to lower an instruction: 0 disable the abort, 1 will "
113 "abort but for args, calls and terminators, 2 will also "
114 "abort for argument lowering, and 3 will never fallback "
115 "to SelectionDAG."));
116
117static cl::opt<bool> EnableFastISelFallbackReport(
118 "fast-isel-report-on-fallback", cl::Hidden,
119 cl::desc("Emit a diagnostic when \"fast\" instruction selection "
120 "falls back to SelectionDAG."));
121
122static cl::opt<bool>
123UseMBPI("use-mbpi",
124 cl::desc("use Machine Branch Probability Info"),
125 cl::init(true), cl::Hidden);
126
127#ifndef NDEBUG
128static cl::opt<std::string>
129FilterDAGBasicBlockName("filter-view-dags", cl::Hidden,
130 cl::desc("Only display the basic block whose name "
131 "matches this for all view-*-dags options"));
132static cl::opt<bool>
133ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden,
134 cl::desc("Pop up a window to show dags before the first "
135 "dag combine pass"));
136static cl::opt<bool>
137ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden,
138 cl::desc("Pop up a window to show dags before legalize types"));
139static cl::opt<bool>
140ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,
141 cl::desc("Pop up a window to show dags before legalize"));
142static cl::opt<bool>
143ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden,
144 cl::desc("Pop up a window to show dags before the second "
145 "dag combine pass"));
146static cl::opt<bool>
147ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden,
148 cl::desc("Pop up a window to show dags before the post legalize types"
149 " dag combine pass"));
150static cl::opt<bool>
151ViewISelDAGs("view-isel-dags", cl::Hidden,
152 cl::desc("Pop up a window to show isel dags as they are selected"));
153static cl::opt<bool>
154ViewSchedDAGs("view-sched-dags", cl::Hidden,
155 cl::desc("Pop up a window to show sched dags as they are processed"));
156static cl::opt<bool>
157ViewSUnitDAGs("view-sunit-dags", cl::Hidden,
158 cl::desc("Pop up a window to show SUnit dags after they are processed"));
159#else
160static const bool ViewDAGCombine1 = false,
161 ViewLegalizeTypesDAGs = false, ViewLegalizeDAGs = false,
162 ViewDAGCombine2 = false,
163 ViewDAGCombineLT = false,
164 ViewISelDAGs = false, ViewSchedDAGs = false,
165 ViewSUnitDAGs = false;
166#endif
167
168//===---------------------------------------------------------------------===//
169///
170/// RegisterScheduler class - Track the registration of instruction schedulers.
171///
172//===---------------------------------------------------------------------===//
173MachinePassRegistry RegisterScheduler::Registry;
174
175//===---------------------------------------------------------------------===//
176///
177/// ISHeuristic command line option for instruction schedulers.
178///
179//===---------------------------------------------------------------------===//
180static cl::opt<RegisterScheduler::FunctionPassCtor, false,
181 RegisterPassParser<RegisterScheduler>>
182ISHeuristic("pre-RA-sched",
183 cl::init(&createDefaultScheduler), cl::Hidden,
184 cl::desc("Instruction schedulers available (before register"
185 " allocation):"));
186
187static RegisterScheduler
188defaultListDAGScheduler("default", "Best scheduler for the target",
189 createDefaultScheduler);
190
191namespace llvm {
192
193 //===--------------------------------------------------------------------===//
194 /// \brief This class is used by SelectionDAGISel to temporarily override
195 /// the optimization level on a per-function basis.
196 class OptLevelChanger {
197 SelectionDAGISel &IS;
198 CodeGenOpt::Level SavedOptLevel;
199 bool SavedFastISel;
200
201 public:
202 OptLevelChanger(SelectionDAGISel &ISel,
203 CodeGenOpt::Level NewOptLevel) : IS(ISel) {
204 SavedOptLevel = IS.OptLevel;
205 if (NewOptLevel == SavedOptLevel)
206 return;
207 IS.OptLevel = NewOptLevel;
208 IS.TM.setOptLevel(NewOptLevel);
209 DEBUG(dbgs() << "\nChanging optimization level for Function "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\nChanging optimization level for Function "
<< IS.MF->getFunction()->getName() << "\n"
; } } while (false)
210 << IS.MF->getFunction()->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\nChanging optimization level for Function "
<< IS.MF->getFunction()->getName() << "\n"
; } } while (false)
;
211 DEBUG(dbgs() << "\tBefore: -O" << SavedOptLeveldo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tBefore: -O" << SavedOptLevel
<< " ; After: -O" << NewOptLevel << "\n"; }
} while (false)
212 << " ; After: -O" << NewOptLevel << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tBefore: -O" << SavedOptLevel
<< " ; After: -O" << NewOptLevel << "\n"; }
} while (false)
;
213 SavedFastISel = IS.TM.Options.EnableFastISel;
214 if (NewOptLevel == CodeGenOpt::None) {
215 IS.TM.setFastISel(IS.TM.getO0WantsFastISel());
216 DEBUG(dbgs() << "\tFastISel is "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tFastISel is " << (IS.TM.
Options.EnableFastISel ? "enabled" : "disabled") << "\n"
; } } while (false)
217 << (IS.TM.Options.EnableFastISel ? "enabled" : "disabled")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tFastISel is " << (IS.TM.
Options.EnableFastISel ? "enabled" : "disabled") << "\n"
; } } while (false)
218 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tFastISel is " << (IS.TM.
Options.EnableFastISel ? "enabled" : "disabled") << "\n"
; } } while (false)
;
219 }
220 }
221
222 ~OptLevelChanger() {
223 if (IS.OptLevel == SavedOptLevel)
224 return;
225 DEBUG(dbgs() << "\nRestoring optimization level for Function "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\nRestoring optimization level for Function "
<< IS.MF->getFunction()->getName() << "\n"
; } } while (false)
226 << IS.MF->getFunction()->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\nRestoring optimization level for Function "
<< IS.MF->getFunction()->getName() << "\n"
; } } while (false)
;
227 DEBUG(dbgs() << "\tBefore: -O" << IS.OptLeveldo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tBefore: -O" << IS.OptLevel
<< " ; After: -O" << SavedOptLevel << "\n"
; } } while (false)
228 << " ; After: -O" << SavedOptLevel << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tBefore: -O" << IS.OptLevel
<< " ; After: -O" << SavedOptLevel << "\n"
; } } while (false)
;
229 IS.OptLevel = SavedOptLevel;
230 IS.TM.setOptLevel(SavedOptLevel);
231 IS.TM.setFastISel(SavedFastISel);
232 }
233 };
234
235 //===--------------------------------------------------------------------===//
236 /// createDefaultScheduler - This creates an instruction scheduler appropriate
237 /// for the target.
238 ScheduleDAGSDNodes* createDefaultScheduler(SelectionDAGISel *IS,
239 CodeGenOpt::Level OptLevel) {
240 const TargetLowering *TLI = IS->TLI;
241 const TargetSubtargetInfo &ST = IS->MF->getSubtarget();
242
243 // Try first to see if the Target has its own way of selecting a scheduler
244 if (auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) {
245 return SchedulerCtor(IS, OptLevel);
246 }
247
248 if (OptLevel == CodeGenOpt::None ||
249 (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) ||
250 TLI->getSchedulingPreference() == Sched::Source)
251 return createSourceListDAGScheduler(IS, OptLevel);
252 if (TLI->getSchedulingPreference() == Sched::RegPressure)
253 return createBURRListDAGScheduler(IS, OptLevel);
254 if (TLI->getSchedulingPreference() == Sched::Hybrid)
255 return createHybridListDAGScheduler(IS, OptLevel);
256 if (TLI->getSchedulingPreference() == Sched::VLIW)
257 return createVLIWDAGScheduler(IS, OptLevel);
258 assert(TLI->getSchedulingPreference() == Sched::ILP &&((TLI->getSchedulingPreference() == Sched::ILP && "Unknown sched type!"
) ? static_cast<void> (0) : __assert_fail ("TLI->getSchedulingPreference() == Sched::ILP && \"Unknown sched type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 259, __PRETTY_FUNCTION__))
259 "Unknown sched type!")((TLI->getSchedulingPreference() == Sched::ILP && "Unknown sched type!"
) ? static_cast<void> (0) : __assert_fail ("TLI->getSchedulingPreference() == Sched::ILP && \"Unknown sched type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 259, __PRETTY_FUNCTION__))
;
260 return createILPListDAGScheduler(IS, OptLevel);
261 }
262
263} // end namespace llvm
264
265// EmitInstrWithCustomInserter - This method should be implemented by targets
266// that mark instructions with the 'usesCustomInserter' flag. These
267// instructions are special in various ways, which require special support to
268// insert. The specified MachineInstr is created but not inserted into any
269// basic blocks, and this method is called to expand it into a sequence of
270// instructions, potentially also creating new basic blocks and control flow.
271// When new basic blocks are inserted and the edges from MBB to its successors
272// are modified, the method should insert pairs of <OldSucc, NewSucc> into the
273// DenseMap.
274MachineBasicBlock *
275TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
276 MachineBasicBlock *MBB) const {
277#ifndef NDEBUG
278 dbgs() << "If a target marks an instruction with "
279 "'usesCustomInserter', it must implement "
280 "TargetLowering::EmitInstrWithCustomInserter!";
281#endif
282 llvm_unreachable(nullptr)::llvm::llvm_unreachable_internal(nullptr, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 282)
;
283}
284
285void TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
286 SDNode *Node) const {
287 assert(!MI.hasPostISelHook() &&((!MI.hasPostISelHook() && "If a target marks an instruction with 'hasPostISelHook', "
"it must implement TargetLowering::AdjustInstrPostInstrSelection!"
) ? static_cast<void> (0) : __assert_fail ("!MI.hasPostISelHook() && \"If a target marks an instruction with 'hasPostISelHook', \" \"it must implement TargetLowering::AdjustInstrPostInstrSelection!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 289, __PRETTY_FUNCTION__))
288 "If a target marks an instruction with 'hasPostISelHook', "((!MI.hasPostISelHook() && "If a target marks an instruction with 'hasPostISelHook', "
"it must implement TargetLowering::AdjustInstrPostInstrSelection!"
) ? static_cast<void> (0) : __assert_fail ("!MI.hasPostISelHook() && \"If a target marks an instruction with 'hasPostISelHook', \" \"it must implement TargetLowering::AdjustInstrPostInstrSelection!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 289, __PRETTY_FUNCTION__))
289 "it must implement TargetLowering::AdjustInstrPostInstrSelection!")((!MI.hasPostISelHook() && "If a target marks an instruction with 'hasPostISelHook', "
"it must implement TargetLowering::AdjustInstrPostInstrSelection!"
) ? static_cast<void> (0) : __assert_fail ("!MI.hasPostISelHook() && \"If a target marks an instruction with 'hasPostISelHook', \" \"it must implement TargetLowering::AdjustInstrPostInstrSelection!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 289, __PRETTY_FUNCTION__))
;
290}
291
292//===----------------------------------------------------------------------===//
293// SelectionDAGISel code
294//===----------------------------------------------------------------------===//
295
296SelectionDAGISel::SelectionDAGISel(TargetMachine &tm,
297 CodeGenOpt::Level OL) :
298 MachineFunctionPass(ID), TM(tm),
299 FuncInfo(new FunctionLoweringInfo()),
300 CurDAG(new SelectionDAG(tm, OL)),
301 SDB(new SelectionDAGBuilder(*CurDAG, *FuncInfo, OL)),
302 GFI(),
303 OptLevel(OL),
304 DAGSize(0) {
305 initializeGCModuleInfoPass(*PassRegistry::getPassRegistry());
306 initializeBranchProbabilityInfoWrapperPassPass(
307 *PassRegistry::getPassRegistry());
308 initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry());
309 initializeTargetLibraryInfoWrapperPassPass(
310 *PassRegistry::getPassRegistry());
311 }
312
313SelectionDAGISel::~SelectionDAGISel() {
314 delete SDB;
315 delete CurDAG;
316 delete FuncInfo;
317}
318
319void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const {
320 AU.addRequired<AAResultsWrapperPass>();
321 AU.addRequired<GCModuleInfo>();
322 AU.addRequired<StackProtector>();
323 AU.addPreserved<StackProtector>();
324 AU.addPreserved<GCModuleInfo>();
325 AU.addRequired<TargetLibraryInfoWrapperPass>();
326 if (UseMBPI && OptLevel != CodeGenOpt::None)
327 AU.addRequired<BranchProbabilityInfoWrapperPass>();
328 MachineFunctionPass::getAnalysisUsage(AU);
329}
330
331/// SplitCriticalSideEffectEdges - Look for critical edges with a PHI value that
332/// may trap on it. In this case we have to split the edge so that the path
333/// through the predecessor block that doesn't go to the phi block doesn't
334/// execute the possibly trapping instruction.
335///
336/// This is required for correctness, so it must be done at -O0.
337///
338static void SplitCriticalSideEffectEdges(Function &Fn) {
339 // Loop for blocks with phi nodes.
340 for (BasicBlock &BB : Fn) {
341 PHINode *PN = dyn_cast<PHINode>(BB.begin());
342 if (!PN) continue;
343
344 ReprocessBlock:
345 // For each block with a PHI node, check to see if any of the input values
346 // are potentially trapping constant expressions. Constant expressions are
347 // the only potentially trapping value that can occur as the argument to a
348 // PHI.
349 for (BasicBlock::iterator I = BB.begin(); (PN = dyn_cast<PHINode>(I)); ++I)
350 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
351 ConstantExpr *CE = dyn_cast<ConstantExpr>(PN->getIncomingValue(i));
352 if (!CE || !CE->canTrap()) continue;
353
354 // The only case we have to worry about is when the edge is critical.
355 // Since this block has a PHI Node, we assume it has multiple input
356 // edges: check to see if the pred has multiple successors.
357 BasicBlock *Pred = PN->getIncomingBlock(i);
358 if (Pred->getTerminator()->getNumSuccessors() == 1)
359 continue;
360
361 // Okay, we have to split this edge.
362 SplitCriticalEdge(
363 Pred->getTerminator(), GetSuccessorNumber(Pred, &BB),
364 CriticalEdgeSplittingOptions().setMergeIdenticalEdges());
365 goto ReprocessBlock;
366 }
367 }
368}
369
370bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
371 // If we already selected that function, we do not need to run SDISel.
372 if (mf.getProperties().hasProperty(
373 MachineFunctionProperties::Property::Selected))
374 return false;
375 // Do some sanity-checking on the command-line options.
376 assert((!EnableFastISelAbort || TM.Options.EnableFastISel) &&(((!EnableFastISelAbort || TM.Options.EnableFastISel) &&
"-fast-isel-abort > 0 requires -fast-isel") ? static_cast
<void> (0) : __assert_fail ("(!EnableFastISelAbort || TM.Options.EnableFastISel) && \"-fast-isel-abort > 0 requires -fast-isel\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 377, __PRETTY_FUNCTION__))
377 "-fast-isel-abort > 0 requires -fast-isel")(((!EnableFastISelAbort || TM.Options.EnableFastISel) &&
"-fast-isel-abort > 0 requires -fast-isel") ? static_cast
<void> (0) : __assert_fail ("(!EnableFastISelAbort || TM.Options.EnableFastISel) && \"-fast-isel-abort > 0 requires -fast-isel\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 377, __PRETTY_FUNCTION__))
;
378
379 const Function &Fn = *mf.getFunction();
380 MF = &mf;
381
382 // Reset the target options before resetting the optimization
383 // level below.
384 // FIXME: This is a horrible hack and should be processed via
385 // codegen looking at the optimization level explicitly when
386 // it wants to look at it.
387 TM.resetTargetOptions(Fn);
388 // Reset OptLevel to None for optnone functions.
389 CodeGenOpt::Level NewOptLevel = OptLevel;
390 if (OptLevel != CodeGenOpt::None && skipFunction(Fn))
391 NewOptLevel = CodeGenOpt::None;
392 OptLevelChanger OLC(*this, NewOptLevel);
393
394 TII = MF->getSubtarget().getInstrInfo();
395 TLI = MF->getSubtarget().getTargetLowering();
396 RegInfo = &MF->getRegInfo();
397 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
398 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
399 GFI = Fn.hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) : nullptr;
400 ORE = make_unique<OptimizationRemarkEmitter>(&Fn);
401
402 DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\n\n\n=== " << Fn.getName(
) << "\n"; } } while (false)
;
403
404 SplitCriticalSideEffectEdges(const_cast<Function &>(Fn));
405
406 CurDAG->init(*MF, *ORE);
407 FuncInfo->set(Fn, *MF, CurDAG);
408
409 if (UseMBPI && OptLevel != CodeGenOpt::None)
410 FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
411 else
412 FuncInfo->BPI = nullptr;
413
414 SDB->init(GFI, *AA, LibInfo);
415
416 MF->setHasInlineAsm(false);
417
418 FuncInfo->SplitCSR = false;
419
420 // We split CSR if the target supports it for the given function
421 // and the function has only return exits.
422 if (OptLevel != CodeGenOpt::None && TLI->supportSplitCSR(MF)) {
423 FuncInfo->SplitCSR = true;
424
425 // Collect all the return blocks.
426 for (const BasicBlock &BB : Fn) {
427 if (!succ_empty(&BB))
428 continue;
429
430 const TerminatorInst *Term = BB.getTerminator();
431 if (isa<UnreachableInst>(Term) || isa<ReturnInst>(Term))
432 continue;
433
434 // Bail out if the exit block is not Return nor Unreachable.
435 FuncInfo->SplitCSR = false;
436 break;
437 }
438 }
439
440 MachineBasicBlock *EntryMBB = &MF->front();
441 if (FuncInfo->SplitCSR)
442 // This performs initialization so lowering for SplitCSR will be correct.
443 TLI->initializeSplitCSR(EntryMBB);
444
445 SelectAllBasicBlocks(Fn);
446 if (FastISelFailed && EnableFastISelFallbackReport) {
447 DiagnosticInfoISelFallback DiagFallback(Fn);
448 Fn.getContext().diagnose(DiagFallback);
449 }
450
451 // If the first basic block in the function has live ins that need to be
452 // copied into vregs, emit the copies into the top of the block before
453 // emitting the code for the block.
454 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
455 RegInfo->EmitLiveInCopies(EntryMBB, TRI, *TII);
456
457 // Insert copies in the entry block and the return blocks.
458 if (FuncInfo->SplitCSR) {
459 SmallVector<MachineBasicBlock*, 4> Returns;
460 // Collect all the return blocks.
461 for (MachineBasicBlock &MBB : mf) {
462 if (!MBB.succ_empty())
463 continue;
464
465 MachineBasicBlock::iterator Term = MBB.getFirstTerminator();
466 if (Term != MBB.end() && Term->isReturn()) {
467 Returns.push_back(&MBB);
468 continue;
469 }
470 }
471 TLI->insertCopiesSplitCSR(EntryMBB, Returns);
472 }
473
474 DenseMap<unsigned, unsigned> LiveInMap;
475 if (!FuncInfo->ArgDbgValues.empty())
476 for (MachineRegisterInfo::livein_iterator LI = RegInfo->livein_begin(),
477 E = RegInfo->livein_end(); LI != E; ++LI)
478 if (LI->second)
479 LiveInMap.insert(std::make_pair(LI->first, LI->second));
480
481 // Insert DBG_VALUE instructions for function arguments to the entry block.
482 for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) {
483 MachineInstr *MI = FuncInfo->ArgDbgValues[e-i-1];
484 bool hasFI = MI->getOperand(0).isFI();
485 unsigned Reg =
486 hasFI ? TRI.getFrameRegister(*MF) : MI->getOperand(0).getReg();
487 if (TargetRegisterInfo::isPhysicalRegister(Reg))
488 EntryMBB->insert(EntryMBB->begin(), MI);
489 else {
490 MachineInstr *Def = RegInfo->getVRegDef(Reg);
491 if (Def) {
492 MachineBasicBlock::iterator InsertPos = Def;
493 // FIXME: VR def may not be in entry block.
494 Def->getParent()->insert(std::next(InsertPos), MI);
495 } else
496 DEBUG(dbgs() << "Dropping debug info for dead vreg"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for dead vreg"
<< TargetRegisterInfo::virtReg2Index(Reg) << "\n"
; } } while (false)
497 << TargetRegisterInfo::virtReg2Index(Reg) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for dead vreg"
<< TargetRegisterInfo::virtReg2Index(Reg) << "\n"
; } } while (false)
;
498 }
499
500 // If Reg is live-in then update debug info to track its copy in a vreg.
501 DenseMap<unsigned, unsigned>::iterator LDI = LiveInMap.find(Reg);
502 if (LDI != LiveInMap.end()) {
503 assert(!hasFI && "There's no handling of frame pointer updating here yet "((!hasFI && "There's no handling of frame pointer updating here yet "
"- add if needed") ? static_cast<void> (0) : __assert_fail
("!hasFI && \"There's no handling of frame pointer updating here yet \" \"- add if needed\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 504, __PRETTY_FUNCTION__))
504 "- add if needed")((!hasFI && "There's no handling of frame pointer updating here yet "
"- add if needed") ? static_cast<void> (0) : __assert_fail
("!hasFI && \"There's no handling of frame pointer updating here yet \" \"- add if needed\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 504, __PRETTY_FUNCTION__))
;
505 MachineInstr *Def = RegInfo->getVRegDef(LDI->second);
506 MachineBasicBlock::iterator InsertPos = Def;
507 const MDNode *Variable = MI->getDebugVariable();
508 const MDNode *Expr = MI->getDebugExpression();
509 DebugLoc DL = MI->getDebugLoc();
510 bool IsIndirect = MI->isIndirectDebugValue();
511 unsigned Offset = IsIndirect ? MI->getOperand(1).getImm() : 0;
512 assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&((cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? static_cast
<void> (0) : __assert_fail ("cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 513, __PRETTY_FUNCTION__))
513 "Expected inlined-at fields to agree")((cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? static_cast
<void> (0) : __assert_fail ("cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 513, __PRETTY_FUNCTION__))
;
514 // Def is never a terminator here, so it is ok to increment InsertPos.
515 BuildMI(*EntryMBB, ++InsertPos, DL, TII->get(TargetOpcode::DBG_VALUE),
516 IsIndirect, LDI->second, Offset, Variable, Expr);
517
518 // If this vreg is directly copied into an exported register then
519 // that COPY instructions also need DBG_VALUE, if it is the only
520 // user of LDI->second.
521 MachineInstr *CopyUseMI = nullptr;
522 for (MachineRegisterInfo::use_instr_iterator
523 UI = RegInfo->use_instr_begin(LDI->second),
524 E = RegInfo->use_instr_end(); UI != E; ) {
525 MachineInstr *UseMI = &*(UI++);
526 if (UseMI->isDebugValue()) continue;
527 if (UseMI->isCopy() && !CopyUseMI && UseMI->getParent() == EntryMBB) {
528 CopyUseMI = UseMI; continue;
529 }
530 // Otherwise this is another use or second copy use.
531 CopyUseMI = nullptr; break;
532 }
533 if (CopyUseMI) {
534 // Use MI's debug location, which describes where Variable was
535 // declared, rather than whatever is attached to CopyUseMI.
536 MachineInstr *NewMI =
537 BuildMI(*MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
538 CopyUseMI->getOperand(0).getReg(), Offset, Variable, Expr);
539 MachineBasicBlock::iterator Pos = CopyUseMI;
540 EntryMBB->insertAfter(Pos, NewMI);
541 }
542 }
543 }
544
545 // Determine if there are any calls in this machine function.
546 MachineFrameInfo &MFI = MF->getFrameInfo();
547 for (const auto &MBB : *MF) {
548 if (MFI.hasCalls() && MF->hasInlineAsm())
549 break;
550
551 for (const auto &MI : MBB) {
552 const MCInstrDesc &MCID = TII->get(MI.getOpcode());
553 if ((MCID.isCall() && !MCID.isReturn()) ||
554 MI.isStackAligningInlineAsm()) {
555 MFI.setHasCalls(true);
556 }
557 if (MI.isInlineAsm()) {
558 MF->setHasInlineAsm(true);
559 }
560 }
561 }
562
563 // Determine if there is a call to setjmp in the machine function.
564 MF->setExposesReturnsTwice(Fn.callsFunctionThatReturnsTwice());
565
566 // Replace forward-declared registers with the registers containing
567 // the desired value.
568 MachineRegisterInfo &MRI = MF->getRegInfo();
569 for (DenseMap<unsigned, unsigned>::iterator
570 I = FuncInfo->RegFixups.begin(), E = FuncInfo->RegFixups.end();
571 I != E; ++I) {
572 unsigned From = I->first;
573 unsigned To = I->second;
574 // If To is also scheduled to be replaced, find what its ultimate
575 // replacement is.
576 while (true) {
577 DenseMap<unsigned, unsigned>::iterator J = FuncInfo->RegFixups.find(To);
578 if (J == E) break;
579 To = J->second;
580 }
581 // Make sure the new register has a sufficiently constrained register class.
582 if (TargetRegisterInfo::isVirtualRegister(From) &&
583 TargetRegisterInfo::isVirtualRegister(To))
584 MRI.constrainRegClass(To, MRI.getRegClass(From));
585 // Replace it.
586
587
588 // Replacing one register with another won't touch the kill flags.
589 // We need to conservatively clear the kill flags as a kill on the old
590 // register might dominate existing uses of the new register.
591 if (!MRI.use_empty(To))
592 MRI.clearKillFlags(From);
593 MRI.replaceRegWith(From, To);
594 }
595
596 TLI->finalizeLowering(*MF);
597
598 // Release function-specific state. SDB and CurDAG are already cleared
599 // at this point.
600 FuncInfo->clear();
601
602 DEBUG(dbgs() << "*** MachineFunction at end of ISel ***\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "*** MachineFunction at end of ISel ***\n"
; } } while (false)
;
603 DEBUG(MF->print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { MF->print(dbgs()); } } while (false)
;
604
605 return true;
606}
607
608static void reportFastISelFailure(MachineFunction &MF,
609 OptimizationRemarkEmitter &ORE,
610 OptimizationRemarkMissed &R,
611 bool ShouldAbort) {
612 // Print the function name explicitly if we don't have a debug location (which
613 // makes the diagnostic less useful) or if we're going to emit a raw error.
614 if (!R.getLocation().isValid() || ShouldAbort)
615 R << (" (in function: " + MF.getName() + ")").str();
616
617 if (ShouldAbort)
618 report_fatal_error(R.getMsg());
619
620 ORE.emit(R);
621}
622
623void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin,
624 BasicBlock::const_iterator End,
625 bool &HadTailCall) {
626 // Lower the instructions. If a call is emitted as a tail call, cease emitting
627 // nodes for this block.
628 for (BasicBlock::const_iterator I = Begin; I != End && !SDB->HasTailCall; ++I) {
629 if (!ElidedArgCopyInstrs.count(&*I))
630 SDB->visit(*I);
631 }
632
633 // Make sure the root of the DAG is up-to-date.
634 CurDAG->setRoot(SDB->getControlRoot());
635 HadTailCall = SDB->HasTailCall;
636 SDB->clear();
637
638 // Final step, emit the lowered DAG as machine code.
639 CodeGenAndEmitDAG();
640}
641
642void SelectionDAGISel::ComputeLiveOutVRegInfo() {
643 SmallPtrSet<SDNode*, 16> VisitedNodes;
644 SmallVector<SDNode*, 128> Worklist;
645
646 Worklist.push_back(CurDAG->getRoot().getNode());
647
648 KnownBits Known;
649
650 do {
651 SDNode *N = Worklist.pop_back_val();
652
653 // If we've already seen this node, ignore it.
654 if (!VisitedNodes.insert(N).second)
655 continue;
656
657 // Otherwise, add all chain operands to the worklist.
658 for (const SDValue &Op : N->op_values())
659 if (Op.getValueType() == MVT::Other)
660 Worklist.push_back(Op.getNode());
661
662 // If this is a CopyToReg with a vreg dest, process it.
663 if (N->getOpcode() != ISD::CopyToReg)
664 continue;
665
666 unsigned DestReg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
667 if (!TargetRegisterInfo::isVirtualRegister(DestReg))
668 continue;
669
670 // Ignore non-scalar or non-integer values.
671 SDValue Src = N->getOperand(2);
672 EVT SrcVT = Src.getValueType();
673 if (!SrcVT.isInteger() || SrcVT.isVector())
674 continue;
675
676 unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src);
677 CurDAG->computeKnownBits(Src, Known);
678 FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, Known);
679 } while (!Worklist.empty());
680}
681
682void SelectionDAGISel::CodeGenAndEmitDAG() {
683 StringRef GroupName = "sdag";
684 StringRef GroupDescription = "Instruction Selection and Scheduling";
685 std::string BlockName;
686 int BlockNumber = -1;
687 (void)BlockNumber;
688 bool MatchFilterBB = false; (void)MatchFilterBB;
689
690 // Pre-type legalization allow creation of any node types.
691 CurDAG->NewNodesMustHaveLegalTypes = false;
692
693#ifndef NDEBUG
694 MatchFilterBB = (FilterDAGBasicBlockName.empty() ||
695 FilterDAGBasicBlockName ==
696 FuncInfo->MBB->getBasicBlock()->getName().str());
697#endif
698#ifdef NDEBUG
699 if (ViewDAGCombine1 || ViewLegalizeTypesDAGs || ViewLegalizeDAGs ||
700 ViewDAGCombine2 || ViewDAGCombineLT || ViewISelDAGs || ViewSchedDAGs ||
701 ViewSUnitDAGs)
702#endif
703 {
704 BlockNumber = FuncInfo->MBB->getNumber();
705 BlockName =
706 (MF->getName() + ":" + FuncInfo->MBB->getBasicBlock()->getName()).str();
707 }
708 DEBUG(dbgs() << "Initial selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Initial selection DAG: BB#" <<
BlockNumber << " '" << BlockName << "'\n";
CurDAG->dump(); } } while (false)
709 << " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Initial selection DAG: BB#" <<
BlockNumber << " '" << BlockName << "'\n";
CurDAG->dump(); } } while (false)
;
710
711 if (ViewDAGCombine1 && MatchFilterBB)
712 CurDAG->viewGraph("dag-combine1 input for " + BlockName);
713
714 // Run the DAG combiner in pre-legalize mode.
715 {
716 NamedRegionTimer T("combine1", "DAG Combining 1", GroupName,
717 GroupDescription, TimePassesIsEnabled);
718 CurDAG->Combine(BeforeLegalizeTypes, *AA, OptLevel);
719 }
720
721 DEBUG(dbgs() << "Optimized lowered selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized lowered selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)
722 << " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized lowered selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)
;
723
724 // Second step, hack on the DAG until it only uses operations and types that
725 // the target supports.
726 if (ViewLegalizeTypesDAGs && MatchFilterBB)
727 CurDAG->viewGraph("legalize-types input for " + BlockName);
728
729 bool Changed;
730 {
731 NamedRegionTimer T("legalize_types", "Type Legalization", GroupName,
732 GroupDescription, TimePassesIsEnabled);
733 Changed = CurDAG->LegalizeTypes();
734 }
735
736 DEBUG(dbgs() << "Type-legalized selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Type-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)
737 << " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Type-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)
;
738
739 // Only allow creation of legal node types.
740 CurDAG->NewNodesMustHaveLegalTypes = true;
741
742 if (Changed) {
743 if (ViewDAGCombineLT && MatchFilterBB)
744 CurDAG->viewGraph("dag-combine-lt input for " + BlockName);
745
746 // Run the DAG combiner in post-type-legalize mode.
747 {
748 NamedRegionTimer T("combine_lt", "DAG Combining after legalize types",
749 GroupName, GroupDescription, TimePassesIsEnabled);
750 CurDAG->Combine(AfterLegalizeTypes, *AA, OptLevel);
751 }
752
753 DEBUG(dbgs() << "Optimized type-legalized selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized type-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)
754 << " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized type-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)
;
755
756 }
757
758 {
759 NamedRegionTimer T("legalize_vec", "Vector Legalization", GroupName,
760 GroupDescription, TimePassesIsEnabled);
761 Changed = CurDAG->LegalizeVectors();
762 }
763
764 if (Changed) {
765 DEBUG(dbgs() << "Vector-legalized selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)
766 << " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)
;
767
768 {
769 NamedRegionTimer T("legalize_types2", "Type Legalization 2", GroupName,
770 GroupDescription, TimePassesIsEnabled);
771 CurDAG->LegalizeTypes();
772 }
773
774 DEBUG(dbgs() << "Vector/type-legalized selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector/type-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)
775 << " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector/type-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)
;
776
777 if (ViewDAGCombineLT && MatchFilterBB)
778 CurDAG->viewGraph("dag-combine-lv input for " + BlockName);
779
780 // Run the DAG combiner in post-type-legalize mode.
781 {
782 NamedRegionTimer T("combine_lv", "DAG Combining after legalize vectors",
783 GroupName, GroupDescription, TimePassesIsEnabled);
784 CurDAG->Combine(AfterLegalizeVectorOps, *AA, OptLevel);
785 }
786
787 DEBUG(dbgs() << "Optimized vector-legalized selection DAG: BB#"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized vector-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)
788 << BlockNumber << " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized vector-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)
;
789 }
790
791 if (ViewLegalizeDAGs && MatchFilterBB)
792 CurDAG->viewGraph("legalize input for " + BlockName);
793
794 {
795 NamedRegionTimer T("legalize", "DAG Legalization", GroupName,
796 GroupDescription, TimePassesIsEnabled);
797 CurDAG->Legalize();
798 }
799
800 DEBUG(dbgs() << "Legalized selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Legalized selection DAG: BB#" <<
BlockNumber << " '" << BlockName << "'\n";
CurDAG->dump(); } } while (false)
801 << " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Legalized selection DAG: BB#" <<
BlockNumber << " '" << BlockName << "'\n";
CurDAG->dump(); } } while (false)
;
802
803 if (ViewDAGCombine2 && MatchFilterBB)
804 CurDAG->viewGraph("dag-combine2 input for " + BlockName);
805
806 // Run the DAG combiner in post-legalize mode.
807 {
808 NamedRegionTimer T("combine2", "DAG Combining 2", GroupName,
809 GroupDescription, TimePassesIsEnabled);
810 CurDAG->Combine(AfterLegalizeDAG, *AA, OptLevel);
811 }
812
813 DEBUG(dbgs() << "Optimized legalized selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)
814 << " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)
;
815
816 if (OptLevel != CodeGenOpt::None)
817 ComputeLiveOutVRegInfo();
818
819 if (ViewISelDAGs && MatchFilterBB)
820 CurDAG->viewGraph("isel input for " + BlockName);
821
822 // Third, instruction select all of the operations to machine code, adding the
823 // code to the MachineBasicBlock.
824 {
825 NamedRegionTimer T("isel", "Instruction Selection", GroupName,
826 GroupDescription, TimePassesIsEnabled);
827 DoInstructionSelection();
828 }
829
830 DEBUG(dbgs() << "Selected selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Selected selection DAG: BB#" <<
BlockNumber << " '" << BlockName << "'\n";
CurDAG->dump(); } } while (false)
831 << " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Selected selection DAG: BB#" <<
BlockNumber << " '" << BlockName << "'\n";
CurDAG->dump(); } } while (false)
;
832
833 if (ViewSchedDAGs && MatchFilterBB)
834 CurDAG->viewGraph("scheduler input for " + BlockName);
835
836 // Schedule machine code.
837 ScheduleDAGSDNodes *Scheduler = CreateScheduler();
838 {
839 NamedRegionTimer T("sched", "Instruction Scheduling", GroupName,
840 GroupDescription, TimePassesIsEnabled);
841 Scheduler->Run(CurDAG, FuncInfo->MBB);
842 }
843
844 if (ViewSUnitDAGs && MatchFilterBB)
845 Scheduler->viewGraph();
846
847 // Emit machine code to BB. This can change 'BB' to the last block being
848 // inserted into.
849 MachineBasicBlock *FirstMBB = FuncInfo->MBB, *LastMBB;
850 {
851 NamedRegionTimer T("emit", "Instruction Creation", GroupName,
852 GroupDescription, TimePassesIsEnabled);
853
854 // FuncInfo->InsertPt is passed by reference and set to the end of the
855 // scheduled instructions.
856 LastMBB = FuncInfo->MBB = Scheduler->EmitSchedule(FuncInfo->InsertPt);
857 }
858
859 // If the block was split, make sure we update any references that are used to
860 // update PHI nodes later on.
861 if (FirstMBB != LastMBB)
862 SDB->UpdateSplitBlock(FirstMBB, LastMBB);
863
864 // Free the scheduler state.
865 {
866 NamedRegionTimer T("cleanup", "Instruction Scheduling Cleanup", GroupName,
867 GroupDescription, TimePassesIsEnabled);
868 delete Scheduler;
869 }
870
871 // Free the SelectionDAG state, now that we're finished with it.
872 CurDAG->clear();
873}
874
875namespace {
876
877/// ISelUpdater - helper class to handle updates of the instruction selection
878/// graph.
879class ISelUpdater : public SelectionDAG::DAGUpdateListener {
880 SelectionDAG::allnodes_iterator &ISelPosition;
881
882public:
883 ISelUpdater(SelectionDAG &DAG, SelectionDAG::allnodes_iterator &isp)
884 : SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}
885
886 /// NodeDeleted - Handle nodes deleted from the graph. If the node being
887 /// deleted is the current ISelPosition node, update ISelPosition.
888 ///
889 void NodeDeleted(SDNode *N, SDNode *E) override {
890 if (ISelPosition == SelectionDAG::allnodes_iterator(N))
891 ++ISelPosition;
892 }
893};
894
895} // end anonymous namespace
896
897static bool isStrictFPOp(SDNode *Node, unsigned &NewOpc) {
898 unsigned OrigOpc = Node->getOpcode();
899 switch (OrigOpc) {
900 case ISD::STRICT_FADD: NewOpc = ISD::FADD; return true;
901 case ISD::STRICT_FSUB: NewOpc = ISD::FSUB; return true;
902 case ISD::STRICT_FMUL: NewOpc = ISD::FMUL; return true;
903 case ISD::STRICT_FDIV: NewOpc = ISD::FDIV; return true;
904 case ISD::STRICT_FREM: NewOpc = ISD::FREM; return true;
905 default: return false;
906 }
907}
908
909SDNode* SelectionDAGISel::MutateStrictFPToFP(SDNode *Node, unsigned NewOpc) {
910 assert(((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) ||((((Node->getOpcode() == ISD::STRICT_FADD && NewOpc
== ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB &&
NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL
&& NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD
::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode
() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&
"Unexpected StrictFP opcode!") ? static_cast<void> (0)
: __assert_fail ("((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) && \"Unexpected StrictFP opcode!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 915, __PRETTY_FUNCTION__))
911 (Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) ||((((Node->getOpcode() == ISD::STRICT_FADD && NewOpc
== ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB &&
NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL
&& NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD
::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode
() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&
"Unexpected StrictFP opcode!") ? static_cast<void> (0)
: __assert_fail ("((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) && \"Unexpected StrictFP opcode!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 915, __PRETTY_FUNCTION__))
912 (Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) ||((((Node->getOpcode() == ISD::STRICT_FADD && NewOpc
== ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB &&
NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL
&& NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD
::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode
() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&
"Unexpected StrictFP opcode!") ? static_cast<void> (0)
: __assert_fail ("((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) && \"Unexpected StrictFP opcode!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 915, __PRETTY_FUNCTION__))
913 (Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) ||((((Node->getOpcode() == ISD::STRICT_FADD && NewOpc
== ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB &&
NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL
&& NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD
::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode
() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&
"Unexpected StrictFP opcode!") ? static_cast<void> (0)
: __assert_fail ("((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) && \"Unexpected StrictFP opcode!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 915, __PRETTY_FUNCTION__))
914 (Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&((((Node->getOpcode() == ISD::STRICT_FADD && NewOpc
== ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB &&
NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL
&& NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD
::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode
() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&
"Unexpected StrictFP opcode!") ? static_cast<void> (0)
: __assert_fail ("((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) && \"Unexpected StrictFP opcode!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 915, __PRETTY_FUNCTION__))
915 "Unexpected StrictFP opcode!")((((Node->getOpcode() == ISD::STRICT_FADD && NewOpc
== ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB &&
NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL
&& NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD
::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode
() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&
"Unexpected StrictFP opcode!") ? static_cast<void> (0)
: __assert_fail ("((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) && \"Unexpected StrictFP opcode!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 915, __PRETTY_FUNCTION__))
;
916
917 // We're taking this node out of the chain, so we need to re-link things.
918 SDValue InputChain = Node->getOperand(0);
919 SDValue OutputChain = SDValue(Node, 1);
920 CurDAG->ReplaceAllUsesOfValueWith(OutputChain, InputChain);
921
922 SDVTList VTs = CurDAG->getVTList(Node->getOperand(1).getValueType());
923 SDValue Ops[2] = { Node->getOperand(1), Node->getOperand(2) };
924 SDNode *Res = CurDAG->MorphNodeTo(Node, NewOpc, VTs, Ops);
925
926 // MorphNodeTo can operate in two ways: if an existing node with the
927 // specified operands exists, it can just return it. Otherwise, it
928 // updates the node in place to have the requested operands.
929 if (Res == Node) {
930 // If we updated the node in place, reset the node ID. To the isel,
931 // this should be just like a newly allocated machine node.
932 Res->setNodeId(-1);
933 } else {
934 CurDAG->ReplaceAllUsesWith(Node, Res);
935 CurDAG->RemoveDeadNode(Node);
936 }
937
938 return Res;
939}
940
941void SelectionDAGISel::DoInstructionSelection() {
942 DEBUG(dbgs() << "===== Instruction selection begins: BB#"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "===== Instruction selection begins: BB#"
<< FuncInfo->MBB->getNumber() << " '" <<
FuncInfo->MBB->getName() << "'\n"; } } while (false
)
943 << FuncInfo->MBB->getNumber()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "===== Instruction selection begins: BB#"
<< FuncInfo->MBB->getNumber() << " '" <<
FuncInfo->MBB->getName() << "'\n"; } } while (false
)
944 << " '" << FuncInfo->MBB->getName() << "'\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "===== Instruction selection begins: BB#"
<< FuncInfo->MBB->getNumber() << " '" <<
FuncInfo->MBB->getName() << "'\n"; } } while (false
)
;
945
946 PreprocessISelDAG();
947
948 // Select target instructions for the DAG.
949 {
950 // Number all nodes with a topological order and set DAGSize.
951 DAGSize = CurDAG->AssignTopologicalOrder();
952
953 // Create a dummy node (which is not added to allnodes), that adds
954 // a reference to the root node, preventing it from being deleted,
955 // and tracking any changes of the root.
956 HandleSDNode Dummy(CurDAG->getRoot());
957 SelectionDAG::allnodes_iterator ISelPosition (CurDAG->getRoot().getNode());
958 ++ISelPosition;
959
960 // Make sure that ISelPosition gets properly updated when nodes are deleted
961 // in calls made from this function.
962 ISelUpdater ISU(*CurDAG, ISelPosition);
963
964 // The AllNodes list is now topological-sorted. Visit the
965 // nodes by starting at the end of the list (the root of the
966 // graph) and preceding back toward the beginning (the entry
967 // node).
968 while (ISelPosition != CurDAG->allnodes_begin()) {
969 SDNode *Node = &*--ISelPosition;
970 // Skip dead nodes. DAGCombiner is expected to eliminate all dead nodes,
971 // but there are currently some corner cases that it misses. Also, this
972 // makes it theoretically possible to disable the DAGCombiner.
973 if (Node->use_empty())
974 continue;
975
976 // When we are using non-default rounding modes or FP exception behavior
977 // FP operations are represented by StrictFP pseudo-operations. They
978 // need to be simplified here so that the target-specific instruction
979 // selectors know how to handle them.
980 //
981 // If the current node is a strict FP pseudo-op, the isStrictFPOp()
982 // function will provide the corresponding normal FP opcode to which the
983 // node should be mutated.
984 unsigned NormalFPOpc = ISD::UNDEF;
985 bool IsStrictFPOp = isStrictFPOp(Node, NormalFPOpc);
986 if (IsStrictFPOp)
987 Node = MutateStrictFPToFP(Node, NormalFPOpc);
988
989 Select(Node);
990
991 // FIXME: Add code here to attach an implicit def and use of
992 // target-specific FP environment registers.
993 }
994
995 CurDAG->setRoot(Dummy.getValue());
996 }
997
998 DEBUG(dbgs() << "===== Instruction selection ends:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "===== Instruction selection ends:\n"
; } } while (false)
;
999
1000 PostprocessISelDAG();
1001}
1002
1003static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI) {
1004 for (const User *U : CPI->users()) {
1005 if (const IntrinsicInst *EHPtrCall = dyn_cast<IntrinsicInst>(U)) {
1006 Intrinsic::ID IID = EHPtrCall->getIntrinsicID();
1007 if (IID == Intrinsic::eh_exceptionpointer ||
1008 IID == Intrinsic::eh_exceptioncode)
1009 return true;
1010 }
1011 }
1012 return false;
1013}
1014
1015/// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and
1016/// do other setup for EH landing-pad blocks.
1017bool SelectionDAGISel::PrepareEHLandingPad() {
1018 MachineBasicBlock *MBB = FuncInfo->MBB;
1019 const Constant *PersonalityFn = FuncInfo->Fn->getPersonalityFn();
1020 const BasicBlock *LLVMBB = MBB->getBasicBlock();
1021 const TargetRegisterClass *PtrRC =
1022 TLI->getRegClassFor(TLI->getPointerTy(CurDAG->getDataLayout()));
1023
1024 // Catchpads have one live-in register, which typically holds the exception
1025 // pointer or code.
1026 if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI())) {
1027 if (hasExceptionPointerOrCodeUser(CPI)) {
1028 // Get or create the virtual register to hold the pointer or code. Mark
1029 // the live in physreg and copy into the vreg.
1030 MCPhysReg EHPhysReg = TLI->getExceptionPointerRegister(PersonalityFn);
1031 assert(EHPhysReg && "target lacks exception pointer register")((EHPhysReg && "target lacks exception pointer register"
) ? static_cast<void> (0) : __assert_fail ("EHPhysReg && \"target lacks exception pointer register\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1031, __PRETTY_FUNCTION__))
;
1032 MBB->addLiveIn(EHPhysReg);
1033 unsigned VReg = FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC);
1034 BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(),
1035 TII->get(TargetOpcode::COPY), VReg)
1036 .addReg(EHPhysReg, RegState::Kill);
1037 }
1038 return true;
1039 }
1040
1041 if (!LLVMBB->isLandingPad())
1042 return true;
1043
1044 // Add a label to mark the beginning of the landing pad. Deletion of the
1045 // landing pad can thus be detected via the MachineModuleInfo.
1046 MCSymbol *Label = MF->addLandingPad(MBB);
1047
1048 // Assign the call site to the landing pad's begin label.
1049 MF->setCallSiteLandingPad(Label, SDB->LPadToCallSiteMap[MBB]);
1050
1051 const MCInstrDesc &II = TII->get(TargetOpcode::EH_LABEL);
1052 BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II)
1053 .addSym(Label);
1054
1055 // Mark exception register as live in.
1056 if (unsigned Reg = TLI->getExceptionPointerRegister(PersonalityFn))
1057 FuncInfo->ExceptionPointerVirtReg = MBB->addLiveIn(Reg, PtrRC);
1058
1059 // Mark exception selector register as live in.
1060 if (unsigned Reg = TLI->getExceptionSelectorRegister(PersonalityFn))
1061 FuncInfo->ExceptionSelectorVirtReg = MBB->addLiveIn(Reg, PtrRC);
1062
1063 return true;
1064}
1065
1066/// isFoldedOrDeadInstruction - Return true if the specified instruction is
1067/// side-effect free and is either dead or folded into a generated instruction.
1068/// Return false if it needs to be emitted.
1069static bool isFoldedOrDeadInstruction(const Instruction *I,
1070 FunctionLoweringInfo *FuncInfo) {
1071 return !I->mayWriteToMemory() && // Side-effecting instructions aren't folded.
1072 !isa<TerminatorInst>(I) && // Terminators aren't folded.
1073 !isa<DbgInfoIntrinsic>(I) && // Debug instructions aren't folded.
1074 !I->isEHPad() && // EH pad instructions aren't folded.
1075 !FuncInfo->isExportedInst(I); // Exported instrs must be computed.
1076}
1077
1078/// Set up SwiftErrorVals by going through the function. If the function has
1079/// swifterror argument, it will be the first entry.
1080static void setupSwiftErrorVals(const Function &Fn, const TargetLowering *TLI,
1081 FunctionLoweringInfo *FuncInfo) {
1082 if (!TLI->supportSwiftError())
1083 return;
1084
1085 FuncInfo->SwiftErrorVals.clear();
1086 FuncInfo->SwiftErrorVRegDefMap.clear();
1087 FuncInfo->SwiftErrorVRegUpwardsUse.clear();
1088 FuncInfo->SwiftErrorArg = nullptr;
1089
1090 // Check if function has a swifterror argument.
1091 bool HaveSeenSwiftErrorArg = false;
1092 for (Function::const_arg_iterator AI = Fn.arg_begin(), AE = Fn.arg_end();
1093 AI != AE; ++AI)
1094 if (AI->hasSwiftErrorAttr()) {
1095 assert(!HaveSeenSwiftErrorArg &&((!HaveSeenSwiftErrorArg && "Must have only one swifterror parameter"
) ? static_cast<void> (0) : __assert_fail ("!HaveSeenSwiftErrorArg && \"Must have only one swifterror parameter\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1096, __PRETTY_FUNCTION__))
1096 "Must have only one swifterror parameter")((!HaveSeenSwiftErrorArg && "Must have only one swifterror parameter"
) ? static_cast<void> (0) : __assert_fail ("!HaveSeenSwiftErrorArg && \"Must have only one swifterror parameter\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1096, __PRETTY_FUNCTION__))
;
1097 (void)HaveSeenSwiftErrorArg; // silence warning.
1098 HaveSeenSwiftErrorArg = true;
1099 FuncInfo->SwiftErrorArg = &*AI;
1100 FuncInfo->SwiftErrorVals.push_back(&*AI);
1101 }
1102
1103 for (const auto &LLVMBB : Fn)
1104 for (const auto &Inst : LLVMBB) {
1105 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(&Inst))
1106 if (Alloca->isSwiftError())
1107 FuncInfo->SwiftErrorVals.push_back(Alloca);
1108 }
1109}
1110
1111static void createSwiftErrorEntriesInEntryBlock(FunctionLoweringInfo *FuncInfo,
1112 FastISel *FastIS,
1113 const TargetLowering *TLI,
1114 const TargetInstrInfo *TII,
1115 SelectionDAGBuilder *SDB) {
1116 if (!TLI->supportSwiftError())
1117 return;
1118
1119 // We only need to do this when we have swifterror parameter or swifterror
1120 // alloc.
1121 if (FuncInfo->SwiftErrorVals.empty())
1122 return;
1123
1124 assert(FuncInfo->MBB == &*FuncInfo->MF->begin() &&((FuncInfo->MBB == &*FuncInfo->MF->begin() &&
"expected to insert into entry block") ? static_cast<void
> (0) : __assert_fail ("FuncInfo->MBB == &*FuncInfo->MF->begin() && \"expected to insert into entry block\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1125, __PRETTY_FUNCTION__))
1125 "expected to insert into entry block")((FuncInfo->MBB == &*FuncInfo->MF->begin() &&
"expected to insert into entry block") ? static_cast<void
> (0) : __assert_fail ("FuncInfo->MBB == &*FuncInfo->MF->begin() && \"expected to insert into entry block\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1125, __PRETTY_FUNCTION__))
;
1126 auto &DL = FuncInfo->MF->getDataLayout();
1127 auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL));
1128 for (const auto *SwiftErrorVal : FuncInfo->SwiftErrorVals) {
1129 // We will always generate a copy from the argument. It is always used at
1130 // least by the 'return' of the swifterror.
1131 if (FuncInfo->SwiftErrorArg && FuncInfo->SwiftErrorArg == SwiftErrorVal)
1132 continue;
1133 unsigned VReg = FuncInfo->MF->getRegInfo().createVirtualRegister(RC);
1134 // Assign Undef to Vreg. We construct MI directly to make sure it works
1135 // with FastISel.
1136 BuildMI(*FuncInfo->MBB, FuncInfo->MBB->getFirstNonPHI(),
1137 SDB->getCurDebugLoc(), TII->get(TargetOpcode::IMPLICIT_DEF),
1138 VReg);
1139
1140 // Keep FastIS informed about the value we just inserted.
1141 if (FastIS)
1142 FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));
1143
1144 FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB, SwiftErrorVal, VReg);
1145 }
1146}
1147
1148/// Propagate swifterror values through the machine function CFG.
1149static void propagateSwiftErrorVRegs(FunctionLoweringInfo *FuncInfo) {
1150 auto *TLI = FuncInfo->TLI;
1151 if (!TLI->supportSwiftError())
1152 return;
1153
1154 // We only need to do this when we have swifterror parameter or swifterror
1155 // alloc.
1156 if (FuncInfo->SwiftErrorVals.empty())
1157 return;
1158
1159 // For each machine basic block in reverse post order.
1160 ReversePostOrderTraversal<MachineFunction *> RPOT(FuncInfo->MF);
1161 for (ReversePostOrderTraversal<MachineFunction *>::rpo_iterator
1162 It = RPOT.begin(),
1163 E = RPOT.end();
1164 It != E; ++It) {
1165 MachineBasicBlock *MBB = *It;
1166
1167 // For each swifterror value in the function.
1168 for(const auto *SwiftErrorVal : FuncInfo->SwiftErrorVals) {
1169 auto Key = std::make_pair(MBB, SwiftErrorVal);
1170 auto UUseIt = FuncInfo->SwiftErrorVRegUpwardsUse.find(Key);
1171 auto VRegDefIt = FuncInfo->SwiftErrorVRegDefMap.find(Key);
1172 bool UpwardsUse = UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end();
1173 unsigned UUseVReg = UpwardsUse ? UUseIt->second : 0;
1174 bool DownwardDef = VRegDefIt != FuncInfo->SwiftErrorVRegDefMap.end();
1175 assert(!(UpwardsUse && !DownwardDef) &&((!(UpwardsUse && !DownwardDef) && "We can't have an upwards use but no downwards def"
) ? static_cast<void> (0) : __assert_fail ("!(UpwardsUse && !DownwardDef) && \"We can't have an upwards use but no downwards def\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1176, __PRETTY_FUNCTION__))
1176 "We can't have an upwards use but no downwards def")((!(UpwardsUse && !DownwardDef) && "We can't have an upwards use but no downwards def"
) ? static_cast<void> (0) : __assert_fail ("!(UpwardsUse && !DownwardDef) && \"We can't have an upwards use but no downwards def\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1176, __PRETTY_FUNCTION__))
;
1177
1178 // If there is no upwards exposed use and an entry for the swifterror in
1179 // the def map for this value we don't need to do anything: We already
1180 // have a downward def for this basic block.
1181 if (!UpwardsUse && DownwardDef)
1182 continue;
1183
1184 // Otherwise we either have an upwards exposed use vreg that we need to
1185 // materialize or need to forward the downward def from predecessors.
1186
1187 // Check whether we have a single vreg def from all predecessors.
1188 // Otherwise we need a phi.
1189 SmallVector<std::pair<MachineBasicBlock *, unsigned>, 4> VRegs;
1190 SmallSet<const MachineBasicBlock*, 8> Visited;
1191 for (auto *Pred : MBB->predecessors()) {
1192 if (!Visited.insert(Pred).second)
1193 continue;
1194 VRegs.push_back(std::make_pair(
1195 Pred, FuncInfo->getOrCreateSwiftErrorVReg(Pred, SwiftErrorVal)));
1196 if (Pred != MBB)
1197 continue;
1198 // We have a self-edge.
1199 // If there was no upwards use in this basic block there is now one: the
1200 // phi needs to use it self.
1201 if (!UpwardsUse) {
1202 UpwardsUse = true;
1203 UUseIt = FuncInfo->SwiftErrorVRegUpwardsUse.find(Key);
1204 assert(UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end())((UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end()) ? static_cast
<void> (0) : __assert_fail ("UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end()"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1204, __PRETTY_FUNCTION__))
;
1205 UUseVReg = UUseIt->second;
1206 }
1207 }
1208
1209 // We need a phi node if we have more than one predecessor with different
1210 // downward defs.
1211 bool needPHI =
1212 VRegs.size() >= 1 &&
1213 std::find_if(
1214 VRegs.begin(), VRegs.end(),
1215 [&](const std::pair<const MachineBasicBlock *, unsigned> &V)
1216 -> bool { return V.second != VRegs[0].second; }) !=
1217 VRegs.end();
1218
1219 // If there is no upwards exposed used and we don't need a phi just
1220 // forward the swifterror vreg from the predecessor(s).
1221 if (!UpwardsUse && !needPHI) {
1222 assert(!VRegs.empty() &&((!VRegs.empty() && "No predecessors? The entry block should bail out earlier"
) ? static_cast<void> (0) : __assert_fail ("!VRegs.empty() && \"No predecessors? The entry block should bail out earlier\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1223, __PRETTY_FUNCTION__))
1223 "No predecessors? The entry block should bail out earlier")((!VRegs.empty() && "No predecessors? The entry block should bail out earlier"
) ? static_cast<void> (0) : __assert_fail ("!VRegs.empty() && \"No predecessors? The entry block should bail out earlier\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1223, __PRETTY_FUNCTION__))
;
1224 // Just forward the swifterror vreg from the predecessor(s).
1225 FuncInfo->setCurrentSwiftErrorVReg(MBB, SwiftErrorVal, VRegs[0].second);
1226 continue;
1227 }
1228
1229 auto DLoc = isa<Instruction>(SwiftErrorVal)
1230 ? dyn_cast<Instruction>(SwiftErrorVal)->getDebugLoc()
1231 : DebugLoc();
1232 const auto *TII = FuncInfo->MF->getSubtarget().getInstrInfo();
1233
1234 // If we don't need a phi create a copy to the upward exposed vreg.
1235 if (!needPHI) {
1236 assert(UpwardsUse)((UpwardsUse) ? static_cast<void> (0) : __assert_fail (
"UpwardsUse", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1236, __PRETTY_FUNCTION__))
;
1237 unsigned DestReg = UUseVReg;
1238 BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc, TII->get(TargetOpcode::COPY),
1239 DestReg)
1240 .addReg(VRegs[0].second);
1241 continue;
1242 }
1243
1244 // We need a phi: if there is an upwards exposed use we already have a
1245 // destination virtual register number otherwise we generate a new one.
1246 auto &DL = FuncInfo->MF->getDataLayout();
1247 auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL));
1248 unsigned PHIVReg =
1249 UpwardsUse ? UUseVReg
1250 : FuncInfo->MF->getRegInfo().createVirtualRegister(RC);
1251 MachineInstrBuilder SwiftErrorPHI =
1252 BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc,
1253 TII->get(TargetOpcode::PHI), PHIVReg);
1254 for (auto BBRegPair : VRegs) {
1255 SwiftErrorPHI.addReg(BBRegPair.second).addMBB(BBRegPair.first);
1256 }
1257
1258 // We did not have a definition in this block before: store the phi's vreg
1259 // as this block downward exposed def.
1260 if (!UpwardsUse)
1261 FuncInfo->setCurrentSwiftErrorVReg(MBB, SwiftErrorVal, PHIVReg);
1262 }
1263 }
1264}
1265
1266void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
1267 FastISelFailed = false;
1268 // Initialize the Fast-ISel state, if needed.
1269 FastISel *FastIS = nullptr;
1
'FastIS' initialized to a null pointer value
1270 if (TM.Options.EnableFastISel)
2
Assuming the condition is false
3
Taking false branch
1271 FastIS = TLI->createFastISel(*FuncInfo, LibInfo);
1272
1273 setupSwiftErrorVals(Fn, TLI, FuncInfo);
1274
1275 ReversePostOrderTraversal<const Function*> RPOT(&Fn);
1276
1277 // Lower arguments up front. An RPO iteration always visits the entry block
1278 // first.
1279 assert(*RPOT.begin() == &Fn.getEntryBlock())((*RPOT.begin() == &Fn.getEntryBlock()) ? static_cast<
void> (0) : __assert_fail ("*RPOT.begin() == &Fn.getEntryBlock()"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1279, __PRETTY_FUNCTION__))
;
1280 ++NumEntryBlocks;
1281
1282 // Set up FuncInfo for ISel. Entry blocks never have PHIs.
1283 FuncInfo->MBB = FuncInfo->MBBMap[&Fn.getEntryBlock()];
1284 FuncInfo->InsertPt = FuncInfo->MBB->begin();
1285
1286 if (!FastIS) {
4
Taking true branch
1287 LowerArguments(Fn);
1288 } else {
1289 // See if fast isel can lower the arguments.
1290 FastIS->startNewBlock();
1291 if (!FastIS->lowerArguments()) {
1292 FastISelFailed = true;
1293 // Fast isel failed to lower these arguments
1294 ++NumFastIselFailLowerArguments;
1295
1296 OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1297 Fn.getSubprogram(),
1298 &Fn.getEntryBlock());
1299 R << "FastISel didn't lower all arguments: "
1300 << ore::NV("Prototype", Fn.getType());
1301 reportFastISelFailure(*MF, *ORE, R, EnableFastISelAbort > 1);
1302
1303 // Use SelectionDAG argument lowering
1304 LowerArguments(Fn);
1305 CurDAG->setRoot(SDB->getControlRoot());
1306 SDB->clear();
1307 CodeGenAndEmitDAG();
1308 }
1309
1310 // If we inserted any instructions at the beginning, make a note of
1311 // where they are, so we can be sure to emit subsequent instructions
1312 // after them.
1313 if (FuncInfo->InsertPt != FuncInfo->MBB->begin())
1314 FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));
1315 else
1316 FastIS->setLastLocalValue(nullptr);
1317 }
1318 createSwiftErrorEntriesInEntryBlock(FuncInfo, FastIS, TLI, TII, SDB);
1319
1320 // Iterate over all basic blocks in the function.
1321 for (const BasicBlock *LLVMBB : RPOT) {
1322 if (OptLevel != CodeGenOpt::None) {
5
Assuming the condition is false
6
Taking false branch
16
Assuming the condition is false
17
Taking false branch
27
Assuming the condition is false
28
Taking false branch
38
Assuming the condition is false
39
Taking false branch
1323 bool AllPredsVisited = true;
1324 for (const_pred_iterator PI = pred_begin(LLVMBB), PE = pred_end(LLVMBB);
1325 PI != PE; ++PI) {
1326 if (!FuncInfo->VisitedBBs.count(*PI)) {
1327 AllPredsVisited = false;
1328 break;
1329 }
1330 }
1331
1332 if (AllPredsVisited) {
1333 for (BasicBlock::const_iterator I = LLVMBB->begin();
1334 const PHINode *PN = dyn_cast<PHINode>(I); ++I)
1335 FuncInfo->ComputePHILiveOutRegInfo(PN);
1336 } else {
1337 for (BasicBlock::const_iterator I = LLVMBB->begin();
1338 const PHINode *PN = dyn_cast<PHINode>(I); ++I)
1339 FuncInfo->InvalidatePHILiveOutRegInfo(PN);
1340 }
1341
1342 FuncInfo->VisitedBBs.insert(LLVMBB);
1343 }
1344
1345 BasicBlock::const_iterator const Begin =
1346 LLVMBB->getFirstNonPHI()->getIterator();
1347 BasicBlock::const_iterator const End = LLVMBB->end();
1348 BasicBlock::const_iterator BI = End;
1349
1350 FuncInfo->MBB = FuncInfo->MBBMap[LLVMBB];
1351 if (!FuncInfo->MBB)
7
Assuming the condition is false
8
Taking false branch
18
Assuming the condition is false
19
Taking false branch
29
Assuming the condition is false
30
Taking false branch
40
Assuming the condition is false
41
Taking false branch
1352 continue; // Some blocks like catchpads have no code or MBB.
1353
1354 // Insert new instructions after any phi or argument setup code.
1355 FuncInfo->InsertPt = FuncInfo->MBB->end();
1356
1357 // Setup an EH landing-pad block.
1358 FuncInfo->ExceptionPointerVirtReg = 0;
1359 FuncInfo->ExceptionSelectorVirtReg = 0;
1360 if (LLVMBB->isEHPad())
9
Assuming the condition is false
10
Taking false branch
20
Assuming the condition is false
21
Taking false branch
31
Assuming the condition is false
32
Taking false branch
42
Assuming the condition is false
43
Taking false branch
1361 if (!PrepareEHLandingPad())
1362 continue;
1363
1364 // Before doing SelectionDAG ISel, see if FastISel has been requested.
1365 if (FastIS) {
11
Taking false branch
22
Taking false branch
33
Taking false branch
44
Taking false branch
1366 if (LLVMBB != &Fn.getEntryBlock())
1367 FastIS->startNewBlock();
1368
1369 unsigned NumFastIselRemaining = std::distance(Begin, End);
1370 // Do FastISel on as many instructions as possible.
1371 for (; BI != Begin; --BI) {
1372 const Instruction *Inst = &*std::prev(BI);
1373
1374 // If we no longer require this instruction, skip it.
1375 if (isFoldedOrDeadInstruction(Inst, FuncInfo) ||
1376 ElidedArgCopyInstrs.count(Inst)) {
1377 --NumFastIselRemaining;
1378 continue;
1379 }
1380
1381 // Bottom-up: reset the insert pos at the top, after any local-value
1382 // instructions.
1383 FastIS->recomputeInsertPt();
1384
1385 // Try to select the instruction with FastISel.
1386 if (FastIS->selectInstruction(Inst)) {
1387 FastISelFailed = true;
1388 --NumFastIselRemaining;
1389 ++NumFastIselSuccess;
1390 // If fast isel succeeded, skip over all the folded instructions, and
1391 // then see if there is a load right before the selected instructions.
1392 // Try to fold the load if so.
1393 const Instruction *BeforeInst = Inst;
1394 while (BeforeInst != &*Begin) {
1395 BeforeInst = &*std::prev(BasicBlock::const_iterator(BeforeInst));
1396 if (!isFoldedOrDeadInstruction(BeforeInst, FuncInfo))
1397 break;
1398 }
1399 if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&
1400 BeforeInst->hasOneUse() &&
1401 FastIS->tryToFoldLoad(cast<LoadInst>(BeforeInst), Inst)) {
1402 // If we succeeded, don't re-select the load.
1403 BI = std::next(BasicBlock::const_iterator(BeforeInst));
1404 --NumFastIselRemaining;
1405 ++NumFastIselSuccess;
1406 }
1407 continue;
1408 }
1409
1410 // Then handle certain instructions as single-LLVM-Instruction blocks.
1411 if (isa<CallInst>(Inst)) {
1412 OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1413 Inst->getDebugLoc(), LLVMBB);
1414
1415 R << "FastISel missed call";
1416
1417 if (R.isEnabled() || EnableFastISelAbort) {
1418 std::string InstStrStorage;
1419 raw_string_ostream InstStr(InstStrStorage);
1420 InstStr << *Inst;
1421
1422 R << ": " << InstStr.str();
1423 }
1424
1425 reportFastISelFailure(*MF, *ORE, R, EnableFastISelAbort > 2);
1426
1427 if (!Inst->getType()->isVoidTy() && !Inst->getType()->isTokenTy() &&
1428 !Inst->use_empty()) {
1429 unsigned &R = FuncInfo->ValueMap[Inst];
1430 if (!R)
1431 R = FuncInfo->CreateRegs(Inst->getType());
1432 }
1433
1434 bool HadTailCall = false;
1435 MachineBasicBlock::iterator SavedInsertPt = FuncInfo->InsertPt;
1436 SelectBasicBlock(Inst->getIterator(), BI, HadTailCall);
1437
1438 // If the call was emitted as a tail call, we're done with the block.
1439 // We also need to delete any previously emitted instructions.
1440 if (HadTailCall) {
1441 FastIS->removeDeadCode(SavedInsertPt, FuncInfo->MBB->end());
1442 --BI;
1443 break;
1444 }
1445
1446 // Recompute NumFastIselRemaining as Selection DAG instruction
1447 // selection may have handled the call, input args, etc.
1448 unsigned RemainingNow = std::distance(Begin, BI);
1449 NumFastIselFailures += NumFastIselRemaining - RemainingNow;
1450 NumFastIselRemaining = RemainingNow;
1451 continue;
1452 }
1453
1454 OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1455 Inst->getDebugLoc(), LLVMBB);
1456
1457 bool ShouldAbort = EnableFastISelAbort;
1458 if (isa<TerminatorInst>(Inst)) {
1459 // Use a different message for terminator misses.
1460 R << "FastISel missed terminator";
1461 // Don't abort for terminator unless the level is really high
1462 ShouldAbort = (EnableFastISelAbort > 2);
1463 } else {
1464 R << "FastISel missed";
1465 }
1466
1467 if (R.isEnabled() || EnableFastISelAbort) {
1468 std::string InstStrStorage;
1469 raw_string_ostream InstStr(InstStrStorage);
1470 InstStr << *Inst;
1471 R << ": " << InstStr.str();
1472 }
1473
1474 reportFastISelFailure(*MF, *ORE, R, ShouldAbort);
1475
1476 NumFastIselFailures += NumFastIselRemaining;
1477 break;
1478 }
1479
1480 FastIS->recomputeInsertPt();
1481 }
1482
1483 if (getAnalysis<StackProtector>().shouldEmitSDCheck(*LLVMBB)) {
12
Assuming the condition is false
13
Taking false branch
23
Assuming the condition is false
24
Taking false branch
34
Assuming the condition is false
35
Taking false branch
45
Assuming the condition is false
46
Taking false branch
1484 bool FunctionBasedInstrumentation =
1485 TLI->getSSPStackGuardCheck(*Fn.getParent());
1486 SDB->SPDescriptor.initialize(LLVMBB, FuncInfo->MBBMap[LLVMBB],
1487 FunctionBasedInstrumentation);
1488 }
1489
1490 if (Begin != BI)
14
Taking false branch
25
Taking false branch
36
Taking false branch
47
Taking true branch
1491 ++NumDAGBlocks;
1492 else
1493 ++NumFastIselBlocks;
1494
1495 if (Begin != BI) {
15
Taking false branch
26
Taking false branch
37
Taking false branch
48
Taking true branch
1496 // Run SelectionDAG instruction selection on the remainder of the block
1497 // not handled by FastISel. If FastISel is not run, this is the entire
1498 // block.
1499 bool HadTailCall;
1500 SelectBasicBlock(Begin, BI, HadTailCall);
1501
1502 // But if FastISel was run, we already selected some of the block.
1503 // If we emitted a tail-call, we need to delete any previously emitted
1504 // instruction that follows it.
1505 if (HadTailCall && FuncInfo->InsertPt != FuncInfo->MBB->end())
49
Assuming 'HadTailCall' is not equal to 0
50
Taking true branch
1506 FastIS->removeDeadCode(FuncInfo->InsertPt, FuncInfo->MBB->end());
51
Called C++ object pointer is null
1507 }
1508
1509 FinishBasicBlock();
1510 FuncInfo->PHINodesToUpdate.clear();
1511 ElidedArgCopyInstrs.clear();
1512 }
1513
1514 propagateSwiftErrorVRegs(FuncInfo);
1515
1516 delete FastIS;
1517 SDB->clearDanglingDebugInfo();
1518 SDB->SPDescriptor.resetPerFunctionState();
1519}
1520
1521/// Given that the input MI is before a partial terminator sequence TSeq, return
1522/// true if M + TSeq also a partial terminator sequence.
1523///
1524/// A Terminator sequence is a sequence of MachineInstrs which at this point in
1525/// lowering copy vregs into physical registers, which are then passed into
1526/// terminator instructors so we can satisfy ABI constraints. A partial
1527/// terminator sequence is an improper subset of a terminator sequence (i.e. it
1528/// may be the whole terminator sequence).
1529static bool MIIsInTerminatorSequence(const MachineInstr &MI) {
1530 // If we do not have a copy or an implicit def, we return true if and only if
1531 // MI is a debug value.
1532 if (!MI.isCopy() && !MI.isImplicitDef())
1533 // Sometimes DBG_VALUE MI sneak in between the copies from the vregs to the
1534 // physical registers if there is debug info associated with the terminator
1535 // of our mbb. We want to include said debug info in our terminator
1536 // sequence, so we return true in that case.
1537 return MI.isDebugValue();
1538
1539 // We have left the terminator sequence if we are not doing one of the
1540 // following:
1541 //
1542 // 1. Copying a vreg into a physical register.
1543 // 2. Copying a vreg into a vreg.
1544 // 3. Defining a register via an implicit def.
1545
1546 // OPI should always be a register definition...
1547 MachineInstr::const_mop_iterator OPI = MI.operands_begin();
1548 if (!OPI->isReg() || !OPI->isDef())
1549 return false;
1550
1551 // Defining any register via an implicit def is always ok.
1552 if (MI.isImplicitDef())
1553 return true;
1554
1555 // Grab the copy source...
1556 MachineInstr::const_mop_iterator OPI2 = OPI;
1557 ++OPI2;
1558 assert(OPI2 != MI.operands_end()((OPI2 != MI.operands_end() && "Should have a copy implying we should have 2 arguments."
) ? static_cast<void> (0) : __assert_fail ("OPI2 != MI.operands_end() && \"Should have a copy implying we should have 2 arguments.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1559, __PRETTY_FUNCTION__))
1559 && "Should have a copy implying we should have 2 arguments.")((OPI2 != MI.operands_end() && "Should have a copy implying we should have 2 arguments."
) ? static_cast<void> (0) : __assert_fail ("OPI2 != MI.operands_end() && \"Should have a copy implying we should have 2 arguments.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1559, __PRETTY_FUNCTION__))
;
1560
1561 // Make sure that the copy dest is not a vreg when the copy source is a
1562 // physical register.
1563 if (!OPI2->isReg() ||
1564 (!TargetRegisterInfo::isPhysicalRegister(OPI->getReg()) &&
1565 TargetRegisterInfo::isPhysicalRegister(OPI2->getReg())))
1566 return false;
1567
1568 return true;
1569}
1570
1571/// Find the split point at which to splice the end of BB into its success stack
1572/// protector check machine basic block.
1573///
1574/// On many platforms, due to ABI constraints, terminators, even before register
1575/// allocation, use physical registers. This creates an issue for us since
1576/// physical registers at this point can not travel across basic
1577/// blocks. Luckily, selectiondag always moves physical registers into vregs
1578/// when they enter functions and moves them through a sequence of copies back
1579/// into the physical registers right before the terminator creating a
1580/// ``Terminator Sequence''. This function is searching for the beginning of the
1581/// terminator sequence so that we can ensure that we splice off not just the
1582/// terminator, but additionally the copies that move the vregs into the
1583/// physical registers.
1584static MachineBasicBlock::iterator
1585FindSplitPointForStackProtector(MachineBasicBlock *BB) {
1586 MachineBasicBlock::iterator SplitPoint = BB->getFirstTerminator();
1587 //
1588 if (SplitPoint == BB->begin())
1589 return SplitPoint;
1590
1591 MachineBasicBlock::iterator Start = BB->begin();
1592 MachineBasicBlock::iterator Previous = SplitPoint;
1593 --Previous;
1594
1595 while (MIIsInTerminatorSequence(*Previous)) {
1596 SplitPoint = Previous;
1597 if (Previous == Start)
1598 break;
1599 --Previous;
1600 }
1601
1602 return SplitPoint;
1603}
1604
1605void
1606SelectionDAGISel::FinishBasicBlock() {
1607 DEBUG(dbgs() << "Total amount of phi nodes to update: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " << i << " : ("
<< FuncInfo->PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)
1608 << FuncInfo->PHINodesToUpdate.size() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " << i << " : ("
<< FuncInfo->PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)
1609 for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " << i << " : ("
<< FuncInfo->PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)
1610 dbgs() << "Node " << i << " : ("do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " << i << " : ("
<< FuncInfo->PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)
1611 << FuncInfo->PHINodesToUpdate[i].firstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " << i << " : ("
<< FuncInfo->PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)
1612 << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " << i << " : ("
<< FuncInfo->PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)
;
1613
1614 // Next, now that we know what the last MBB the LLVM BB expanded is, update
1615 // PHI nodes in successors.
1616 for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
1617 MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first);
1618 assert(PHI->isPHI() &&((PHI->isPHI() && "This is not a machine PHI node that we are updating!"
) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1619, __PRETTY_FUNCTION__))
1619 "This is not a machine PHI node that we are updating!")((PHI->isPHI() && "This is not a machine PHI node that we are updating!"
) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1619, __PRETTY_FUNCTION__))
;
1620 if (!FuncInfo->MBB->isSuccessor(PHI->getParent()))
1621 continue;
1622 PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB);
1623 }
1624
1625 // Handle stack protector.
1626 if (SDB->SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
1627 // The target provides a guard check function. There is no need to
1628 // generate error handling code or to split current basic block.
1629 MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1630
1631 // Add load and check to the basicblock.
1632 FuncInfo->MBB = ParentMBB;
1633 FuncInfo->InsertPt =
1634 FindSplitPointForStackProtector(ParentMBB);
1635 SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
1636 CurDAG->setRoot(SDB->getRoot());
1637 SDB->clear();
1638 CodeGenAndEmitDAG();
1639
1640 // Clear the Per-BB State.
1641 SDB->SPDescriptor.resetPerBBState();
1642 } else if (SDB->SPDescriptor.shouldEmitStackProtector()) {
1643 MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1644 MachineBasicBlock *SuccessMBB = SDB->SPDescriptor.getSuccessMBB();
1645
1646 // Find the split point to split the parent mbb. At the same time copy all
1647 // physical registers used in the tail of parent mbb into virtual registers
1648 // before the split point and back into physical registers after the split
1649 // point. This prevents us needing to deal with Live-ins and many other
1650 // register allocation issues caused by us splitting the parent mbb. The
1651 // register allocator will clean up said virtual copies later on.
1652 MachineBasicBlock::iterator SplitPoint =
1653 FindSplitPointForStackProtector(ParentMBB);
1654
1655 // Splice the terminator of ParentMBB into SuccessMBB.
1656 SuccessMBB->splice(SuccessMBB->end(), ParentMBB,
1657 SplitPoint,
1658 ParentMBB->end());
1659
1660 // Add compare/jump on neq/jump to the parent BB.
1661 FuncInfo->MBB = ParentMBB;
1662 FuncInfo->InsertPt = ParentMBB->end();
1663 SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
1664 CurDAG->setRoot(SDB->getRoot());
1665 SDB->clear();
1666 CodeGenAndEmitDAG();
1667
1668 // CodeGen Failure MBB if we have not codegened it yet.
1669 MachineBasicBlock *FailureMBB = SDB->SPDescriptor.getFailureMBB();
1670 if (FailureMBB->empty()) {
1671 FuncInfo->MBB = FailureMBB;
1672 FuncInfo->InsertPt = FailureMBB->end();
1673 SDB->visitSPDescriptorFailure(SDB->SPDescriptor);
1674 CurDAG->setRoot(SDB->getRoot());
1675 SDB->clear();
1676 CodeGenAndEmitDAG();
1677 }
1678
1679 // Clear the Per-BB State.
1680 SDB->SPDescriptor.resetPerBBState();
1681 }
1682
1683 // Lower each BitTestBlock.
1684 for (auto &BTB : SDB->BitTestCases) {
1685 // Lower header first, if it wasn't already lowered
1686 if (!BTB.Emitted) {
1687 // Set the current basic block to the mbb we wish to insert the code into
1688 FuncInfo->MBB = BTB.Parent;
1689 FuncInfo->InsertPt = FuncInfo->MBB->end();
1690 // Emit the code
1691 SDB->visitBitTestHeader(BTB, FuncInfo->MBB);
1692 CurDAG->setRoot(SDB->getRoot());
1693 SDB->clear();
1694 CodeGenAndEmitDAG();
1695 }
1696
1697 BranchProbability UnhandledProb = BTB.Prob;
1698 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
1699 UnhandledProb -= BTB.Cases[j].ExtraProb;
1700 // Set the current basic block to the mbb we wish to insert the code into
1701 FuncInfo->MBB = BTB.Cases[j].ThisBB;
1702 FuncInfo->InsertPt = FuncInfo->MBB->end();
1703 // Emit the code
1704
1705 // If all cases cover a contiguous range, it is not necessary to jump to
1706 // the default block after the last bit test fails. This is because the
1707 // range check during bit test header creation has guaranteed that every
1708 // case here doesn't go outside the range. In this case, there is no need
1709 // to perform the last bit test, as it will always be true. Instead, make
1710 // the second-to-last bit-test fall through to the target of the last bit
1711 // test, and delete the last bit test.
1712
1713 MachineBasicBlock *NextMBB;
1714 if (BTB.ContiguousRange && j + 2 == ej) {
1715 // Second-to-last bit-test with contiguous range: fall through to the
1716 // target of the final bit test.
1717 NextMBB = BTB.Cases[j + 1].TargetBB;
1718 } else if (j + 1 == ej) {
1719 // For the last bit test, fall through to Default.
1720 NextMBB = BTB.Default;
1721 } else {
1722 // Otherwise, fall through to the next bit test.
1723 NextMBB = BTB.Cases[j + 1].ThisBB;
1724 }
1725
1726 SDB->visitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j],
1727 FuncInfo->MBB);
1728
1729 CurDAG->setRoot(SDB->getRoot());
1730 SDB->clear();
1731 CodeGenAndEmitDAG();
1732
1733 if (BTB.ContiguousRange && j + 2 == ej) {
1734 // Since we're not going to use the final bit test, remove it.
1735 BTB.Cases.pop_back();
1736 break;
1737 }
1738 }
1739
1740 // Update PHI Nodes
1741 for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
1742 pi != pe; ++pi) {
1743 MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);
1744 MachineBasicBlock *PHIBB = PHI->getParent();
1745 assert(PHI->isPHI() &&((PHI->isPHI() && "This is not a machine PHI node that we are updating!"
) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1746, __PRETTY_FUNCTION__))
1746 "This is not a machine PHI node that we are updating!")((PHI->isPHI() && "This is not a machine PHI node that we are updating!"
) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1746, __PRETTY_FUNCTION__))
;
1747 // This is "default" BB. We have two jumps to it. From "header" BB and
1748 // from last "case" BB, unless the latter was skipped.
1749 if (PHIBB == BTB.Default) {
1750 PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(BTB.Parent);
1751 if (!BTB.ContiguousRange) {
1752 PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
1753 .addMBB(BTB.Cases.back().ThisBB);
1754 }
1755 }
1756 // One of "cases" BB.
1757 for (unsigned j = 0, ej = BTB.Cases.size();
1758 j != ej; ++j) {
1759 MachineBasicBlock* cBB = BTB.Cases[j].ThisBB;
1760 if (cBB->isSuccessor(PHIBB))
1761 PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(cBB);
1762 }
1763 }
1764 }
1765 SDB->BitTestCases.clear();
1766
1767 // If the JumpTable record is filled in, then we need to emit a jump table.
1768 // Updating the PHI nodes is tricky in this case, since we need to determine
1769 // whether the PHI is a successor of the range check MBB or the jump table MBB
1770 for (unsigned i = 0, e = SDB->JTCases.size(); i != e; ++i) {
1771 // Lower header first, if it wasn't already lowered
1772 if (!SDB->JTCases[i].first.Emitted) {
1773 // Set the current basic block to the mbb we wish to insert the code into
1774 FuncInfo->MBB = SDB->JTCases[i].first.HeaderBB;
1775 FuncInfo->InsertPt = FuncInfo->MBB->end();
1776 // Emit the code
1777 SDB->visitJumpTableHeader(SDB->JTCases[i].second, SDB->JTCases[i].first,
1778 FuncInfo->MBB);
1779 CurDAG->setRoot(SDB->getRoot());
1780 SDB->clear();
1781 CodeGenAndEmitDAG();
1782 }
1783
1784 // Set the current basic block to the mbb we wish to insert the code into
1785 FuncInfo->MBB = SDB->JTCases[i].second.MBB;
1786 FuncInfo->InsertPt = FuncInfo->MBB->end();
1787 // Emit the code
1788 SDB->visitJumpTable(SDB->JTCases[i].second);
1789 CurDAG->setRoot(SDB->getRoot());
1790 SDB->clear();
1791 CodeGenAndEmitDAG();
1792
1793 // Update PHI Nodes
1794 for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
1795 pi != pe; ++pi) {
1796 MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);
1797 MachineBasicBlock *PHIBB = PHI->getParent();
1798 assert(PHI->isPHI() &&((PHI->isPHI() && "This is not a machine PHI node that we are updating!"
) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1799, __PRETTY_FUNCTION__))
1799 "This is not a machine PHI node that we are updating!")((PHI->isPHI() && "This is not a machine PHI node that we are updating!"
) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1799, __PRETTY_FUNCTION__))
;
1800 // "default" BB. We can go there only from header BB.
1801 if (PHIBB == SDB->JTCases[i].second.Default)
1802 PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
1803 .addMBB(SDB->JTCases[i].first.HeaderBB);
1804 // JT BB. Just iterate over successors here
1805 if (FuncInfo->MBB->isSuccessor(PHIBB))
1806 PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(FuncInfo->MBB);
1807 }
1808 }
1809 SDB->JTCases.clear();
1810
1811 // If we generated any switch lowering information, build and codegen any
1812 // additional DAGs necessary.
1813 for (unsigned i = 0, e = SDB->SwitchCases.size(); i != e; ++i) {
1814 // Set the current basic block to the mbb we wish to insert the code into
1815 FuncInfo->MBB = SDB->SwitchCases[i].ThisBB;
1816 FuncInfo->InsertPt = FuncInfo->MBB->end();
1817
1818 // Determine the unique successors.
1819 SmallVector<MachineBasicBlock *, 2> Succs;
1820 Succs.push_back(SDB->SwitchCases[i].TrueBB);
1821 if (SDB->SwitchCases[i].TrueBB != SDB->SwitchCases[i].FalseBB)
1822 Succs.push_back(SDB->SwitchCases[i].FalseBB);
1823
1824 // Emit the code. Note that this could result in FuncInfo->MBB being split.
1825 SDB->visitSwitchCase(SDB->SwitchCases[i], FuncInfo->MBB);
1826 CurDAG->setRoot(SDB->getRoot());
1827 SDB->clear();
1828 CodeGenAndEmitDAG();
1829
1830 // Remember the last block, now that any splitting is done, for use in
1831 // populating PHI nodes in successors.
1832 MachineBasicBlock *ThisBB = FuncInfo->MBB;
1833
1834 // Handle any PHI nodes in successors of this chunk, as if we were coming
1835 // from the original BB before switch expansion. Note that PHI nodes can
1836 // occur multiple times in PHINodesToUpdate. We have to be very careful to
1837 // handle them the right number of times.
1838 for (unsigned i = 0, e = Succs.size(); i != e; ++i) {
1839 FuncInfo->MBB = Succs[i];
1840 FuncInfo->InsertPt = FuncInfo->MBB->end();
1841 // FuncInfo->MBB may have been removed from the CFG if a branch was
1842 // constant folded.
1843 if (ThisBB->isSuccessor(FuncInfo->MBB)) {
1844 for (MachineBasicBlock::iterator
1845 MBBI = FuncInfo->MBB->begin(), MBBE = FuncInfo->MBB->end();
1846 MBBI != MBBE && MBBI->isPHI(); ++MBBI) {
1847 MachineInstrBuilder PHI(*MF, MBBI);
1848 // This value for this PHI node is recorded in PHINodesToUpdate.
1849 for (unsigned pn = 0; ; ++pn) {
1850 assert(pn != FuncInfo->PHINodesToUpdate.size() &&((pn != FuncInfo->PHINodesToUpdate.size() && "Didn't find PHI entry!"
) ? static_cast<void> (0) : __assert_fail ("pn != FuncInfo->PHINodesToUpdate.size() && \"Didn't find PHI entry!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1851, __PRETTY_FUNCTION__))
1851 "Didn't find PHI entry!")((pn != FuncInfo->PHINodesToUpdate.size() && "Didn't find PHI entry!"
) ? static_cast<void> (0) : __assert_fail ("pn != FuncInfo->PHINodesToUpdate.size() && \"Didn't find PHI entry!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1851, __PRETTY_FUNCTION__))
;
1852 if (FuncInfo->PHINodesToUpdate[pn].first == PHI) {
1853 PHI.addReg(FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB);
1854 break;
1855 }
1856 }
1857 }
1858 }
1859 }
1860 }
1861 SDB->SwitchCases.clear();
1862}
1863
1864/// Create the scheduler. If a specific scheduler was specified
1865/// via the SchedulerRegistry, use it, otherwise select the
1866/// one preferred by the target.
1867///
1868ScheduleDAGSDNodes *SelectionDAGISel::CreateScheduler() {
1869 return ISHeuristic(this, OptLevel);
1870}
1871
1872//===----------------------------------------------------------------------===//
1873// Helper functions used by the generated instruction selector.
1874//===----------------------------------------------------------------------===//
1875// Calls to these methods are generated by tblgen.
1876
1877/// CheckAndMask - The isel is trying to match something like (and X, 255). If
1878/// the dag combiner simplified the 255, we still want to match. RHS is the
1879/// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value
1880/// specified in the .td file (e.g. 255).
1881bool SelectionDAGISel::CheckAndMask(SDValue LHS, ConstantSDNode *RHS,
1882 int64_t DesiredMaskS) const {
1883 const APInt &ActualMask = RHS->getAPIntValue();
1884 const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
1885
1886 // If the actual mask exactly matches, success!
1887 if (ActualMask == DesiredMask)
1888 return true;
1889
1890 // If the actual AND mask is allowing unallowed bits, this doesn't match.
1891 if (ActualMask.intersects(~DesiredMask))
1892 return false;
1893
1894 // Otherwise, the DAG Combiner may have proven that the value coming in is
1895 // either already zero or is not demanded. Check for known zero input bits.
1896 APInt NeededMask = DesiredMask & ~ActualMask;
1897 if (CurDAG->MaskedValueIsZero(LHS, NeededMask))
1898 return true;
1899
1900 // TODO: check to see if missing bits are just not demanded.
1901
1902 // Otherwise, this pattern doesn't match.
1903 return false;
1904}
1905
1906/// CheckOrMask - The isel is trying to match something like (or X, 255). If
1907/// the dag combiner simplified the 255, we still want to match. RHS is the
1908/// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value
1909/// specified in the .td file (e.g. 255).
1910bool SelectionDAGISel::CheckOrMask(SDValue LHS, ConstantSDNode *RHS,
1911 int64_t DesiredMaskS) const {
1912 const APInt &ActualMask = RHS->getAPIntValue();
1913 const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
1914
1915 // If the actual mask exactly matches, success!
1916 if (ActualMask == DesiredMask)
1917 return true;
1918
1919 // If the actual AND mask is allowing unallowed bits, this doesn't match.
1920 if (ActualMask.intersects(~DesiredMask))
1921 return false;
1922
1923 // Otherwise, the DAG Combiner may have proven that the value coming in is
1924 // either already zero or is not demanded. Check for known zero input bits.
1925 APInt NeededMask = DesiredMask & ~ActualMask;
1926
1927 KnownBits Known;
1928 CurDAG->computeKnownBits(LHS, Known);
1929
1930 // If all the missing bits in the or are already known to be set, match!
1931 if (NeededMask.isSubsetOf(Known.One))
1932 return true;
1933
1934 // TODO: check to see if missing bits are just not demanded.
1935
1936 // Otherwise, this pattern doesn't match.
1937 return false;
1938}
1939
1940/// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
1941/// by tblgen. Others should not call it.
1942void SelectionDAGISel::SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops,
1943 const SDLoc &DL) {
1944 std::vector<SDValue> InOps;
1945 std::swap(InOps, Ops);
1946
1947 Ops.push_back(InOps[InlineAsm::Op_InputChain]); // 0
1948 Ops.push_back(InOps[InlineAsm::Op_AsmString]); // 1
1949 Ops.push_back(InOps[InlineAsm::Op_MDNode]); // 2, !srcloc
1950 Ops.push_back(InOps[InlineAsm::Op_ExtraInfo]); // 3 (SideEffect, AlignStack)
1951
1952 unsigned i = InlineAsm::Op_FirstOperand, e = InOps.size();
1953 if (InOps[e-1].getValueType() == MVT::Glue)
1954 --e; // Don't process a glue operand if it is here.
1955
1956 while (i != e) {
1957 unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue();
1958 if (!InlineAsm::isMemKind(Flags)) {
1959 // Just skip over this operand, copying the operands verbatim.
1960 Ops.insert(Ops.end(), InOps.begin()+i,
1961 InOps.begin()+i+InlineAsm::getNumOperandRegisters(Flags) + 1);
1962 i += InlineAsm::getNumOperandRegisters(Flags) + 1;
1963 } else {
1964 assert(InlineAsm::getNumOperandRegisters(Flags) == 1 &&((InlineAsm::getNumOperandRegisters(Flags) == 1 && "Memory operand with multiple values?"
) ? static_cast<void> (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(Flags) == 1 && \"Memory operand with multiple values?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1965, __PRETTY_FUNCTION__))
1965 "Memory operand with multiple values?")((InlineAsm::getNumOperandRegisters(Flags) == 1 && "Memory operand with multiple values?"
) ? static_cast<void> (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(Flags) == 1 && \"Memory operand with multiple values?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1965, __PRETTY_FUNCTION__))
;
1966
1967 unsigned TiedToOperand;
1968 if (InlineAsm::isUseOperandTiedToDef(Flags, TiedToOperand)) {
1969 // We need the constraint ID from the operand this is tied to.
1970 unsigned CurOp = InlineAsm::Op_FirstOperand;
1971 Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
1972 for (; TiedToOperand; --TiedToOperand) {
1973 CurOp += InlineAsm::getNumOperandRegisters(Flags)+1;
1974 Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
1975 }
1976 }
1977
1978 // Otherwise, this is a memory operand. Ask the target to select it.
1979 std::vector<SDValue> SelOps;
1980 unsigned ConstraintID = InlineAsm::getMemoryConstraintID(Flags);
1981 if (SelectInlineAsmMemoryOperand(InOps[i+1], ConstraintID, SelOps))
1982 report_fatal_error("Could not match memory address. Inline asm"
1983 " failure!");
1984
1985 // Add this to the output node.
1986 unsigned NewFlags =
1987 InlineAsm::getFlagWord(InlineAsm::Kind_Mem, SelOps.size());
1988 NewFlags = InlineAsm::getFlagWordForMem(NewFlags, ConstraintID);
1989 Ops.push_back(CurDAG->getTargetConstant(NewFlags, DL, MVT::i32));
1990 Ops.insert(Ops.end(), SelOps.begin(), SelOps.end());
1991 i += 2;
1992 }
1993 }
1994
1995 // Add the glue input back if present.
1996 if (e != InOps.size())
1997 Ops.push_back(InOps.back());
1998}
1999
2000/// findGlueUse - Return use of MVT::Glue value produced by the specified
2001/// SDNode.
2002///
2003static SDNode *findGlueUse(SDNode *N) {
2004 unsigned FlagResNo = N->getNumValues()-1;
2005 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
2006 SDUse &Use = I.getUse();
2007 if (Use.getResNo() == FlagResNo)
2008 return Use.getUser();
2009 }
2010 return nullptr;
2011}
2012
2013/// findNonImmUse - Return true if "Use" is a non-immediate use of "Def".
2014/// This function recursively traverses up the operand chain, ignoring
2015/// certain nodes.
2016static bool findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse,
2017 SDNode *Root, SmallPtrSetImpl<SDNode*> &Visited,
2018 bool IgnoreChains) {
2019 // The NodeID's are given uniques ID's where a node ID is guaranteed to be
2020 // greater than all of its (recursive) operands. If we scan to a point where
2021 // 'use' is smaller than the node we're scanning for, then we know we will
2022 // never find it.
2023 //
2024 // The Use may be -1 (unassigned) if it is a newly allocated node. This can
2025 // happen because we scan down to newly selected nodes in the case of glue
2026 // uses.
2027 if ((Use->getNodeId() < Def->getNodeId() && Use->getNodeId() != -1))
2028 return false;
2029
2030 // Don't revisit nodes if we already scanned it and didn't fail, we know we
2031 // won't fail if we scan it again.
2032 if (!Visited.insert(Use).second)
2033 return false;
2034
2035 for (const SDValue &Op : Use->op_values()) {
2036 // Ignore chain uses, they are validated by HandleMergeInputChains.
2037 if (Op.getValueType() == MVT::Other && IgnoreChains)
2038 continue;
2039
2040 SDNode *N = Op.getNode();
2041 if (N == Def) {
2042 if (Use == ImmedUse || Use == Root)
2043 continue; // We are not looking for immediate use.
2044 assert(N != Root)((N != Root) ? static_cast<void> (0) : __assert_fail ("N != Root"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2044, __PRETTY_FUNCTION__))
;
2045 return true;
2046 }
2047
2048 // Traverse up the operand chain.
2049 if (findNonImmUse(N, Def, ImmedUse, Root, Visited, IgnoreChains))
2050 return true;
2051 }
2052 return false;
2053}
2054
2055/// IsProfitableToFold - Returns true if it's profitable to fold the specific
2056/// operand node N of U during instruction selection that starts at Root.
2057bool SelectionDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
2058 SDNode *Root) const {
2059 if (OptLevel == CodeGenOpt::None) return false;
2060 return N.hasOneUse();
2061}
2062
2063/// IsLegalToFold - Returns true if the specific operand node N of
2064/// U can be folded during instruction selection that starts at Root.
2065bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root,
2066 CodeGenOpt::Level OptLevel,
2067 bool IgnoreChains) {
2068 if (OptLevel == CodeGenOpt::None) return false;
2069
2070 // If Root use can somehow reach N through a path that that doesn't contain
2071 // U then folding N would create a cycle. e.g. In the following
2072 // diagram, Root can reach N through X. If N is folded into into Root, then
2073 // X is both a predecessor and a successor of U.
2074 //
2075 // [N*] //
2076 // ^ ^ //
2077 // / \ //
2078 // [U*] [X]? //
2079 // ^ ^ //
2080 // \ / //
2081 // \ / //
2082 // [Root*] //
2083 //
2084 // * indicates nodes to be folded together.
2085 //
2086 // If Root produces glue, then it gets (even more) interesting. Since it
2087 // will be "glued" together with its glue use in the scheduler, we need to
2088 // check if it might reach N.
2089 //
2090 // [N*] //
2091 // ^ ^ //
2092 // / \ //
2093 // [U*] [X]? //
2094 // ^ ^ //
2095 // \ \ //
2096 // \ | //
2097 // [Root*] | //
2098 // ^ | //
2099 // f | //
2100 // | / //
2101 // [Y] / //
2102 // ^ / //
2103 // f / //
2104 // | / //
2105 // [GU] //
2106 //
2107 // If GU (glue use) indirectly reaches N (the load), and Root folds N
2108 // (call it Fold), then X is a predecessor of GU and a successor of
2109 // Fold. But since Fold and GU are glued together, this will create
2110 // a cycle in the scheduling graph.
2111
2112 // If the node has glue, walk down the graph to the "lowest" node in the
2113 // glueged set.
2114 EVT VT = Root->getValueType(Root->getNumValues()-1);
2115 while (VT == MVT::Glue) {
2116 SDNode *GU = findGlueUse(Root);
2117 if (!GU)
2118 break;
2119 Root = GU;
2120 VT = Root->getValueType(Root->getNumValues()-1);
2121
2122 // If our query node has a glue result with a use, we've walked up it. If
2123 // the user (which has already been selected) has a chain or indirectly uses
2124 // the chain, our WalkChainUsers predicate will not consider it. Because of
2125 // this, we cannot ignore chains in this predicate.
2126 IgnoreChains = false;
2127 }
2128
2129 SmallPtrSet<SDNode*, 16> Visited;
2130 return !findNonImmUse(Root, N.getNode(), U, Root, Visited, IgnoreChains);
2131}
2132
2133void SelectionDAGISel::Select_INLINEASM(SDNode *N) {
2134 SDLoc DL(N);
2135
2136 std::vector<SDValue> Ops(N->op_begin(), N->op_end());
2137 SelectInlineAsmMemoryOperands(Ops, DL);
2138
2139 const EVT VTs[] = {MVT::Other, MVT::Glue};
2140 SDValue New = CurDAG->getNode(ISD::INLINEASM, DL, VTs, Ops);
2141 New->setNodeId(-1);
2142 ReplaceUses(N, New.getNode());
2143 CurDAG->RemoveDeadNode(N);
2144}
2145
2146void SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) {
2147 SDLoc dl(Op);
2148 MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(Op->getOperand(1));
2149 const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));
2150 unsigned Reg =
2151 TLI->getRegisterByName(RegStr->getString().data(), Op->getValueType(0),
2152 *CurDAG);
2153 SDValue New = CurDAG->getCopyFromReg(
2154 Op->getOperand(0), dl, Reg, Op->getValueType(0));
2155 New->setNodeId(-1);
2156 ReplaceUses(Op, New.getNode());
2157 CurDAG->RemoveDeadNode(Op);
2158}
2159
2160void SelectionDAGISel::Select_WRITE_REGISTER(SDNode *Op) {
2161 SDLoc dl(Op);
2162 MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(Op->getOperand(1));
2163 const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));
2164 unsigned Reg = TLI->getRegisterByName(RegStr->getString().data(),
2165 Op->getOperand(2).getValueType(),
2166 *CurDAG);
2167 SDValue New = CurDAG->getCopyToReg(
2168 Op->getOperand(0), dl, Reg, Op->getOperand(2));
2169 New->setNodeId(-1);
2170 ReplaceUses(Op, New.getNode());
2171 CurDAG->RemoveDeadNode(Op);
2172}
2173
2174void SelectionDAGISel::Select_UNDEF(SDNode *N) {
2175 CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0));
2176}
2177
2178/// GetVBR - decode a vbr encoding whose top bit is set.
2179LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline uint64_t
2180GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) {
2181 assert(Val >= 128 && "Not a VBR")((Val >= 128 && "Not a VBR") ? static_cast<void
> (0) : __assert_fail ("Val >= 128 && \"Not a VBR\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2181, __PRETTY_FUNCTION__))
;
2182 Val &= 127; // Remove first vbr bit.
2183
2184 unsigned Shift = 7;
2185 uint64_t NextBits;
2186 do {
2187 NextBits = MatcherTable[Idx++];
2188 Val |= (NextBits&127) << Shift;
2189 Shift += 7;
2190 } while (NextBits & 128);
2191
2192 return Val;
2193}
2194
2195/// When a match is complete, this method updates uses of interior chain results
2196/// to use the new results.
2197void SelectionDAGISel::UpdateChains(
2198 SDNode *NodeToMatch, SDValue InputChain,
2199 SmallVectorImpl<SDNode *> &ChainNodesMatched, bool isMorphNodeTo) {
2200 SmallVector<SDNode*, 4> NowDeadNodes;
2201
2202 // Now that all the normal results are replaced, we replace the chain and
2203 // glue results if present.
2204 if (!ChainNodesMatched.empty()) {
2205 assert(InputChain.getNode() &&((InputChain.getNode() && "Matched input chains but didn't produce a chain"
) ? static_cast<void> (0) : __assert_fail ("InputChain.getNode() && \"Matched input chains but didn't produce a chain\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2206, __PRETTY_FUNCTION__))
2206 "Matched input chains but didn't produce a chain")((InputChain.getNode() && "Matched input chains but didn't produce a chain"
) ? static_cast<void> (0) : __assert_fail ("InputChain.getNode() && \"Matched input chains but didn't produce a chain\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2206, __PRETTY_FUNCTION__))
;
2207 // Loop over all of the nodes we matched that produced a chain result.
2208 // Replace all the chain results with the final chain we ended up with.
2209 for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
2210 SDNode *ChainNode = ChainNodesMatched[i];
2211 // If ChainNode is null, it's because we replaced it on a previous
2212 // iteration and we cleared it out of the map. Just skip it.
2213 if (!ChainNode)
2214 continue;
2215
2216 assert(ChainNode->getOpcode() != ISD::DELETED_NODE &&((ChainNode->getOpcode() != ISD::DELETED_NODE && "Deleted node left in chain"
) ? static_cast<void> (0) : __assert_fail ("ChainNode->getOpcode() != ISD::DELETED_NODE && \"Deleted node left in chain\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2217, __PRETTY_FUNCTION__))
2217 "Deleted node left in chain")((ChainNode->getOpcode() != ISD::DELETED_NODE && "Deleted node left in chain"
) ? static_cast<void> (0) : __assert_fail ("ChainNode->getOpcode() != ISD::DELETED_NODE && \"Deleted node left in chain\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2217, __PRETTY_FUNCTION__))
;
2218
2219 // Don't replace the results of the root node if we're doing a
2220 // MorphNodeTo.
2221 if (ChainNode == NodeToMatch && isMorphNodeTo)
2222 continue;
2223
2224 SDValue ChainVal = SDValue(ChainNode, ChainNode->getNumValues()-1);
2225 if (ChainVal.getValueType() == MVT::Glue)
2226 ChainVal = ChainVal.getValue(ChainVal->getNumValues()-2);
2227 assert(ChainVal.getValueType() == MVT::Other && "Not a chain?")((ChainVal.getValueType() == MVT::Other && "Not a chain?"
) ? static_cast<void> (0) : __assert_fail ("ChainVal.getValueType() == MVT::Other && \"Not a chain?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2227, __PRETTY_FUNCTION__))
;
2228 SelectionDAG::DAGNodeDeletedListener NDL(
2229 *CurDAG, [&](SDNode *N, SDNode *E) {
2230 std::replace(ChainNodesMatched.begin(), ChainNodesMatched.end(), N,
2231 static_cast<SDNode *>(nullptr));
2232 });
2233 CurDAG->ReplaceAllUsesOfValueWith(ChainVal, InputChain);
2234
2235 // If the node became dead and we haven't already seen it, delete it.
2236 if (ChainNode != NodeToMatch && ChainNode->use_empty() &&
2237 !std::count(NowDeadNodes.begin(), NowDeadNodes.end(), ChainNode))
2238 NowDeadNodes.push_back(ChainNode);
2239 }
2240 }
2241
2242 if (!NowDeadNodes.empty())
2243 CurDAG->RemoveDeadNodes(NowDeadNodes);
2244
2245 DEBUG(dbgs() << "ISEL: Match complete!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "ISEL: Match complete!\n"; } } while
(false)
;
2246}
2247
2248enum ChainResult {
2249 CR_Simple,
2250 CR_InducesCycle,
2251 CR_LeadsToInteriorNode
2252};
2253
2254/// WalkChainUsers - Walk down the users of the specified chained node that is
2255/// part of the pattern we're matching, looking at all of the users we find.
2256/// This determines whether something is an interior node, whether we have a
2257/// non-pattern node in between two pattern nodes (which prevent folding because
2258/// it would induce a cycle) and whether we have a TokenFactor node sandwiched
2259/// between pattern nodes (in which case the TF becomes part of the pattern).
2260///
2261/// The walk we do here is guaranteed to be small because we quickly get down to
2262/// already selected nodes "below" us.
2263static ChainResult
2264WalkChainUsers(const SDNode *ChainedNode,
2265 SmallVectorImpl<SDNode *> &ChainedNodesInPattern,
2266 DenseMap<const SDNode *, ChainResult> &TokenFactorResult,
2267 SmallVectorImpl<SDNode *> &InteriorChainedNodes) {
2268 ChainResult Result = CR_Simple;
2269
2270 for (SDNode::use_iterator UI = ChainedNode->use_begin(),
2271 E = ChainedNode->use_end(); UI != E; ++UI) {
2272 // Make sure the use is of the chain, not some other value we produce.
2273 if (UI.getUse().getValueType() != MVT::Other) continue;
2274
2275 SDNode *User = *UI;
2276
2277 if (User->getOpcode() == ISD::HANDLENODE) // Root of the graph.
2278 continue;
2279
2280 // If we see an already-selected machine node, then we've gone beyond the
2281 // pattern that we're selecting down into the already selected chunk of the
2282 // DAG.
2283 unsigned UserOpcode = User->getOpcode();
2284 if (User->isMachineOpcode() ||
2285 UserOpcode == ISD::CopyToReg ||
2286 UserOpcode == ISD::CopyFromReg ||
2287 UserOpcode == ISD::INLINEASM ||
2288 UserOpcode == ISD::EH_LABEL ||
2289 UserOpcode == ISD::LIFETIME_START ||
2290 UserOpcode == ISD::LIFETIME_END) {
2291 // If their node ID got reset to -1 then they've already been selected.
2292 // Treat them like a MachineOpcode.
2293 if (User->getNodeId() == -1)
2294 continue;
2295 }
2296
2297 // If we have a TokenFactor, we handle it specially.
2298 if (User->getOpcode() != ISD::TokenFactor) {
2299 // If the node isn't a token factor and isn't part of our pattern, then it
2300 // must be a random chained node in between two nodes we're selecting.
2301 // This happens when we have something like:
2302 // x = load ptr
2303 // call
2304 // y = x+4
2305 // store y -> ptr
2306 // Because we structurally match the load/store as a read/modify/write,
2307 // but the call is chained between them. We cannot fold in this case
2308 // because it would induce a cycle in the graph.
2309 if (!std::count(ChainedNodesInPattern.begin(),
2310 ChainedNodesInPattern.end(), User))
2311 return CR_InducesCycle;
2312
2313 // Otherwise we found a node that is part of our pattern. For example in:
2314 // x = load ptr
2315 // y = x+4
2316 // store y -> ptr
2317 // This would happen when we're scanning down from the load and see the
2318 // store as a user. Record that there is a use of ChainedNode that is
2319 // part of the pattern and keep scanning uses.
2320 Result = CR_LeadsToInteriorNode;
2321 InteriorChainedNodes.push_back(User);
2322 continue;
2323 }
2324
2325 // If we found a TokenFactor, there are two cases to consider: first if the
2326 // TokenFactor is just hanging "below" the pattern we're matching (i.e. no
2327 // uses of the TF are in our pattern) we just want to ignore it. Second,
2328 // the TokenFactor can be sandwiched in between two chained nodes, like so:
2329 // [Load chain]
2330 // ^
2331 // |
2332 // [Load]
2333 // ^ ^
2334 // | \ DAG's like cheese
2335 // / \ do you?
2336 // / |
2337 // [TokenFactor] [Op]
2338 // ^ ^
2339 // | |
2340 // \ /
2341 // \ /
2342 // [Store]
2343 //
2344 // In this case, the TokenFactor becomes part of our match and we rewrite it
2345 // as a new TokenFactor.
2346 //
2347 // To distinguish these two cases, do a recursive walk down the uses.
2348 auto MemoizeResult = TokenFactorResult.find(User);
2349 bool Visited = MemoizeResult != TokenFactorResult.end();
2350 // Recursively walk chain users only if the result is not memoized.
2351 if (!Visited) {
2352 auto Res = WalkChainUsers(User, ChainedNodesInPattern, TokenFactorResult,
2353 InteriorChainedNodes);
2354 MemoizeResult = TokenFactorResult.insert(std::make_pair(User, Res)).first;
2355 }
2356 switch (MemoizeResult->second) {
2357 case CR_Simple:
2358 // If the uses of the TokenFactor are just already-selected nodes, ignore
2359 // it, it is "below" our pattern.
2360 continue;
2361 case CR_InducesCycle:
2362 // If the uses of the TokenFactor lead to nodes that are not part of our
2363 // pattern that are not selected, folding would turn this into a cycle,
2364 // bail out now.
2365 return CR_InducesCycle;
2366 case CR_LeadsToInteriorNode:
2367 break; // Otherwise, keep processing.
2368 }
2369
2370 // Okay, we know we're in the interesting interior case. The TokenFactor
2371 // is now going to be considered part of the pattern so that we rewrite its
2372 // uses (it may have uses that are not part of the pattern) with the
2373 // ultimate chain result of the generated code. We will also add its chain
2374 // inputs as inputs to the ultimate TokenFactor we create.
2375 Result = CR_LeadsToInteriorNode;
2376 if (!Visited) {
2377 ChainedNodesInPattern.push_back(User);
2378 InteriorChainedNodes.push_back(User);
2379 }
2380 }
2381
2382 return Result;
2383}
2384
2385/// HandleMergeInputChains - This implements the OPC_EmitMergeInputChains
2386/// operation for when the pattern matched at least one node with a chains. The
2387/// input vector contains a list of all of the chained nodes that we match. We
2388/// must determine if this is a valid thing to cover (i.e. matching it won't
2389/// induce cycles in the DAG) and if so, creating a TokenFactor node. that will
2390/// be used as the input node chain for the generated nodes.
2391static SDValue
2392HandleMergeInputChains(SmallVectorImpl<SDNode*> &ChainNodesMatched,
2393 SelectionDAG *CurDAG) {
2394 // Used for memoization. Without it WalkChainUsers could take exponential
2395 // time to run.
2396 DenseMap<const SDNode *, ChainResult> TokenFactorResult;
2397 // Walk all of the chained nodes we've matched, recursively scanning down the
2398 // users of the chain result. This adds any TokenFactor nodes that are caught
2399 // in between chained nodes to the chained and interior nodes list.
2400 SmallVector<SDNode*, 3> InteriorChainedNodes;
2401 for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
2402 if (WalkChainUsers(ChainNodesMatched[i], ChainNodesMatched,
2403 TokenFactorResult,
2404 InteriorChainedNodes) == CR_InducesCycle)
2405 return SDValue(); // Would induce a cycle.
2406 }
2407
2408 // Okay, we have walked all the matched nodes and collected TokenFactor nodes
2409 // that we are interested in. Form our input TokenFactor node.
2410 SmallVector<SDValue, 3> InputChains;
2411 for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
2412 // Add the input chain of this node to the InputChains list (which will be
2413 // the operands of the generated TokenFactor) if it's not an interior node.
2414 SDNode *N = ChainNodesMatched[i];
2415 if (N->getOpcode() != ISD::TokenFactor) {
2416 if (std::count(InteriorChainedNodes.begin(),InteriorChainedNodes.end(),N))
2417 continue;
2418
2419 // Otherwise, add the input chain.
2420 SDValue InChain = ChainNodesMatched[i]->getOperand(0);
2421 assert(InChain.getValueType() == MVT::Other && "Not a chain")((InChain.getValueType() == MVT::Other && "Not a chain"
) ? static_cast<void> (0) : __assert_fail ("InChain.getValueType() == MVT::Other && \"Not a chain\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2421, __PRETTY_FUNCTION__))
;
2422 InputChains.push_back(InChain);
2423 continue;
2424 }
2425
2426 // If we have a token factor, we want to add all inputs of the token factor
2427 // that are not part of the pattern we're matching.
2428 for (const SDValue &Op : N->op_values()) {
2429 if (!std::count(ChainNodesMatched.begin(), ChainNodesMatched.end(),
2430 Op.getNode()))
2431 InputChains.push_back(Op);
2432 }
2433 }
2434
2435 if (InputChains.size() == 1)
2436 return InputChains[0];
2437 return CurDAG->getNode(ISD::TokenFactor, SDLoc(ChainNodesMatched[0]),
2438 MVT::Other, InputChains);
2439}
2440
2441/// MorphNode - Handle morphing a node in place for the selector.
2442SDNode *SelectionDAGISel::
2443MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
2444 ArrayRef<SDValue> Ops, unsigned EmitNodeInfo) {
2445 // It is possible we're using MorphNodeTo to replace a node with no
2446 // normal results with one that has a normal result (or we could be
2447 // adding a chain) and the input could have glue and chains as well.
2448 // In this case we need to shift the operands down.
2449 // FIXME: This is a horrible hack and broken in obscure cases, no worse
2450 // than the old isel though.
2451 int OldGlueResultNo = -1, OldChainResultNo = -1;
2452
2453 unsigned NTMNumResults = Node->getNumValues();
2454 if (Node->getValueType(NTMNumResults-1) == MVT::Glue) {
2455 OldGlueResultNo = NTMNumResults-1;
2456 if (NTMNumResults != 1 &&
2457 Node->getValueType(NTMNumResults-2) == MVT::Other)
2458 OldChainResultNo = NTMNumResults-2;
2459 } else if (Node->getValueType(NTMNumResults-1) == MVT::Other)
2460 OldChainResultNo = NTMNumResults-1;
2461
2462 // Call the underlying SelectionDAG routine to do the transmogrification. Note
2463 // that this deletes operands of the old node that become dead.
2464 SDNode *Res = CurDAG->MorphNodeTo(Node, ~TargetOpc, VTList, Ops);
2465
2466 // MorphNodeTo can operate in two ways: if an existing node with the
2467 // specified operands exists, it can just return it. Otherwise, it
2468 // updates the node in place to have the requested operands.
2469 if (Res == Node) {
2470 // If we updated the node in place, reset the node ID. To the isel,
2471 // this should be just like a newly allocated machine node.
2472 Res->setNodeId(-1);
2473 }
2474
2475 unsigned ResNumResults = Res->getNumValues();
2476 // Move the glue if needed.
2477 if ((EmitNodeInfo & OPFL_GlueOutput) && OldGlueResultNo != -1 &&
2478 (unsigned)OldGlueResultNo != ResNumResults-1)
2479 CurDAG->ReplaceAllUsesOfValueWith(SDValue(Node, OldGlueResultNo),
2480 SDValue(Res, ResNumResults-1));
2481
2482 if ((EmitNodeInfo & OPFL_GlueOutput) != 0)
2483 --ResNumResults;
2484
2485 // Move the chain reference if needed.
2486 if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 &&
2487 (unsigned)OldChainResultNo != ResNumResults-1)
2488 CurDAG->ReplaceAllUsesOfValueWith(SDValue(Node, OldChainResultNo),
2489 SDValue(Res, ResNumResults-1));
2490
2491 // Otherwise, no replacement happened because the node already exists. Replace
2492 // Uses of the old node with the new one.
2493 if (Res != Node) {
2494 CurDAG->ReplaceAllUsesWith(Node, Res);
2495 CurDAG->RemoveDeadNode(Node);
2496 }
2497
2498 return Res;
2499}
2500
2501/// CheckSame - Implements OP_CheckSame.
2502LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2503CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2504 SDValue N,
2505 const SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) {
2506 // Accept if it is exactly the same as a previously recorded node.
2507 unsigned RecNo = MatcherTable[MatcherIndex++];
2508 assert(RecNo < RecordedNodes.size() && "Invalid CheckSame")((RecNo < RecordedNodes.size() && "Invalid CheckSame"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid CheckSame\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2508, __PRETTY_FUNCTION__))
;
2509 return N == RecordedNodes[RecNo].first;
2510}
2511
2512/// CheckChildSame - Implements OP_CheckChildXSame.
2513LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2514CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2515 SDValue N,
2516 const SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes,
2517 unsigned ChildNo) {
2518 if (ChildNo >= N.getNumOperands())
2519 return false; // Match fails if out of range child #.
2520 return ::CheckSame(MatcherTable, MatcherIndex, N.getOperand(ChildNo),
2521 RecordedNodes);
2522}
2523
2524/// CheckPatternPredicate - Implements OP_CheckPatternPredicate.
2525LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2526CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2527 const SelectionDAGISel &SDISel) {
2528 return SDISel.CheckPatternPredicate(MatcherTable[MatcherIndex++]);
2529}
2530
2531/// CheckNodePredicate - Implements OP_CheckNodePredicate.
2532LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2533CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2534 const SelectionDAGISel &SDISel, SDNode *N) {
2535 return SDISel.CheckNodePredicate(N, MatcherTable[MatcherIndex++]);
2536}
2537
2538LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2539CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2540 SDNode *N) {
2541 uint16_t Opc = MatcherTable[MatcherIndex++];
2542 Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
2543 return N->getOpcode() == Opc;
2544}
2545
2546LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2547CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2548 const TargetLowering *TLI, const DataLayout &DL) {
2549 MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2550 if (N.getValueType() == VT) return true;
2551
2552 // Handle the case when VT is iPTR.
2553 return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(DL);
2554}
2555
2556LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2557CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2558 SDValue N, const TargetLowering *TLI, const DataLayout &DL,
2559 unsigned ChildNo) {
2560 if (ChildNo >= N.getNumOperands())
2561 return false; // Match fails if out of range child #.
2562 return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI,
2563 DL);
2564}
2565
2566LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2567CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2568 SDValue N) {
2569 return cast<CondCodeSDNode>(N)->get() ==
2570 (ISD::CondCode)MatcherTable[MatcherIndex++];
2571}
2572
2573LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2574CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2575 SDValue N, const TargetLowering *TLI, const DataLayout &DL) {
2576 MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2577 if (cast<VTSDNode>(N)->getVT() == VT)
2578 return true;
2579
2580 // Handle the case when VT is iPTR.
2581 return VT == MVT::iPTR && cast<VTSDNode>(N)->getVT() == TLI->getPointerTy(DL);
2582}
2583
2584LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2585CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2586 SDValue N) {
2587 int64_t Val = MatcherTable[MatcherIndex++];
2588 if (Val & 128)
2589 Val = GetVBR(Val, MatcherTable, MatcherIndex);
2590
2591 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N);
2592 return C && C->getSExtValue() == Val;
2593}
2594
2595LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2596CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2597 SDValue N, unsigned ChildNo) {
2598 if (ChildNo >= N.getNumOperands())
2599 return false; // Match fails if out of range child #.
2600 return ::CheckInteger(MatcherTable, MatcherIndex, N.getOperand(ChildNo));
2601}
2602
2603LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2604CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2605 SDValue N, const SelectionDAGISel &SDISel) {
2606 int64_t Val = MatcherTable[MatcherIndex++];
2607 if (Val & 128)
2608 Val = GetVBR(Val, MatcherTable, MatcherIndex);
2609
2610 if (N->getOpcode() != ISD::AND) return false;
2611
2612 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2613 return C && SDISel.CheckAndMask(N.getOperand(0), C, Val);
2614}
2615
2616LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2617CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2618 SDValue N, const SelectionDAGISel &SDISel) {
2619 int64_t Val = MatcherTable[MatcherIndex++];
2620 if (Val & 128)
2621 Val = GetVBR(Val, MatcherTable, MatcherIndex);
2622
2623 if (N->getOpcode() != ISD::OR) return false;
2624
2625 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2626 return C && SDISel.CheckOrMask(N.getOperand(0), C, Val);
2627}
2628
2629/// IsPredicateKnownToFail - If we know how and can do so without pushing a
2630/// scope, evaluate the current node. If the current predicate is known to
2631/// fail, set Result=true and return anything. If the current predicate is
2632/// known to pass, set Result=false and return the MatcherIndex to continue
2633/// with. If the current predicate is unknown, set Result=false and return the
2634/// MatcherIndex to continue with.
2635static unsigned IsPredicateKnownToFail(const unsigned char *Table,
2636 unsigned Index, SDValue N,
2637 bool &Result,
2638 const SelectionDAGISel &SDISel,
2639 SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) {
2640 switch (Table[Index++]) {
2641 default:
2642 Result = false;
2643 return Index-1; // Could not evaluate this predicate.
2644 case SelectionDAGISel::OPC_CheckSame:
2645 Result = !::CheckSame(Table, Index, N, RecordedNodes);
2646 return Index;
2647 case SelectionDAGISel::OPC_CheckChild0Same:
2648 case SelectionDAGISel::OPC_CheckChild1Same:
2649 case SelectionDAGISel::OPC_CheckChild2Same:
2650 case SelectionDAGISel::OPC_CheckChild3Same:
2651 Result = !::CheckChildSame(Table, Index, N, RecordedNodes,
2652 Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Same);
2653 return Index;
2654 case SelectionDAGISel::OPC_CheckPatternPredicate:
2655 Result = !::CheckPatternPredicate(Table, Index, SDISel);
2656 return Index;
2657 case SelectionDAGISel::OPC_CheckPredicate:
2658 Result = !::CheckNodePredicate(Table, Index, SDISel, N.getNode());
2659 return Index;
2660 case SelectionDAGISel::OPC_CheckOpcode:
2661 Result = !::CheckOpcode(Table, Index, N.getNode());
2662 return Index;
2663 case SelectionDAGISel::OPC_CheckType:
2664 Result = !::CheckType(Table, Index, N, SDISel.TLI,
2665 SDISel.CurDAG->getDataLayout());
2666 return Index;
2667 case SelectionDAGISel::OPC_CheckChild0Type:
2668 case SelectionDAGISel::OPC_CheckChild1Type:
2669 case SelectionDAGISel::OPC_CheckChild2Type:
2670 case SelectionDAGISel::OPC_CheckChild3Type:
2671 case SelectionDAGISel::OPC_CheckChild4Type:
2672 case SelectionDAGISel::OPC_CheckChild5Type:
2673 case SelectionDAGISel::OPC_CheckChild6Type:
2674 case SelectionDAGISel::OPC_CheckChild7Type:
2675 Result = !::CheckChildType(
2676 Table, Index, N, SDISel.TLI, SDISel.CurDAG->getDataLayout(),
2677 Table[Index - 1] - SelectionDAGISel::OPC_CheckChild0Type);
2678 return Index;
2679 case SelectionDAGISel::OPC_CheckCondCode:
2680 Result = !::CheckCondCode(Table, Index, N);
2681 return Index;
2682 case SelectionDAGISel::OPC_CheckValueType:
2683 Result = !::CheckValueType(Table, Index, N, SDISel.TLI,
2684 SDISel.CurDAG->getDataLayout());
2685 return Index;
2686 case SelectionDAGISel::OPC_CheckInteger:
2687 Result = !::CheckInteger(Table, Index, N);
2688 return Index;
2689 case SelectionDAGISel::OPC_CheckChild0Integer:
2690 case SelectionDAGISel::OPC_CheckChild1Integer:
2691 case SelectionDAGISel::OPC_CheckChild2Integer:
2692 case SelectionDAGISel::OPC_CheckChild3Integer:
2693 case SelectionDAGISel::OPC_CheckChild4Integer:
2694 Result = !::CheckChildInteger(Table, Index, N,
2695 Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Integer);
2696 return Index;
2697 case SelectionDAGISel::OPC_CheckAndImm:
2698 Result = !::CheckAndImm(Table, Index, N, SDISel);
2699 return Index;
2700 case SelectionDAGISel::OPC_CheckOrImm:
2701 Result = !::CheckOrImm(Table, Index, N, SDISel);
2702 return Index;
2703 }
2704}
2705
2706namespace {
2707
2708struct MatchScope {
2709 /// FailIndex - If this match fails, this is the index to continue with.
2710 unsigned FailIndex;
2711
2712 /// NodeStack - The node stack when the scope was formed.
2713 SmallVector<SDValue, 4> NodeStack;
2714
2715 /// NumRecordedNodes - The number of recorded nodes when the scope was formed.
2716 unsigned NumRecordedNodes;
2717
2718 /// NumMatchedMemRefs - The number of matched memref entries.
2719 unsigned NumMatchedMemRefs;
2720
2721 /// InputChain/InputGlue - The current chain/glue
2722 SDValue InputChain, InputGlue;
2723
2724 /// HasChainNodesMatched - True if the ChainNodesMatched list is non-empty.
2725 bool HasChainNodesMatched;
2726};
2727
2728/// \\brief A DAG update listener to keep the matching state
2729/// (i.e. RecordedNodes and MatchScope) uptodate if the target is allowed to
2730/// change the DAG while matching. X86 addressing mode matcher is an example
2731/// for this.
2732class MatchStateUpdater : public SelectionDAG::DAGUpdateListener
2733{
2734 SDNode **NodeToMatch;
2735 SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes;
2736 SmallVectorImpl<MatchScope> &MatchScopes;
2737
2738public:
2739 MatchStateUpdater(SelectionDAG &DAG, SDNode **NodeToMatch,
2740 SmallVectorImpl<std::pair<SDValue, SDNode *>> &RN,
2741 SmallVectorImpl<MatchScope> &MS)
2742 : SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch),
2743 RecordedNodes(RN), MatchScopes(MS) {}
2744
2745 void NodeDeleted(SDNode *N, SDNode *E) override {
2746 // Some early-returns here to avoid the search if we deleted the node or
2747 // if the update comes from MorphNodeTo (MorphNodeTo is the last thing we
2748 // do, so it's unnecessary to update matching state at that point).
2749 // Neither of these can occur currently because we only install this
2750 // update listener during matching a complex patterns.
2751 if (!E || E->isMachineOpcode())
2752 return;
2753 // Check if NodeToMatch was updated.
2754 if (N == *NodeToMatch)
2755 *NodeToMatch = E;
2756 // Performing linear search here does not matter because we almost never
2757 // run this code. You'd have to have a CSE during complex pattern
2758 // matching.
2759 for (auto &I : RecordedNodes)
2760 if (I.first.getNode() == N)
2761 I.first.setNode(E);
2762
2763 for (auto &I : MatchScopes)
2764 for (auto &J : I.NodeStack)
2765 if (J.getNode() == N)
2766 J.setNode(E);
2767 }
2768};
2769
2770} // end anonymous namespace
2771
2772void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
2773 const unsigned char *MatcherTable,
2774 unsigned TableSize) {
2775 // FIXME: Should these even be selected? Handle these cases in the caller?
2776 switch (NodeToMatch->getOpcode()) {
2777 default:
2778 break;
2779 case ISD::EntryToken: // These nodes remain the same.
2780 case ISD::BasicBlock:
2781 case ISD::Register:
2782 case ISD::RegisterMask:
2783 case ISD::HANDLENODE:
2784 case ISD::MDNODE_SDNODE:
2785 case ISD::TargetConstant:
2786 case ISD::TargetConstantFP:
2787 case ISD::TargetConstantPool:
2788 case ISD::TargetFrameIndex:
2789 case ISD::TargetExternalSymbol:
2790 case ISD::MCSymbol:
2791 case ISD::TargetBlockAddress:
2792 case ISD::TargetJumpTable:
2793 case ISD::TargetGlobalTLSAddress:
2794 case ISD::TargetGlobalAddress:
2795 case ISD::TokenFactor:
2796 case ISD::CopyFromReg:
2797 case ISD::CopyToReg:
2798 case ISD::EH_LABEL:
2799 case ISD::LIFETIME_START:
2800 case ISD::LIFETIME_END:
2801 NodeToMatch->setNodeId(-1); // Mark selected.
2802 return;
2803 case ISD::AssertSext:
2804 case ISD::AssertZext:
2805 CurDAG->ReplaceAllUsesOfValueWith(SDValue(NodeToMatch, 0),
2806 NodeToMatch->getOperand(0));
2807 CurDAG->RemoveDeadNode(NodeToMatch);
2808 return;
2809 case ISD::INLINEASM:
2810 Select_INLINEASM(NodeToMatch);
2811 return;
2812 case ISD::READ_REGISTER:
2813 Select_READ_REGISTER(NodeToMatch);
2814 return;
2815 case ISD::WRITE_REGISTER:
2816 Select_WRITE_REGISTER(NodeToMatch);
2817 return;
2818 case ISD::UNDEF:
2819 Select_UNDEF(NodeToMatch);
2820 return;
2821 }
2822
2823 assert(!NodeToMatch->isMachineOpcode() && "Node already selected!")((!NodeToMatch->isMachineOpcode() && "Node already selected!"
) ? static_cast<void> (0) : __assert_fail ("!NodeToMatch->isMachineOpcode() && \"Node already selected!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2823, __PRETTY_FUNCTION__))
;
2824
2825 // Set up the node stack with NodeToMatch as the only node on the stack.
2826 SmallVector<SDValue, 8> NodeStack;
2827 SDValue N = SDValue(NodeToMatch, 0);
2828 NodeStack.push_back(N);
2829
2830 // MatchScopes - Scopes used when matching, if a match failure happens, this
2831 // indicates where to continue checking.
2832 SmallVector<MatchScope, 8> MatchScopes;
2833
2834 // RecordedNodes - This is the set of nodes that have been recorded by the
2835 // state machine. The second value is the parent of the node, or null if the
2836 // root is recorded.
2837 SmallVector<std::pair<SDValue, SDNode*>, 8> RecordedNodes;
2838
2839 // MatchedMemRefs - This is the set of MemRef's we've seen in the input
2840 // pattern.
2841 SmallVector<MachineMemOperand*, 2> MatchedMemRefs;
2842
2843 // These are the current input chain and glue for use when generating nodes.
2844 // Various Emit operations change these. For example, emitting a copytoreg
2845 // uses and updates these.
2846 SDValue InputChain, InputGlue;
2847
2848 // ChainNodesMatched - If a pattern matches nodes that have input/output
2849 // chains, the OPC_EmitMergeInputChains operation is emitted which indicates
2850 // which ones they are. The result is captured into this list so that we can
2851 // update the chain results when the pattern is complete.
2852 SmallVector<SDNode*, 3> ChainNodesMatched;
2853
2854 DEBUG(dbgs() << "ISEL: Starting pattern match on root node: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "ISEL: Starting pattern match on root node: "
; NodeToMatch->dump(CurDAG); dbgs() << '\n'; } } while
(false)
2855 NodeToMatch->dump(CurDAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "ISEL: Starting pattern match on root node: "
; NodeToMatch->dump(CurDAG); dbgs() << '\n'; } } while
(false)
2856 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "ISEL: Starting pattern match on root node: "
; NodeToMatch->dump(CurDAG); dbgs() << '\n'; } } while
(false)
;
2857
2858 // Determine where to start the interpreter. Normally we start at opcode #0,
2859 // but if the state machine starts with an OPC_SwitchOpcode, then we
2860 // accelerate the first lookup (which is guaranteed to be hot) with the
2861 // OpcodeOffset table.
2862 unsigned MatcherIndex = 0;
2863
2864 if (!OpcodeOffset.empty()) {
2865 // Already computed the OpcodeOffset table, just index into it.
2866 if (N.getOpcode() < OpcodeOffset.size())
2867 MatcherIndex = OpcodeOffset[N.getOpcode()];
2868 DEBUG(dbgs() << " Initial Opcode index to " << MatcherIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Initial Opcode index to " <<
MatcherIndex << "\n"; } } while (false)
;
2869
2870 } else if (MatcherTable[0] == OPC_SwitchOpcode) {
2871 // Otherwise, the table isn't computed, but the state machine does start
2872 // with an OPC_SwitchOpcode instruction. Populate the table now, since this
2873 // is the first time we're selecting an instruction.
2874 unsigned Idx = 1;
2875 while (true) {
2876 // Get the size of this case.
2877 unsigned CaseSize = MatcherTable[Idx++];
2878 if (CaseSize & 128)
2879 CaseSize = GetVBR(CaseSize, MatcherTable, Idx);
2880 if (CaseSize == 0) break;
2881
2882 // Get the opcode, add the index to the table.
2883 uint16_t Opc = MatcherTable[Idx++];
2884 Opc |= (unsigned short)MatcherTable[Idx++] << 8;
2885 if (Opc >= OpcodeOffset.size())
2886 OpcodeOffset.resize((Opc+1)*2);
2887 OpcodeOffset[Opc] = Idx;
2888 Idx += CaseSize;
2889 }
2890
2891 // Okay, do the lookup for the first opcode.
2892 if (N.getOpcode() < OpcodeOffset.size())
2893 MatcherIndex = OpcodeOffset[N.getOpcode()];
2894 }
2895
2896 while (true) {
2897 assert(MatcherIndex < TableSize && "Invalid index")((MatcherIndex < TableSize && "Invalid index") ? static_cast
<void> (0) : __assert_fail ("MatcherIndex < TableSize && \"Invalid index\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2897, __PRETTY_FUNCTION__))
;
2898#ifndef NDEBUG
2899 unsigned CurrentOpcodeIndex = MatcherIndex;
2900#endif
2901 BuiltinOpcodes Opcode = (BuiltinOpcodes)MatcherTable[MatcherIndex++];
2902 switch (Opcode) {
2903 case OPC_Scope: {
2904 // Okay, the semantics of this operation are that we should push a scope
2905 // then evaluate the first child. However, pushing a scope only to have
2906 // the first check fail (which then pops it) is inefficient. If we can
2907 // determine immediately that the first check (or first several) will
2908 // immediately fail, don't even bother pushing a scope for them.
2909 unsigned FailIndex;
2910
2911 while (true) {
2912 unsigned NumToSkip = MatcherTable[MatcherIndex++];
2913 if (NumToSkip & 128)
2914 NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
2915 // Found the end of the scope with no match.
2916 if (NumToSkip == 0) {
2917 FailIndex = 0;
2918 break;
2919 }
2920
2921 FailIndex = MatcherIndex+NumToSkip;
2922
2923 unsigned MatcherIndexOfPredicate = MatcherIndex;
2924 (void)MatcherIndexOfPredicate; // silence warning.
2925
2926 // If we can't evaluate this predicate without pushing a scope (e.g. if
2927 // it is a 'MoveParent') or if the predicate succeeds on this node, we
2928 // push the scope and evaluate the full predicate chain.
2929 bool Result;
2930 MatcherIndex = IsPredicateKnownToFail(MatcherTable, MatcherIndex, N,
2931 Result, *this, RecordedNodes);
2932 if (!Result)
2933 break;
2934
2935 DEBUG(dbgs() << " Skipped scope entry (due to false predicate) at "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Skipped scope entry (due to false predicate) at "
<< "index " << MatcherIndexOfPredicate << ", continuing at "
<< FailIndex << "\n"; } } while (false)
2936 << "index " << MatcherIndexOfPredicatedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Skipped scope entry (due to false predicate) at "
<< "index " << MatcherIndexOfPredicate << ", continuing at "
<< FailIndex << "\n"; } } while (false)
2937 << ", continuing at " << FailIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Skipped scope entry (due to false predicate) at "
<< "index " << MatcherIndexOfPredicate << ", continuing at "
<< FailIndex << "\n"; } } while (false)
;
2938 ++NumDAGIselRetries;
2939
2940 // Otherwise, we know that this case of the Scope is guaranteed to fail,
2941 // move to the next case.
2942 MatcherIndex = FailIndex;
2943 }
2944
2945 // If the whole scope failed to match, bail.
2946 if (FailIndex == 0) break;
2947
2948 // Push a MatchScope which indicates where to go if the first child fails
2949 // to match.
2950 MatchScope NewEntry;
2951 NewEntry.FailIndex = FailIndex;
2952 NewEntry.NodeStack.append(NodeStack.begin(), NodeStack.end());
2953 NewEntry.NumRecordedNodes = RecordedNodes.size();
2954 NewEntry.NumMatchedMemRefs = MatchedMemRefs.size();
2955 NewEntry.InputChain = InputChain;
2956 NewEntry.InputGlue = InputGlue;
2957 NewEntry.HasChainNodesMatched = !ChainNodesMatched.empty();
2958 MatchScopes.push_back(NewEntry);
2959 continue;
2960 }
2961 case OPC_RecordNode: {
2962 // Remember this node, it may end up being an operand in the pattern.
2963 SDNode *Parent = nullptr;
2964 if (NodeStack.size() > 1)
2965 Parent = NodeStack[NodeStack.size()-2].getNode();
2966 RecordedNodes.push_back(std::make_pair(N, Parent));
2967 continue;
2968 }
2969
2970 case OPC_RecordChild0: case OPC_RecordChild1:
2971 case OPC_RecordChild2: case OPC_RecordChild3:
2972 case OPC_RecordChild4: case OPC_RecordChild5:
2973 case OPC_RecordChild6: case OPC_RecordChild7: {
2974 unsigned ChildNo = Opcode-OPC_RecordChild0;
2975 if (ChildNo >= N.getNumOperands())
2976 break; // Match fails if out of range child #.
2977
2978 RecordedNodes.push_back(std::make_pair(N->getOperand(ChildNo),
2979 N.getNode()));
2980 continue;
2981 }
2982 case OPC_RecordMemRef:
2983 MatchedMemRefs.push_back(cast<MemSDNode>(N)->getMemOperand());
2984 continue;
2985
2986 case OPC_CaptureGlueInput:
2987 // If the current node has an input glue, capture it in InputGlue.
2988 if (N->getNumOperands() != 0 &&
2989 N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue)
2990 InputGlue = N->getOperand(N->getNumOperands()-1);
2991 continue;
2992
2993 case OPC_MoveChild: {
2994 unsigned ChildNo = MatcherTable[MatcherIndex++];
2995 if (ChildNo >= N.getNumOperands())
2996 break; // Match fails if out of range child #.
2997 N = N.getOperand(ChildNo);
2998 NodeStack.push_back(N);
2999 continue;
3000 }
3001
3002 case OPC_MoveChild0: case OPC_MoveChild1:
3003 case OPC_MoveChild2: case OPC_MoveChild3:
3004 case OPC_MoveChild4: case OPC_MoveChild5:
3005 case OPC_MoveChild6: case OPC_MoveChild7: {
3006 unsigned ChildNo = Opcode-OPC_MoveChild0;
3007 if (ChildNo >= N.getNumOperands())
3008 break; // Match fails if out of range child #.
3009 N = N.getOperand(ChildNo);
3010 NodeStack.push_back(N);
3011 continue;
3012 }
3013
3014 case OPC_MoveParent:
3015 // Pop the current node off the NodeStack.
3016 NodeStack.pop_back();
3017 assert(!NodeStack.empty() && "Node stack imbalance!")((!NodeStack.empty() && "Node stack imbalance!") ? static_cast
<void> (0) : __assert_fail ("!NodeStack.empty() && \"Node stack imbalance!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3017, __PRETTY_FUNCTION__))
;
3018 N = NodeStack.back();
3019 continue;
3020
3021 case OPC_CheckSame:
3022 if (!::CheckSame(MatcherTable, MatcherIndex, N, RecordedNodes)) break;
3023 continue;
3024
3025 case OPC_CheckChild0Same: case OPC_CheckChild1Same:
3026 case OPC_CheckChild2Same: case OPC_CheckChild3Same:
3027 if (!::CheckChildSame(MatcherTable, MatcherIndex, N, RecordedNodes,
3028 Opcode-OPC_CheckChild0Same))
3029 break;
3030 continue;
3031
3032 case OPC_CheckPatternPredicate:
3033 if (!::CheckPatternPredicate(MatcherTable, MatcherIndex, *this)) break;
3034 continue;
3035 case OPC_CheckPredicate:
3036 if (!::CheckNodePredicate(MatcherTable, MatcherIndex, *this,
3037 N.getNode()))
3038 break;
3039 continue;
3040 case OPC_CheckComplexPat: {
3041 unsigned CPNum = MatcherTable[MatcherIndex++];
3042 unsigned RecNo = MatcherTable[MatcherIndex++];
3043 assert(RecNo < RecordedNodes.size() && "Invalid CheckComplexPat")((RecNo < RecordedNodes.size() && "Invalid CheckComplexPat"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid CheckComplexPat\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3043, __PRETTY_FUNCTION__))
;
3044
3045 // If target can modify DAG during matching, keep the matching state
3046 // consistent.
3047 std::unique_ptr<MatchStateUpdater> MSU;
3048 if (ComplexPatternFuncMutatesDAG())
3049 MSU.reset(new MatchStateUpdater(*CurDAG, &NodeToMatch, RecordedNodes,
3050 MatchScopes));
3051
3052 if (!CheckComplexPattern(NodeToMatch, RecordedNodes[RecNo].second,
3053 RecordedNodes[RecNo].first, CPNum,
3054 RecordedNodes))
3055 break;
3056 continue;
3057 }
3058 case OPC_CheckOpcode:
3059 if (!::CheckOpcode(MatcherTable, MatcherIndex, N.getNode())) break;
3060 continue;
3061
3062 case OPC_CheckType:
3063 if (!::CheckType(MatcherTable, MatcherIndex, N, TLI,
3064 CurDAG->getDataLayout()))
3065 break;
3066 continue;
3067
3068 case OPC_SwitchOpcode: {
3069 unsigned CurNodeOpcode = N.getOpcode();
3070 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3071 unsigned CaseSize;
3072 while (true) {
3073 // Get the size of this case.
3074 CaseSize = MatcherTable[MatcherIndex++];
3075 if (CaseSize & 128)
3076 CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3077 if (CaseSize == 0) break;
3078
3079 uint16_t Opc = MatcherTable[MatcherIndex++];
3080 Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3081
3082 // If the opcode matches, then we will execute this case.
3083 if (CurNodeOpcode == Opc)
3084 break;
3085
3086 // Otherwise, skip over this case.
3087 MatcherIndex += CaseSize;
3088 }
3089
3090 // If no cases matched, bail out.
3091 if (CaseSize == 0) break;
3092
3093 // Otherwise, execute the case we found.
3094 DEBUG(dbgs() << " OpcodeSwitch from " << SwitchStartdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " OpcodeSwitch from " << SwitchStart
<< " to " << MatcherIndex << "\n"; } } while
(false)
3095 << " to " << MatcherIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " OpcodeSwitch from " << SwitchStart
<< " to " << MatcherIndex << "\n"; } } while
(false)
;
3096 continue;
3097 }
3098
3099 case OPC_SwitchType: {
3100 MVT CurNodeVT = N.getSimpleValueType();
3101 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3102 unsigned CaseSize;
3103 while (true) {
3104 // Get the size of this case.
3105 CaseSize = MatcherTable[MatcherIndex++];
3106 if (CaseSize & 128)
3107 CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3108 if (CaseSize == 0) break;
3109
3110 MVT CaseVT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3111 if (CaseVT == MVT::iPTR)
3112 CaseVT = TLI->getPointerTy(CurDAG->getDataLayout());
3113
3114 // If the VT matches, then we will execute this case.
3115 if (CurNodeVT == CaseVT)
3116 break;
3117
3118 // Otherwise, skip over this case.
3119 MatcherIndex += CaseSize;
3120 }
3121
3122 // If no cases matched, bail out.
3123 if (CaseSize == 0) break;
3124
3125 // Otherwise, execute the case we found.
3126 DEBUG(dbgs() << " TypeSwitch[" << EVT(CurNodeVT).getEVTString()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " TypeSwitch[" << EVT(CurNodeVT
).getEVTString() << "] from " << SwitchStart <<
" to " << MatcherIndex<<'\n'; } } while (false)
3127 << "] from " << SwitchStart << " to " << MatcherIndex<<'\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " TypeSwitch[" << EVT(CurNodeVT
).getEVTString() << "] from " << SwitchStart <<
" to " << MatcherIndex<<'\n'; } } while (false)
;
3128 continue;
3129 }
3130 case OPC_CheckChild0Type: case OPC_CheckChild1Type:
3131 case OPC_CheckChild2Type: case OPC_CheckChild3Type:
3132 case OPC_CheckChild4Type: case OPC_CheckChild5Type:
3133 case OPC_CheckChild6Type: case OPC_CheckChild7Type:
3134 if (!::CheckChildType(MatcherTable, MatcherIndex, N, TLI,
3135 CurDAG->getDataLayout(),
3136 Opcode - OPC_CheckChild0Type))
3137 break;
3138 continue;
3139 case OPC_CheckCondCode:
3140 if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break;
3141 continue;
3142 case OPC_CheckValueType:
3143 if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI,
3144 CurDAG->getDataLayout()))
3145 break;
3146 continue;
3147 case OPC_CheckInteger:
3148 if (!::CheckInteger(MatcherTable, MatcherIndex, N)) break;
3149 continue;
3150 case OPC_CheckChild0Integer: case OPC_CheckChild1Integer:
3151 case OPC_CheckChild2Integer: case OPC_CheckChild3Integer:
3152 case OPC_CheckChild4Integer:
3153 if (!::CheckChildInteger(MatcherTable, MatcherIndex, N,
3154 Opcode-OPC_CheckChild0Integer)) break;
3155 continue;
3156 case OPC_CheckAndImm:
3157 if (!::CheckAndImm(MatcherTable, MatcherIndex, N, *this)) break;
3158 continue;
3159 case OPC_CheckOrImm:
3160 if (!::CheckOrImm(MatcherTable, MatcherIndex, N, *this)) break;
3161 continue;
3162
3163 case OPC_CheckFoldableChainNode: {
3164 assert(NodeStack.size() != 1 && "No parent node")((NodeStack.size() != 1 && "No parent node") ? static_cast
<void> (0) : __assert_fail ("NodeStack.size() != 1 && \"No parent node\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3164, __PRETTY_FUNCTION__))
;
3165 // Verify that all intermediate nodes between the root and this one have
3166 // a single use.
3167 bool HasMultipleUses = false;
3168 for (unsigned i = 1, e = NodeStack.size()-1; i != e; ++i)
3169 if (!NodeStack[i].getNode()->hasOneUse()) {
3170 HasMultipleUses = true;
3171 break;
3172 }
3173 if (HasMultipleUses) break;
3174
3175 // Check to see that the target thinks this is profitable to fold and that
3176 // we can fold it without inducing cycles in the graph.
3177 if (!IsProfitableToFold(N, NodeStack[NodeStack.size()-2].getNode(),
3178 NodeToMatch) ||
3179 !IsLegalToFold(N, NodeStack[NodeStack.size()-2].getNode(),
3180 NodeToMatch, OptLevel,
3181 true/*We validate our own chains*/))
3182 break;
3183
3184 continue;
3185 }
3186 case OPC_EmitInteger: {
3187 MVT::SimpleValueType VT =
3188 (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3189 int64_t Val = MatcherTable[MatcherIndex++];
3190 if (Val & 128)
3191 Val = GetVBR(Val, MatcherTable, MatcherIndex);
3192 RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3193 CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch),
3194 VT), nullptr));
3195 continue;
3196 }
3197 case OPC_EmitRegister: {
3198 MVT::SimpleValueType VT =
3199 (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3200 unsigned RegNo = MatcherTable[MatcherIndex++];
3201 RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3202 CurDAG->getRegister(RegNo, VT), nullptr));
3203 continue;
3204 }
3205 case OPC_EmitRegister2: {
3206 // For targets w/ more than 256 register names, the register enum
3207 // values are stored in two bytes in the matcher table (just like
3208 // opcodes).
3209 MVT::SimpleValueType VT =
3210 (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3211 unsigned RegNo = MatcherTable[MatcherIndex++];
3212 RegNo |= MatcherTable[MatcherIndex++] << 8;
3213 RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3214 CurDAG->getRegister(RegNo, VT), nullptr));
3215 continue;
3216 }
3217
3218 case OPC_EmitConvertToTarget: {
3219 // Convert from IMM/FPIMM to target version.
3220 unsigned RecNo = MatcherTable[MatcherIndex++];
3221 assert(RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget")((RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitConvertToTarget\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3221, __PRETTY_FUNCTION__))
;
3222 SDValue Imm = RecordedNodes[RecNo].first;
3223
3224 if (Imm->getOpcode() == ISD::Constant) {
3225 const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();
3226 Imm = CurDAG->getTargetConstant(*Val, SDLoc(NodeToMatch),
3227 Imm.getValueType());
3228 } else if (Imm->getOpcode() == ISD::ConstantFP) {
3229 const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();
3230 Imm = CurDAG->getTargetConstantFP(*Val, SDLoc(NodeToMatch),
3231 Imm.getValueType());
3232 }
3233
3234 RecordedNodes.push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
3235 continue;
3236 }
3237
3238 case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0
3239 case OPC_EmitMergeInputChains1_1: // OPC_EmitMergeInputChains, 1, 1
3240 case OPC_EmitMergeInputChains1_2: { // OPC_EmitMergeInputChains, 1, 2
3241 // These are space-optimized forms of OPC_EmitMergeInputChains.
3242 assert(!InputChain.getNode() &&((!InputChain.getNode() && "EmitMergeInputChains should be the first chain producing node"
) ? static_cast<void> (0) : __assert_fail ("!InputChain.getNode() && \"EmitMergeInputChains should be the first chain producing node\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3243, __PRETTY_FUNCTION__))
3243 "EmitMergeInputChains should be the first chain producing node")((!InputChain.getNode() && "EmitMergeInputChains should be the first chain producing node"
) ? static_cast<void> (0) : __assert_fail ("!InputChain.getNode() && \"EmitMergeInputChains should be the first chain producing node\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3243, __PRETTY_FUNCTION__))
;
3244 assert(ChainNodesMatched.empty() &&((ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match"
) ? static_cast<void> (0) : __assert_fail ("ChainNodesMatched.empty() && \"Should only have one EmitMergeInputChains per match\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3245, __PRETTY_FUNCTION__))
3245 "Should only have one EmitMergeInputChains per match")((ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match"
) ? static_cast<void> (0) : __assert_fail ("ChainNodesMatched.empty() && \"Should only have one EmitMergeInputChains per match\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3245, __PRETTY_FUNCTION__))
;
3246
3247 // Read all of the chained nodes.
3248 unsigned RecNo = Opcode - OPC_EmitMergeInputChains1_0;
3249 assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains")((RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitMergeInputChains\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3249, __PRETTY_FUNCTION__))
;
3250 ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3251
3252 // FIXME: What if other value results of the node have uses not matched
3253 // by this pattern?
3254 if (ChainNodesMatched.back() != NodeToMatch &&
3255 !RecordedNodes[RecNo].first.hasOneUse()) {
3256 ChainNodesMatched.clear();
3257 break;
3258 }
3259
3260 // Merge the input chains if they are not intra-pattern references.
3261 InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3262
3263 if (!InputChain.getNode())
3264 break; // Failed to merge.
3265 continue;
3266 }
3267
3268 case OPC_EmitMergeInputChains: {
3269 assert(!InputChain.getNode() &&((!InputChain.getNode() && "EmitMergeInputChains should be the first chain producing node"
) ? static_cast<void> (0) : __assert_fail ("!InputChain.getNode() && \"EmitMergeInputChains should be the first chain producing node\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3270, __PRETTY_FUNCTION__))
3270 "EmitMergeInputChains should be the first chain producing node")((!InputChain.getNode() && "EmitMergeInputChains should be the first chain producing node"
) ? static_cast<void> (0) : __assert_fail ("!InputChain.getNode() && \"EmitMergeInputChains should be the first chain producing node\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3270, __PRETTY_FUNCTION__))
;
3271 // This node gets a list of nodes we matched in the input that have
3272 // chains. We want to token factor all of the input chains to these nodes
3273 // together. However, if any of the input chains is actually one of the
3274 // nodes matched in this pattern, then we have an intra-match reference.
3275 // Ignore these because the newly token factored chain should not refer to
3276 // the old nodes.
3277 unsigned NumChains = MatcherTable[MatcherIndex++];
3278 assert(NumChains != 0 && "Can't TF zero chains")((NumChains != 0 && "Can't TF zero chains") ? static_cast
<void> (0) : __assert_fail ("NumChains != 0 && \"Can't TF zero chains\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3278, __PRETTY_FUNCTION__))
;
3279
3280 assert(ChainNodesMatched.empty() &&((ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match"
) ? static_cast<void> (0) : __assert_fail ("ChainNodesMatched.empty() && \"Should only have one EmitMergeInputChains per match\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3281, __PRETTY_FUNCTION__))
3281 "Should only have one EmitMergeInputChains per match")((ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match"
) ? static_cast<void> (0) : __assert_fail ("ChainNodesMatched.empty() && \"Should only have one EmitMergeInputChains per match\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3281, __PRETTY_FUNCTION__))
;
3282
3283 // Read all of the chained nodes.
3284 for (unsigned i = 0; i != NumChains; ++i) {
3285 unsigned RecNo = MatcherTable[MatcherIndex++];
3286 assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains")((RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitMergeInputChains\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3286, __PRETTY_FUNCTION__))
;
3287 ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3288
3289 // FIXME: What if other value results of the node have uses not matched
3290 // by this pattern?
3291 if (ChainNodesMatched.back() != NodeToMatch &&
3292 !RecordedNodes[RecNo].first.hasOneUse()) {
3293 ChainNodesMatched.clear();
3294 break;
3295 }
3296 }
3297
3298 // If the inner loop broke out, the match fails.
3299 if (ChainNodesMatched.empty())
3300 break;
3301
3302 // Merge the input chains if they are not intra-pattern references.
3303 InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3304
3305 if (!InputChain.getNode())
3306 break; // Failed to merge.
3307
3308 continue;
3309 }
3310
3311 case OPC_EmitCopyToReg: {
3312 unsigned RecNo = MatcherTable[MatcherIndex++];
3313 assert(RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg")((RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitCopyToReg\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3313, __PRETTY_FUNCTION__))
;
3314 unsigned DestPhysReg = MatcherTable[MatcherIndex++];
3315
3316 if (!InputChain.getNode())
3317 InputChain = CurDAG->getEntryNode();
3318
3319 InputChain = CurDAG->getCopyToReg(InputChain, SDLoc(NodeToMatch),
3320 DestPhysReg, RecordedNodes[RecNo].first,
3321 InputGlue);
3322
3323 InputGlue = InputChain.getValue(1);
3324 continue;
3325 }
3326
3327 case OPC_EmitNodeXForm: {
3328 unsigned XFormNo = MatcherTable[MatcherIndex++];
3329 unsigned RecNo = MatcherTable[MatcherIndex++];
3330 assert(RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm")((RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitNodeXForm\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3330, __PRETTY_FUNCTION__))
;
3331 SDValue Res = RunSDNodeXForm(RecordedNodes[RecNo].first, XFormNo);
3332 RecordedNodes.push_back(std::pair<SDValue,SDNode*>(Res, nullptr));
3333 continue;
3334 }
3335 case OPC_Coverage: {
3336 // This is emitted right before MorphNode/EmitNode.
3337 // So it should be safe to assume that this node has been selected
3338 unsigned index = MatcherTable[MatcherIndex++];
3339 index |= (MatcherTable[MatcherIndex++] << 8);
3340 dbgs() << "COVERED: " << getPatternForIndex(index) << "\n";
3341 dbgs() << "INCLUDED: " << getIncludePathForIndex(index) << "\n";
3342 continue;
3343 }
3344
3345 case OPC_EmitNode: case OPC_MorphNodeTo:
3346 case OPC_EmitNode0: case OPC_EmitNode1: case OPC_EmitNode2:
3347 case OPC_MorphNodeTo0: case OPC_MorphNodeTo1: case OPC_MorphNodeTo2: {
3348 uint16_t TargetOpc = MatcherTable[MatcherIndex++];
3349 TargetOpc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3350 unsigned EmitNodeInfo = MatcherTable[MatcherIndex++];
3351 // Get the result VT list.
3352 unsigned NumVTs;
3353 // If this is one of the compressed forms, get the number of VTs based
3354 // on the Opcode. Otherwise read the next byte from the table.
3355 if (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2)
3356 NumVTs = Opcode - OPC_MorphNodeTo0;
3357 else if (Opcode >= OPC_EmitNode0 && Opcode <= OPC_EmitNode2)
3358 NumVTs = Opcode - OPC_EmitNode0;
3359 else
3360 NumVTs = MatcherTable[MatcherIndex++];
3361 SmallVector<EVT, 4> VTs;
3362 for (unsigned i = 0; i != NumVTs; ++i) {
3363 MVT::SimpleValueType VT =
3364 (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3365 if (VT == MVT::iPTR)
3366 VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy;
3367 VTs.push_back(VT);
3368 }
3369
3370 if (EmitNodeInfo & OPFL_Chain)
3371 VTs.push_back(MVT::Other);
3372 if (EmitNodeInfo & OPFL_GlueOutput)
3373 VTs.push_back(MVT::Glue);
3374
3375 // This is hot code, so optimize the two most common cases of 1 and 2
3376 // results.
3377 SDVTList VTList;
3378 if (VTs.size() == 1)
3379 VTList = CurDAG->getVTList(VTs[0]);
3380 else if (VTs.size() == 2)
3381 VTList = CurDAG->getVTList(VTs[0], VTs[1]);
3382 else
3383 VTList = CurDAG->getVTList(VTs);
3384
3385 // Get the operand list.
3386 unsigned NumOps = MatcherTable[MatcherIndex++];
3387 SmallVector<SDValue, 8> Ops;
3388 for (unsigned i = 0; i != NumOps; ++i) {
3389 unsigned RecNo = MatcherTable[MatcherIndex++];
3390 if (RecNo & 128)
3391 RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex);
3392
3393 assert(RecNo < RecordedNodes.size() && "Invalid EmitNode")((RecNo < RecordedNodes.size() && "Invalid EmitNode"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitNode\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3393, __PRETTY_FUNCTION__))
;
3394 Ops.push_back(RecordedNodes[RecNo].first);
3395 }
3396
3397 // If there are variadic operands to add, handle them now.
3398 if (EmitNodeInfo & OPFL_VariadicInfo) {
3399 // Determine the start index to copy from.
3400 unsigned FirstOpToCopy = getNumFixedFromVariadicInfo(EmitNodeInfo);
3401 FirstOpToCopy += (EmitNodeInfo & OPFL_Chain) ? 1 : 0;
3402 assert(NodeToMatch->getNumOperands() >= FirstOpToCopy &&((NodeToMatch->getNumOperands() >= FirstOpToCopy &&
"Invalid variadic node") ? static_cast<void> (0) : __assert_fail
("NodeToMatch->getNumOperands() >= FirstOpToCopy && \"Invalid variadic node\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3403, __PRETTY_FUNCTION__))
3403 "Invalid variadic node")((NodeToMatch->getNumOperands() >= FirstOpToCopy &&
"Invalid variadic node") ? static_cast<void> (0) : __assert_fail
("NodeToMatch->getNumOperands() >= FirstOpToCopy && \"Invalid variadic node\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3403, __PRETTY_FUNCTION__))
;
3404 // Copy all of the variadic operands, not including a potential glue
3405 // input.
3406 for (unsigned i = FirstOpToCopy, e = NodeToMatch->getNumOperands();
3407 i != e; ++i) {
3408 SDValue V = NodeToMatch->getOperand(i);
3409 if (V.getValueType() == MVT::Glue) break;
3410 Ops.push_back(V);
3411 }
3412 }
3413
3414 // If this has chain/glue inputs, add them.
3415 if (EmitNodeInfo & OPFL_Chain)
3416 Ops.push_back(InputChain);
3417 if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr)
3418 Ops.push_back(InputGlue);
3419
3420 // Create the node.
3421 SDNode *Res = nullptr;
3422 bool IsMorphNodeTo = Opcode == OPC_MorphNodeTo ||
3423 (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2);
3424 if (!IsMorphNodeTo) {
3425 // If this is a normal EmitNode command, just create the new node and
3426 // add the results to the RecordedNodes list.
3427 Res = CurDAG->getMachineNode(TargetOpc, SDLoc(NodeToMatch),
3428 VTList, Ops);
3429
3430 // Add all the non-glue/non-chain results to the RecordedNodes list.
3431 for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
3432 if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue) break;
3433 RecordedNodes.push_back(std::pair<SDValue,SDNode*>(SDValue(Res, i),
3434 nullptr));
3435 }
3436 } else {
3437 assert(NodeToMatch->getOpcode() != ISD::DELETED_NODE &&((NodeToMatch->getOpcode() != ISD::DELETED_NODE &&
"NodeToMatch was removed partway through selection") ? static_cast
<void> (0) : __assert_fail ("NodeToMatch->getOpcode() != ISD::DELETED_NODE && \"NodeToMatch was removed partway through selection\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3438, __PRETTY_FUNCTION__))
3438 "NodeToMatch was removed partway through selection")((NodeToMatch->getOpcode() != ISD::DELETED_NODE &&
"NodeToMatch was removed partway through selection") ? static_cast
<void> (0) : __assert_fail ("NodeToMatch->getOpcode() != ISD::DELETED_NODE && \"NodeToMatch was removed partway through selection\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3438, __PRETTY_FUNCTION__))
;
3439 SelectionDAG::DAGNodeDeletedListener NDL(*CurDAG, [&](SDNode *N,
3440 SDNode *E) {
3441 auto &Chain = ChainNodesMatched;
3442 assert((!E || !is_contained(Chain, N)) &&(((!E || !is_contained(Chain, N)) && "Chain node replaced during MorphNode"
) ? static_cast<void> (0) : __assert_fail ("(!E || !is_contained(Chain, N)) && \"Chain node replaced during MorphNode\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3443, __PRETTY_FUNCTION__))
3443 "Chain node replaced during MorphNode")(((!E || !is_contained(Chain, N)) && "Chain node replaced during MorphNode"
) ? static_cast<void> (0) : __assert_fail ("(!E || !is_contained(Chain, N)) && \"Chain node replaced during MorphNode\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3443, __PRETTY_FUNCTION__))
;
3444 Chain.erase(std::remove(Chain.begin(), Chain.end(), N), Chain.end());
3445 });
3446 Res = MorphNode(NodeToMatch, TargetOpc, VTList, Ops, EmitNodeInfo);
3447 }
3448
3449 // If the node had chain/glue results, update our notion of the current
3450 // chain and glue.
3451 if (EmitNodeInfo & OPFL_GlueOutput) {
3452 InputGlue = SDValue(Res, VTs.size()-1);
3453 if (EmitNodeInfo & OPFL_Chain)
3454 InputChain = SDValue(Res, VTs.size()-2);
3455 } else if (EmitNodeInfo & OPFL_Chain)
3456 InputChain = SDValue(Res, VTs.size()-1);
3457
3458 // If the OPFL_MemRefs glue is set on this node, slap all of the
3459 // accumulated memrefs onto it.
3460 //
3461 // FIXME: This is vastly incorrect for patterns with multiple outputs
3462 // instructions that access memory and for ComplexPatterns that match
3463 // loads.
3464 if (EmitNodeInfo & OPFL_MemRefs) {
3465 // Only attach load or store memory operands if the generated
3466 // instruction may load or store.
3467 const MCInstrDesc &MCID = TII->get(TargetOpc);
3468 bool mayLoad = MCID.mayLoad();
3469 bool mayStore = MCID.mayStore();
3470
3471 unsigned NumMemRefs = 0;
3472 for (SmallVectorImpl<MachineMemOperand *>::const_iterator I =
3473 MatchedMemRefs.begin(), E = MatchedMemRefs.end(); I != E; ++I) {
3474 if ((*I)->isLoad()) {
3475 if (mayLoad)
3476 ++NumMemRefs;
3477 } else if ((*I)->isStore()) {
3478 if (mayStore)
3479 ++NumMemRefs;
3480 } else {
3481 ++NumMemRefs;
3482 }
3483 }
3484
3485 MachineSDNode::mmo_iterator MemRefs =
3486 MF->allocateMemRefsArray(NumMemRefs);
3487
3488 MachineSDNode::mmo_iterator MemRefsPos = MemRefs;
3489 for (SmallVectorImpl<MachineMemOperand *>::const_iterator I =
3490 MatchedMemRefs.begin(), E = MatchedMemRefs.end(); I != E; ++I) {
3491 if ((*I)->isLoad()) {
3492 if (mayLoad)
3493 *MemRefsPos++ = *I;
3494 } else if ((*I)->isStore()) {
3495 if (mayStore)
3496 *MemRefsPos++ = *I;
3497 } else {
3498 *MemRefsPos++ = *I;
3499 }
3500 }
3501
3502 cast<MachineSDNode>(Res)
3503 ->setMemRefs(MemRefs, MemRefs + NumMemRefs);
3504 }
3505
3506 DEBUG(dbgs() << " "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " " << (IsMorphNodeTo ? "Morphed"
: "Created") << " node: "; Res->dump(CurDAG); dbgs(
) << "\n"; } } while (false)
3507 << (IsMorphNodeTo ? "Morphed" : "Created")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " " << (IsMorphNodeTo ? "Morphed"
: "Created") << " node: "; Res->dump(CurDAG); dbgs(
) << "\n"; } } while (false)
3508 << " node: "; Res->dump(CurDAG); dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " " << (IsMorphNodeTo ? "Morphed"
: "Created") << " node: "; Res->dump(CurDAG); dbgs(
) << "\n"; } } while (false)
;
3509
3510 // If this was a MorphNodeTo then we're completely done!
3511 if (IsMorphNodeTo) {
3512 // Update chain uses.
3513 UpdateChains(Res, InputChain, ChainNodesMatched, true);
3514 return;
3515 }
3516 continue;
3517 }
3518
3519 case OPC_CompleteMatch: {
3520 // The match has been completed, and any new nodes (if any) have been
3521 // created. Patch up references to the matched dag to use the newly
3522 // created nodes.
3523 unsigned NumResults = MatcherTable[MatcherIndex++];
3524
3525 for (unsigned i = 0; i != NumResults; ++i) {
3526 unsigned ResSlot = MatcherTable[MatcherIndex++];
3527 if (ResSlot & 128)
3528 ResSlot = GetVBR(ResSlot, MatcherTable, MatcherIndex);
3529
3530 assert(ResSlot < RecordedNodes.size() && "Invalid CompleteMatch")((ResSlot < RecordedNodes.size() && "Invalid CompleteMatch"
) ? static_cast<void> (0) : __assert_fail ("ResSlot < RecordedNodes.size() && \"Invalid CompleteMatch\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3530, __PRETTY_FUNCTION__))
;
3531 SDValue Res = RecordedNodes[ResSlot].first;
3532
3533 assert(i < NodeToMatch->getNumValues() &&((i < NodeToMatch->getNumValues() && NodeToMatch
->getValueType(i) != MVT::Other && NodeToMatch->
getValueType(i) != MVT::Glue && "Invalid number of results to complete!"
) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3536, __PRETTY_FUNCTION__))
3534 NodeToMatch->getValueType(i) != MVT::Other &&((i < NodeToMatch->getNumValues() && NodeToMatch
->getValueType(i) != MVT::Other && NodeToMatch->
getValueType(i) != MVT::Glue && "Invalid number of results to complete!"
) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3536, __PRETTY_FUNCTION__))
3535 NodeToMatch->getValueType(i) != MVT::Glue &&((i < NodeToMatch->getNumValues() && NodeToMatch
->getValueType(i) != MVT::Other && NodeToMatch->
getValueType(i) != MVT::Glue && "Invalid number of results to complete!"
) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3536, __PRETTY_FUNCTION__))
3536 "Invalid number of results to complete!")((i < NodeToMatch->getNumValues() && NodeToMatch
->getValueType(i) != MVT::Other && NodeToMatch->
getValueType(i) != MVT::Glue && "Invalid number of results to complete!"
) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3536, __PRETTY_FUNCTION__))
;
3537 assert((NodeToMatch->getValueType(i) == Res.getValueType() ||(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3542, __PRETTY_FUNCTION__))
3538 NodeToMatch->getValueType(i) == MVT::iPTR ||(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3542, __PRETTY_FUNCTION__))
3539 Res.getValueType() == MVT::iPTR ||(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3542, __PRETTY_FUNCTION__))
3540 NodeToMatch->getValueType(i).getSizeInBits() ==(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3542, __PRETTY_FUNCTION__))
3541 Res.getValueSizeInBits()) &&(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3542, __PRETTY_FUNCTION__))
3542 "invalid replacement")(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3542, __PRETTY_FUNCTION__))
;
3543 CurDAG->ReplaceAllUsesOfValueWith(SDValue(NodeToMatch, i), Res);
3544 }
3545
3546 // Update chain uses.
3547 UpdateChains(NodeToMatch, InputChain, ChainNodesMatched, false);
3548
3549 // If the root node defines glue, we need to update it to the glue result.
3550 // TODO: This never happens in our tests and I think it can be removed /
3551 // replaced with an assert, but if we do it this the way the change is
3552 // NFC.
3553 if (NodeToMatch->getValueType(NodeToMatch->getNumValues() - 1) ==
3554 MVT::Glue &&
3555 InputGlue.getNode())
3556 CurDAG->ReplaceAllUsesOfValueWith(
3557 SDValue(NodeToMatch, NodeToMatch->getNumValues() - 1), InputGlue);
3558
3559 assert(NodeToMatch->use_empty() &&((NodeToMatch->use_empty() && "Didn't replace all uses of the node?"
) ? static_cast<void> (0) : __assert_fail ("NodeToMatch->use_empty() && \"Didn't replace all uses of the node?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3560, __PRETTY_FUNCTION__))
3560 "Didn't replace all uses of the node?")((NodeToMatch->use_empty() && "Didn't replace all uses of the node?"
) ? static_cast<void> (0) : __assert_fail ("NodeToMatch->use_empty() && \"Didn't replace all uses of the node?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn301769/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3560, __PRETTY_FUNCTION__))
;
3561 CurDAG->RemoveDeadNode(NodeToMatch);
3562
3563 return;
3564 }
3565 }
3566
3567 // If the code reached this point, then the match failed. See if there is
3568 // another child to try in the current 'Scope', otherwise pop it until we
3569 // find a case to check.
3570 DEBUG(dbgs() << " Match failed at index " << CurrentOpcodeIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Match failed at index " <<
CurrentOpcodeIndex << "\n"; } } while (false)
;
3571 ++NumDAGIselRetries;
3572 while (true) {
3573 if (MatchScopes.empty()) {
3574 CannotYetSelect(NodeToMatch);
3575 return;
3576 }
3577
3578 // Restore the interpreter state back to the point where the scope was
3579 // formed.
3580 MatchScope &LastScope = MatchScopes.back();
3581 RecordedNodes.resize(LastScope.NumRecordedNodes);
3582 NodeStack.clear();
3583 NodeStack.append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());
3584 N = NodeStack.back();
3585
3586 if (LastScope.NumMatchedMemRefs != MatchedMemRefs.size())
3587 MatchedMemRefs.resize(LastScope.NumMatchedMemRefs);
3588 MatcherIndex = LastScope.FailIndex;
3589
3590 DEBUG(dbgs() << " Continuing at " << MatcherIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Continuing at " << MatcherIndex
<< "\n"; } } while (false)
;
3591
3592 InputChain = LastScope.InputChain;
3593 InputGlue = LastScope.InputGlue;
3594 if (!LastScope.HasChainNodesMatched)
3595 ChainNodesMatched.clear();
3596
3597 // Check to see what the offset is at the new MatcherIndex. If it is zero
3598 // we have reached the end of this scope, otherwise we have another child
3599 // in the current scope to try.
3600 unsigned NumToSkip = MatcherTable[MatcherIndex++];
3601 if (NumToSkip & 128)
3602 NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3603
3604 // If we have another child in this scope to match, update FailIndex and
3605 // try it.
3606 if (NumToSkip != 0) {
3607 LastScope.FailIndex = MatcherIndex+NumToSkip;
3608 break;
3609 }
3610
3611 // End of this scope, pop it and try the next child in the containing
3612 // scope.
3613 MatchScopes.pop_back();
3614 }
3615 }
3616}
3617
3618void SelectionDAGISel::CannotYetSelect(SDNode *N) {
3619 std::string msg;
3620 raw_string_ostream Msg(msg);
3621 Msg << "Cannot select: ";
3622
3623 if (N->getOpcode() != ISD::INTRINSIC_W_CHAIN &&
3624 N->getOpcode() != ISD::INTRINSIC_WO_CHAIN &&
3625 N->getOpcode() != ISD::INTRINSIC_VOID) {
3626 N->printrFull(Msg, CurDAG);
3627 Msg << "\nIn function: " << MF->getName();
3628 } else {
3629 bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other;
3630 unsigned iid =
3631 cast<ConstantSDNode>(N->getOperand(HasInputChain))->getZExtValue();
3632 if (iid < Intrinsic::num_intrinsics)
3633 Msg << "intrinsic %" << Intrinsic::getName((Intrinsic::ID)iid, None);
3634 else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo())
3635 Msg << "target intrinsic %" << TII->getName(iid);
3636 else
3637 Msg << "unknown intrinsic #" << iid;
3638 }
3639 report_fatal_error(Msg.str());
3640}
3641
3642char SelectionDAGISel::ID = 0;