Bug Summary

File:lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
Warning:line 240, column 7
1st function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SelectionDAGISel.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-8/lib/clang/8.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/include -I /build/llvm-toolchain-snapshot-8~svn350071/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/8.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-8/lib/clang/8.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/lib/CodeGen/SelectionDAG -fdebug-prefix-map=/build/llvm-toolchain-snapshot-8~svn350071=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-12-27-042839-1215-1 -x c++ /build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp -faddrsig
1//===- SelectionDAGISel.cpp - Implement the SelectionDAGISel class --------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This implements the SelectionDAGISel class.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/SelectionDAGISel.h"
15#include "ScheduleDAGSDNodes.h"
16#include "SelectionDAGBuilder.h"
17#include "llvm/ADT/APInt.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/None.h"
20#include "llvm/ADT/PostOrderIterator.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/ADT/SmallSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Statistic.h"
26#include "llvm/ADT/StringRef.h"
27#include "llvm/Analysis/AliasAnalysis.h"
28#include "llvm/Analysis/BranchProbabilityInfo.h"
29#include "llvm/Analysis/CFG.h"
30#include "llvm/Analysis/EHPersonalities.h"
31#include "llvm/Analysis/OptimizationRemarkEmitter.h"
32#include "llvm/Analysis/TargetLibraryInfo.h"
33#include "llvm/Analysis/TargetTransformInfo.h"
34#include "llvm/CodeGen/FastISel.h"
35#include "llvm/CodeGen/FunctionLoweringInfo.h"
36#include "llvm/CodeGen/GCMetadata.h"
37#include "llvm/CodeGen/ISDOpcodes.h"
38#include "llvm/CodeGen/MachineBasicBlock.h"
39#include "llvm/CodeGen/MachineFrameInfo.h"
40#include "llvm/CodeGen/MachineFunction.h"
41#include "llvm/CodeGen/MachineFunctionPass.h"
42#include "llvm/CodeGen/MachineInstr.h"
43#include "llvm/CodeGen/MachineInstrBuilder.h"
44#include "llvm/CodeGen/MachineMemOperand.h"
45#include "llvm/CodeGen/MachineOperand.h"
46#include "llvm/CodeGen/MachinePassRegistry.h"
47#include "llvm/CodeGen/MachineRegisterInfo.h"
48#include "llvm/CodeGen/SchedulerRegistry.h"
49#include "llvm/CodeGen/SelectionDAG.h"
50#include "llvm/CodeGen/SelectionDAGNodes.h"
51#include "llvm/CodeGen/StackProtector.h"
52#include "llvm/CodeGen/TargetInstrInfo.h"
53#include "llvm/CodeGen/TargetLowering.h"
54#include "llvm/CodeGen/TargetRegisterInfo.h"
55#include "llvm/CodeGen/TargetSubtargetInfo.h"
56#include "llvm/CodeGen/ValueTypes.h"
57#include "llvm/IR/BasicBlock.h"
58#include "llvm/IR/Constants.h"
59#include "llvm/IR/DataLayout.h"
60#include "llvm/IR/DebugInfoMetadata.h"
61#include "llvm/IR/DebugLoc.h"
62#include "llvm/IR/DiagnosticInfo.h"
63#include "llvm/IR/Dominators.h"
64#include "llvm/IR/Function.h"
65#include "llvm/IR/InlineAsm.h"
66#include "llvm/IR/InstrTypes.h"
67#include "llvm/IR/Instruction.h"
68#include "llvm/IR/Instructions.h"
69#include "llvm/IR/IntrinsicInst.h"
70#include "llvm/IR/Intrinsics.h"
71#include "llvm/IR/Metadata.h"
72#include "llvm/IR/Type.h"
73#include "llvm/IR/User.h"
74#include "llvm/IR/Value.h"
75#include "llvm/MC/MCInstrDesc.h"
76#include "llvm/MC/MCRegisterInfo.h"
77#include "llvm/Pass.h"
78#include "llvm/Support/BranchProbability.h"
79#include "llvm/Support/Casting.h"
80#include "llvm/Support/CodeGen.h"
81#include "llvm/Support/CommandLine.h"
82#include "llvm/Support/Compiler.h"
83#include "llvm/Support/Debug.h"
84#include "llvm/Support/ErrorHandling.h"
85#include "llvm/Support/KnownBits.h"
86#include "llvm/Support/MachineValueType.h"
87#include "llvm/Support/Timer.h"
88#include "llvm/Support/raw_ostream.h"
89#include "llvm/Target/TargetIntrinsicInfo.h"
90#include "llvm/Target/TargetMachine.h"
91#include "llvm/Target/TargetOptions.h"
92#include "llvm/Transforms/Utils/BasicBlockUtils.h"
93#include <algorithm>
94#include <cassert>
95#include <cstdint>
96#include <iterator>
97#include <limits>
98#include <memory>
99#include <string>
100#include <utility>
101#include <vector>
102
103using namespace llvm;
104
105#define DEBUG_TYPE"isel" "isel"
106
107STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on")static llvm::Statistic NumFastIselFailures = {"isel", "NumFastIselFailures"
, "Number of instructions fast isel failed on", {0}, {false}}
;
108STATISTIC(NumFastIselSuccess, "Number of instructions fast isel selected")static llvm::Statistic NumFastIselSuccess = {"isel", "NumFastIselSuccess"
, "Number of instructions fast isel selected", {0}, {false}}
;
109STATISTIC(NumFastIselBlocks, "Number of blocks selected entirely by fast isel")static llvm::Statistic NumFastIselBlocks = {"isel", "NumFastIselBlocks"
, "Number of blocks selected entirely by fast isel", {0}, {false
}}
;
110STATISTIC(NumDAGBlocks, "Number of blocks selected using DAG")static llvm::Statistic NumDAGBlocks = {"isel", "NumDAGBlocks"
, "Number of blocks selected using DAG", {0}, {false}}
;
111STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path")static llvm::Statistic NumDAGIselRetries = {"isel", "NumDAGIselRetries"
, "Number of times dag isel has to try another path", {0}, {false
}}
;
112STATISTIC(NumEntryBlocks, "Number of entry blocks encountered")static llvm::Statistic NumEntryBlocks = {"isel", "NumEntryBlocks"
, "Number of entry blocks encountered", {0}, {false}}
;
113STATISTIC(NumFastIselFailLowerArguments,static llvm::Statistic NumFastIselFailLowerArguments = {"isel"
, "NumFastIselFailLowerArguments", "Number of entry blocks where fast isel failed to lower arguments"
, {0}, {false}}
114 "Number of entry blocks where fast isel failed to lower arguments")static llvm::Statistic NumFastIselFailLowerArguments = {"isel"
, "NumFastIselFailLowerArguments", "Number of entry blocks where fast isel failed to lower arguments"
, {0}, {false}}
;
115
116static cl::opt<int> EnableFastISelAbort(
117 "fast-isel-abort", cl::Hidden,
118 cl::desc("Enable abort calls when \"fast\" instruction selection "
119 "fails to lower an instruction: 0 disable the abort, 1 will "
120 "abort but for args, calls and terminators, 2 will also "
121 "abort for argument lowering, and 3 will never fallback "
122 "to SelectionDAG."));
123
124static cl::opt<bool> EnableFastISelFallbackReport(
125 "fast-isel-report-on-fallback", cl::Hidden,
126 cl::desc("Emit a diagnostic when \"fast\" instruction selection "
127 "falls back to SelectionDAG."));
128
129static cl::opt<bool>
130UseMBPI("use-mbpi",
131 cl::desc("use Machine Branch Probability Info"),
132 cl::init(true), cl::Hidden);
133
134#ifndef NDEBUG
135static cl::opt<std::string>
136FilterDAGBasicBlockName("filter-view-dags", cl::Hidden,
137 cl::desc("Only display the basic block whose name "
138 "matches this for all view-*-dags options"));
139static cl::opt<bool>
140ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden,
141 cl::desc("Pop up a window to show dags before the first "
142 "dag combine pass"));
143static cl::opt<bool>
144ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden,
145 cl::desc("Pop up a window to show dags before legalize types"));
146static cl::opt<bool>
147ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,
148 cl::desc("Pop up a window to show dags before legalize"));
149static cl::opt<bool>
150ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden,
151 cl::desc("Pop up a window to show dags before the second "
152 "dag combine pass"));
153static cl::opt<bool>
154ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden,
155 cl::desc("Pop up a window to show dags before the post legalize types"
156 " dag combine pass"));
157static cl::opt<bool>
158ViewISelDAGs("view-isel-dags", cl::Hidden,
159 cl::desc("Pop up a window to show isel dags as they are selected"));
160static cl::opt<bool>
161ViewSchedDAGs("view-sched-dags", cl::Hidden,
162 cl::desc("Pop up a window to show sched dags as they are processed"));
163static cl::opt<bool>
164ViewSUnitDAGs("view-sunit-dags", cl::Hidden,
165 cl::desc("Pop up a window to show SUnit dags after they are processed"));
166#else
167static const bool ViewDAGCombine1 = false,
168 ViewLegalizeTypesDAGs = false, ViewLegalizeDAGs = false,
169 ViewDAGCombine2 = false,
170 ViewDAGCombineLT = false,
171 ViewISelDAGs = false, ViewSchedDAGs = false,
172 ViewSUnitDAGs = false;
173#endif
174
175//===---------------------------------------------------------------------===//
176///
177/// RegisterScheduler class - Track the registration of instruction schedulers.
178///
179//===---------------------------------------------------------------------===//
180MachinePassRegistry<RegisterScheduler::FunctionPassCtor>
181 RegisterScheduler::Registry;
182
183//===---------------------------------------------------------------------===//
184///
185/// ISHeuristic command line option for instruction schedulers.
186///
187//===---------------------------------------------------------------------===//
188static cl::opt<RegisterScheduler::FunctionPassCtor, false,
189 RegisterPassParser<RegisterScheduler>>
190ISHeuristic("pre-RA-sched",
191 cl::init(&createDefaultScheduler), cl::Hidden,
192 cl::desc("Instruction schedulers available (before register"
193 " allocation):"));
194
195static RegisterScheduler
196defaultListDAGScheduler("default", "Best scheduler for the target",
197 createDefaultScheduler);
198
199namespace llvm {
200
201 //===--------------------------------------------------------------------===//
202 /// This class is used by SelectionDAGISel to temporarily override
203 /// the optimization level on a per-function basis.
204 class OptLevelChanger {
205 SelectionDAGISel &IS;
206 CodeGenOpt::Level SavedOptLevel;
207 bool SavedFastISel;
208
209 public:
210 OptLevelChanger(SelectionDAGISel &ISel,
211 CodeGenOpt::Level NewOptLevel) : IS(ISel) {
212 SavedOptLevel = IS.OptLevel;
213 if (NewOptLevel == SavedOptLevel)
7
Taking true branch
214 return;
8
Returning without writing to 'this->SavedFastISel'
215 IS.OptLevel = NewOptLevel;
216 IS.TM.setOptLevel(NewOptLevel);
217 LLVM_DEBUG(dbgs() << "\nChanging optimization level for Function "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\nChanging optimization level for Function "
<< IS.MF->getFunction().getName() << "\n"; } }
while (false)
218 << IS.MF->getFunction().getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\nChanging optimization level for Function "
<< IS.MF->getFunction().getName() << "\n"; } }
while (false)
;
219 LLVM_DEBUG(dbgs() << "\tBefore: -O" << SavedOptLevel << " ; After: -O"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tBefore: -O" << SavedOptLevel
<< " ; After: -O" << NewOptLevel << "\n"; }
} while (false)
220 << NewOptLevel << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tBefore: -O" << SavedOptLevel
<< " ; After: -O" << NewOptLevel << "\n"; }
} while (false)
;
221 SavedFastISel = IS.TM.Options.EnableFastISel;
222 if (NewOptLevel == CodeGenOpt::None) {
223 IS.TM.setFastISel(IS.TM.getO0WantsFastISel());
224 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tFastISel is " << (IS.TM.
Options.EnableFastISel ? "enabled" : "disabled") << "\n"
; } } while (false)
225 dbgs() << "\tFastISel is "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tFastISel is " << (IS.TM.
Options.EnableFastISel ? "enabled" : "disabled") << "\n"
; } } while (false)
226 << (IS.TM.Options.EnableFastISel ? "enabled" : "disabled")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tFastISel is " << (IS.TM.
Options.EnableFastISel ? "enabled" : "disabled") << "\n"
; } } while (false)
227 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tFastISel is " << (IS.TM.
Options.EnableFastISel ? "enabled" : "disabled") << "\n"
; } } while (false)
;
228 }
229 }
230
231 ~OptLevelChanger() {
232 if (IS.OptLevel == SavedOptLevel)
37
Assuming the condition is false
38
Taking false branch
233 return;
234 LLVM_DEBUG(dbgs() << "\nRestoring optimization level for Function "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\nRestoring optimization level for Function "
<< IS.MF->getFunction().getName() << "\n"; } }
while (false)
39
Assuming 'DebugFlag' is 0
40
Loop condition is false. Exiting loop
235 << IS.MF->getFunction().getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\nRestoring optimization level for Function "
<< IS.MF->getFunction().getName() << "\n"; } }
while (false)
;
236 LLVM_DEBUG(dbgs() << "\tBefore: -O" << IS.OptLevel << " ; After: -O"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tBefore: -O" << IS.OptLevel
<< " ; After: -O" << SavedOptLevel << "\n"
; } } while (false)
41
Loop condition is false. Exiting loop
237 << SavedOptLevel << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tBefore: -O" << IS.OptLevel
<< " ; After: -O" << SavedOptLevel << "\n"
; } } while (false)
;
238 IS.OptLevel = SavedOptLevel;
239 IS.TM.setOptLevel(SavedOptLevel);
240 IS.TM.setFastISel(SavedFastISel);
42
1st function call argument is an uninitialized value
241 }
242 };
243
244 //===--------------------------------------------------------------------===//
245 /// createDefaultScheduler - This creates an instruction scheduler appropriate
246 /// for the target.
247 ScheduleDAGSDNodes* createDefaultScheduler(SelectionDAGISel *IS,
248 CodeGenOpt::Level OptLevel) {
249 const TargetLowering *TLI = IS->TLI;
250 const TargetSubtargetInfo &ST = IS->MF->getSubtarget();
251
252 // Try first to see if the Target has its own way of selecting a scheduler
253 if (auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) {
254 return SchedulerCtor(IS, OptLevel);
255 }
256
257 if (OptLevel == CodeGenOpt::None ||
258 (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) ||
259 TLI->getSchedulingPreference() == Sched::Source)
260 return createSourceListDAGScheduler(IS, OptLevel);
261 if (TLI->getSchedulingPreference() == Sched::RegPressure)
262 return createBURRListDAGScheduler(IS, OptLevel);
263 if (TLI->getSchedulingPreference() == Sched::Hybrid)
264 return createHybridListDAGScheduler(IS, OptLevel);
265 if (TLI->getSchedulingPreference() == Sched::VLIW)
266 return createVLIWDAGScheduler(IS, OptLevel);
267 assert(TLI->getSchedulingPreference() == Sched::ILP &&((TLI->getSchedulingPreference() == Sched::ILP && "Unknown sched type!"
) ? static_cast<void> (0) : __assert_fail ("TLI->getSchedulingPreference() == Sched::ILP && \"Unknown sched type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 268, __PRETTY_FUNCTION__))
268 "Unknown sched type!")((TLI->getSchedulingPreference() == Sched::ILP && "Unknown sched type!"
) ? static_cast<void> (0) : __assert_fail ("TLI->getSchedulingPreference() == Sched::ILP && \"Unknown sched type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 268, __PRETTY_FUNCTION__))
;
269 return createILPListDAGScheduler(IS, OptLevel);
270 }
271
272} // end namespace llvm
273
274// EmitInstrWithCustomInserter - This method should be implemented by targets
275// that mark instructions with the 'usesCustomInserter' flag. These
276// instructions are special in various ways, which require special support to
277// insert. The specified MachineInstr is created but not inserted into any
278// basic blocks, and this method is called to expand it into a sequence of
279// instructions, potentially also creating new basic blocks and control flow.
280// When new basic blocks are inserted and the edges from MBB to its successors
281// are modified, the method should insert pairs of <OldSucc, NewSucc> into the
282// DenseMap.
283MachineBasicBlock *
284TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
285 MachineBasicBlock *MBB) const {
286#ifndef NDEBUG
287 dbgs() << "If a target marks an instruction with "
288 "'usesCustomInserter', it must implement "
289 "TargetLowering::EmitInstrWithCustomInserter!";
290#endif
291 llvm_unreachable(nullptr)::llvm::llvm_unreachable_internal(nullptr, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 291)
;
292}
293
294void TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
295 SDNode *Node) const {
296 assert(!MI.hasPostISelHook() &&((!MI.hasPostISelHook() && "If a target marks an instruction with 'hasPostISelHook', "
"it must implement TargetLowering::AdjustInstrPostInstrSelection!"
) ? static_cast<void> (0) : __assert_fail ("!MI.hasPostISelHook() && \"If a target marks an instruction with 'hasPostISelHook', \" \"it must implement TargetLowering::AdjustInstrPostInstrSelection!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 298, __PRETTY_FUNCTION__))
297 "If a target marks an instruction with 'hasPostISelHook', "((!MI.hasPostISelHook() && "If a target marks an instruction with 'hasPostISelHook', "
"it must implement TargetLowering::AdjustInstrPostInstrSelection!"
) ? static_cast<void> (0) : __assert_fail ("!MI.hasPostISelHook() && \"If a target marks an instruction with 'hasPostISelHook', \" \"it must implement TargetLowering::AdjustInstrPostInstrSelection!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 298, __PRETTY_FUNCTION__))
298 "it must implement TargetLowering::AdjustInstrPostInstrSelection!")((!MI.hasPostISelHook() && "If a target marks an instruction with 'hasPostISelHook', "
"it must implement TargetLowering::AdjustInstrPostInstrSelection!"
) ? static_cast<void> (0) : __assert_fail ("!MI.hasPostISelHook() && \"If a target marks an instruction with 'hasPostISelHook', \" \"it must implement TargetLowering::AdjustInstrPostInstrSelection!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 298, __PRETTY_FUNCTION__))
;
299}
300
301//===----------------------------------------------------------------------===//
302// SelectionDAGISel code
303//===----------------------------------------------------------------------===//
304
305SelectionDAGISel::SelectionDAGISel(TargetMachine &tm,
306 CodeGenOpt::Level OL) :
307 MachineFunctionPass(ID), TM(tm),
308 FuncInfo(new FunctionLoweringInfo()),
309 CurDAG(new SelectionDAG(tm, OL)),
310 SDB(new SelectionDAGBuilder(*CurDAG, *FuncInfo, OL)),
311 AA(), GFI(),
312 OptLevel(OL),
313 DAGSize(0) {
314 initializeGCModuleInfoPass(*PassRegistry::getPassRegistry());
315 initializeBranchProbabilityInfoWrapperPassPass(
316 *PassRegistry::getPassRegistry());
317 initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry());
318 initializeTargetLibraryInfoWrapperPassPass(
319 *PassRegistry::getPassRegistry());
320 }
321
322SelectionDAGISel::~SelectionDAGISel() {
323 delete SDB;
324 delete CurDAG;
325 delete FuncInfo;
326}
327
328void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const {
329 if (OptLevel != CodeGenOpt::None)
330 AU.addRequired<AAResultsWrapperPass>();
331 AU.addRequired<GCModuleInfo>();
332 AU.addRequired<StackProtector>();
333 AU.addPreserved<GCModuleInfo>();
334 AU.addRequired<TargetLibraryInfoWrapperPass>();
335 AU.addRequired<TargetTransformInfoWrapperPass>();
336 if (UseMBPI && OptLevel != CodeGenOpt::None)
337 AU.addRequired<BranchProbabilityInfoWrapperPass>();
338 MachineFunctionPass::getAnalysisUsage(AU);
339}
340
341/// SplitCriticalSideEffectEdges - Look for critical edges with a PHI value that
342/// may trap on it. In this case we have to split the edge so that the path
343/// through the predecessor block that doesn't go to the phi block doesn't
344/// execute the possibly trapping instruction. If available, we pass domtree
345/// and loop info to be updated when we split critical edges. This is because
346/// SelectionDAGISel preserves these analyses.
347/// This is required for correctness, so it must be done at -O0.
348///
349static void SplitCriticalSideEffectEdges(Function &Fn, DominatorTree *DT,
350 LoopInfo *LI) {
351 // Loop for blocks with phi nodes.
352 for (BasicBlock &BB : Fn) {
353 PHINode *PN = dyn_cast<PHINode>(BB.begin());
354 if (!PN) continue;
355
356 ReprocessBlock:
357 // For each block with a PHI node, check to see if any of the input values
358 // are potentially trapping constant expressions. Constant expressions are
359 // the only potentially trapping value that can occur as the argument to a
360 // PHI.
361 for (BasicBlock::iterator I = BB.begin(); (PN = dyn_cast<PHINode>(I)); ++I)
362 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
363 ConstantExpr *CE = dyn_cast<ConstantExpr>(PN->getIncomingValue(i));
364 if (!CE || !CE->canTrap()) continue;
365
366 // The only case we have to worry about is when the edge is critical.
367 // Since this block has a PHI Node, we assume it has multiple input
368 // edges: check to see if the pred has multiple successors.
369 BasicBlock *Pred = PN->getIncomingBlock(i);
370 if (Pred->getTerminator()->getNumSuccessors() == 1)
371 continue;
372
373 // Okay, we have to split this edge.
374 SplitCriticalEdge(
375 Pred->getTerminator(), GetSuccessorNumber(Pred, &BB),
376 CriticalEdgeSplittingOptions(DT, LI).setMergeIdenticalEdges());
377 goto ReprocessBlock;
378 }
379 }
380}
381
382bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
383 // If we already selected that function, we do not need to run SDISel.
384 if (mf.getProperties().hasProperty(
1
Taking false branch
385 MachineFunctionProperties::Property::Selected))
386 return false;
387 // Do some sanity-checking on the command-line options.
388 assert((!EnableFastISelAbort || TM.Options.EnableFastISel) &&(((!EnableFastISelAbort || TM.Options.EnableFastISel) &&
"-fast-isel-abort > 0 requires -fast-isel") ? static_cast
<void> (0) : __assert_fail ("(!EnableFastISelAbort || TM.Options.EnableFastISel) && \"-fast-isel-abort > 0 requires -fast-isel\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 389, __PRETTY_FUNCTION__))
2
Assuming the condition is false
3
Assuming the condition is true
4
'?' condition is true
389 "-fast-isel-abort > 0 requires -fast-isel")(((!EnableFastISelAbort || TM.Options.EnableFastISel) &&
"-fast-isel-abort > 0 requires -fast-isel") ? static_cast
<void> (0) : __assert_fail ("(!EnableFastISelAbort || TM.Options.EnableFastISel) && \"-fast-isel-abort > 0 requires -fast-isel\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 389, __PRETTY_FUNCTION__))
;
390
391 const Function &Fn = mf.getFunction();
392 MF = &mf;
393
394 // Reset the target options before resetting the optimization
395 // level below.
396 // FIXME: This is a horrible hack and should be processed via
397 // codegen looking at the optimization level explicitly when
398 // it wants to look at it.
399 TM.resetTargetOptions(Fn);
400 // Reset OptLevel to None for optnone functions.
401 CodeGenOpt::Level NewOptLevel = OptLevel;
402 if (OptLevel != CodeGenOpt::None && skipFunction(Fn))
5
Assuming the condition is false
403 NewOptLevel = CodeGenOpt::None;
404 OptLevelChanger OLC(*this, NewOptLevel);
6
Calling constructor for 'OptLevelChanger'
9
Returning from constructor for 'OptLevelChanger'
405
406 TII = MF->getSubtarget().getInstrInfo();
407 TLI = MF->getSubtarget().getTargetLowering();
408 RegInfo = &MF->getRegInfo();
409 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
410 GFI = Fn.hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) : nullptr;
10
Assuming the condition is false
11
'?' condition is false
411 ORE = make_unique<OptimizationRemarkEmitter>(&Fn);
412 auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
413 DominatorTree *DT = DTWP ? &DTWP->getDomTree() : nullptr;
12
'?' condition is false
414 auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>();
415 LoopInfo *LI = LIWP ? &LIWP->getLoopInfo() : nullptr;
13
'?' condition is false
416
417 LLVM_DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\n\n\n=== " << Fn.getName(
) << "\n"; } } while (false)
;
14
Assuming 'DebugFlag' is 0
15
Loop condition is false. Exiting loop
418
419 SplitCriticalSideEffectEdges(const_cast<Function &>(Fn), DT, LI);
420
421 CurDAG->init(*MF, *ORE, this, LibInfo,
422 getAnalysisIfAvailable<LegacyDivergenceAnalysis>());
423 FuncInfo->set(Fn, *MF, CurDAG);
424
425 // Now get the optional analyzes if we want to.
426 // This is based on the possibly changed OptLevel (after optnone is taken
427 // into account). That's unfortunate but OK because it just means we won't
428 // ask for passes that have been required anyway.
429
430 if (UseMBPI && OptLevel != CodeGenOpt::None)
16
Assuming the condition is false
431 FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
432 else
433 FuncInfo->BPI = nullptr;
434
435 if (OptLevel != CodeGenOpt::None)
17
Assuming the condition is false
18
Taking false branch
436 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
437 else
438 AA = nullptr;
439
440 SDB->init(GFI, AA, LibInfo);
441
442 MF->setHasInlineAsm(false);
443
444 FuncInfo->SplitCSR = false;
445
446 // We split CSR if the target supports it for the given function
447 // and the function has only return exits.
448 if (OptLevel != CodeGenOpt::None && TLI->supportSplitCSR(MF)) {
449 FuncInfo->SplitCSR = true;
450
451 // Collect all the return blocks.
452 for (const BasicBlock &BB : Fn) {
453 if (!succ_empty(&BB))
454 continue;
455
456 const Instruction *Term = BB.getTerminator();
457 if (isa<UnreachableInst>(Term) || isa<ReturnInst>(Term))
458 continue;
459
460 // Bail out if the exit block is not Return nor Unreachable.
461 FuncInfo->SplitCSR = false;
462 break;
463 }
464 }
465
466 MachineBasicBlock *EntryMBB = &MF->front();
467 if (FuncInfo->SplitCSR)
19
Taking false branch
468 // This performs initialization so lowering for SplitCSR will be correct.
469 TLI->initializeSplitCSR(EntryMBB);
470
471 SelectAllBasicBlocks(Fn);
472 if (FastISelFailed && EnableFastISelFallbackReport) {
20
Assuming the condition is false
473 DiagnosticInfoISelFallback DiagFallback(Fn);
474 Fn.getContext().diagnose(DiagFallback);
475 }
476
477 // If the first basic block in the function has live ins that need to be
478 // copied into vregs, emit the copies into the top of the block before
479 // emitting the code for the block.
480 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
481 RegInfo->EmitLiveInCopies(EntryMBB, TRI, *TII);
482
483 // Insert copies in the entry block and the return blocks.
484 if (FuncInfo->SplitCSR) {
21
Assuming the condition is false
22
Taking false branch
485 SmallVector<MachineBasicBlock*, 4> Returns;
486 // Collect all the return blocks.
487 for (MachineBasicBlock &MBB : mf) {
488 if (!MBB.succ_empty())
489 continue;
490
491 MachineBasicBlock::iterator Term = MBB.getFirstTerminator();
492 if (Term != MBB.end() && Term->isReturn()) {
493 Returns.push_back(&MBB);
494 continue;
495 }
496 }
497 TLI->insertCopiesSplitCSR(EntryMBB, Returns);
498 }
499
500 DenseMap<unsigned, unsigned> LiveInMap;
501 if (!FuncInfo->ArgDbgValues.empty())
23
Taking true branch
502 for (std::pair<unsigned, unsigned> LI : RegInfo->liveins())
24
Assuming '__begin2' is equal to '__end2'
503 if (LI.second)
504 LiveInMap.insert(LI);
505
506 // Insert DBG_VALUE instructions for function arguments to the entry block.
507 for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) {
25
Loop condition is true. Entering loop body
30
Assuming 'i' is equal to 'e'
31
Loop condition is false. Execution continues on line 573
508 MachineInstr *MI = FuncInfo->ArgDbgValues[e-i-1];
509 bool hasFI = MI->getOperand(0).isFI();
510 unsigned Reg =
511 hasFI ? TRI.getFrameRegister(*MF) : MI->getOperand(0).getReg();
26
'?' condition is true
512 if (TargetRegisterInfo::isPhysicalRegister(Reg))
27
Taking true branch
513 EntryMBB->insert(EntryMBB->begin(), MI);
514 else {
515 MachineInstr *Def = RegInfo->getVRegDef(Reg);
516 if (Def) {
517 MachineBasicBlock::iterator InsertPos = Def;
518 // FIXME: VR def may not be in entry block.
519 Def->getParent()->insert(std::next(InsertPos), MI);
520 } else
521 LLVM_DEBUG(dbgs() << "Dropping debug info for dead vreg"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for dead vreg"
<< TargetRegisterInfo::virtReg2Index(Reg) << "\n"
; } } while (false)
522 << TargetRegisterInfo::virtReg2Index(Reg) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for dead vreg"
<< TargetRegisterInfo::virtReg2Index(Reg) << "\n"
; } } while (false)
;
523 }
524
525 // If Reg is live-in then update debug info to track its copy in a vreg.
526 DenseMap<unsigned, unsigned>::iterator LDI = LiveInMap.find(Reg);
527 if (LDI != LiveInMap.end()) {
28
Assuming the condition is false
29
Taking false branch
528 assert(!hasFI && "There's no handling of frame pointer updating here yet "((!hasFI && "There's no handling of frame pointer updating here yet "
"- add if needed") ? static_cast<void> (0) : __assert_fail
("!hasFI && \"There's no handling of frame pointer updating here yet \" \"- add if needed\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 529, __PRETTY_FUNCTION__))
529 "- add if needed")((!hasFI && "There's no handling of frame pointer updating here yet "
"- add if needed") ? static_cast<void> (0) : __assert_fail
("!hasFI && \"There's no handling of frame pointer updating here yet \" \"- add if needed\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 529, __PRETTY_FUNCTION__))
;
530 MachineInstr *Def = RegInfo->getVRegDef(LDI->second);
531 MachineBasicBlock::iterator InsertPos = Def;
532 const MDNode *Variable = MI->getDebugVariable();
533 const MDNode *Expr = MI->getDebugExpression();
534 DebugLoc DL = MI->getDebugLoc();
535 bool IsIndirect = MI->isIndirectDebugValue();
536 if (IsIndirect)
537 assert(MI->getOperand(1).getImm() == 0 &&((MI->getOperand(1).getImm() == 0 && "DBG_VALUE with nonzero offset"
) ? static_cast<void> (0) : __assert_fail ("MI->getOperand(1).getImm() == 0 && \"DBG_VALUE with nonzero offset\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 538, __PRETTY_FUNCTION__))
538 "DBG_VALUE with nonzero offset")((MI->getOperand(1).getImm() == 0 && "DBG_VALUE with nonzero offset"
) ? static_cast<void> (0) : __assert_fail ("MI->getOperand(1).getImm() == 0 && \"DBG_VALUE with nonzero offset\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 538, __PRETTY_FUNCTION__))
;
539 assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&((cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? static_cast
<void> (0) : __assert_fail ("cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 540, __PRETTY_FUNCTION__))
540 "Expected inlined-at fields to agree")((cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? static_cast
<void> (0) : __assert_fail ("cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 540, __PRETTY_FUNCTION__))
;
541 // Def is never a terminator here, so it is ok to increment InsertPos.
542 BuildMI(*EntryMBB, ++InsertPos, DL, TII->get(TargetOpcode::DBG_VALUE),
543 IsIndirect, LDI->second, Variable, Expr);
544
545 // If this vreg is directly copied into an exported register then
546 // that COPY instructions also need DBG_VALUE, if it is the only
547 // user of LDI->second.
548 MachineInstr *CopyUseMI = nullptr;
549 for (MachineRegisterInfo::use_instr_iterator
550 UI = RegInfo->use_instr_begin(LDI->second),
551 E = RegInfo->use_instr_end(); UI != E; ) {
552 MachineInstr *UseMI = &*(UI++);
553 if (UseMI->isDebugValue()) continue;
554 if (UseMI->isCopy() && !CopyUseMI && UseMI->getParent() == EntryMBB) {
555 CopyUseMI = UseMI; continue;
556 }
557 // Otherwise this is another use or second copy use.
558 CopyUseMI = nullptr; break;
559 }
560 if (CopyUseMI) {
561 // Use MI's debug location, which describes where Variable was
562 // declared, rather than whatever is attached to CopyUseMI.
563 MachineInstr *NewMI =
564 BuildMI(*MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
565 CopyUseMI->getOperand(0).getReg(), Variable, Expr);
566 MachineBasicBlock::iterator Pos = CopyUseMI;
567 EntryMBB->insertAfter(Pos, NewMI);
568 }
569 }
570 }
571
572 // Determine if there are any calls in this machine function.
573 MachineFrameInfo &MFI = MF->getFrameInfo();
574 for (const auto &MBB : *MF) {
575 if (MFI.hasCalls() && MF->hasInlineAsm())
576 break;
577
578 for (const auto &MI : MBB) {
579 const MCInstrDesc &MCID = TII->get(MI.getOpcode());
580 if ((MCID.isCall() && !MCID.isReturn()) ||
581 MI.isStackAligningInlineAsm()) {
582 MFI.setHasCalls(true);
583 }
584 if (MI.isInlineAsm()) {
585 MF->setHasInlineAsm(true);
586 }
587 }
588 }
589
590 // Determine if there is a call to setjmp in the machine function.
591 MF->setExposesReturnsTwice(Fn.callsFunctionThatReturnsTwice());
592
593 // Replace forward-declared registers with the registers containing
594 // the desired value.
595 MachineRegisterInfo &MRI = MF->getRegInfo();
596 for (DenseMap<unsigned, unsigned>::iterator
32
Loop condition is false. Execution continues on line 623
597 I = FuncInfo->RegFixups.begin(), E = FuncInfo->RegFixups.end();
598 I != E; ++I) {
599 unsigned From = I->first;
600 unsigned To = I->second;
601 // If To is also scheduled to be replaced, find what its ultimate
602 // replacement is.
603 while (true) {
604 DenseMap<unsigned, unsigned>::iterator J = FuncInfo->RegFixups.find(To);
605 if (J == E) break;
606 To = J->second;
607 }
608 // Make sure the new register has a sufficiently constrained register class.
609 if (TargetRegisterInfo::isVirtualRegister(From) &&
610 TargetRegisterInfo::isVirtualRegister(To))
611 MRI.constrainRegClass(To, MRI.getRegClass(From));
612 // Replace it.
613
614
615 // Replacing one register with another won't touch the kill flags.
616 // We need to conservatively clear the kill flags as a kill on the old
617 // register might dominate existing uses of the new register.
618 if (!MRI.use_empty(To))
619 MRI.clearKillFlags(From);
620 MRI.replaceRegWith(From, To);
621 }
622
623 TLI->finalizeLowering(*MF);
624
625 // Release function-specific state. SDB and CurDAG are already cleared
626 // at this point.
627 FuncInfo->clear();
628
629 LLVM_DEBUG(dbgs() << "*** MachineFunction at end of ISel ***\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "*** MachineFunction at end of ISel ***\n"
; } } while (false)
;
33
Assuming 'DebugFlag' is 0
34
Loop condition is false. Exiting loop
630 LLVM_DEBUG(MF->print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { MF->print(dbgs()); } } while (false)
;
35
Loop condition is false. Exiting loop
631
632 return true;
36
Calling '~OptLevelChanger'
633}
634
635static void reportFastISelFailure(MachineFunction &MF,
636 OptimizationRemarkEmitter &ORE,
637 OptimizationRemarkMissed &R,
638 bool ShouldAbort) {
639 // Print the function name explicitly if we don't have a debug location (which
640 // makes the diagnostic less useful) or if we're going to emit a raw error.
641 if (!R.getLocation().isValid() || ShouldAbort)
642 R << (" (in function: " + MF.getName() + ")").str();
643
644 if (ShouldAbort)
645 report_fatal_error(R.getMsg());
646
647 ORE.emit(R);
648}
649
650void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin,
651 BasicBlock::const_iterator End,
652 bool &HadTailCall) {
653 // Allow creating illegal types during DAG building for the basic block.
654 CurDAG->NewNodesMustHaveLegalTypes = false;
655
656 // Lower the instructions. If a call is emitted as a tail call, cease emitting
657 // nodes for this block.
658 for (BasicBlock::const_iterator I = Begin; I != End && !SDB->HasTailCall; ++I) {
659 if (!ElidedArgCopyInstrs.count(&*I))
660 SDB->visit(*I);
661 }
662
663 // Make sure the root of the DAG is up-to-date.
664 CurDAG->setRoot(SDB->getControlRoot());
665 HadTailCall = SDB->HasTailCall;
666 SDB->clear();
667
668 // Final step, emit the lowered DAG as machine code.
669 CodeGenAndEmitDAG();
670}
671
672void SelectionDAGISel::ComputeLiveOutVRegInfo() {
673 SmallPtrSet<SDNode*, 16> VisitedNodes;
674 SmallVector<SDNode*, 128> Worklist;
675
676 Worklist.push_back(CurDAG->getRoot().getNode());
677
678 KnownBits Known;
679
680 do {
681 SDNode *N = Worklist.pop_back_val();
682
683 // If we've already seen this node, ignore it.
684 if (!VisitedNodes.insert(N).second)
685 continue;
686
687 // Otherwise, add all chain operands to the worklist.
688 for (const SDValue &Op : N->op_values())
689 if (Op.getValueType() == MVT::Other)
690 Worklist.push_back(Op.getNode());
691
692 // If this is a CopyToReg with a vreg dest, process it.
693 if (N->getOpcode() != ISD::CopyToReg)
694 continue;
695
696 unsigned DestReg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
697 if (!TargetRegisterInfo::isVirtualRegister(DestReg))
698 continue;
699
700 // Ignore non-integer values.
701 SDValue Src = N->getOperand(2);
702 EVT SrcVT = Src.getValueType();
703 if (!SrcVT.isInteger())
704 continue;
705
706 unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src);
707 Known = CurDAG->computeKnownBits(Src);
708 FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, Known);
709 } while (!Worklist.empty());
710}
711
712void SelectionDAGISel::CodeGenAndEmitDAG() {
713 StringRef GroupName = "sdag";
714 StringRef GroupDescription = "Instruction Selection and Scheduling";
715 std::string BlockName;
716 int BlockNumber = -1;
717 (void)BlockNumber;
718 bool MatchFilterBB = false; (void)MatchFilterBB;
719#ifndef NDEBUG
720 TargetTransformInfo &TTI =
721 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(*FuncInfo->Fn);
722#endif
723
724 // Pre-type legalization allow creation of any node types.
725 CurDAG->NewNodesMustHaveLegalTypes = false;
726
727#ifndef NDEBUG
728 MatchFilterBB = (FilterDAGBasicBlockName.empty() ||
729 FilterDAGBasicBlockName ==
730 FuncInfo->MBB->getBasicBlock()->getName());
731#endif
732#ifdef NDEBUG
733 if (ViewDAGCombine1 || ViewLegalizeTypesDAGs || ViewLegalizeDAGs ||
734 ViewDAGCombine2 || ViewDAGCombineLT || ViewISelDAGs || ViewSchedDAGs ||
735 ViewSUnitDAGs)
736#endif
737 {
738 BlockNumber = FuncInfo->MBB->getNumber();
739 BlockName =
740 (MF->getName() + ":" + FuncInfo->MBB->getBasicBlock()->getName()).str();
741 }
742 LLVM_DEBUG(dbgs() << "Initial selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Initial selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
743 << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Initial selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
744 << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Initial selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
745 CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Initial selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
;
746
747 if (ViewDAGCombine1 && MatchFilterBB)
748 CurDAG->viewGraph("dag-combine1 input for " + BlockName);
749
750 // Run the DAG combiner in pre-legalize mode.
751 {
752 NamedRegionTimer T("combine1", "DAG Combining 1", GroupName,
753 GroupDescription, TimePassesIsEnabled);
754 CurDAG->Combine(BeforeLegalizeTypes, AA, OptLevel);
755 }
756
757#ifndef NDEBUG
758 if (TTI.hasBranchDivergence())
759 CurDAG->VerifyDAGDiverence();
760#endif
761
762 LLVM_DEBUG(dbgs() << "Optimized lowered selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized lowered selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
763 << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized lowered selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
764 << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized lowered selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
765 CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized lowered selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
;
766
767 // Second step, hack on the DAG until it only uses operations and types that
768 // the target supports.
769 if (ViewLegalizeTypesDAGs && MatchFilterBB)
770 CurDAG->viewGraph("legalize-types input for " + BlockName);
771
772 bool Changed;
773 {
774 NamedRegionTimer T("legalize_types", "Type Legalization", GroupName,
775 GroupDescription, TimePassesIsEnabled);
776 Changed = CurDAG->LegalizeTypes();
777 }
778
779#ifndef NDEBUG
780 if (TTI.hasBranchDivergence())
781 CurDAG->VerifyDAGDiverence();
782#endif
783
784 LLVM_DEBUG(dbgs() << "Type-legalized selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Type-legalized selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
785 << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Type-legalized selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
786 << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Type-legalized selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
787 CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Type-legalized selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
;
788
789 // Only allow creation of legal node types.
790 CurDAG->NewNodesMustHaveLegalTypes = true;
791
792 if (Changed) {
793 if (ViewDAGCombineLT && MatchFilterBB)
794 CurDAG->viewGraph("dag-combine-lt input for " + BlockName);
795
796 // Run the DAG combiner in post-type-legalize mode.
797 {
798 NamedRegionTimer T("combine_lt", "DAG Combining after legalize types",
799 GroupName, GroupDescription, TimePassesIsEnabled);
800 CurDAG->Combine(AfterLegalizeTypes, AA, OptLevel);
801 }
802
803#ifndef NDEBUG
804 if (TTI.hasBranchDivergence())
805 CurDAG->VerifyDAGDiverence();
806#endif
807
808 LLVM_DEBUG(dbgs() << "Optimized type-legalized selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized type-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
809 << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized type-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
810 << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized type-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
811 CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized type-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
;
812 }
813
814 {
815 NamedRegionTimer T("legalize_vec", "Vector Legalization", GroupName,
816 GroupDescription, TimePassesIsEnabled);
817 Changed = CurDAG->LegalizeVectors();
818 }
819
820 if (Changed) {
821 LLVM_DEBUG(dbgs() << "Vector-legalized selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
822 << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
823 << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
824 CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
;
825
826 {
827 NamedRegionTimer T("legalize_types2", "Type Legalization 2", GroupName,
828 GroupDescription, TimePassesIsEnabled);
829 CurDAG->LegalizeTypes();
830 }
831
832 LLVM_DEBUG(dbgs() << "Vector/type-legalized selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector/type-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
833 << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector/type-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
834 << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector/type-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
835 CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector/type-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
;
836
837 if (ViewDAGCombineLT && MatchFilterBB)
838 CurDAG->viewGraph("dag-combine-lv input for " + BlockName);
839
840 // Run the DAG combiner in post-type-legalize mode.
841 {
842 NamedRegionTimer T("combine_lv", "DAG Combining after legalize vectors",
843 GroupName, GroupDescription, TimePassesIsEnabled);
844 CurDAG->Combine(AfterLegalizeVectorOps, AA, OptLevel);
845 }
846
847 LLVM_DEBUG(dbgs() << "Optimized vector-legalized selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized vector-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
848 << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized vector-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
849 << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized vector-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
850 CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized vector-legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
;
851
852#ifndef NDEBUG
853 if (TTI.hasBranchDivergence())
854 CurDAG->VerifyDAGDiverence();
855#endif
856 }
857
858 if (ViewLegalizeDAGs && MatchFilterBB)
859 CurDAG->viewGraph("legalize input for " + BlockName);
860
861 {
862 NamedRegionTimer T("legalize", "DAG Legalization", GroupName,
863 GroupDescription, TimePassesIsEnabled);
864 CurDAG->Legalize();
865 }
866
867#ifndef NDEBUG
868 if (TTI.hasBranchDivergence())
869 CurDAG->VerifyDAGDiverence();
870#endif
871
872 LLVM_DEBUG(dbgs() << "Legalized selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Legalized selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
873 << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Legalized selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
874 << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Legalized selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
875 CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Legalized selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
;
876
877 if (ViewDAGCombine2 && MatchFilterBB)
878 CurDAG->viewGraph("dag-combine2 input for " + BlockName);
879
880 // Run the DAG combiner in post-legalize mode.
881 {
882 NamedRegionTimer T("combine2", "DAG Combining 2", GroupName,
883 GroupDescription, TimePassesIsEnabled);
884 CurDAG->Combine(AfterLegalizeDAG, AA, OptLevel);
885 }
886
887#ifndef NDEBUG
888 if (TTI.hasBranchDivergence())
889 CurDAG->VerifyDAGDiverence();
890#endif
891
892 LLVM_DEBUG(dbgs() << "Optimized legalized selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
893 << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
894 << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
895 CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized legalized selection DAG: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
BlockName << "'\n"; CurDAG->dump(); } } while (false
)
;
896
897 if (OptLevel != CodeGenOpt::None)
898 ComputeLiveOutVRegInfo();
899
900 if (ViewISelDAGs && MatchFilterBB)
901 CurDAG->viewGraph("isel input for " + BlockName);
902
903 // Third, instruction select all of the operations to machine code, adding the
904 // code to the MachineBasicBlock.
905 {
906 NamedRegionTimer T("isel", "Instruction Selection", GroupName,
907 GroupDescription, TimePassesIsEnabled);
908 DoInstructionSelection();
909 }
910
911 LLVM_DEBUG(dbgs() << "Selected selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Selected selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
912 << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Selected selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
913 << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Selected selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
914 CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Selected selection DAG: " <<
printMBBReference(*FuncInfo->MBB) << " '" << BlockName
<< "'\n"; CurDAG->dump(); } } while (false)
;
915
916 if (ViewSchedDAGs && MatchFilterBB)
917 CurDAG->viewGraph("scheduler input for " + BlockName);
918
919 // Schedule machine code.
920 ScheduleDAGSDNodes *Scheduler = CreateScheduler();
921 {
922 NamedRegionTimer T("sched", "Instruction Scheduling", GroupName,
923 GroupDescription, TimePassesIsEnabled);
924 Scheduler->Run(CurDAG, FuncInfo->MBB);
925 }
926
927 if (ViewSUnitDAGs && MatchFilterBB)
928 Scheduler->viewGraph();
929
930 // Emit machine code to BB. This can change 'BB' to the last block being
931 // inserted into.
932 MachineBasicBlock *FirstMBB = FuncInfo->MBB, *LastMBB;
933 {
934 NamedRegionTimer T("emit", "Instruction Creation", GroupName,
935 GroupDescription, TimePassesIsEnabled);
936
937 // FuncInfo->InsertPt is passed by reference and set to the end of the
938 // scheduled instructions.
939 LastMBB = FuncInfo->MBB = Scheduler->EmitSchedule(FuncInfo->InsertPt);
940 }
941
942 // If the block was split, make sure we update any references that are used to
943 // update PHI nodes later on.
944 if (FirstMBB != LastMBB)
945 SDB->UpdateSplitBlock(FirstMBB, LastMBB);
946
947 // Free the scheduler state.
948 {
949 NamedRegionTimer T("cleanup", "Instruction Scheduling Cleanup", GroupName,
950 GroupDescription, TimePassesIsEnabled);
951 delete Scheduler;
952 }
953
954 // Free the SelectionDAG state, now that we're finished with it.
955 CurDAG->clear();
956}
957
958namespace {
959
960/// ISelUpdater - helper class to handle updates of the instruction selection
961/// graph.
962class ISelUpdater : public SelectionDAG::DAGUpdateListener {
963 SelectionDAG::allnodes_iterator &ISelPosition;
964
965public:
966 ISelUpdater(SelectionDAG &DAG, SelectionDAG::allnodes_iterator &isp)
967 : SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}
968
969 /// NodeDeleted - Handle nodes deleted from the graph. If the node being
970 /// deleted is the current ISelPosition node, update ISelPosition.
971 ///
972 void NodeDeleted(SDNode *N, SDNode *E) override {
973 if (ISelPosition == SelectionDAG::allnodes_iterator(N))
974 ++ISelPosition;
975 }
976};
977
978} // end anonymous namespace
979
980// This function is used to enforce the topological node id property
981// property leveraged during Instruction selection. Before selection all
982// nodes are given a non-negative id such that all nodes have a larger id than
983// their operands. As this holds transitively we can prune checks that a node N
984// is a predecessor of M another by not recursively checking through M's
985// operands if N's ID is larger than M's ID. This is significantly improves
986// performance of for various legality checks (e.g. IsLegalToFold /
987// UpdateChains).
988
989// However, when we fuse multiple nodes into a single node
990// during selection we may induce a predecessor relationship between inputs and
991// outputs of distinct nodes being merged violating the topological property.
992// Should a fused node have a successor which has yet to be selected, our
993// legality checks would be incorrect. To avoid this we mark all unselected
994// sucessor nodes, i.e. id != -1 as invalid for pruning by bit-negating (x =>
995// (-(x+1))) the ids and modify our pruning check to ignore negative Ids of M.
996// We use bit-negation to more clearly enforce that node id -1 can only be
997// achieved by selected nodes). As the conversion is reversable the original Id,
998// topological pruning can still be leveraged when looking for unselected nodes.
999// This method is call internally in all ISel replacement calls.
1000void SelectionDAGISel::EnforceNodeIdInvariant(SDNode *Node) {
1001 SmallVector<SDNode *, 4> Nodes;
1002 Nodes.push_back(Node);
1003
1004 while (!Nodes.empty()) {
1005 SDNode *N = Nodes.pop_back_val();
1006 for (auto *U : N->uses()) {
1007 auto UId = U->getNodeId();
1008 if (UId > 0) {
1009 InvalidateNodeId(U);
1010 Nodes.push_back(U);
1011 }
1012 }
1013 }
1014}
1015
1016// InvalidateNodeId - As discusses in EnforceNodeIdInvariant, mark a
1017// NodeId with the equivalent node id which is invalid for topological
1018// pruning.
1019void SelectionDAGISel::InvalidateNodeId(SDNode *N) {
1020 int InvalidId = -(N->getNodeId() + 1);
1021 N->setNodeId(InvalidId);
1022}
1023
1024// getUninvalidatedNodeId - get original uninvalidated node id.
1025int SelectionDAGISel::getUninvalidatedNodeId(SDNode *N) {
1026 int Id = N->getNodeId();
1027 if (Id < -1)
1028 return -(Id + 1);
1029 return Id;
1030}
1031
1032void SelectionDAGISel::DoInstructionSelection() {
1033 LLVM_DEBUG(dbgs() << "===== Instruction selection begins: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "===== Instruction selection begins: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
FuncInfo->MBB->getName() << "'\n"; } } while (false
)
1034 << printMBBReference(*FuncInfo->MBB) << " '"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "===== Instruction selection begins: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
FuncInfo->MBB->getName() << "'\n"; } } while (false
)
1035 << FuncInfo->MBB->getName() << "'\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "===== Instruction selection begins: "
<< printMBBReference(*FuncInfo->MBB) << " '" <<
FuncInfo->MBB->getName() << "'\n"; } } while (false
)
;
1036
1037 PreprocessISelDAG();
1038
1039 // Select target instructions for the DAG.
1040 {
1041 // Number all nodes with a topological order and set DAGSize.
1042 DAGSize = CurDAG->AssignTopologicalOrder();
1043
1044 // Create a dummy node (which is not added to allnodes), that adds
1045 // a reference to the root node, preventing it from being deleted,
1046 // and tracking any changes of the root.
1047 HandleSDNode Dummy(CurDAG->getRoot());
1048 SelectionDAG::allnodes_iterator ISelPosition (CurDAG->getRoot().getNode());
1049 ++ISelPosition;
1050
1051 // Make sure that ISelPosition gets properly updated when nodes are deleted
1052 // in calls made from this function.
1053 ISelUpdater ISU(*CurDAG, ISelPosition);
1054
1055 // The AllNodes list is now topological-sorted. Visit the
1056 // nodes by starting at the end of the list (the root of the
1057 // graph) and preceding back toward the beginning (the entry
1058 // node).
1059 while (ISelPosition != CurDAG->allnodes_begin()) {
1060 SDNode *Node = &*--ISelPosition;
1061 // Skip dead nodes. DAGCombiner is expected to eliminate all dead nodes,
1062 // but there are currently some corner cases that it misses. Also, this
1063 // makes it theoretically possible to disable the DAGCombiner.
1064 if (Node->use_empty())
1065 continue;
1066
1067#ifndef NDEBUG
1068 SmallVector<SDNode *, 4> Nodes;
1069 Nodes.push_back(Node);
1070
1071 while (!Nodes.empty()) {
1072 auto N = Nodes.pop_back_val();
1073 if (N->getOpcode() == ISD::TokenFactor || N->getNodeId() < 0)
1074 continue;
1075 for (const SDValue &Op : N->op_values()) {
1076 if (Op->getOpcode() == ISD::TokenFactor)
1077 Nodes.push_back(Op.getNode());
1078 else {
1079 // We rely on topological ordering of node ids for checking for
1080 // cycles when fusing nodes during selection. All unselected nodes
1081 // successors of an already selected node should have a negative id.
1082 // This assertion will catch such cases. If this assertion triggers
1083 // it is likely you using DAG-level Value/Node replacement functions
1084 // (versus equivalent ISEL replacement) in backend-specific
1085 // selections. See comment in EnforceNodeIdInvariant for more
1086 // details.
1087 assert(Op->getNodeId() != -1 &&((Op->getNodeId() != -1 && "Node has already selected predecessor node"
) ? static_cast<void> (0) : __assert_fail ("Op->getNodeId() != -1 && \"Node has already selected predecessor node\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1088, __PRETTY_FUNCTION__))
1088 "Node has already selected predecessor node")((Op->getNodeId() != -1 && "Node has already selected predecessor node"
) ? static_cast<void> (0) : __assert_fail ("Op->getNodeId() != -1 && \"Node has already selected predecessor node\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1088, __PRETTY_FUNCTION__))
;
1089 }
1090 }
1091 }
1092#endif
1093
1094 // When we are using non-default rounding modes or FP exception behavior
1095 // FP operations are represented by StrictFP pseudo-operations. They
1096 // need to be simplified here so that the target-specific instruction
1097 // selectors know how to handle them.
1098 //
1099 // If the current node is a strict FP pseudo-op, the isStrictFPOp()
1100 // function will provide the corresponding normal FP opcode to which the
1101 // node should be mutated.
1102 //
1103 // FIXME: The backends need a way to handle FP constraints.
1104 if (Node->isStrictFPOpcode())
1105 Node = CurDAG->mutateStrictFPToFP(Node);
1106
1107 LLVM_DEBUG(dbgs() << "\nISEL: Starting selection on root node: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\nISEL: Starting selection on root node: "
; Node->dump(CurDAG); } } while (false)
1108 Node->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\nISEL: Starting selection on root node: "
; Node->dump(CurDAG); } } while (false)
;
1109
1110 Select(Node);
1111 }
1112
1113 CurDAG->setRoot(Dummy.getValue());
1114 }
1115
1116 LLVM_DEBUG(dbgs() << "\n===== Instruction selection ends:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\n===== Instruction selection ends:\n"
; } } while (false)
;
1117
1118 PostprocessISelDAG();
1119}
1120
1121static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI) {
1122 for (const User *U : CPI->users()) {
1123 if (const IntrinsicInst *EHPtrCall = dyn_cast<IntrinsicInst>(U)) {
1124 Intrinsic::ID IID = EHPtrCall->getIntrinsicID();
1125 if (IID == Intrinsic::eh_exceptionpointer ||
1126 IID == Intrinsic::eh_exceptioncode)
1127 return true;
1128 }
1129 }
1130 return false;
1131}
1132
1133// wasm.landingpad.index intrinsic is for associating a landing pad index number
1134// with a catchpad instruction. Retrieve the landing pad index in the intrinsic
1135// and store the mapping in the function.
1136static void mapWasmLandingPadIndex(MachineBasicBlock *MBB,
1137 const CatchPadInst *CPI) {
1138 MachineFunction *MF = MBB->getParent();
1139 // In case of single catch (...), we don't emit LSDA, so we don't need
1140 // this information.
1141 bool IsSingleCatchAllClause =
1142 CPI->getNumArgOperands() == 1 &&
1143 cast<Constant>(CPI->getArgOperand(0))->isNullValue();
1144 if (!IsSingleCatchAllClause) {
1145 // Create a mapping from landing pad label to landing pad index.
1146 bool IntrFound = false;
1147 for (const User *U : CPI->users()) {
1148 if (const auto *Call = dyn_cast<IntrinsicInst>(U)) {
1149 Intrinsic::ID IID = Call->getIntrinsicID();
1150 if (IID == Intrinsic::wasm_landingpad_index) {
1151 Value *IndexArg = Call->getArgOperand(1);
1152 int Index = cast<ConstantInt>(IndexArg)->getZExtValue();
1153 MF->setWasmLandingPadIndex(MBB, Index);
1154 IntrFound = true;
1155 break;
1156 }
1157 }
1158 }
1159 assert(IntrFound && "wasm.landingpad.index intrinsic not found!")((IntrFound && "wasm.landingpad.index intrinsic not found!"
) ? static_cast<void> (0) : __assert_fail ("IntrFound && \"wasm.landingpad.index intrinsic not found!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1159, __PRETTY_FUNCTION__))
;
1160 (void)IntrFound;
1161 }
1162}
1163
1164/// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and
1165/// do other setup for EH landing-pad blocks.
1166bool SelectionDAGISel::PrepareEHLandingPad() {
1167 MachineBasicBlock *MBB = FuncInfo->MBB;
1168 const Constant *PersonalityFn = FuncInfo->Fn->getPersonalityFn();
1169 const BasicBlock *LLVMBB = MBB->getBasicBlock();
1170 const TargetRegisterClass *PtrRC =
1171 TLI->getRegClassFor(TLI->getPointerTy(CurDAG->getDataLayout()));
1172
1173 auto Pers = classifyEHPersonality(PersonalityFn);
1174
1175 // Catchpads have one live-in register, which typically holds the exception
1176 // pointer or code.
1177 if (isFuncletEHPersonality(Pers)) {
1178 if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI())) {
1179 if (hasExceptionPointerOrCodeUser(CPI)) {
1180 // Get or create the virtual register to hold the pointer or code. Mark
1181 // the live in physreg and copy into the vreg.
1182 MCPhysReg EHPhysReg = TLI->getExceptionPointerRegister(PersonalityFn);
1183 assert(EHPhysReg && "target lacks exception pointer register")((EHPhysReg && "target lacks exception pointer register"
) ? static_cast<void> (0) : __assert_fail ("EHPhysReg && \"target lacks exception pointer register\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1183, __PRETTY_FUNCTION__))
;
1184 MBB->addLiveIn(EHPhysReg);
1185 unsigned VReg = FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC);
1186 BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(),
1187 TII->get(TargetOpcode::COPY), VReg)
1188 .addReg(EHPhysReg, RegState::Kill);
1189 }
1190 }
1191 return true;
1192 }
1193
1194 // Add a label to mark the beginning of the landing pad. Deletion of the
1195 // landing pad can thus be detected via the MachineModuleInfo.
1196 MCSymbol *Label = MF->addLandingPad(MBB);
1197
1198 const MCInstrDesc &II = TII->get(TargetOpcode::EH_LABEL);
1199 BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II)
1200 .addSym(Label);
1201
1202 if (Pers == EHPersonality::Wasm_CXX) {
1203 if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI()))
1204 mapWasmLandingPadIndex(MBB, CPI);
1205 } else {
1206 // Assign the call site to the landing pad's begin label.
1207 MF->setCallSiteLandingPad(Label, SDB->LPadToCallSiteMap[MBB]);
1208 // Mark exception register as live in.
1209 if (unsigned Reg = TLI->getExceptionPointerRegister(PersonalityFn))
1210 FuncInfo->ExceptionPointerVirtReg = MBB->addLiveIn(Reg, PtrRC);
1211 // Mark exception selector register as live in.
1212 if (unsigned Reg = TLI->getExceptionSelectorRegister(PersonalityFn))
1213 FuncInfo->ExceptionSelectorVirtReg = MBB->addLiveIn(Reg, PtrRC);
1214 }
1215
1216 return true;
1217}
1218
1219/// isFoldedOrDeadInstruction - Return true if the specified instruction is
1220/// side-effect free and is either dead or folded into a generated instruction.
1221/// Return false if it needs to be emitted.
1222static bool isFoldedOrDeadInstruction(const Instruction *I,
1223 FunctionLoweringInfo *FuncInfo) {
1224 return !I->mayWriteToMemory() && // Side-effecting instructions aren't folded.
1225 !I->isTerminator() && // Terminators aren't folded.
1226 !isa<DbgInfoIntrinsic>(I) && // Debug instructions aren't folded.
1227 !I->isEHPad() && // EH pad instructions aren't folded.
1228 !FuncInfo->isExportedInst(I); // Exported instrs must be computed.
1229}
1230
1231/// Set up SwiftErrorVals by going through the function. If the function has
1232/// swifterror argument, it will be the first entry.
1233static void setupSwiftErrorVals(const Function &Fn, const TargetLowering *TLI,
1234 FunctionLoweringInfo *FuncInfo) {
1235 if (!TLI->supportSwiftError())
1236 return;
1237
1238 FuncInfo->SwiftErrorVals.clear();
1239 FuncInfo->SwiftErrorVRegDefMap.clear();
1240 FuncInfo->SwiftErrorVRegUpwardsUse.clear();
1241 FuncInfo->SwiftErrorVRegDefUses.clear();
1242 FuncInfo->SwiftErrorArg = nullptr;
1243
1244 // Check if function has a swifterror argument.
1245 bool HaveSeenSwiftErrorArg = false;
1246 for (Function::const_arg_iterator AI = Fn.arg_begin(), AE = Fn.arg_end();
1247 AI != AE; ++AI)
1248 if (AI->hasSwiftErrorAttr()) {
1249 assert(!HaveSeenSwiftErrorArg &&((!HaveSeenSwiftErrorArg && "Must have only one swifterror parameter"
) ? static_cast<void> (0) : __assert_fail ("!HaveSeenSwiftErrorArg && \"Must have only one swifterror parameter\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1250, __PRETTY_FUNCTION__))
1250 "Must have only one swifterror parameter")((!HaveSeenSwiftErrorArg && "Must have only one swifterror parameter"
) ? static_cast<void> (0) : __assert_fail ("!HaveSeenSwiftErrorArg && \"Must have only one swifterror parameter\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1250, __PRETTY_FUNCTION__))
;
1251 (void)HaveSeenSwiftErrorArg; // silence warning.
1252 HaveSeenSwiftErrorArg = true;
1253 FuncInfo->SwiftErrorArg = &*AI;
1254 FuncInfo->SwiftErrorVals.push_back(&*AI);
1255 }
1256
1257 for (const auto &LLVMBB : Fn)
1258 for (const auto &Inst : LLVMBB) {
1259 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(&Inst))
1260 if (Alloca->isSwiftError())
1261 FuncInfo->SwiftErrorVals.push_back(Alloca);
1262 }
1263}
1264
1265static void createSwiftErrorEntriesInEntryBlock(FunctionLoweringInfo *FuncInfo,
1266 FastISel *FastIS,
1267 const TargetLowering *TLI,
1268 const TargetInstrInfo *TII,
1269 SelectionDAGBuilder *SDB) {
1270 if (!TLI->supportSwiftError())
1271 return;
1272
1273 // We only need to do this when we have swifterror parameter or swifterror
1274 // alloc.
1275 if (FuncInfo->SwiftErrorVals.empty())
1276 return;
1277
1278 assert(FuncInfo->MBB == &*FuncInfo->MF->begin() &&((FuncInfo->MBB == &*FuncInfo->MF->begin() &&
"expected to insert into entry block") ? static_cast<void
> (0) : __assert_fail ("FuncInfo->MBB == &*FuncInfo->MF->begin() && \"expected to insert into entry block\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1279, __PRETTY_FUNCTION__))
1279 "expected to insert into entry block")((FuncInfo->MBB == &*FuncInfo->MF->begin() &&
"expected to insert into entry block") ? static_cast<void
> (0) : __assert_fail ("FuncInfo->MBB == &*FuncInfo->MF->begin() && \"expected to insert into entry block\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1279, __PRETTY_FUNCTION__))
;
1280 auto &DL = FuncInfo->MF->getDataLayout();
1281 auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL));
1282 for (const auto *SwiftErrorVal : FuncInfo->SwiftErrorVals) {
1283 // We will always generate a copy from the argument. It is always used at
1284 // least by the 'return' of the swifterror.
1285 if (FuncInfo->SwiftErrorArg && FuncInfo->SwiftErrorArg == SwiftErrorVal)
1286 continue;
1287 unsigned VReg = FuncInfo->MF->getRegInfo().createVirtualRegister(RC);
1288 // Assign Undef to Vreg. We construct MI directly to make sure it works
1289 // with FastISel.
1290 BuildMI(*FuncInfo->MBB, FuncInfo->MBB->getFirstNonPHI(),
1291 SDB->getCurDebugLoc(), TII->get(TargetOpcode::IMPLICIT_DEF),
1292 VReg);
1293
1294 // Keep FastIS informed about the value we just inserted.
1295 if (FastIS)
1296 FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));
1297
1298 FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB, SwiftErrorVal, VReg);
1299 }
1300}
1301
1302/// Collect llvm.dbg.declare information. This is done after argument lowering
1303/// in case the declarations refer to arguments.
1304static void processDbgDeclares(FunctionLoweringInfo *FuncInfo) {
1305 MachineFunction *MF = FuncInfo->MF;
1306 const DataLayout &DL = MF->getDataLayout();
1307 for (const BasicBlock &BB : *FuncInfo->Fn) {
1308 for (const Instruction &I : BB) {
1309 const DbgDeclareInst *DI = dyn_cast<DbgDeclareInst>(&I);
1310 if (!DI)
1311 continue;
1312
1313 assert(DI->getVariable() && "Missing variable")((DI->getVariable() && "Missing variable") ? static_cast
<void> (0) : __assert_fail ("DI->getVariable() && \"Missing variable\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1313, __PRETTY_FUNCTION__))
;
1314 assert(DI->getDebugLoc() && "Missing location")((DI->getDebugLoc() && "Missing location") ? static_cast
<void> (0) : __assert_fail ("DI->getDebugLoc() && \"Missing location\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1314, __PRETTY_FUNCTION__))
;
1315 const Value *Address = DI->getAddress();
1316 if (!Address)
1317 continue;
1318
1319 // Look through casts and constant offset GEPs. These mostly come from
1320 // inalloca.
1321 APInt Offset(DL.getTypeSizeInBits(Address->getType()), 0);
1322 Address = Address->stripAndAccumulateInBoundsConstantOffsets(DL, Offset);
1323
1324 // Check if the variable is a static alloca or a byval or inalloca
1325 // argument passed in memory. If it is not, then we will ignore this
1326 // intrinsic and handle this during isel like dbg.value.
1327 int FI = std::numeric_limits<int>::max();
1328 if (const auto *AI = dyn_cast<AllocaInst>(Address)) {
1329 auto SI = FuncInfo->StaticAllocaMap.find(AI);
1330 if (SI != FuncInfo->StaticAllocaMap.end())
1331 FI = SI->second;
1332 } else if (const auto *Arg = dyn_cast<Argument>(Address))
1333 FI = FuncInfo->getArgumentFrameIndex(Arg);
1334
1335 if (FI == std::numeric_limits<int>::max())
1336 continue;
1337
1338 DIExpression *Expr = DI->getExpression();
1339 if (Offset.getBoolValue())
1340 Expr = DIExpression::prepend(Expr, DIExpression::NoDeref,
1341 Offset.getZExtValue());
1342 MF->setVariableDbgInfo(DI->getVariable(), Expr, FI, DI->getDebugLoc());
1343 }
1344 }
1345}
1346
1347/// Propagate swifterror values through the machine function CFG.
1348static void propagateSwiftErrorVRegs(FunctionLoweringInfo *FuncInfo) {
1349 auto *TLI = FuncInfo->TLI;
1350 if (!TLI->supportSwiftError())
1351 return;
1352
1353 // We only need to do this when we have swifterror parameter or swifterror
1354 // alloc.
1355 if (FuncInfo->SwiftErrorVals.empty())
1356 return;
1357
1358 // For each machine basic block in reverse post order.
1359 ReversePostOrderTraversal<MachineFunction *> RPOT(FuncInfo->MF);
1360 for (MachineBasicBlock *MBB : RPOT) {
1361 // For each swifterror value in the function.
1362 for(const auto *SwiftErrorVal : FuncInfo->SwiftErrorVals) {
1363 auto Key = std::make_pair(MBB, SwiftErrorVal);
1364 auto UUseIt = FuncInfo->SwiftErrorVRegUpwardsUse.find(Key);
1365 auto VRegDefIt = FuncInfo->SwiftErrorVRegDefMap.find(Key);
1366 bool UpwardsUse = UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end();
1367 unsigned UUseVReg = UpwardsUse ? UUseIt->second : 0;
1368 bool DownwardDef = VRegDefIt != FuncInfo->SwiftErrorVRegDefMap.end();
1369 assert(!(UpwardsUse && !DownwardDef) &&((!(UpwardsUse && !DownwardDef) && "We can't have an upwards use but no downwards def"
) ? static_cast<void> (0) : __assert_fail ("!(UpwardsUse && !DownwardDef) && \"We can't have an upwards use but no downwards def\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1370, __PRETTY_FUNCTION__))
1370 "We can't have an upwards use but no downwards def")((!(UpwardsUse && !DownwardDef) && "We can't have an upwards use but no downwards def"
) ? static_cast<void> (0) : __assert_fail ("!(UpwardsUse && !DownwardDef) && \"We can't have an upwards use but no downwards def\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1370, __PRETTY_FUNCTION__))
;
1371
1372 // If there is no upwards exposed use and an entry for the swifterror in
1373 // the def map for this value we don't need to do anything: We already
1374 // have a downward def for this basic block.
1375 if (!UpwardsUse && DownwardDef)
1376 continue;
1377
1378 // Otherwise we either have an upwards exposed use vreg that we need to
1379 // materialize or need to forward the downward def from predecessors.
1380
1381 // Check whether we have a single vreg def from all predecessors.
1382 // Otherwise we need a phi.
1383 SmallVector<std::pair<MachineBasicBlock *, unsigned>, 4> VRegs;
1384 SmallSet<const MachineBasicBlock*, 8> Visited;
1385 for (auto *Pred : MBB->predecessors()) {
1386 if (!Visited.insert(Pred).second)
1387 continue;
1388 VRegs.push_back(std::make_pair(
1389 Pred, FuncInfo->getOrCreateSwiftErrorVReg(Pred, SwiftErrorVal)));
1390 if (Pred != MBB)
1391 continue;
1392 // We have a self-edge.
1393 // If there was no upwards use in this basic block there is now one: the
1394 // phi needs to use it self.
1395 if (!UpwardsUse) {
1396 UpwardsUse = true;
1397 UUseIt = FuncInfo->SwiftErrorVRegUpwardsUse.find(Key);
1398 assert(UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end())((UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end()) ? static_cast
<void> (0) : __assert_fail ("UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end()"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1398, __PRETTY_FUNCTION__))
;
1399 UUseVReg = UUseIt->second;
1400 }
1401 }
1402
1403 // We need a phi node if we have more than one predecessor with different
1404 // downward defs.
1405 bool needPHI =
1406 VRegs.size() >= 1 &&
1407 std::find_if(
1408 VRegs.begin(), VRegs.end(),
1409 [&](const std::pair<const MachineBasicBlock *, unsigned> &V)
1410 -> bool { return V.second != VRegs[0].second; }) !=
1411 VRegs.end();
1412
1413 // If there is no upwards exposed used and we don't need a phi just
1414 // forward the swifterror vreg from the predecessor(s).
1415 if (!UpwardsUse && !needPHI) {
1416 assert(!VRegs.empty() &&((!VRegs.empty() && "No predecessors? The entry block should bail out earlier"
) ? static_cast<void> (0) : __assert_fail ("!VRegs.empty() && \"No predecessors? The entry block should bail out earlier\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1417, __PRETTY_FUNCTION__))
1417 "No predecessors? The entry block should bail out earlier")((!VRegs.empty() && "No predecessors? The entry block should bail out earlier"
) ? static_cast<void> (0) : __assert_fail ("!VRegs.empty() && \"No predecessors? The entry block should bail out earlier\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1417, __PRETTY_FUNCTION__))
;
1418 // Just forward the swifterror vreg from the predecessor(s).
1419 FuncInfo->setCurrentSwiftErrorVReg(MBB, SwiftErrorVal, VRegs[0].second);
1420 continue;
1421 }
1422
1423 auto DLoc = isa<Instruction>(SwiftErrorVal)
1424 ? cast<Instruction>(SwiftErrorVal)->getDebugLoc()
1425 : DebugLoc();
1426 const auto *TII = FuncInfo->MF->getSubtarget().getInstrInfo();
1427
1428 // If we don't need a phi create a copy to the upward exposed vreg.
1429 if (!needPHI) {
1430 assert(UpwardsUse)((UpwardsUse) ? static_cast<void> (0) : __assert_fail (
"UpwardsUse", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1430, __PRETTY_FUNCTION__))
;
1431 assert(!VRegs.empty() &&((!VRegs.empty() && "No predecessors? Is the Calling Convention correct?"
) ? static_cast<void> (0) : __assert_fail ("!VRegs.empty() && \"No predecessors? Is the Calling Convention correct?\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1432, __PRETTY_FUNCTION__))
1432 "No predecessors? Is the Calling Convention correct?")((!VRegs.empty() && "No predecessors? Is the Calling Convention correct?"
) ? static_cast<void> (0) : __assert_fail ("!VRegs.empty() && \"No predecessors? Is the Calling Convention correct?\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1432, __PRETTY_FUNCTION__))
;
1433 unsigned DestReg = UUseVReg;
1434 BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc, TII->get(TargetOpcode::COPY),
1435 DestReg)
1436 .addReg(VRegs[0].second);
1437 continue;
1438 }
1439
1440 // We need a phi: if there is an upwards exposed use we already have a
1441 // destination virtual register number otherwise we generate a new one.
1442 auto &DL = FuncInfo->MF->getDataLayout();
1443 auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL));
1444 unsigned PHIVReg =
1445 UpwardsUse ? UUseVReg
1446 : FuncInfo->MF->getRegInfo().createVirtualRegister(RC);
1447 MachineInstrBuilder SwiftErrorPHI =
1448 BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc,
1449 TII->get(TargetOpcode::PHI), PHIVReg);
1450 for (auto BBRegPair : VRegs) {
1451 SwiftErrorPHI.addReg(BBRegPair.second).addMBB(BBRegPair.first);
1452 }
1453
1454 // We did not have a definition in this block before: store the phi's vreg
1455 // as this block downward exposed def.
1456 if (!UpwardsUse)
1457 FuncInfo->setCurrentSwiftErrorVReg(MBB, SwiftErrorVal, PHIVReg);
1458 }
1459 }
1460}
1461
1462static void preassignSwiftErrorRegs(const TargetLowering *TLI,
1463 FunctionLoweringInfo *FuncInfo,
1464 BasicBlock::const_iterator Begin,
1465 BasicBlock::const_iterator End) {
1466 if (!TLI->supportSwiftError() || FuncInfo->SwiftErrorVals.empty())
1467 return;
1468
1469 // Iterator over instructions and assign vregs to swifterror defs and uses.
1470 for (auto It = Begin; It != End; ++It) {
1471 ImmutableCallSite CS(&*It);
1472 if (CS) {
1473 // A call-site with a swifterror argument is both use and def.
1474 const Value *SwiftErrorAddr = nullptr;
1475 for (auto &Arg : CS.args()) {
1476 if (!Arg->isSwiftError())
1477 continue;
1478 // Use of swifterror.
1479 assert(!SwiftErrorAddr && "Cannot have multiple swifterror arguments")((!SwiftErrorAddr && "Cannot have multiple swifterror arguments"
) ? static_cast<void> (0) : __assert_fail ("!SwiftErrorAddr && \"Cannot have multiple swifterror arguments\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1479, __PRETTY_FUNCTION__))
;
1480 SwiftErrorAddr = &*Arg;
1481 assert(SwiftErrorAddr->isSwiftError() &&((SwiftErrorAddr->isSwiftError() && "Must have a swifterror value argument"
) ? static_cast<void> (0) : __assert_fail ("SwiftErrorAddr->isSwiftError() && \"Must have a swifterror value argument\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1482, __PRETTY_FUNCTION__))
1482 "Must have a swifterror value argument")((SwiftErrorAddr->isSwiftError() && "Must have a swifterror value argument"
) ? static_cast<void> (0) : __assert_fail ("SwiftErrorAddr->isSwiftError() && \"Must have a swifterror value argument\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1482, __PRETTY_FUNCTION__))
;
1483 unsigned VReg; bool CreatedReg;
1484 std::tie(VReg, CreatedReg) = FuncInfo->getOrCreateSwiftErrorVRegUseAt(
1485 &*It, FuncInfo->MBB, SwiftErrorAddr);
1486 assert(CreatedReg)((CreatedReg) ? static_cast<void> (0) : __assert_fail (
"CreatedReg", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1486, __PRETTY_FUNCTION__))
;
1487 }
1488 if (!SwiftErrorAddr)
1489 continue;
1490
1491 // Def of swifterror.
1492 unsigned VReg; bool CreatedReg;
1493 std::tie(VReg, CreatedReg) =
1494 FuncInfo->getOrCreateSwiftErrorVRegDefAt(&*It);
1495 assert(CreatedReg)((CreatedReg) ? static_cast<void> (0) : __assert_fail (
"CreatedReg", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1495, __PRETTY_FUNCTION__))
;
1496 FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB, SwiftErrorAddr, VReg);
1497
1498 // A load is a use.
1499 } else if (const LoadInst *LI = dyn_cast<const LoadInst>(&*It)) {
1500 const Value *V = LI->getOperand(0);
1501 if (!V->isSwiftError())
1502 continue;
1503
1504 unsigned VReg; bool CreatedReg;
1505 std::tie(VReg, CreatedReg) =
1506 FuncInfo->getOrCreateSwiftErrorVRegUseAt(LI, FuncInfo->MBB, V);
1507 assert(CreatedReg)((CreatedReg) ? static_cast<void> (0) : __assert_fail (
"CreatedReg", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1507, __PRETTY_FUNCTION__))
;
1508
1509 // A store is a def.
1510 } else if (const StoreInst *SI = dyn_cast<const StoreInst>(&*It)) {
1511 const Value *SwiftErrorAddr = SI->getOperand(1);
1512 if (!SwiftErrorAddr->isSwiftError())
1513 continue;
1514
1515 // Def of swifterror.
1516 unsigned VReg; bool CreatedReg;
1517 std::tie(VReg, CreatedReg) =
1518 FuncInfo->getOrCreateSwiftErrorVRegDefAt(&*It);
1519 assert(CreatedReg)((CreatedReg) ? static_cast<void> (0) : __assert_fail (
"CreatedReg", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1519, __PRETTY_FUNCTION__))
;
1520 FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB, SwiftErrorAddr, VReg);
1521
1522 // A return in a swiferror returning function is a use.
1523 } else if (const ReturnInst *R = dyn_cast<const ReturnInst>(&*It)) {
1524 const Function *F = R->getParent()->getParent();
1525 if(!F->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
1526 continue;
1527
1528 unsigned VReg; bool CreatedReg;
1529 std::tie(VReg, CreatedReg) = FuncInfo->getOrCreateSwiftErrorVRegUseAt(
1530 R, FuncInfo->MBB, FuncInfo->SwiftErrorArg);
1531 assert(CreatedReg)((CreatedReg) ? static_cast<void> (0) : __assert_fail (
"CreatedReg", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1531, __PRETTY_FUNCTION__))
;
1532 }
1533 }
1534}
1535
1536void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
1537 FastISelFailed = false;
1538 // Initialize the Fast-ISel state, if needed.
1539 FastISel *FastIS = nullptr;
1540 if (TM.Options.EnableFastISel) {
1541 LLVM_DEBUG(dbgs() << "Enabling fast-isel\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Enabling fast-isel\n"; } } while
(false)
;
1542 FastIS = TLI->createFastISel(*FuncInfo, LibInfo);
1543 }
1544
1545 setupSwiftErrorVals(Fn, TLI, FuncInfo);
1546
1547 ReversePostOrderTraversal<const Function*> RPOT(&Fn);
1548
1549 // Lower arguments up front. An RPO iteration always visits the entry block
1550 // first.
1551 assert(*RPOT.begin() == &Fn.getEntryBlock())((*RPOT.begin() == &Fn.getEntryBlock()) ? static_cast<
void> (0) : __assert_fail ("*RPOT.begin() == &Fn.getEntryBlock()"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1551, __PRETTY_FUNCTION__))
;
1552 ++NumEntryBlocks;
1553
1554 // Set up FuncInfo for ISel. Entry blocks never have PHIs.
1555 FuncInfo->MBB = FuncInfo->MBBMap[&Fn.getEntryBlock()];
1556 FuncInfo->InsertPt = FuncInfo->MBB->begin();
1557
1558 CurDAG->setFunctionLoweringInfo(FuncInfo);
1559
1560 if (!FastIS) {
1561 LowerArguments(Fn);
1562 } else {
1563 // See if fast isel can lower the arguments.
1564 FastIS->startNewBlock();
1565 if (!FastIS->lowerArguments()) {
1566 FastISelFailed = true;
1567 // Fast isel failed to lower these arguments
1568 ++NumFastIselFailLowerArguments;
1569
1570 OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1571 Fn.getSubprogram(),
1572 &Fn.getEntryBlock());
1573 R << "FastISel didn't lower all arguments: "
1574 << ore::NV("Prototype", Fn.getType());
1575 reportFastISelFailure(*MF, *ORE, R, EnableFastISelAbort > 1);
1576
1577 // Use SelectionDAG argument lowering
1578 LowerArguments(Fn);
1579 CurDAG->setRoot(SDB->getControlRoot());
1580 SDB->clear();
1581 CodeGenAndEmitDAG();
1582 }
1583
1584 // If we inserted any instructions at the beginning, make a note of
1585 // where they are, so we can be sure to emit subsequent instructions
1586 // after them.
1587 if (FuncInfo->InsertPt != FuncInfo->MBB->begin())
1588 FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));
1589 else
1590 FastIS->setLastLocalValue(nullptr);
1591 }
1592 createSwiftErrorEntriesInEntryBlock(FuncInfo, FastIS, TLI, TII, SDB);
1593
1594 processDbgDeclares(FuncInfo);
1595
1596 // Iterate over all basic blocks in the function.
1597 StackProtector &SP = getAnalysis<StackProtector>();
1598 for (const BasicBlock *LLVMBB : RPOT) {
1599 if (OptLevel != CodeGenOpt::None) {
1600 bool AllPredsVisited = true;
1601 for (const_pred_iterator PI = pred_begin(LLVMBB), PE = pred_end(LLVMBB);
1602 PI != PE; ++PI) {
1603 if (!FuncInfo->VisitedBBs.count(*PI)) {
1604 AllPredsVisited = false;
1605 break;
1606 }
1607 }
1608
1609 if (AllPredsVisited) {
1610 for (const PHINode &PN : LLVMBB->phis())
1611 FuncInfo->ComputePHILiveOutRegInfo(&PN);
1612 } else {
1613 for (const PHINode &PN : LLVMBB->phis())
1614 FuncInfo->InvalidatePHILiveOutRegInfo(&PN);
1615 }
1616
1617 FuncInfo->VisitedBBs.insert(LLVMBB);
1618 }
1619
1620 BasicBlock::const_iterator const Begin =
1621 LLVMBB->getFirstNonPHI()->getIterator();
1622 BasicBlock::const_iterator const End = LLVMBB->end();
1623 BasicBlock::const_iterator BI = End;
1624
1625 FuncInfo->MBB = FuncInfo->MBBMap[LLVMBB];
1626 if (!FuncInfo->MBB)
1627 continue; // Some blocks like catchpads have no code or MBB.
1628
1629 // Insert new instructions after any phi or argument setup code.
1630 FuncInfo->InsertPt = FuncInfo->MBB->end();
1631
1632 // Setup an EH landing-pad block.
1633 FuncInfo->ExceptionPointerVirtReg = 0;
1634 FuncInfo->ExceptionSelectorVirtReg = 0;
1635 if (LLVMBB->isEHPad())
1636 if (!PrepareEHLandingPad())
1637 continue;
1638
1639 // Before doing SelectionDAG ISel, see if FastISel has been requested.
1640 if (FastIS) {
1641 if (LLVMBB != &Fn.getEntryBlock())
1642 FastIS->startNewBlock();
1643
1644 unsigned NumFastIselRemaining = std::distance(Begin, End);
1645
1646 // Pre-assign swifterror vregs.
1647 preassignSwiftErrorRegs(TLI, FuncInfo, Begin, End);
1648
1649 // Do FastISel on as many instructions as possible.
1650 for (; BI != Begin; --BI) {
1651 const Instruction *Inst = &*std::prev(BI);
1652
1653 // If we no longer require this instruction, skip it.
1654 if (isFoldedOrDeadInstruction(Inst, FuncInfo) ||
1655 ElidedArgCopyInstrs.count(Inst)) {
1656 --NumFastIselRemaining;
1657 continue;
1658 }
1659
1660 // Bottom-up: reset the insert pos at the top, after any local-value
1661 // instructions.
1662 FastIS->recomputeInsertPt();
1663
1664 // Try to select the instruction with FastISel.
1665 if (FastIS->selectInstruction(Inst)) {
1666 --NumFastIselRemaining;
1667 ++NumFastIselSuccess;
1668 // If fast isel succeeded, skip over all the folded instructions, and
1669 // then see if there is a load right before the selected instructions.
1670 // Try to fold the load if so.
1671 const Instruction *BeforeInst = Inst;
1672 while (BeforeInst != &*Begin) {
1673 BeforeInst = &*std::prev(BasicBlock::const_iterator(BeforeInst));
1674 if (!isFoldedOrDeadInstruction(BeforeInst, FuncInfo))
1675 break;
1676 }
1677 if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&
1678 BeforeInst->hasOneUse() &&
1679 FastIS->tryToFoldLoad(cast<LoadInst>(BeforeInst), Inst)) {
1680 // If we succeeded, don't re-select the load.
1681 BI = std::next(BasicBlock::const_iterator(BeforeInst));
1682 --NumFastIselRemaining;
1683 ++NumFastIselSuccess;
1684 }
1685 continue;
1686 }
1687
1688 FastISelFailed = true;
1689
1690 // Then handle certain instructions as single-LLVM-Instruction blocks.
1691 // We cannot separate out GCrelocates to their own blocks since we need
1692 // to keep track of gc-relocates for a particular gc-statepoint. This is
1693 // done by SelectionDAGBuilder::LowerAsSTATEPOINT, called before
1694 // visitGCRelocate.
1695 if (isa<CallInst>(Inst) && !isStatepoint(Inst) && !isGCRelocate(Inst)) {
1696 OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1697 Inst->getDebugLoc(), LLVMBB);
1698
1699 R << "FastISel missed call";
1700
1701 if (R.isEnabled() || EnableFastISelAbort) {
1702 std::string InstStrStorage;
1703 raw_string_ostream InstStr(InstStrStorage);
1704 InstStr << *Inst;
1705
1706 R << ": " << InstStr.str();
1707 }
1708
1709 reportFastISelFailure(*MF, *ORE, R, EnableFastISelAbort > 2);
1710
1711 if (!Inst->getType()->isVoidTy() && !Inst->getType()->isTokenTy() &&
1712 !Inst->use_empty()) {
1713 unsigned &R = FuncInfo->ValueMap[Inst];
1714 if (!R)
1715 R = FuncInfo->CreateRegs(Inst->getType());
1716 }
1717
1718 bool HadTailCall = false;
1719 MachineBasicBlock::iterator SavedInsertPt = FuncInfo->InsertPt;
1720 SelectBasicBlock(Inst->getIterator(), BI, HadTailCall);
1721
1722 // If the call was emitted as a tail call, we're done with the block.
1723 // We also need to delete any previously emitted instructions.
1724 if (HadTailCall) {
1725 FastIS->removeDeadCode(SavedInsertPt, FuncInfo->MBB->end());
1726 --BI;
1727 break;
1728 }
1729
1730 // Recompute NumFastIselRemaining as Selection DAG instruction
1731 // selection may have handled the call, input args, etc.
1732 unsigned RemainingNow = std::distance(Begin, BI);
1733 NumFastIselFailures += NumFastIselRemaining - RemainingNow;
1734 NumFastIselRemaining = RemainingNow;
1735 continue;
1736 }
1737
1738 OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1739 Inst->getDebugLoc(), LLVMBB);
1740
1741 bool ShouldAbort = EnableFastISelAbort;
1742 if (Inst->isTerminator()) {
1743 // Use a different message for terminator misses.
1744 R << "FastISel missed terminator";
1745 // Don't abort for terminator unless the level is really high
1746 ShouldAbort = (EnableFastISelAbort > 2);
1747 } else {
1748 R << "FastISel missed";
1749 }
1750
1751 if (R.isEnabled() || EnableFastISelAbort) {
1752 std::string InstStrStorage;
1753 raw_string_ostream InstStr(InstStrStorage);
1754 InstStr << *Inst;
1755 R << ": " << InstStr.str();
1756 }
1757
1758 reportFastISelFailure(*MF, *ORE, R, ShouldAbort);
1759
1760 NumFastIselFailures += NumFastIselRemaining;
1761 break;
1762 }
1763
1764 FastIS->recomputeInsertPt();
1765 }
1766
1767 if (SP.shouldEmitSDCheck(*LLVMBB)) {
1768 bool FunctionBasedInstrumentation =
1769 TLI->getSSPStackGuardCheck(*Fn.getParent());
1770 SDB->SPDescriptor.initialize(LLVMBB, FuncInfo->MBBMap[LLVMBB],
1771 FunctionBasedInstrumentation);
1772 }
1773
1774 if (Begin != BI)
1775 ++NumDAGBlocks;
1776 else
1777 ++NumFastIselBlocks;
1778
1779 if (Begin != BI) {
1780 // Run SelectionDAG instruction selection on the remainder of the block
1781 // not handled by FastISel. If FastISel is not run, this is the entire
1782 // block.
1783 bool HadTailCall;
1784 SelectBasicBlock(Begin, BI, HadTailCall);
1785
1786 // But if FastISel was run, we already selected some of the block.
1787 // If we emitted a tail-call, we need to delete any previously emitted
1788 // instruction that follows it.
1789 if (HadTailCall && FuncInfo->InsertPt != FuncInfo->MBB->end())
1790 FastIS->removeDeadCode(FuncInfo->InsertPt, FuncInfo->MBB->end());
1791 }
1792
1793 if (FastIS)
1794 FastIS->finishBasicBlock();
1795 FinishBasicBlock();
1796 FuncInfo->PHINodesToUpdate.clear();
1797 ElidedArgCopyInstrs.clear();
1798 }
1799
1800 SP.copyToMachineFrameInfo(MF->getFrameInfo());
1801
1802 propagateSwiftErrorVRegs(FuncInfo);
1803
1804 delete FastIS;
1805 SDB->clearDanglingDebugInfo();
1806 SDB->SPDescriptor.resetPerFunctionState();
1807}
1808
1809/// Given that the input MI is before a partial terminator sequence TSeq, return
1810/// true if M + TSeq also a partial terminator sequence.
1811///
1812/// A Terminator sequence is a sequence of MachineInstrs which at this point in
1813/// lowering copy vregs into physical registers, which are then passed into
1814/// terminator instructors so we can satisfy ABI constraints. A partial
1815/// terminator sequence is an improper subset of a terminator sequence (i.e. it
1816/// may be the whole terminator sequence).
1817static bool MIIsInTerminatorSequence(const MachineInstr &MI) {
1818 // If we do not have a copy or an implicit def, we return true if and only if
1819 // MI is a debug value.
1820 if (!MI.isCopy() && !MI.isImplicitDef())
1821 // Sometimes DBG_VALUE MI sneak in between the copies from the vregs to the
1822 // physical registers if there is debug info associated with the terminator
1823 // of our mbb. We want to include said debug info in our terminator
1824 // sequence, so we return true in that case.
1825 return MI.isDebugValue();
1826
1827 // We have left the terminator sequence if we are not doing one of the
1828 // following:
1829 //
1830 // 1. Copying a vreg into a physical register.
1831 // 2. Copying a vreg into a vreg.
1832 // 3. Defining a register via an implicit def.
1833
1834 // OPI should always be a register definition...
1835 MachineInstr::const_mop_iterator OPI = MI.operands_begin();
1836 if (!OPI->isReg() || !OPI->isDef())
1837 return false;
1838
1839 // Defining any register via an implicit def is always ok.
1840 if (MI.isImplicitDef())
1841 return true;
1842
1843 // Grab the copy source...
1844 MachineInstr::const_mop_iterator OPI2 = OPI;
1845 ++OPI2;
1846 assert(OPI2 != MI.operands_end()((OPI2 != MI.operands_end() && "Should have a copy implying we should have 2 arguments."
) ? static_cast<void> (0) : __assert_fail ("OPI2 != MI.operands_end() && \"Should have a copy implying we should have 2 arguments.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1847, __PRETTY_FUNCTION__))
1847 && "Should have a copy implying we should have 2 arguments.")((OPI2 != MI.operands_end() && "Should have a copy implying we should have 2 arguments."
) ? static_cast<void> (0) : __assert_fail ("OPI2 != MI.operands_end() && \"Should have a copy implying we should have 2 arguments.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1847, __PRETTY_FUNCTION__))
;
1848
1849 // Make sure that the copy dest is not a vreg when the copy source is a
1850 // physical register.
1851 if (!OPI2->isReg() ||
1852 (!TargetRegisterInfo::isPhysicalRegister(OPI->getReg()) &&
1853 TargetRegisterInfo::isPhysicalRegister(OPI2->getReg())))
1854 return false;
1855
1856 return true;
1857}
1858
1859/// Find the split point at which to splice the end of BB into its success stack
1860/// protector check machine basic block.
1861///
1862/// On many platforms, due to ABI constraints, terminators, even before register
1863/// allocation, use physical registers. This creates an issue for us since
1864/// physical registers at this point can not travel across basic
1865/// blocks. Luckily, selectiondag always moves physical registers into vregs
1866/// when they enter functions and moves them through a sequence of copies back
1867/// into the physical registers right before the terminator creating a
1868/// ``Terminator Sequence''. This function is searching for the beginning of the
1869/// terminator sequence so that we can ensure that we splice off not just the
1870/// terminator, but additionally the copies that move the vregs into the
1871/// physical registers.
1872static MachineBasicBlock::iterator
1873FindSplitPointForStackProtector(MachineBasicBlock *BB) {
1874 MachineBasicBlock::iterator SplitPoint = BB->getFirstTerminator();
1875 //
1876 if (SplitPoint == BB->begin())
1877 return SplitPoint;
1878
1879 MachineBasicBlock::iterator Start = BB->begin();
1880 MachineBasicBlock::iterator Previous = SplitPoint;
1881 --Previous;
1882
1883 while (MIIsInTerminatorSequence(*Previous)) {
1884 SplitPoint = Previous;
1885 if (Previous == Start)
1886 break;
1887 --Previous;
1888 }
1889
1890 return SplitPoint;
1891}
1892
1893void
1894SelectionDAGISel::FinishBasicBlock() {
1895 LLVM_DEBUG(dbgs() << "Total amount of phi nodes to update: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " << i << " : ("
<< FuncInfo->PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)
1896 << FuncInfo->PHINodesToUpdate.size() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " << i << " : ("
<< FuncInfo->PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)
1897 for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " << i << " : ("
<< FuncInfo->PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)
1898 ++i) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " << i << " : ("
<< FuncInfo->PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)
1899 << "Node " << i << " : (" << FuncInfo->PHINodesToUpdate[i].firstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " << i << " : ("
<< FuncInfo->PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)
1900 << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " << i << " : ("
<< FuncInfo->PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)
;
1901
1902 // Next, now that we know what the last MBB the LLVM BB expanded is, update
1903 // PHI nodes in successors.
1904 for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
1905 MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first);
1906 assert(PHI->isPHI() &&((PHI->isPHI() && "This is not a machine PHI node that we are updating!"
) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1907, __PRETTY_FUNCTION__))
1907 "This is not a machine PHI node that we are updating!")((PHI->isPHI() && "This is not a machine PHI node that we are updating!"
) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1907, __PRETTY_FUNCTION__))
;
1908 if (!FuncInfo->MBB->isSuccessor(PHI->getParent()))
1909 continue;
1910 PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB);
1911 }
1912
1913 // Handle stack protector.
1914 if (SDB->SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
1915 // The target provides a guard check function. There is no need to
1916 // generate error handling code or to split current basic block.
1917 MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1918
1919 // Add load and check to the basicblock.
1920 FuncInfo->MBB = ParentMBB;
1921 FuncInfo->InsertPt =
1922 FindSplitPointForStackProtector(ParentMBB);
1923 SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
1924 CurDAG->setRoot(SDB->getRoot());
1925 SDB->clear();
1926 CodeGenAndEmitDAG();
1927
1928 // Clear the Per-BB State.
1929 SDB->SPDescriptor.resetPerBBState();
1930 } else if (SDB->SPDescriptor.shouldEmitStackProtector()) {
1931 MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1932 MachineBasicBlock *SuccessMBB = SDB->SPDescriptor.getSuccessMBB();
1933
1934 // Find the split point to split the parent mbb. At the same time copy all
1935 // physical registers used in the tail of parent mbb into virtual registers
1936 // before the split point and back into physical registers after the split
1937 // point. This prevents us needing to deal with Live-ins and many other
1938 // register allocation issues caused by us splitting the parent mbb. The
1939 // register allocator will clean up said virtual copies later on.
1940 MachineBasicBlock::iterator SplitPoint =
1941 FindSplitPointForStackProtector(ParentMBB);
1942
1943 // Splice the terminator of ParentMBB into SuccessMBB.
1944 SuccessMBB->splice(SuccessMBB->end(), ParentMBB,
1945 SplitPoint,
1946 ParentMBB->end());
1947
1948 // Add compare/jump on neq/jump to the parent BB.
1949 FuncInfo->MBB = ParentMBB;
1950 FuncInfo->InsertPt = ParentMBB->end();
1951 SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
1952 CurDAG->setRoot(SDB->getRoot());
1953 SDB->clear();
1954 CodeGenAndEmitDAG();
1955
1956 // CodeGen Failure MBB if we have not codegened it yet.
1957 MachineBasicBlock *FailureMBB = SDB->SPDescriptor.getFailureMBB();
1958 if (FailureMBB->empty()) {
1959 FuncInfo->MBB = FailureMBB;
1960 FuncInfo->InsertPt = FailureMBB->end();
1961 SDB->visitSPDescriptorFailure(SDB->SPDescriptor);
1962 CurDAG->setRoot(SDB->getRoot());
1963 SDB->clear();
1964 CodeGenAndEmitDAG();
1965 }
1966
1967 // Clear the Per-BB State.
1968 SDB->SPDescriptor.resetPerBBState();
1969 }
1970
1971 // Lower each BitTestBlock.
1972 for (auto &BTB : SDB->BitTestCases) {
1973 // Lower header first, if it wasn't already lowered
1974 if (!BTB.Emitted) {
1975 // Set the current basic block to the mbb we wish to insert the code into
1976 FuncInfo->MBB = BTB.Parent;
1977 FuncInfo->InsertPt = FuncInfo->MBB->end();
1978 // Emit the code
1979 SDB->visitBitTestHeader(BTB, FuncInfo->MBB);
1980 CurDAG->setRoot(SDB->getRoot());
1981 SDB->clear();
1982 CodeGenAndEmitDAG();
1983 }
1984
1985 BranchProbability UnhandledProb = BTB.Prob;
1986 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
1987 UnhandledProb -= BTB.Cases[j].ExtraProb;
1988 // Set the current basic block to the mbb we wish to insert the code into
1989 FuncInfo->MBB = BTB.Cases[j].ThisBB;
1990 FuncInfo->InsertPt = FuncInfo->MBB->end();
1991 // Emit the code
1992
1993 // If all cases cover a contiguous range, it is not necessary to jump to
1994 // the default block after the last bit test fails. This is because the
1995 // range check during bit test header creation has guaranteed that every
1996 // case here doesn't go outside the range. In this case, there is no need
1997 // to perform the last bit test, as it will always be true. Instead, make
1998 // the second-to-last bit-test fall through to the target of the last bit
1999 // test, and delete the last bit test.
2000
2001 MachineBasicBlock *NextMBB;
2002 if (BTB.ContiguousRange && j + 2 == ej) {
2003 // Second-to-last bit-test with contiguous range: fall through to the
2004 // target of the final bit test.
2005 NextMBB = BTB.Cases[j + 1].TargetBB;
2006 } else if (j + 1 == ej) {
2007 // For the last bit test, fall through to Default.
2008 NextMBB = BTB.Default;
2009 } else {
2010 // Otherwise, fall through to the next bit test.
2011 NextMBB = BTB.Cases[j + 1].ThisBB;
2012 }
2013
2014 SDB->visitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j],
2015 FuncInfo->MBB);
2016
2017 CurDAG->setRoot(SDB->getRoot());
2018 SDB->clear();
2019 CodeGenAndEmitDAG();
2020
2021 if (BTB.ContiguousRange && j + 2 == ej) {
2022 // Since we're not going to use the final bit test, remove it.
2023 BTB.Cases.pop_back();
2024 break;
2025 }
2026 }
2027
2028 // Update PHI Nodes
2029 for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
2030 pi != pe; ++pi) {
2031 MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);
2032 MachineBasicBlock *PHIBB = PHI->getParent();
2033 assert(PHI->isPHI() &&((PHI->isPHI() && "This is not a machine PHI node that we are updating!"
) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2034, __PRETTY_FUNCTION__))
2034 "This is not a machine PHI node that we are updating!")((PHI->isPHI() && "This is not a machine PHI node that we are updating!"
) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2034, __PRETTY_FUNCTION__))
;
2035 // This is "default" BB. We have two jumps to it. From "header" BB and
2036 // from last "case" BB, unless the latter was skipped.
2037 if (PHIBB == BTB.Default) {
2038 PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(BTB.Parent);
2039 if (!BTB.ContiguousRange) {
2040 PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
2041 .addMBB(BTB.Cases.back().ThisBB);
2042 }
2043 }
2044 // One of "cases" BB.
2045 for (unsigned j = 0, ej = BTB.Cases.size();
2046 j != ej; ++j) {
2047 MachineBasicBlock* cBB = BTB.Cases[j].ThisBB;
2048 if (cBB->isSuccessor(PHIBB))
2049 PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(cBB);
2050 }
2051 }
2052 }
2053 SDB->BitTestCases.clear();
2054
2055 // If the JumpTable record is filled in, then we need to emit a jump table.
2056 // Updating the PHI nodes is tricky in this case, since we need to determine
2057 // whether the PHI is a successor of the range check MBB or the jump table MBB
2058 for (unsigned i = 0, e = SDB->JTCases.size(); i != e; ++i) {
2059 // Lower header first, if it wasn't already lowered
2060 if (!SDB->JTCases[i].first.Emitted) {
2061 // Set the current basic block to the mbb we wish to insert the code into
2062 FuncInfo->MBB = SDB->JTCases[i].first.HeaderBB;
2063 FuncInfo->InsertPt = FuncInfo->MBB->end();
2064 // Emit the code
2065 SDB->visitJumpTableHeader(SDB->JTCases[i].second, SDB->JTCases[i].first,
2066 FuncInfo->MBB);
2067 CurDAG->setRoot(SDB->getRoot());
2068 SDB->clear();
2069 CodeGenAndEmitDAG();
2070 }
2071
2072 // Set the current basic block to the mbb we wish to insert the code into
2073 FuncInfo->MBB = SDB->JTCases[i].second.MBB;
2074 FuncInfo->InsertPt = FuncInfo->MBB->end();
2075 // Emit the code
2076 SDB->visitJumpTable(SDB->JTCases[i].second);
2077 CurDAG->setRoot(SDB->getRoot());
2078 SDB->clear();
2079 CodeGenAndEmitDAG();
2080
2081 // Update PHI Nodes
2082 for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
2083 pi != pe; ++pi) {
2084 MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);
2085 MachineBasicBlock *PHIBB = PHI->getParent();
2086 assert(PHI->isPHI() &&((PHI->isPHI() && "This is not a machine PHI node that we are updating!"
) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2087, __PRETTY_FUNCTION__))
2087 "This is not a machine PHI node that we are updating!")((PHI->isPHI() && "This is not a machine PHI node that we are updating!"
) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2087, __PRETTY_FUNCTION__))
;
2088 // "default" BB. We can go there only from header BB.
2089 if (PHIBB == SDB->JTCases[i].second.Default)
2090 PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
2091 .addMBB(SDB->JTCases[i].first.HeaderBB);
2092 // JT BB. Just iterate over successors here
2093 if (FuncInfo->MBB->isSuccessor(PHIBB))
2094 PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(FuncInfo->MBB);
2095 }
2096 }
2097 SDB->JTCases.clear();
2098
2099 // If we generated any switch lowering information, build and codegen any
2100 // additional DAGs necessary.
2101 for (unsigned i = 0, e = SDB->SwitchCases.size(); i != e; ++i) {
2102 // Set the current basic block to the mbb we wish to insert the code into
2103 FuncInfo->MBB = SDB->SwitchCases[i].ThisBB;
2104 FuncInfo->InsertPt = FuncInfo->MBB->end();
2105
2106 // Determine the unique successors.
2107 SmallVector<MachineBasicBlock *, 2> Succs;
2108 Succs.push_back(SDB->SwitchCases[i].TrueBB);
2109 if (SDB->SwitchCases[i].TrueBB != SDB->SwitchCases[i].FalseBB)
2110 Succs.push_back(SDB->SwitchCases[i].FalseBB);
2111
2112 // Emit the code. Note that this could result in FuncInfo->MBB being split.
2113 SDB->visitSwitchCase(SDB->SwitchCases[i], FuncInfo->MBB);
2114 CurDAG->setRoot(SDB->getRoot());
2115 SDB->clear();
2116 CodeGenAndEmitDAG();
2117
2118 // Remember the last block, now that any splitting is done, for use in
2119 // populating PHI nodes in successors.
2120 MachineBasicBlock *ThisBB = FuncInfo->MBB;
2121
2122 // Handle any PHI nodes in successors of this chunk, as if we were coming
2123 // from the original BB before switch expansion. Note that PHI nodes can
2124 // occur multiple times in PHINodesToUpdate. We have to be very careful to
2125 // handle them the right number of times.
2126 for (unsigned i = 0, e = Succs.size(); i != e; ++i) {
2127 FuncInfo->MBB = Succs[i];
2128 FuncInfo->InsertPt = FuncInfo->MBB->end();
2129 // FuncInfo->MBB may have been removed from the CFG if a branch was
2130 // constant folded.
2131 if (ThisBB->isSuccessor(FuncInfo->MBB)) {
2132 for (MachineBasicBlock::iterator
2133 MBBI = FuncInfo->MBB->begin(), MBBE = FuncInfo->MBB->end();
2134 MBBI != MBBE && MBBI->isPHI(); ++MBBI) {
2135 MachineInstrBuilder PHI(*MF, MBBI);
2136 // This value for this PHI node is recorded in PHINodesToUpdate.
2137 for (unsigned pn = 0; ; ++pn) {
2138 assert(pn != FuncInfo->PHINodesToUpdate.size() &&((pn != FuncInfo->PHINodesToUpdate.size() && "Didn't find PHI entry!"
) ? static_cast<void> (0) : __assert_fail ("pn != FuncInfo->PHINodesToUpdate.size() && \"Didn't find PHI entry!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2139, __PRETTY_FUNCTION__))
2139 "Didn't find PHI entry!")((pn != FuncInfo->PHINodesToUpdate.size() && "Didn't find PHI entry!"
) ? static_cast<void> (0) : __assert_fail ("pn != FuncInfo->PHINodesToUpdate.size() && \"Didn't find PHI entry!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2139, __PRETTY_FUNCTION__))
;
2140 if (FuncInfo->PHINodesToUpdate[pn].first == PHI) {
2141 PHI.addReg(FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB);
2142 break;
2143 }
2144 }
2145 }
2146 }
2147 }
2148 }
2149 SDB->SwitchCases.clear();
2150}
2151
2152/// Create the scheduler. If a specific scheduler was specified
2153/// via the SchedulerRegistry, use it, otherwise select the
2154/// one preferred by the target.
2155///
2156ScheduleDAGSDNodes *SelectionDAGISel::CreateScheduler() {
2157 return ISHeuristic(this, OptLevel);
2158}
2159
2160//===----------------------------------------------------------------------===//
2161// Helper functions used by the generated instruction selector.
2162//===----------------------------------------------------------------------===//
2163// Calls to these methods are generated by tblgen.
2164
2165/// CheckAndMask - The isel is trying to match something like (and X, 255). If
2166/// the dag combiner simplified the 255, we still want to match. RHS is the
2167/// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value
2168/// specified in the .td file (e.g. 255).
2169bool SelectionDAGISel::CheckAndMask(SDValue LHS, ConstantSDNode *RHS,
2170 int64_t DesiredMaskS) const {
2171 const APInt &ActualMask = RHS->getAPIntValue();
2172 const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
2173
2174 // If the actual mask exactly matches, success!
2175 if (ActualMask == DesiredMask)
2176 return true;
2177
2178 // If the actual AND mask is allowing unallowed bits, this doesn't match.
2179 if (!ActualMask.isSubsetOf(DesiredMask))
2180 return false;
2181
2182 // Otherwise, the DAG Combiner may have proven that the value coming in is
2183 // either already zero or is not demanded. Check for known zero input bits.
2184 APInt NeededMask = DesiredMask & ~ActualMask;
2185 if (CurDAG->MaskedValueIsZero(LHS, NeededMask))
2186 return true;
2187
2188 // TODO: check to see if missing bits are just not demanded.
2189
2190 // Otherwise, this pattern doesn't match.
2191 return false;
2192}
2193
2194/// CheckOrMask - The isel is trying to match something like (or X, 255). If
2195/// the dag combiner simplified the 255, we still want to match. RHS is the
2196/// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value
2197/// specified in the .td file (e.g. 255).
2198bool SelectionDAGISel::CheckOrMask(SDValue LHS, ConstantSDNode *RHS,
2199 int64_t DesiredMaskS) const {
2200 const APInt &ActualMask = RHS->getAPIntValue();
2201 const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
2202
2203 // If the actual mask exactly matches, success!
2204 if (ActualMask == DesiredMask)
2205 return true;
2206
2207 // If the actual AND mask is allowing unallowed bits, this doesn't match.
2208 if (!ActualMask.isSubsetOf(DesiredMask))
2209 return false;
2210
2211 // Otherwise, the DAG Combiner may have proven that the value coming in is
2212 // either already zero or is not demanded. Check for known zero input bits.
2213 APInt NeededMask = DesiredMask & ~ActualMask;
2214 KnownBits Known = CurDAG->computeKnownBits(LHS);
2215
2216 // If all the missing bits in the or are already known to be set, match!
2217 if (NeededMask.isSubsetOf(Known.One))
2218 return true;
2219
2220 // TODO: check to see if missing bits are just not demanded.
2221
2222 // Otherwise, this pattern doesn't match.
2223 return false;
2224}
2225
2226/// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
2227/// by tblgen. Others should not call it.
2228void SelectionDAGISel::SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops,
2229 const SDLoc &DL) {
2230 std::vector<SDValue> InOps;
2231 std::swap(InOps, Ops);
2232
2233 Ops.push_back(InOps[InlineAsm::Op_InputChain]); // 0
2234 Ops.push_back(InOps[InlineAsm::Op_AsmString]); // 1
2235 Ops.push_back(InOps[InlineAsm::Op_MDNode]); // 2, !srcloc
2236 Ops.push_back(InOps[InlineAsm::Op_ExtraInfo]); // 3 (SideEffect, AlignStack)
2237
2238 unsigned i = InlineAsm::Op_FirstOperand, e = InOps.size();
2239 if (InOps[e-1].getValueType() == MVT::Glue)
2240 --e; // Don't process a glue operand if it is here.
2241
2242 while (i != e) {
2243 unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue();
2244 if (!InlineAsm::isMemKind(Flags)) {
2245 // Just skip over this operand, copying the operands verbatim.
2246 Ops.insert(Ops.end(), InOps.begin()+i,
2247 InOps.begin()+i+InlineAsm::getNumOperandRegisters(Flags) + 1);
2248 i += InlineAsm::getNumOperandRegisters(Flags) + 1;
2249 } else {
2250 assert(InlineAsm::getNumOperandRegisters(Flags) == 1 &&((InlineAsm::getNumOperandRegisters(Flags) == 1 && "Memory operand with multiple values?"
) ? static_cast<void> (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(Flags) == 1 && \"Memory operand with multiple values?\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2251, __PRETTY_FUNCTION__))
2251 "Memory operand with multiple values?")((InlineAsm::getNumOperandRegisters(Flags) == 1 && "Memory operand with multiple values?"
) ? static_cast<void> (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(Flags) == 1 && \"Memory operand with multiple values?\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2251, __PRETTY_FUNCTION__))
;
2252
2253 unsigned TiedToOperand;
2254 if (InlineAsm::isUseOperandTiedToDef(Flags, TiedToOperand)) {
2255 // We need the constraint ID from the operand this is tied to.
2256 unsigned CurOp = InlineAsm::Op_FirstOperand;
2257 Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
2258 for (; TiedToOperand; --TiedToOperand) {
2259 CurOp += InlineAsm::getNumOperandRegisters(Flags)+1;
2260 Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
2261 }
2262 }
2263
2264 // Otherwise, this is a memory operand. Ask the target to select it.
2265 std::vector<SDValue> SelOps;
2266 unsigned ConstraintID = InlineAsm::getMemoryConstraintID(Flags);
2267 if (SelectInlineAsmMemoryOperand(InOps[i+1], ConstraintID, SelOps))
2268 report_fatal_error("Could not match memory address. Inline asm"
2269 " failure!");
2270
2271 // Add this to the output node.
2272 unsigned NewFlags =
2273 InlineAsm::getFlagWord(InlineAsm::Kind_Mem, SelOps.size());
2274 NewFlags = InlineAsm::getFlagWordForMem(NewFlags, ConstraintID);
2275 Ops.push_back(CurDAG->getTargetConstant(NewFlags, DL, MVT::i32));
2276 Ops.insert(Ops.end(), SelOps.begin(), SelOps.end());
2277 i += 2;
2278 }
2279 }
2280
2281 // Add the glue input back if present.
2282 if (e != InOps.size())
2283 Ops.push_back(InOps.back());
2284}
2285
2286/// findGlueUse - Return use of MVT::Glue value produced by the specified
2287/// SDNode.
2288///
2289static SDNode *findGlueUse(SDNode *N) {
2290 unsigned FlagResNo = N->getNumValues()-1;
2291 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
2292 SDUse &Use = I.getUse();
2293 if (Use.getResNo() == FlagResNo)
2294 return Use.getUser();
2295 }
2296 return nullptr;
2297}
2298
2299/// findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path
2300/// beyond "ImmedUse". We may ignore chains as they are checked separately.
2301static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse,
2302 bool IgnoreChains) {
2303 SmallPtrSet<const SDNode *, 16> Visited;
2304 SmallVector<const SDNode *, 16> WorkList;
2305 // Only check if we have non-immediate uses of Def.
2306 if (ImmedUse->isOnlyUserOf(Def))
2307 return false;
2308
2309 // We don't care about paths to Def that go through ImmedUse so mark it
2310 // visited and mark non-def operands as used.
2311 Visited.insert(ImmedUse);
2312 for (const SDValue &Op : ImmedUse->op_values()) {
2313 SDNode *N = Op.getNode();
2314 // Ignore chain deps (they are validated by
2315 // HandleMergeInputChains) and immediate uses
2316 if ((Op.getValueType() == MVT::Other && IgnoreChains) || N == Def)
2317 continue;
2318 if (!Visited.insert(N).second)
2319 continue;
2320 WorkList.push_back(N);
2321 }
2322
2323 // Initialize worklist to operands of Root.
2324 if (Root != ImmedUse) {
2325 for (const SDValue &Op : Root->op_values()) {
2326 SDNode *N = Op.getNode();
2327 // Ignore chains (they are validated by HandleMergeInputChains)
2328 if ((Op.getValueType() == MVT::Other && IgnoreChains) || N == Def)
2329 continue;
2330 if (!Visited.insert(N).second)
2331 continue;
2332 WorkList.push_back(N);
2333 }
2334 }
2335
2336 return SDNode::hasPredecessorHelper(Def, Visited, WorkList, 0, true);
2337}
2338
2339/// IsProfitableToFold - Returns true if it's profitable to fold the specific
2340/// operand node N of U during instruction selection that starts at Root.
2341bool SelectionDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
2342 SDNode *Root) const {
2343 if (OptLevel == CodeGenOpt::None) return false;
2344 return N.hasOneUse();
2345}
2346
2347/// IsLegalToFold - Returns true if the specific operand node N of
2348/// U can be folded during instruction selection that starts at Root.
2349bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root,
2350 CodeGenOpt::Level OptLevel,
2351 bool IgnoreChains) {
2352 if (OptLevel == CodeGenOpt::None) return false;
2353
2354 // If Root use can somehow reach N through a path that that doesn't contain
2355 // U then folding N would create a cycle. e.g. In the following
2356 // diagram, Root can reach N through X. If N is folded into Root, then
2357 // X is both a predecessor and a successor of U.
2358 //
2359 // [N*] //
2360 // ^ ^ //
2361 // / \ //
2362 // [U*] [X]? //
2363 // ^ ^ //
2364 // \ / //
2365 // \ / //
2366 // [Root*] //
2367 //
2368 // * indicates nodes to be folded together.
2369 //
2370 // If Root produces glue, then it gets (even more) interesting. Since it
2371 // will be "glued" together with its glue use in the scheduler, we need to
2372 // check if it might reach N.
2373 //
2374 // [N*] //
2375 // ^ ^ //
2376 // / \ //
2377 // [U*] [X]? //
2378 // ^ ^ //
2379 // \ \ //
2380 // \ | //
2381 // [Root*] | //
2382 // ^ | //
2383 // f | //
2384 // | / //
2385 // [Y] / //
2386 // ^ / //
2387 // f / //
2388 // | / //
2389 // [GU] //
2390 //
2391 // If GU (glue use) indirectly reaches N (the load), and Root folds N
2392 // (call it Fold), then X is a predecessor of GU and a successor of
2393 // Fold. But since Fold and GU are glued together, this will create
2394 // a cycle in the scheduling graph.
2395
2396 // If the node has glue, walk down the graph to the "lowest" node in the
2397 // glueged set.
2398 EVT VT = Root->getValueType(Root->getNumValues()-1);
2399 while (VT == MVT::Glue) {
2400 SDNode *GU = findGlueUse(Root);
2401 if (!GU)
2402 break;
2403 Root = GU;
2404 VT = Root->getValueType(Root->getNumValues()-1);
2405
2406 // If our query node has a glue result with a use, we've walked up it. If
2407 // the user (which has already been selected) has a chain or indirectly uses
2408 // the chain, HandleMergeInputChains will not consider it. Because of
2409 // this, we cannot ignore chains in this predicate.
2410 IgnoreChains = false;
2411 }
2412
2413 return !findNonImmUse(Root, N.getNode(), U, IgnoreChains);
2414}
2415
2416void SelectionDAGISel::Select_INLINEASM(SDNode *N) {
2417 SDLoc DL(N);
2418
2419 std::vector<SDValue> Ops(N->op_begin(), N->op_end());
2420 SelectInlineAsmMemoryOperands(Ops, DL);
2421
2422 const EVT VTs[] = {MVT::Other, MVT::Glue};
2423 SDValue New = CurDAG->getNode(ISD::INLINEASM, DL, VTs, Ops);
2424 New->setNodeId(-1);
2425 ReplaceUses(N, New.getNode());
2426 CurDAG->RemoveDeadNode(N);
2427}
2428
2429void SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) {
2430 SDLoc dl(Op);
2431 MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(Op->getOperand(1));
2432 const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));
2433 unsigned Reg =
2434 TLI->getRegisterByName(RegStr->getString().data(), Op->getValueType(0),
2435 *CurDAG);
2436 SDValue New = CurDAG->getCopyFromReg(
2437 Op->getOperand(0), dl, Reg, Op->getValueType(0));
2438 New->setNodeId(-1);
2439 ReplaceUses(Op, New.getNode());
2440 CurDAG->RemoveDeadNode(Op);
2441}
2442
2443void SelectionDAGISel::Select_WRITE_REGISTER(SDNode *Op) {
2444 SDLoc dl(Op);
2445 MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(Op->getOperand(1));
2446 const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));
2447 unsigned Reg = TLI->getRegisterByName(RegStr->getString().data(),
2448 Op->getOperand(2).getValueType(),
2449 *CurDAG);
2450 SDValue New = CurDAG->getCopyToReg(
2451 Op->getOperand(0), dl, Reg, Op->getOperand(2));
2452 New->setNodeId(-1);
2453 ReplaceUses(Op, New.getNode());
2454 CurDAG->RemoveDeadNode(Op);
2455}
2456
2457void SelectionDAGISel::Select_UNDEF(SDNode *N) {
2458 CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0));
2459}
2460
2461/// GetVBR - decode a vbr encoding whose top bit is set.
2462LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline uint64_t
2463GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) {
2464 assert(Val >= 128 && "Not a VBR")((Val >= 128 && "Not a VBR") ? static_cast<void
> (0) : __assert_fail ("Val >= 128 && \"Not a VBR\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2464, __PRETTY_FUNCTION__))
;
2465 Val &= 127; // Remove first vbr bit.
2466
2467 unsigned Shift = 7;
2468 uint64_t NextBits;
2469 do {
2470 NextBits = MatcherTable[Idx++];
2471 Val |= (NextBits&127) << Shift;
2472 Shift += 7;
2473 } while (NextBits & 128);
2474
2475 return Val;
2476}
2477
2478/// When a match is complete, this method updates uses of interior chain results
2479/// to use the new results.
2480void SelectionDAGISel::UpdateChains(
2481 SDNode *NodeToMatch, SDValue InputChain,
2482 SmallVectorImpl<SDNode *> &ChainNodesMatched, bool isMorphNodeTo) {
2483 SmallVector<SDNode*, 4> NowDeadNodes;
2484
2485 // Now that all the normal results are replaced, we replace the chain and
2486 // glue results if present.
2487 if (!ChainNodesMatched.empty()) {
2488 assert(InputChain.getNode() &&((InputChain.getNode() && "Matched input chains but didn't produce a chain"
) ? static_cast<void> (0) : __assert_fail ("InputChain.getNode() && \"Matched input chains but didn't produce a chain\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2489, __PRETTY_FUNCTION__))
2489 "Matched input chains but didn't produce a chain")((InputChain.getNode() && "Matched input chains but didn't produce a chain"
) ? static_cast<void> (0) : __assert_fail ("InputChain.getNode() && \"Matched input chains but didn't produce a chain\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2489, __PRETTY_FUNCTION__))
;
2490 // Loop over all of the nodes we matched that produced a chain result.
2491 // Replace all the chain results with the final chain we ended up with.
2492 for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
2493 SDNode *ChainNode = ChainNodesMatched[i];
2494 // If ChainNode is null, it's because we replaced it on a previous
2495 // iteration and we cleared it out of the map. Just skip it.
2496 if (!ChainNode)
2497 continue;
2498
2499 assert(ChainNode->getOpcode() != ISD::DELETED_NODE &&((ChainNode->getOpcode() != ISD::DELETED_NODE && "Deleted node left in chain"
) ? static_cast<void> (0) : __assert_fail ("ChainNode->getOpcode() != ISD::DELETED_NODE && \"Deleted node left in chain\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2500, __PRETTY_FUNCTION__))
2500 "Deleted node left in chain")((ChainNode->getOpcode() != ISD::DELETED_NODE && "Deleted node left in chain"
) ? static_cast<void> (0) : __assert_fail ("ChainNode->getOpcode() != ISD::DELETED_NODE && \"Deleted node left in chain\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2500, __PRETTY_FUNCTION__))
;
2501
2502 // Don't replace the results of the root node if we're doing a
2503 // MorphNodeTo.
2504 if (ChainNode == NodeToMatch && isMorphNodeTo)
2505 continue;
2506
2507 SDValue ChainVal = SDValue(ChainNode, ChainNode->getNumValues()-1);
2508 if (ChainVal.getValueType() == MVT::Glue)
2509 ChainVal = ChainVal.getValue(ChainVal->getNumValues()-2);
2510 assert(ChainVal.getValueType() == MVT::Other && "Not a chain?")((ChainVal.getValueType() == MVT::Other && "Not a chain?"
) ? static_cast<void> (0) : __assert_fail ("ChainVal.getValueType() == MVT::Other && \"Not a chain?\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2510, __PRETTY_FUNCTION__))
;
2511 SelectionDAG::DAGNodeDeletedListener NDL(
2512 *CurDAG, [&](SDNode *N, SDNode *E) {
2513 std::replace(ChainNodesMatched.begin(), ChainNodesMatched.end(), N,
2514 static_cast<SDNode *>(nullptr));
2515 });
2516 if (ChainNode->getOpcode() != ISD::TokenFactor)
2517 ReplaceUses(ChainVal, InputChain);
2518
2519 // If the node became dead and we haven't already seen it, delete it.
2520 if (ChainNode != NodeToMatch && ChainNode->use_empty() &&
2521 !std::count(NowDeadNodes.begin(), NowDeadNodes.end(), ChainNode))
2522 NowDeadNodes.push_back(ChainNode);
2523 }
2524 }
2525
2526 if (!NowDeadNodes.empty())
2527 CurDAG->RemoveDeadNodes(NowDeadNodes);
2528
2529 LLVM_DEBUG(dbgs() << "ISEL: Match complete!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "ISEL: Match complete!\n"; } } while
(false)
;
2530}
2531
2532/// HandleMergeInputChains - This implements the OPC_EmitMergeInputChains
2533/// operation for when the pattern matched at least one node with a chains. The
2534/// input vector contains a list of all of the chained nodes that we match. We
2535/// must determine if this is a valid thing to cover (i.e. matching it won't
2536/// induce cycles in the DAG) and if so, creating a TokenFactor node. that will
2537/// be used as the input node chain for the generated nodes.
2538static SDValue
2539HandleMergeInputChains(SmallVectorImpl<SDNode*> &ChainNodesMatched,
2540 SelectionDAG *CurDAG) {
2541
2542 SmallPtrSet<const SDNode *, 16> Visited;
2543 SmallVector<const SDNode *, 8> Worklist;
2544 SmallVector<SDValue, 3> InputChains;
2545 unsigned int Max = 8192;
2546
2547 // Quick exit on trivial merge.
2548 if (ChainNodesMatched.size() == 1)
2549 return ChainNodesMatched[0]->getOperand(0);
2550
2551 // Add chains that aren't already added (internal). Peek through
2552 // token factors.
2553 std::function<void(const SDValue)> AddChains = [&](const SDValue V) {
2554 if (V.getValueType() != MVT::Other)
2555 return;
2556 if (V->getOpcode() == ISD::EntryToken)
2557 return;
2558 if (!Visited.insert(V.getNode()).second)
2559 return;
2560 if (V->getOpcode() == ISD::TokenFactor) {
2561 for (const SDValue &Op : V->op_values())
2562 AddChains(Op);
2563 } else
2564 InputChains.push_back(V);
2565 };
2566
2567 for (auto *N : ChainNodesMatched) {
2568 Worklist.push_back(N);
2569 Visited.insert(N);
2570 }
2571
2572 while (!Worklist.empty())
2573 AddChains(Worklist.pop_back_val()->getOperand(0));
2574
2575 // Skip the search if there are no chain dependencies.
2576 if (InputChains.size() == 0)
2577 return CurDAG->getEntryNode();
2578
2579 // If one of these chains is a successor of input, we must have a
2580 // node that is both the predecessor and successor of the
2581 // to-be-merged nodes. Fail.
2582 Visited.clear();
2583 for (SDValue V : InputChains)
2584 Worklist.push_back(V.getNode());
2585
2586 for (auto *N : ChainNodesMatched)
2587 if (SDNode::hasPredecessorHelper(N, Visited, Worklist, Max, true))
2588 return SDValue();
2589
2590 // Return merged chain.
2591 if (InputChains.size() == 1)
2592 return InputChains[0];
2593 return CurDAG->getNode(ISD::TokenFactor, SDLoc(ChainNodesMatched[0]),
2594 MVT::Other, InputChains);
2595}
2596
2597/// MorphNode - Handle morphing a node in place for the selector.
2598SDNode *SelectionDAGISel::
2599MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
2600 ArrayRef<SDValue> Ops, unsigned EmitNodeInfo) {
2601 // It is possible we're using MorphNodeTo to replace a node with no
2602 // normal results with one that has a normal result (or we could be
2603 // adding a chain) and the input could have glue and chains as well.
2604 // In this case we need to shift the operands down.
2605 // FIXME: This is a horrible hack and broken in obscure cases, no worse
2606 // than the old isel though.
2607 int OldGlueResultNo = -1, OldChainResultNo = -1;
2608
2609 unsigned NTMNumResults = Node->getNumValues();
2610 if (Node->getValueType(NTMNumResults-1) == MVT::Glue) {
2611 OldGlueResultNo = NTMNumResults-1;
2612 if (NTMNumResults != 1 &&
2613 Node->getValueType(NTMNumResults-2) == MVT::Other)
2614 OldChainResultNo = NTMNumResults-2;
2615 } else if (Node->getValueType(NTMNumResults-1) == MVT::Other)
2616 OldChainResultNo = NTMNumResults-1;
2617
2618 // Call the underlying SelectionDAG routine to do the transmogrification. Note
2619 // that this deletes operands of the old node that become dead.
2620 SDNode *Res = CurDAG->MorphNodeTo(Node, ~TargetOpc, VTList, Ops);
2621
2622 // MorphNodeTo can operate in two ways: if an existing node with the
2623 // specified operands exists, it can just return it. Otherwise, it
2624 // updates the node in place to have the requested operands.
2625 if (Res == Node) {
2626 // If we updated the node in place, reset the node ID. To the isel,
2627 // this should be just like a newly allocated machine node.
2628 Res->setNodeId(-1);
2629 }
2630
2631 unsigned ResNumResults = Res->getNumValues();
2632 // Move the glue if needed.
2633 if ((EmitNodeInfo & OPFL_GlueOutput) && OldGlueResultNo != -1 &&
2634 (unsigned)OldGlueResultNo != ResNumResults-1)
2635 ReplaceUses(SDValue(Node, OldGlueResultNo),
2636 SDValue(Res, ResNumResults - 1));
2637
2638 if ((EmitNodeInfo & OPFL_GlueOutput) != 0)
2639 --ResNumResults;
2640
2641 // Move the chain reference if needed.
2642 if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 &&
2643 (unsigned)OldChainResultNo != ResNumResults-1)
2644 ReplaceUses(SDValue(Node, OldChainResultNo),
2645 SDValue(Res, ResNumResults - 1));
2646
2647 // Otherwise, no replacement happened because the node already exists. Replace
2648 // Uses of the old node with the new one.
2649 if (Res != Node) {
2650 ReplaceNode(Node, Res);
2651 } else {
2652 EnforceNodeIdInvariant(Res);
2653 }
2654
2655 return Res;
2656}
2657
2658/// CheckSame - Implements OP_CheckSame.
2659LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2660CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2661 SDValue N,
2662 const SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) {
2663 // Accept if it is exactly the same as a previously recorded node.
2664 unsigned RecNo = MatcherTable[MatcherIndex++];
2665 assert(RecNo < RecordedNodes.size() && "Invalid CheckSame")((RecNo < RecordedNodes.size() && "Invalid CheckSame"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid CheckSame\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2665, __PRETTY_FUNCTION__))
;
2666 return N == RecordedNodes[RecNo].first;
2667}
2668
2669/// CheckChildSame - Implements OP_CheckChildXSame.
2670LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2671CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2672 SDValue N,
2673 const SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes,
2674 unsigned ChildNo) {
2675 if (ChildNo >= N.getNumOperands())
2676 return false; // Match fails if out of range child #.
2677 return ::CheckSame(MatcherTable, MatcherIndex, N.getOperand(ChildNo),
2678 RecordedNodes);
2679}
2680
2681/// CheckPatternPredicate - Implements OP_CheckPatternPredicate.
2682LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2683CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2684 const SelectionDAGISel &SDISel) {
2685 return SDISel.CheckPatternPredicate(MatcherTable[MatcherIndex++]);
2686}
2687
2688/// CheckNodePredicate - Implements OP_CheckNodePredicate.
2689LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2690CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2691 const SelectionDAGISel &SDISel, SDNode *N) {
2692 return SDISel.CheckNodePredicate(N, MatcherTable[MatcherIndex++]);
2693}
2694
2695LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2696CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2697 SDNode *N) {
2698 uint16_t Opc = MatcherTable[MatcherIndex++];
2699 Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
2700 return N->getOpcode() == Opc;
2701}
2702
2703LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2704CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2705 const TargetLowering *TLI, const DataLayout &DL) {
2706 MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2707 if (N.getValueType() == VT) return true;
2708
2709 // Handle the case when VT is iPTR.
2710 return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(DL);
2711}
2712
2713LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2714CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2715 SDValue N, const TargetLowering *TLI, const DataLayout &DL,
2716 unsigned ChildNo) {
2717 if (ChildNo >= N.getNumOperands())
2718 return false; // Match fails if out of range child #.
2719 return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI,
2720 DL);
2721}
2722
2723LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2724CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2725 SDValue N) {
2726 return cast<CondCodeSDNode>(N)->get() ==
2727 (ISD::CondCode)MatcherTable[MatcherIndex++];
2728}
2729
2730LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2731CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2732 SDValue N, const TargetLowering *TLI, const DataLayout &DL) {
2733 MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2734 if (cast<VTSDNode>(N)->getVT() == VT)
2735 return true;
2736
2737 // Handle the case when VT is iPTR.
2738 return VT == MVT::iPTR && cast<VTSDNode>(N)->getVT() == TLI->getPointerTy(DL);
2739}
2740
2741LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2742CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2743 SDValue N) {
2744 int64_t Val = MatcherTable[MatcherIndex++];
2745 if (Val & 128)
2746 Val = GetVBR(Val, MatcherTable, MatcherIndex);
2747
2748 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N);
2749 return C && C->getSExtValue() == Val;
2750}
2751
2752LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2753CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2754 SDValue N, unsigned ChildNo) {
2755 if (ChildNo >= N.getNumOperands())
2756 return false; // Match fails if out of range child #.
2757 return ::CheckInteger(MatcherTable, MatcherIndex, N.getOperand(ChildNo));
2758}
2759
2760LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2761CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2762 SDValue N, const SelectionDAGISel &SDISel) {
2763 int64_t Val = MatcherTable[MatcherIndex++];
2764 if (Val & 128)
2765 Val = GetVBR(Val, MatcherTable, MatcherIndex);
2766
2767 if (N->getOpcode() != ISD::AND) return false;
2768
2769 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2770 return C && SDISel.CheckAndMask(N.getOperand(0), C, Val);
2771}
2772
2773LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool
2774CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2775 SDValue N, const SelectionDAGISel &SDISel) {
2776 int64_t Val = MatcherTable[MatcherIndex++];
2777 if (Val & 128)
2778 Val = GetVBR(Val, MatcherTable, MatcherIndex);
2779
2780 if (N->getOpcode() != ISD::OR) return false;
2781
2782 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2783 return C && SDISel.CheckOrMask(N.getOperand(0), C, Val);
2784}
2785
2786/// IsPredicateKnownToFail - If we know how and can do so without pushing a
2787/// scope, evaluate the current node. If the current predicate is known to
2788/// fail, set Result=true and return anything. If the current predicate is
2789/// known to pass, set Result=false and return the MatcherIndex to continue
2790/// with. If the current predicate is unknown, set Result=false and return the
2791/// MatcherIndex to continue with.
2792static unsigned IsPredicateKnownToFail(const unsigned char *Table,
2793 unsigned Index, SDValue N,
2794 bool &Result,
2795 const SelectionDAGISel &SDISel,
2796 SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) {
2797 switch (Table[Index++]) {
2798 default:
2799 Result = false;
2800 return Index-1; // Could not evaluate this predicate.
2801 case SelectionDAGISel::OPC_CheckSame:
2802 Result = !::CheckSame(Table, Index, N, RecordedNodes);
2803 return Index;
2804 case SelectionDAGISel::OPC_CheckChild0Same:
2805 case SelectionDAGISel::OPC_CheckChild1Same:
2806 case SelectionDAGISel::OPC_CheckChild2Same:
2807 case SelectionDAGISel::OPC_CheckChild3Same:
2808 Result = !::CheckChildSame(Table, Index, N, RecordedNodes,
2809 Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Same);
2810 return Index;
2811 case SelectionDAGISel::OPC_CheckPatternPredicate:
2812 Result = !::CheckPatternPredicate(Table, Index, SDISel);
2813 return Index;
2814 case SelectionDAGISel::OPC_CheckPredicate:
2815 Result = !::CheckNodePredicate(Table, Index, SDISel, N.getNode());
2816 return Index;
2817 case SelectionDAGISel::OPC_CheckOpcode:
2818 Result = !::CheckOpcode(Table, Index, N.getNode());
2819 return Index;
2820 case SelectionDAGISel::OPC_CheckType:
2821 Result = !::CheckType(Table, Index, N, SDISel.TLI,
2822 SDISel.CurDAG->getDataLayout());
2823 return Index;
2824 case SelectionDAGISel::OPC_CheckTypeRes: {
2825 unsigned Res = Table[Index++];
2826 Result = !::CheckType(Table, Index, N.getValue(Res), SDISel.TLI,
2827 SDISel.CurDAG->getDataLayout());
2828 return Index;
2829 }
2830 case SelectionDAGISel::OPC_CheckChild0Type:
2831 case SelectionDAGISel::OPC_CheckChild1Type:
2832 case SelectionDAGISel::OPC_CheckChild2Type:
2833 case SelectionDAGISel::OPC_CheckChild3Type:
2834 case SelectionDAGISel::OPC_CheckChild4Type:
2835 case SelectionDAGISel::OPC_CheckChild5Type:
2836 case SelectionDAGISel::OPC_CheckChild6Type:
2837 case SelectionDAGISel::OPC_CheckChild7Type:
2838 Result = !::CheckChildType(
2839 Table, Index, N, SDISel.TLI, SDISel.CurDAG->getDataLayout(),
2840 Table[Index - 1] - SelectionDAGISel::OPC_CheckChild0Type);
2841 return Index;
2842 case SelectionDAGISel::OPC_CheckCondCode:
2843 Result = !::CheckCondCode(Table, Index, N);
2844 return Index;
2845 case SelectionDAGISel::OPC_CheckValueType:
2846 Result = !::CheckValueType(Table, Index, N, SDISel.TLI,
2847 SDISel.CurDAG->getDataLayout());
2848 return Index;
2849 case SelectionDAGISel::OPC_CheckInteger:
2850 Result = !::CheckInteger(Table, Index, N);
2851 return Index;
2852 case SelectionDAGISel::OPC_CheckChild0Integer:
2853 case SelectionDAGISel::OPC_CheckChild1Integer:
2854 case SelectionDAGISel::OPC_CheckChild2Integer:
2855 case SelectionDAGISel::OPC_CheckChild3Integer:
2856 case SelectionDAGISel::OPC_CheckChild4Integer:
2857 Result = !::CheckChildInteger(Table, Index, N,
2858 Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Integer);
2859 return Index;
2860 case SelectionDAGISel::OPC_CheckAndImm:
2861 Result = !::CheckAndImm(Table, Index, N, SDISel);
2862 return Index;
2863 case SelectionDAGISel::OPC_CheckOrImm:
2864 Result = !::CheckOrImm(Table, Index, N, SDISel);
2865 return Index;
2866 }
2867}
2868
2869namespace {
2870
2871struct MatchScope {
2872 /// FailIndex - If this match fails, this is the index to continue with.
2873 unsigned FailIndex;
2874
2875 /// NodeStack - The node stack when the scope was formed.
2876 SmallVector<SDValue, 4> NodeStack;
2877
2878 /// NumRecordedNodes - The number of recorded nodes when the scope was formed.
2879 unsigned NumRecordedNodes;
2880
2881 /// NumMatchedMemRefs - The number of matched memref entries.
2882 unsigned NumMatchedMemRefs;
2883
2884 /// InputChain/InputGlue - The current chain/glue
2885 SDValue InputChain, InputGlue;
2886
2887 /// HasChainNodesMatched - True if the ChainNodesMatched list is non-empty.
2888 bool HasChainNodesMatched;
2889};
2890
2891/// \A DAG update listener to keep the matching state
2892/// (i.e. RecordedNodes and MatchScope) uptodate if the target is allowed to
2893/// change the DAG while matching. X86 addressing mode matcher is an example
2894/// for this.
2895class MatchStateUpdater : public SelectionDAG::DAGUpdateListener
2896{
2897 SDNode **NodeToMatch;
2898 SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes;
2899 SmallVectorImpl<MatchScope> &MatchScopes;
2900
2901public:
2902 MatchStateUpdater(SelectionDAG &DAG, SDNode **NodeToMatch,
2903 SmallVectorImpl<std::pair<SDValue, SDNode *>> &RN,
2904 SmallVectorImpl<MatchScope> &MS)
2905 : SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch),
2906 RecordedNodes(RN), MatchScopes(MS) {}
2907
2908 void NodeDeleted(SDNode *N, SDNode *E) override {
2909 // Some early-returns here to avoid the search if we deleted the node or
2910 // if the update comes from MorphNodeTo (MorphNodeTo is the last thing we
2911 // do, so it's unnecessary to update matching state at that point).
2912 // Neither of these can occur currently because we only install this
2913 // update listener during matching a complex patterns.
2914 if (!E || E->isMachineOpcode())
2915 return;
2916 // Check if NodeToMatch was updated.
2917 if (N == *NodeToMatch)
2918 *NodeToMatch = E;
2919 // Performing linear search here does not matter because we almost never
2920 // run this code. You'd have to have a CSE during complex pattern
2921 // matching.
2922 for (auto &I : RecordedNodes)
2923 if (I.first.getNode() == N)
2924 I.first.setNode(E);
2925
2926 for (auto &I : MatchScopes)
2927 for (auto &J : I.NodeStack)
2928 if (J.getNode() == N)
2929 J.setNode(E);
2930 }
2931};
2932
2933} // end anonymous namespace
2934
2935void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
2936 const unsigned char *MatcherTable,
2937 unsigned TableSize) {
2938 // FIXME: Should these even be selected? Handle these cases in the caller?
2939 switch (NodeToMatch->getOpcode()) {
2940 default:
2941 break;
2942 case ISD::EntryToken: // These nodes remain the same.
2943 case ISD::BasicBlock:
2944 case ISD::Register:
2945 case ISD::RegisterMask:
2946 case ISD::HANDLENODE:
2947 case ISD::MDNODE_SDNODE:
2948 case ISD::TargetConstant:
2949 case ISD::TargetConstantFP:
2950 case ISD::TargetConstantPool:
2951 case ISD::TargetFrameIndex:
2952 case ISD::TargetExternalSymbol:
2953 case ISD::MCSymbol:
2954 case ISD::TargetBlockAddress:
2955 case ISD::TargetJumpTable:
2956 case ISD::TargetGlobalTLSAddress:
2957 case ISD::TargetGlobalAddress:
2958 case ISD::TokenFactor:
2959 case ISD::CopyFromReg:
2960 case ISD::CopyToReg:
2961 case ISD::EH_LABEL:
2962 case ISD::ANNOTATION_LABEL:
2963 case ISD::LIFETIME_START:
2964 case ISD::LIFETIME_END:
2965 NodeToMatch->setNodeId(-1); // Mark selected.
2966 return;
2967 case ISD::AssertSext:
2968 case ISD::AssertZext:
2969 ReplaceUses(SDValue(NodeToMatch, 0), NodeToMatch->getOperand(0));
2970 CurDAG->RemoveDeadNode(NodeToMatch);
2971 return;
2972 case ISD::INLINEASM:
2973 Select_INLINEASM(NodeToMatch);
2974 return;
2975 case ISD::READ_REGISTER:
2976 Select_READ_REGISTER(NodeToMatch);
2977 return;
2978 case ISD::WRITE_REGISTER:
2979 Select_WRITE_REGISTER(NodeToMatch);
2980 return;
2981 case ISD::UNDEF:
2982 Select_UNDEF(NodeToMatch);
2983 return;
2984 }
2985
2986 assert(!NodeToMatch->isMachineOpcode() && "Node already selected!")((!NodeToMatch->isMachineOpcode() && "Node already selected!"
) ? static_cast<void> (0) : __assert_fail ("!NodeToMatch->isMachineOpcode() && \"Node already selected!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2986, __PRETTY_FUNCTION__))
;
2987
2988 // Set up the node stack with NodeToMatch as the only node on the stack.
2989 SmallVector<SDValue, 8> NodeStack;
2990 SDValue N = SDValue(NodeToMatch, 0);
2991 NodeStack.push_back(N);
2992
2993 // MatchScopes - Scopes used when matching, if a match failure happens, this
2994 // indicates where to continue checking.
2995 SmallVector<MatchScope, 8> MatchScopes;
2996
2997 // RecordedNodes - This is the set of nodes that have been recorded by the
2998 // state machine. The second value is the parent of the node, or null if the
2999 // root is recorded.
3000 SmallVector<std::pair<SDValue, SDNode*>, 8> RecordedNodes;
3001
3002 // MatchedMemRefs - This is the set of MemRef's we've seen in the input
3003 // pattern.
3004 SmallVector<MachineMemOperand*, 2> MatchedMemRefs;
3005
3006 // These are the current input chain and glue for use when generating nodes.
3007 // Various Emit operations change these. For example, emitting a copytoreg
3008 // uses and updates these.
3009 SDValue InputChain, InputGlue;
3010
3011 // ChainNodesMatched - If a pattern matches nodes that have input/output
3012 // chains, the OPC_EmitMergeInputChains operation is emitted which indicates
3013 // which ones they are. The result is captured into this list so that we can
3014 // update the chain results when the pattern is complete.
3015 SmallVector<SDNode*, 3> ChainNodesMatched;
3016
3017 LLVM_DEBUG(dbgs() << "ISEL: Starting pattern match\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "ISEL: Starting pattern match\n";
} } while (false)
;
3018
3019 // Determine where to start the interpreter. Normally we start at opcode #0,
3020 // but if the state machine starts with an OPC_SwitchOpcode, then we
3021 // accelerate the first lookup (which is guaranteed to be hot) with the
3022 // OpcodeOffset table.
3023 unsigned MatcherIndex = 0;
3024
3025 if (!OpcodeOffset.empty()) {
3026 // Already computed the OpcodeOffset table, just index into it.
3027 if (N.getOpcode() < OpcodeOffset.size())
3028 MatcherIndex = OpcodeOffset[N.getOpcode()];
3029 LLVM_DEBUG(dbgs() << " Initial Opcode index to " << MatcherIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Initial Opcode index to " <<
MatcherIndex << "\n"; } } while (false)
;
3030
3031 } else if (MatcherTable[0] == OPC_SwitchOpcode) {
3032 // Otherwise, the table isn't computed, but the state machine does start
3033 // with an OPC_SwitchOpcode instruction. Populate the table now, since this
3034 // is the first time we're selecting an instruction.
3035 unsigned Idx = 1;
3036 while (true) {
3037 // Get the size of this case.
3038 unsigned CaseSize = MatcherTable[Idx++];
3039 if (CaseSize & 128)
3040 CaseSize = GetVBR(CaseSize, MatcherTable, Idx);
3041 if (CaseSize == 0) break;
3042
3043 // Get the opcode, add the index to the table.
3044 uint16_t Opc = MatcherTable[Idx++];
3045 Opc |= (unsigned short)MatcherTable[Idx++] << 8;
3046 if (Opc >= OpcodeOffset.size())
3047 OpcodeOffset.resize((Opc+1)*2);
3048 OpcodeOffset[Opc] = Idx;
3049 Idx += CaseSize;
3050 }
3051
3052 // Okay, do the lookup for the first opcode.
3053 if (N.getOpcode() < OpcodeOffset.size())
3054 MatcherIndex = OpcodeOffset[N.getOpcode()];
3055 }
3056
3057 while (true) {
3058 assert(MatcherIndex < TableSize && "Invalid index")((MatcherIndex < TableSize && "Invalid index") ? static_cast
<void> (0) : __assert_fail ("MatcherIndex < TableSize && \"Invalid index\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3058, __PRETTY_FUNCTION__))
;
3059#ifndef NDEBUG
3060 unsigned CurrentOpcodeIndex = MatcherIndex;
3061#endif
3062 BuiltinOpcodes Opcode = (BuiltinOpcodes)MatcherTable[MatcherIndex++];
3063 switch (Opcode) {
3064 case OPC_Scope: {
3065 // Okay, the semantics of this operation are that we should push a scope
3066 // then evaluate the first child. However, pushing a scope only to have
3067 // the first check fail (which then pops it) is inefficient. If we can
3068 // determine immediately that the first check (or first several) will
3069 // immediately fail, don't even bother pushing a scope for them.
3070 unsigned FailIndex;
3071
3072 while (true) {
3073 unsigned NumToSkip = MatcherTable[MatcherIndex++];
3074 if (NumToSkip & 128)
3075 NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3076 // Found the end of the scope with no match.
3077 if (NumToSkip == 0) {
3078 FailIndex = 0;
3079 break;
3080 }
3081
3082 FailIndex = MatcherIndex+NumToSkip;
3083
3084 unsigned MatcherIndexOfPredicate = MatcherIndex;
3085 (void)MatcherIndexOfPredicate; // silence warning.
3086
3087 // If we can't evaluate this predicate without pushing a scope (e.g. if
3088 // it is a 'MoveParent') or if the predicate succeeds on this node, we
3089 // push the scope and evaluate the full predicate chain.
3090 bool Result;
3091 MatcherIndex = IsPredicateKnownToFail(MatcherTable, MatcherIndex, N,
3092 Result, *this, RecordedNodes);
3093 if (!Result)
3094 break;
3095
3096 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Skipped scope entry (due to false predicate) at "
<< "index " << MatcherIndexOfPredicate << ", continuing at "
<< FailIndex << "\n"; } } while (false)
3097 dbgs() << " Skipped scope entry (due to false predicate) at "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Skipped scope entry (due to false predicate) at "
<< "index " << MatcherIndexOfPredicate << ", continuing at "
<< FailIndex << "\n"; } } while (false)
3098 << "index " << MatcherIndexOfPredicate << ", continuing at "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Skipped scope entry (due to false predicate) at "
<< "index " << MatcherIndexOfPredicate << ", continuing at "
<< FailIndex << "\n"; } } while (false)
3099 << FailIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Skipped scope entry (due to false predicate) at "
<< "index " << MatcherIndexOfPredicate << ", continuing at "
<< FailIndex << "\n"; } } while (false)
;
3100 ++NumDAGIselRetries;
3101
3102 // Otherwise, we know that this case of the Scope is guaranteed to fail,
3103 // move to the next case.
3104 MatcherIndex = FailIndex;
3105 }
3106
3107 // If the whole scope failed to match, bail.
3108 if (FailIndex == 0) break;
3109
3110 // Push a MatchScope which indicates where to go if the first child fails
3111 // to match.
3112 MatchScope NewEntry;
3113 NewEntry.FailIndex = FailIndex;
3114 NewEntry.NodeStack.append(NodeStack.begin(), NodeStack.end());
3115 NewEntry.NumRecordedNodes = RecordedNodes.size();
3116 NewEntry.NumMatchedMemRefs = MatchedMemRefs.size();
3117 NewEntry.InputChain = InputChain;
3118 NewEntry.InputGlue = InputGlue;
3119 NewEntry.HasChainNodesMatched = !ChainNodesMatched.empty();
3120 MatchScopes.push_back(NewEntry);
3121 continue;
3122 }
3123 case OPC_RecordNode: {
3124 // Remember this node, it may end up being an operand in the pattern.
3125 SDNode *Parent = nullptr;
3126 if (NodeStack.size() > 1)
3127 Parent = NodeStack[NodeStack.size()-2].getNode();
3128 RecordedNodes.push_back(std::make_pair(N, Parent));
3129 continue;
3130 }
3131
3132 case OPC_RecordChild0: case OPC_RecordChild1:
3133 case OPC_RecordChild2: case OPC_RecordChild3:
3134 case OPC_RecordChild4: case OPC_RecordChild5:
3135 case OPC_RecordChild6: case OPC_RecordChild7: {
3136 unsigned ChildNo = Opcode-OPC_RecordChild0;
3137 if (ChildNo >= N.getNumOperands())
3138 break; // Match fails if out of range child #.
3139
3140 RecordedNodes.push_back(std::make_pair(N->getOperand(ChildNo),
3141 N.getNode()));
3142 continue;
3143 }
3144 case OPC_RecordMemRef:
3145 if (auto *MN = dyn_cast<MemSDNode>(N))
3146 MatchedMemRefs.push_back(MN->getMemOperand());
3147 else {
3148 LLVM_DEBUG(dbgs() << "Expected MemSDNode "; N->dump(CurDAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Expected MemSDNode "; N->dump
(CurDAG); dbgs() << '\n'; } } while (false)
3149 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Expected MemSDNode "; N->dump
(CurDAG); dbgs() << '\n'; } } while (false)
;
3150 }
3151
3152 continue;
3153
3154 case OPC_CaptureGlueInput:
3155 // If the current node has an input glue, capture it in InputGlue.
3156 if (N->getNumOperands() != 0 &&
3157 N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue)
3158 InputGlue = N->getOperand(N->getNumOperands()-1);
3159 continue;
3160
3161 case OPC_MoveChild: {
3162 unsigned ChildNo = MatcherTable[MatcherIndex++];
3163 if (ChildNo >= N.getNumOperands())
3164 break; // Match fails if out of range child #.
3165 N = N.getOperand(ChildNo);
3166 NodeStack.push_back(N);
3167 continue;
3168 }
3169
3170 case OPC_MoveChild0: case OPC_MoveChild1:
3171 case OPC_MoveChild2: case OPC_MoveChild3:
3172 case OPC_MoveChild4: case OPC_MoveChild5:
3173 case OPC_MoveChild6: case OPC_MoveChild7: {
3174 unsigned ChildNo = Opcode-OPC_MoveChild0;
3175 if (ChildNo >= N.getNumOperands())
3176 break; // Match fails if out of range child #.
3177 N = N.getOperand(ChildNo);
3178 NodeStack.push_back(N);
3179 continue;
3180 }
3181
3182 case OPC_MoveParent:
3183 // Pop the current node off the NodeStack.
3184 NodeStack.pop_back();
3185 assert(!NodeStack.empty() && "Node stack imbalance!")((!NodeStack.empty() && "Node stack imbalance!") ? static_cast
<void> (0) : __assert_fail ("!NodeStack.empty() && \"Node stack imbalance!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3185, __PRETTY_FUNCTION__))
;
3186 N = NodeStack.back();
3187 continue;
3188
3189 case OPC_CheckSame:
3190 if (!::CheckSame(MatcherTable, MatcherIndex, N, RecordedNodes)) break;
3191 continue;
3192
3193 case OPC_CheckChild0Same: case OPC_CheckChild1Same:
3194 case OPC_CheckChild2Same: case OPC_CheckChild3Same:
3195 if (!::CheckChildSame(MatcherTable, MatcherIndex, N, RecordedNodes,
3196 Opcode-OPC_CheckChild0Same))
3197 break;
3198 continue;
3199
3200 case OPC_CheckPatternPredicate:
3201 if (!::CheckPatternPredicate(MatcherTable, MatcherIndex, *this)) break;
3202 continue;
3203 case OPC_CheckPredicate:
3204 if (!::CheckNodePredicate(MatcherTable, MatcherIndex, *this,
3205 N.getNode()))
3206 break;
3207 continue;
3208 case OPC_CheckPredicateWithOperands: {
3209 unsigned OpNum = MatcherTable[MatcherIndex++];
3210 SmallVector<SDValue, 8> Operands;
3211
3212 for (unsigned i = 0; i < OpNum; ++i)
3213 Operands.push_back(RecordedNodes[MatcherTable[MatcherIndex++]].first);
3214
3215 unsigned PredNo = MatcherTable[MatcherIndex++];
3216 if (!CheckNodePredicateWithOperands(N.getNode(), PredNo, Operands))
3217 break;
3218 continue;
3219 }
3220 case OPC_CheckComplexPat: {
3221 unsigned CPNum = MatcherTable[MatcherIndex++];
3222 unsigned RecNo = MatcherTable[MatcherIndex++];
3223 assert(RecNo < RecordedNodes.size() && "Invalid CheckComplexPat")((RecNo < RecordedNodes.size() && "Invalid CheckComplexPat"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid CheckComplexPat\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3223, __PRETTY_FUNCTION__))
;
3224
3225 // If target can modify DAG during matching, keep the matching state
3226 // consistent.
3227 std::unique_ptr<MatchStateUpdater> MSU;
3228 if (ComplexPatternFuncMutatesDAG())
3229 MSU.reset(new MatchStateUpdater(*CurDAG, &NodeToMatch, RecordedNodes,
3230 MatchScopes));
3231
3232 if (!CheckComplexPattern(NodeToMatch, RecordedNodes[RecNo].second,
3233 RecordedNodes[RecNo].first, CPNum,
3234 RecordedNodes))
3235 break;
3236 continue;
3237 }
3238 case OPC_CheckOpcode:
3239 if (!::CheckOpcode(MatcherTable, MatcherIndex, N.getNode())) break;
3240 continue;
3241
3242 case OPC_CheckType:
3243 if (!::CheckType(MatcherTable, MatcherIndex, N, TLI,
3244 CurDAG->getDataLayout()))
3245 break;
3246 continue;
3247
3248 case OPC_CheckTypeRes: {
3249 unsigned Res = MatcherTable[MatcherIndex++];
3250 if (!::CheckType(MatcherTable, MatcherIndex, N.getValue(Res), TLI,
3251 CurDAG->getDataLayout()))
3252 break;
3253 continue;
3254 }
3255
3256 case OPC_SwitchOpcode: {
3257 unsigned CurNodeOpcode = N.getOpcode();
3258 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3259 unsigned CaseSize;
3260 while (true) {
3261 // Get the size of this case.
3262 CaseSize = MatcherTable[MatcherIndex++];
3263 if (CaseSize & 128)
3264 CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3265 if (CaseSize == 0) break;
3266
3267 uint16_t Opc = MatcherTable[MatcherIndex++];
3268 Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3269
3270 // If the opcode matches, then we will execute this case.
3271 if (CurNodeOpcode == Opc)
3272 break;
3273
3274 // Otherwise, skip over this case.
3275 MatcherIndex += CaseSize;
3276 }
3277
3278 // If no cases matched, bail out.
3279 if (CaseSize == 0) break;
3280
3281 // Otherwise, execute the case we found.
3282 LLVM_DEBUG(dbgs() << " OpcodeSwitch from " << SwitchStart << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " OpcodeSwitch from " << SwitchStart
<< " to " << MatcherIndex << "\n"; } } while
(false)
3283 << MatcherIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " OpcodeSwitch from " << SwitchStart
<< " to " << MatcherIndex << "\n"; } } while
(false)
;
3284 continue;
3285 }
3286
3287 case OPC_SwitchType: {
3288 MVT CurNodeVT = N.getSimpleValueType();
3289 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3290 unsigned CaseSize;
3291 while (true) {
3292 // Get the size of this case.
3293 CaseSize = MatcherTable[MatcherIndex++];
3294 if (CaseSize & 128)
3295 CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3296 if (CaseSize == 0) break;
3297
3298 MVT CaseVT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3299 if (CaseVT == MVT::iPTR)
3300 CaseVT = TLI->getPointerTy(CurDAG->getDataLayout());
3301
3302 // If the VT matches, then we will execute this case.
3303 if (CurNodeVT == CaseVT)
3304 break;
3305
3306 // Otherwise, skip over this case.
3307 MatcherIndex += CaseSize;
3308 }
3309
3310 // If no cases matched, bail out.
3311 if (CaseSize == 0) break;
3312
3313 // Otherwise, execute the case we found.
3314 LLVM_DEBUG(dbgs() << " TypeSwitch[" << EVT(CurNodeVT).getEVTString()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " TypeSwitch[" << EVT(CurNodeVT
).getEVTString() << "] from " << SwitchStart <<
" to " << MatcherIndex << '\n'; } } while (false
)
3315 << "] from " << SwitchStart << " to " << MatcherIndexdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " TypeSwitch[" << EVT(CurNodeVT
).getEVTString() << "] from " << SwitchStart <<
" to " << MatcherIndex << '\n'; } } while (false
)
3316 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " TypeSwitch[" << EVT(CurNodeVT
).getEVTString() << "] from " << SwitchStart <<
" to " << MatcherIndex << '\n'; } } while (false
)
;
3317 continue;
3318 }
3319 case OPC_CheckChild0Type: case OPC_CheckChild1Type:
3320 case OPC_CheckChild2Type: case OPC_CheckChild3Type:
3321 case OPC_CheckChild4Type: case OPC_CheckChild5Type:
3322 case OPC_CheckChild6Type: case OPC_CheckChild7Type:
3323 if (!::CheckChildType(MatcherTable, MatcherIndex, N, TLI,
3324 CurDAG->getDataLayout(),
3325 Opcode - OPC_CheckChild0Type))
3326 break;
3327 continue;
3328 case OPC_CheckCondCode:
3329 if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break;
3330 continue;
3331 case OPC_CheckValueType:
3332 if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI,
3333 CurDAG->getDataLayout()))
3334 break;
3335 continue;
3336 case OPC_CheckInteger:
3337 if (!::CheckInteger(MatcherTable, MatcherIndex, N)) break;
3338 continue;
3339 case OPC_CheckChild0Integer: case OPC_CheckChild1Integer:
3340 case OPC_CheckChild2Integer: case OPC_CheckChild3Integer:
3341 case OPC_CheckChild4Integer:
3342 if (!::CheckChildInteger(MatcherTable, MatcherIndex, N,
3343 Opcode-OPC_CheckChild0Integer)) break;
3344 continue;
3345 case OPC_CheckAndImm:
3346 if (!::CheckAndImm(MatcherTable, MatcherIndex, N, *this)) break;
3347 continue;
3348 case OPC_CheckOrImm:
3349 if (!::CheckOrImm(MatcherTable, MatcherIndex, N, *this)) break;
3350 continue;
3351
3352 case OPC_CheckFoldableChainNode: {
3353 assert(NodeStack.size() != 1 && "No parent node")((NodeStack.size() != 1 && "No parent node") ? static_cast
<void> (0) : __assert_fail ("NodeStack.size() != 1 && \"No parent node\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3353, __PRETTY_FUNCTION__))
;
3354 // Verify that all intermediate nodes between the root and this one have
3355 // a single use.
3356 bool HasMultipleUses = false;
3357 for (unsigned i = 1, e = NodeStack.size()-1; i != e; ++i)
3358 if (!NodeStack[i].getNode()->hasOneUse()) {
3359 HasMultipleUses = true;
3360 break;
3361 }
3362 if (HasMultipleUses) break;
3363
3364 // Check to see that the target thinks this is profitable to fold and that
3365 // we can fold it without inducing cycles in the graph.
3366 if (!IsProfitableToFold(N, NodeStack[NodeStack.size()-2].getNode(),
3367 NodeToMatch) ||
3368 !IsLegalToFold(N, NodeStack[NodeStack.size()-2].getNode(),
3369 NodeToMatch, OptLevel,
3370 true/*We validate our own chains*/))
3371 break;
3372
3373 continue;
3374 }
3375 case OPC_EmitInteger: {
3376 MVT::SimpleValueType VT =
3377 (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3378 int64_t Val = MatcherTable[MatcherIndex++];
3379 if (Val & 128)
3380 Val = GetVBR(Val, MatcherTable, MatcherIndex);
3381 RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3382 CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch),
3383 VT), nullptr));
3384 continue;
3385 }
3386 case OPC_EmitRegister: {
3387 MVT::SimpleValueType VT =
3388 (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3389 unsigned RegNo = MatcherTable[MatcherIndex++];
3390 RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3391 CurDAG->getRegister(RegNo, VT), nullptr));
3392 continue;
3393 }
3394 case OPC_EmitRegister2: {
3395 // For targets w/ more than 256 register names, the register enum
3396 // values are stored in two bytes in the matcher table (just like
3397 // opcodes).
3398 MVT::SimpleValueType VT =
3399 (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3400 unsigned RegNo = MatcherTable[MatcherIndex++];
3401 RegNo |= MatcherTable[MatcherIndex++] << 8;
3402 RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3403 CurDAG->getRegister(RegNo, VT), nullptr));
3404 continue;
3405 }
3406
3407 case OPC_EmitConvertToTarget: {
3408 // Convert from IMM/FPIMM to target version.
3409 unsigned RecNo = MatcherTable[MatcherIndex++];
3410 assert(RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget")((RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitConvertToTarget\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3410, __PRETTY_FUNCTION__))
;
3411 SDValue Imm = RecordedNodes[RecNo].first;
3412
3413 if (Imm->getOpcode() == ISD::Constant) {
3414 const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();
3415 Imm = CurDAG->getTargetConstant(*Val, SDLoc(NodeToMatch),
3416 Imm.getValueType());
3417 } else if (Imm->getOpcode() == ISD::ConstantFP) {
3418 const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();
3419 Imm = CurDAG->getTargetConstantFP(*Val, SDLoc(NodeToMatch),
3420 Imm.getValueType());
3421 }
3422
3423 RecordedNodes.push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
3424 continue;
3425 }
3426
3427 case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0
3428 case OPC_EmitMergeInputChains1_1: // OPC_EmitMergeInputChains, 1, 1
3429 case OPC_EmitMergeInputChains1_2: { // OPC_EmitMergeInputChains, 1, 2
3430 // These are space-optimized forms of OPC_EmitMergeInputChains.
3431 assert(!InputChain.getNode() &&((!InputChain.getNode() && "EmitMergeInputChains should be the first chain producing node"
) ? static_cast<void> (0) : __assert_fail ("!InputChain.getNode() && \"EmitMergeInputChains should be the first chain producing node\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3432, __PRETTY_FUNCTION__))
3432 "EmitMergeInputChains should be the first chain producing node")((!InputChain.getNode() && "EmitMergeInputChains should be the first chain producing node"
) ? static_cast<void> (0) : __assert_fail ("!InputChain.getNode() && \"EmitMergeInputChains should be the first chain producing node\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3432, __PRETTY_FUNCTION__))
;
3433 assert(ChainNodesMatched.empty() &&((ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match"
) ? static_cast<void> (0) : __assert_fail ("ChainNodesMatched.empty() && \"Should only have one EmitMergeInputChains per match\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3434, __PRETTY_FUNCTION__))
3434 "Should only have one EmitMergeInputChains per match")((ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match"
) ? static_cast<void> (0) : __assert_fail ("ChainNodesMatched.empty() && \"Should only have one EmitMergeInputChains per match\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3434, __PRETTY_FUNCTION__))
;
3435
3436 // Read all of the chained nodes.
3437 unsigned RecNo = Opcode - OPC_EmitMergeInputChains1_0;
3438 assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains")((RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitMergeInputChains\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3438, __PRETTY_FUNCTION__))
;
3439 ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3440
3441 // FIXME: What if other value results of the node have uses not matched
3442 // by this pattern?
3443 if (ChainNodesMatched.back() != NodeToMatch &&
3444 !RecordedNodes[RecNo].first.hasOneUse()) {
3445 ChainNodesMatched.clear();
3446 break;
3447 }
3448
3449 // Merge the input chains if they are not intra-pattern references.
3450 InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3451
3452 if (!InputChain.getNode())
3453 break; // Failed to merge.
3454 continue;
3455 }
3456
3457 case OPC_EmitMergeInputChains: {
3458 assert(!InputChain.getNode() &&((!InputChain.getNode() && "EmitMergeInputChains should be the first chain producing node"
) ? static_cast<void> (0) : __assert_fail ("!InputChain.getNode() && \"EmitMergeInputChains should be the first chain producing node\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3459, __PRETTY_FUNCTION__))
3459 "EmitMergeInputChains should be the first chain producing node")((!InputChain.getNode() && "EmitMergeInputChains should be the first chain producing node"
) ? static_cast<void> (0) : __assert_fail ("!InputChain.getNode() && \"EmitMergeInputChains should be the first chain producing node\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3459, __PRETTY_FUNCTION__))
;
3460 // This node gets a list of nodes we matched in the input that have
3461 // chains. We want to token factor all of the input chains to these nodes
3462 // together. However, if any of the input chains is actually one of the
3463 // nodes matched in this pattern, then we have an intra-match reference.
3464 // Ignore these because the newly token factored chain should not refer to
3465 // the old nodes.
3466 unsigned NumChains = MatcherTable[MatcherIndex++];
3467 assert(NumChains != 0 && "Can't TF zero chains")((NumChains != 0 && "Can't TF zero chains") ? static_cast
<void> (0) : __assert_fail ("NumChains != 0 && \"Can't TF zero chains\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3467, __PRETTY_FUNCTION__))
;
3468
3469 assert(ChainNodesMatched.empty() &&((ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match"
) ? static_cast<void> (0) : __assert_fail ("ChainNodesMatched.empty() && \"Should only have one EmitMergeInputChains per match\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3470, __PRETTY_FUNCTION__))
3470 "Should only have one EmitMergeInputChains per match")((ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match"
) ? static_cast<void> (0) : __assert_fail ("ChainNodesMatched.empty() && \"Should only have one EmitMergeInputChains per match\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3470, __PRETTY_FUNCTION__))
;
3471
3472 // Read all of the chained nodes.
3473 for (unsigned i = 0; i != NumChains; ++i) {
3474 unsigned RecNo = MatcherTable[MatcherIndex++];
3475 assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains")((RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitMergeInputChains\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3475, __PRETTY_FUNCTION__))
;
3476 ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3477
3478 // FIXME: What if other value results of the node have uses not matched
3479 // by this pattern?
3480 if (ChainNodesMatched.back() != NodeToMatch &&
3481 !RecordedNodes[RecNo].first.hasOneUse()) {
3482 ChainNodesMatched.clear();
3483 break;
3484 }
3485 }
3486
3487 // If the inner loop broke out, the match fails.
3488 if (ChainNodesMatched.empty())
3489 break;
3490
3491 // Merge the input chains if they are not intra-pattern references.
3492 InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3493
3494 if (!InputChain.getNode())
3495 break; // Failed to merge.
3496
3497 continue;
3498 }
3499
3500 case OPC_EmitCopyToReg: {
3501 unsigned RecNo = MatcherTable[MatcherIndex++];
3502 assert(RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg")((RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitCopyToReg\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3502, __PRETTY_FUNCTION__))
;
3503 unsigned DestPhysReg = MatcherTable[MatcherIndex++];
3504
3505 if (!InputChain.getNode())
3506 InputChain = CurDAG->getEntryNode();
3507
3508 InputChain = CurDAG->getCopyToReg(InputChain, SDLoc(NodeToMatch),
3509 DestPhysReg, RecordedNodes[RecNo].first,
3510 InputGlue);
3511
3512 InputGlue = InputChain.getValue(1);
3513 continue;
3514 }
3515
3516 case OPC_EmitNodeXForm: {
3517 unsigned XFormNo = MatcherTable[MatcherIndex++];
3518 unsigned RecNo = MatcherTable[MatcherIndex++];
3519 assert(RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm")((RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitNodeXForm\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3519, __PRETTY_FUNCTION__))
;
3520 SDValue Res = RunSDNodeXForm(RecordedNodes[RecNo].first, XFormNo);
3521 RecordedNodes.push_back(std::pair<SDValue,SDNode*>(Res, nullptr));
3522 continue;
3523 }
3524 case OPC_Coverage: {
3525 // This is emitted right before MorphNode/EmitNode.
3526 // So it should be safe to assume that this node has been selected
3527 unsigned index = MatcherTable[MatcherIndex++];
3528 index |= (MatcherTable[MatcherIndex++] << 8);
3529 dbgs() << "COVERED: " << getPatternForIndex(index) << "\n";
3530 dbgs() << "INCLUDED: " << getIncludePathForIndex(index) << "\n";
3531 continue;
3532 }
3533
3534 case OPC_EmitNode: case OPC_MorphNodeTo:
3535 case OPC_EmitNode0: case OPC_EmitNode1: case OPC_EmitNode2:
3536 case OPC_MorphNodeTo0: case OPC_MorphNodeTo1: case OPC_MorphNodeTo2: {
3537 uint16_t TargetOpc = MatcherTable[MatcherIndex++];
3538 TargetOpc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3539 unsigned EmitNodeInfo = MatcherTable[MatcherIndex++];
3540 // Get the result VT list.
3541 unsigned NumVTs;
3542 // If this is one of the compressed forms, get the number of VTs based
3543 // on the Opcode. Otherwise read the next byte from the table.
3544 if (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2)
3545 NumVTs = Opcode - OPC_MorphNodeTo0;
3546 else if (Opcode >= OPC_EmitNode0 && Opcode <= OPC_EmitNode2)
3547 NumVTs = Opcode - OPC_EmitNode0;
3548 else
3549 NumVTs = MatcherTable[MatcherIndex++];
3550 SmallVector<EVT, 4> VTs;
3551 for (unsigned i = 0; i != NumVTs; ++i) {
3552 MVT::SimpleValueType VT =
3553 (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3554 if (VT == MVT::iPTR)
3555 VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy;
3556 VTs.push_back(VT);
3557 }
3558
3559 if (EmitNodeInfo & OPFL_Chain)
3560 VTs.push_back(MVT::Other);
3561 if (EmitNodeInfo & OPFL_GlueOutput)
3562 VTs.push_back(MVT::Glue);
3563
3564 // This is hot code, so optimize the two most common cases of 1 and 2
3565 // results.
3566 SDVTList VTList;
3567 if (VTs.size() == 1)
3568 VTList = CurDAG->getVTList(VTs[0]);
3569 else if (VTs.size() == 2)
3570 VTList = CurDAG->getVTList(VTs[0], VTs[1]);
3571 else
3572 VTList = CurDAG->getVTList(VTs);
3573
3574 // Get the operand list.
3575 unsigned NumOps = MatcherTable[MatcherIndex++];
3576 SmallVector<SDValue, 8> Ops;
3577 for (unsigned i = 0; i != NumOps; ++i) {
3578 unsigned RecNo = MatcherTable[MatcherIndex++];
3579 if (RecNo & 128)
3580 RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex);
3581
3582 assert(RecNo < RecordedNodes.size() && "Invalid EmitNode")((RecNo < RecordedNodes.size() && "Invalid EmitNode"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitNode\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3582, __PRETTY_FUNCTION__))
;
3583 Ops.push_back(RecordedNodes[RecNo].first);
3584 }
3585
3586 // If there are variadic operands to add, handle them now.
3587 if (EmitNodeInfo & OPFL_VariadicInfo) {
3588 // Determine the start index to copy from.
3589 unsigned FirstOpToCopy = getNumFixedFromVariadicInfo(EmitNodeInfo);
3590 FirstOpToCopy += (EmitNodeInfo & OPFL_Chain) ? 1 : 0;
3591 assert(NodeToMatch->getNumOperands() >= FirstOpToCopy &&((NodeToMatch->getNumOperands() >= FirstOpToCopy &&
"Invalid variadic node") ? static_cast<void> (0) : __assert_fail
("NodeToMatch->getNumOperands() >= FirstOpToCopy && \"Invalid variadic node\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3592, __PRETTY_FUNCTION__))
3592 "Invalid variadic node")((NodeToMatch->getNumOperands() >= FirstOpToCopy &&
"Invalid variadic node") ? static_cast<void> (0) : __assert_fail
("NodeToMatch->getNumOperands() >= FirstOpToCopy && \"Invalid variadic node\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3592, __PRETTY_FUNCTION__))
;
3593 // Copy all of the variadic operands, not including a potential glue
3594 // input.
3595 for (unsigned i = FirstOpToCopy, e = NodeToMatch->getNumOperands();
3596 i != e; ++i) {
3597 SDValue V = NodeToMatch->getOperand(i);
3598 if (V.getValueType() == MVT::Glue) break;
3599 Ops.push_back(V);
3600 }
3601 }
3602
3603 // If this has chain/glue inputs, add them.
3604 if (EmitNodeInfo & OPFL_Chain)
3605 Ops.push_back(InputChain);
3606 if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr)
3607 Ops.push_back(InputGlue);
3608
3609 // Create the node.
3610 MachineSDNode *Res = nullptr;
3611 bool IsMorphNodeTo = Opcode == OPC_MorphNodeTo ||
3612 (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2);
3613 if (!IsMorphNodeTo) {
3614 // If this is a normal EmitNode command, just create the new node and
3615 // add the results to the RecordedNodes list.
3616 Res = CurDAG->getMachineNode(TargetOpc, SDLoc(NodeToMatch),
3617 VTList, Ops);
3618
3619 // Add all the non-glue/non-chain results to the RecordedNodes list.
3620 for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
3621 if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue) break;
3622 RecordedNodes.push_back(std::pair<SDValue,SDNode*>(SDValue(Res, i),
3623 nullptr));
3624 }
3625 } else {
3626 assert(NodeToMatch->getOpcode() != ISD::DELETED_NODE &&((NodeToMatch->getOpcode() != ISD::DELETED_NODE &&
"NodeToMatch was removed partway through selection") ? static_cast
<void> (0) : __assert_fail ("NodeToMatch->getOpcode() != ISD::DELETED_NODE && \"NodeToMatch was removed partway through selection\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3627, __PRETTY_FUNCTION__))
3627 "NodeToMatch was removed partway through selection")((NodeToMatch->getOpcode() != ISD::DELETED_NODE &&
"NodeToMatch was removed partway through selection") ? static_cast
<void> (0) : __assert_fail ("NodeToMatch->getOpcode() != ISD::DELETED_NODE && \"NodeToMatch was removed partway through selection\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3627, __PRETTY_FUNCTION__))
;
3628 SelectionDAG::DAGNodeDeletedListener NDL(*CurDAG, [&](SDNode *N,
3629 SDNode *E) {
3630 CurDAG->salvageDebugInfo(*N);
3631 auto &Chain = ChainNodesMatched;
3632 assert((!E || !is_contained(Chain, N)) &&(((!E || !is_contained(Chain, N)) && "Chain node replaced during MorphNode"
) ? static_cast<void> (0) : __assert_fail ("(!E || !is_contained(Chain, N)) && \"Chain node replaced during MorphNode\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3633, __PRETTY_FUNCTION__))
3633 "Chain node replaced during MorphNode")(((!E || !is_contained(Chain, N)) && "Chain node replaced during MorphNode"
) ? static_cast<void> (0) : __assert_fail ("(!E || !is_contained(Chain, N)) && \"Chain node replaced during MorphNode\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3633, __PRETTY_FUNCTION__))
;
3634 Chain.erase(std::remove(Chain.begin(), Chain.end(), N), Chain.end());
3635 });
3636 Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList,
3637 Ops, EmitNodeInfo));
3638 }
3639
3640 // If the node had chain/glue results, update our notion of the current
3641 // chain and glue.
3642 if (EmitNodeInfo & OPFL_GlueOutput) {
3643 InputGlue = SDValue(Res, VTs.size()-1);
3644 if (EmitNodeInfo & OPFL_Chain)
3645 InputChain = SDValue(Res, VTs.size()-2);
3646 } else if (EmitNodeInfo & OPFL_Chain)
3647 InputChain = SDValue(Res, VTs.size()-1);
3648
3649 // If the OPFL_MemRefs glue is set on this node, slap all of the
3650 // accumulated memrefs onto it.
3651 //
3652 // FIXME: This is vastly incorrect for patterns with multiple outputs
3653 // instructions that access memory and for ComplexPatterns that match
3654 // loads.
3655 if (EmitNodeInfo & OPFL_MemRefs) {
3656 // Only attach load or store memory operands if the generated
3657 // instruction may load or store.
3658 const MCInstrDesc &MCID = TII->get(TargetOpc);
3659 bool mayLoad = MCID.mayLoad();
3660 bool mayStore = MCID.mayStore();
3661
3662 // We expect to have relatively few of these so just filter them into a
3663 // temporary buffer so that we can easily add them to the instruction.
3664 SmallVector<MachineMemOperand *, 4> FilteredMemRefs;
3665 for (MachineMemOperand *MMO : MatchedMemRefs) {
3666 if (MMO->isLoad()) {
3667 if (mayLoad)
3668 FilteredMemRefs.push_back(MMO);
3669 } else if (MMO->isStore()) {
3670 if (mayStore)
3671 FilteredMemRefs.push_back(MMO);
3672 } else {
3673 FilteredMemRefs.push_back(MMO);
3674 }
3675 }
3676
3677 CurDAG->setNodeMemRefs(Res, FilteredMemRefs);
3678 }
3679
3680 LLVM_DEBUG(if (!MatchedMemRefs.empty() && Res->memoperands_empty()) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (!MatchedMemRefs.empty() && Res->memoperands_empty
()) dbgs() << " Dropping mem operands\n"; dbgs() <<
" " << (IsMorphNodeTo ? "Morphed" : "Created") <<
" node: "; Res->dump(CurDAG);; } } while (false)
3681 << " Dropping mem operands\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (!MatchedMemRefs.empty() && Res->memoperands_empty
()) dbgs() << " Dropping mem operands\n"; dbgs() <<
" " << (IsMorphNodeTo ? "Morphed" : "Created") <<
" node: "; Res->dump(CurDAG);; } } while (false)
3682 dbgs() << " " << (IsMorphNodeTo ? "Morphed" : "Created")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (!MatchedMemRefs.empty() && Res->memoperands_empty
()) dbgs() << " Dropping mem operands\n"; dbgs() <<
" " << (IsMorphNodeTo ? "Morphed" : "Created") <<
" node: "; Res->dump(CurDAG);; } } while (false)
3683 << " node: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (!MatchedMemRefs.empty() && Res->memoperands_empty
()) dbgs() << " Dropping mem operands\n"; dbgs() <<
" " << (IsMorphNodeTo ? "Morphed" : "Created") <<
" node: "; Res->dump(CurDAG);; } } while (false)
3684 Res->dump(CurDAG);)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (!MatchedMemRefs.empty() && Res->memoperands_empty
()) dbgs() << " Dropping mem operands\n"; dbgs() <<
" " << (IsMorphNodeTo ? "Morphed" : "Created") <<
" node: "; Res->dump(CurDAG);; } } while (false)
;
3685
3686 // If this was a MorphNodeTo then we're completely done!
3687 if (IsMorphNodeTo) {
3688 // Update chain uses.
3689 UpdateChains(Res, InputChain, ChainNodesMatched, true);
3690 return;
3691 }
3692 continue;
3693 }
3694
3695 case OPC_CompleteMatch: {
3696 // The match has been completed, and any new nodes (if any) have been
3697 // created. Patch up references to the matched dag to use the newly
3698 // created nodes.
3699 unsigned NumResults = MatcherTable[MatcherIndex++];
3700
3701 for (unsigned i = 0; i != NumResults; ++i) {
3702 unsigned ResSlot = MatcherTable[MatcherIndex++];
3703 if (ResSlot & 128)
3704 ResSlot = GetVBR(ResSlot, MatcherTable, MatcherIndex);
3705
3706 assert(ResSlot < RecordedNodes.size() && "Invalid CompleteMatch")((ResSlot < RecordedNodes.size() && "Invalid CompleteMatch"
) ? static_cast<void> (0) : __assert_fail ("ResSlot < RecordedNodes.size() && \"Invalid CompleteMatch\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3706, __PRETTY_FUNCTION__))
;
3707 SDValue Res = RecordedNodes[ResSlot].first;
3708
3709 assert(i < NodeToMatch->getNumValues() &&((i < NodeToMatch->getNumValues() && NodeToMatch
->getValueType(i) != MVT::Other && NodeToMatch->
getValueType(i) != MVT::Glue && "Invalid number of results to complete!"
) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3712, __PRETTY_FUNCTION__))
3710 NodeToMatch->getValueType(i) != MVT::Other &&((i < NodeToMatch->getNumValues() && NodeToMatch
->getValueType(i) != MVT::Other && NodeToMatch->
getValueType(i) != MVT::Glue && "Invalid number of results to complete!"
) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3712, __PRETTY_FUNCTION__))
3711 NodeToMatch->getValueType(i) != MVT::Glue &&((i < NodeToMatch->getNumValues() && NodeToMatch
->getValueType(i) != MVT::Other && NodeToMatch->
getValueType(i) != MVT::Glue && "Invalid number of results to complete!"
) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3712, __PRETTY_FUNCTION__))
3712 "Invalid number of results to complete!")((i < NodeToMatch->getNumValues() && NodeToMatch
->getValueType(i) != MVT::Other && NodeToMatch->
getValueType(i) != MVT::Glue && "Invalid number of results to complete!"
) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3712, __PRETTY_FUNCTION__))
;
3713 assert((NodeToMatch->getValueType(i) == Res.getValueType() ||(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3718, __PRETTY_FUNCTION__))
3714 NodeToMatch->getValueType(i) == MVT::iPTR ||(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3718, __PRETTY_FUNCTION__))
3715 Res.getValueType() == MVT::iPTR ||(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3718, __PRETTY_FUNCTION__))
3716 NodeToMatch->getValueType(i).getSizeInBits() ==(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3718, __PRETTY_FUNCTION__))
3717 Res.getValueSizeInBits()) &&(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3718, __PRETTY_FUNCTION__))
3718 "invalid replacement")(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3718, __PRETTY_FUNCTION__))
;
3719 ReplaceUses(SDValue(NodeToMatch, i), Res);
3720 }
3721
3722 // Update chain uses.
3723 UpdateChains(NodeToMatch, InputChain, ChainNodesMatched, false);
3724
3725 // If the root node defines glue, we need to update it to the glue result.
3726 // TODO: This never happens in our tests and I think it can be removed /
3727 // replaced with an assert, but if we do it this the way the change is
3728 // NFC.
3729 if (NodeToMatch->getValueType(NodeToMatch->getNumValues() - 1) ==
3730 MVT::Glue &&
3731 InputGlue.getNode())
3732 ReplaceUses(SDValue(NodeToMatch, NodeToMatch->getNumValues() - 1),
3733 InputGlue);
3734
3735 assert(NodeToMatch->use_empty() &&((NodeToMatch->use_empty() && "Didn't replace all uses of the node?"
) ? static_cast<void> (0) : __assert_fail ("NodeToMatch->use_empty() && \"Didn't replace all uses of the node?\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3736, __PRETTY_FUNCTION__))
3736 "Didn't replace all uses of the node?")((NodeToMatch->use_empty() && "Didn't replace all uses of the node?"
) ? static_cast<void> (0) : __assert_fail ("NodeToMatch->use_empty() && \"Didn't replace all uses of the node?\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3736, __PRETTY_FUNCTION__))
;
3737 CurDAG->RemoveDeadNode(NodeToMatch);
3738
3739 return;
3740 }
3741 }
3742
3743 // If the code reached this point, then the match failed. See if there is
3744 // another child to try in the current 'Scope', otherwise pop it until we
3745 // find a case to check.
3746 LLVM_DEBUG(dbgs() << " Match failed at index " << CurrentOpcodeIndexdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Match failed at index " <<
CurrentOpcodeIndex << "\n"; } } while (false)
3747 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Match failed at index " <<
CurrentOpcodeIndex << "\n"; } } while (false)
;
3748 ++NumDAGIselRetries;
3749 while (true) {
3750 if (MatchScopes.empty()) {
3751 CannotYetSelect(NodeToMatch);
3752 return;
3753 }
3754
3755 // Restore the interpreter state back to the point where the scope was
3756 // formed.
3757 MatchScope &LastScope = MatchScopes.back();
3758 RecordedNodes.resize(LastScope.NumRecordedNodes);
3759 NodeStack.clear();
3760 NodeStack.append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());
3761 N = NodeStack.back();
3762
3763 if (LastScope.NumMatchedMemRefs != MatchedMemRefs.size())
3764 MatchedMemRefs.resize(LastScope.NumMatchedMemRefs);
3765 MatcherIndex = LastScope.FailIndex;
3766
3767 LLVM_DEBUG(dbgs() << " Continuing at " << MatcherIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Continuing at " << MatcherIndex
<< "\n"; } } while (false)
;
3768
3769 InputChain = LastScope.InputChain;
3770 InputGlue = LastScope.InputGlue;
3771 if (!LastScope.HasChainNodesMatched)
3772 ChainNodesMatched.clear();
3773
3774 // Check to see what the offset is at the new MatcherIndex. If it is zero
3775 // we have reached the end of this scope, otherwise we have another child
3776 // in the current scope to try.
3777 unsigned NumToSkip = MatcherTable[MatcherIndex++];
3778 if (NumToSkip & 128)
3779 NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3780
3781 // If we have another child in this scope to match, update FailIndex and
3782 // try it.
3783 if (NumToSkip != 0) {
3784 LastScope.FailIndex = MatcherIndex+NumToSkip;
3785 break;
3786 }
3787
3788 // End of this scope, pop it and try the next child in the containing
3789 // scope.
3790 MatchScopes.pop_back();
3791 }
3792 }
3793}
3794
3795bool SelectionDAGISel::isOrEquivalentToAdd(const SDNode *N) const {
3796 assert(N->getOpcode() == ISD::OR && "Unexpected opcode")((N->getOpcode() == ISD::OR && "Unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::OR && \"Unexpected opcode\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3796, __PRETTY_FUNCTION__))
;
3797 auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
3798 if (!C)
3799 return false;
3800
3801 // Detect when "or" is used to add an offset to a stack object.
3802 if (auto *FN = dyn_cast<FrameIndexSDNode>(N->getOperand(0))) {
3803 MachineFrameInfo &MFI = MF->getFrameInfo();
3804 unsigned A = MFI.getObjectAlignment(FN->getIndex());
3805 assert(isPowerOf2_32(A) && "Unexpected alignment")((isPowerOf2_32(A) && "Unexpected alignment") ? static_cast
<void> (0) : __assert_fail ("isPowerOf2_32(A) && \"Unexpected alignment\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3805, __PRETTY_FUNCTION__))
;
3806 int32_t Off = C->getSExtValue();
3807 // If the alleged offset fits in the zero bits guaranteed by
3808 // the alignment, then this or is really an add.
3809 return (Off >= 0) && (((A - 1) & Off) == unsigned(Off));
3810 }
3811 return false;
3812}
3813
3814void SelectionDAGISel::CannotYetSelect(SDNode *N) {
3815 std::string msg;
3816 raw_string_ostream Msg(msg);
3817 Msg << "Cannot select: ";
3818
3819 if (N->getOpcode() != ISD::INTRINSIC_W_CHAIN &&
3820 N->getOpcode() != ISD::INTRINSIC_WO_CHAIN &&
3821 N->getOpcode() != ISD::INTRINSIC_VOID) {
3822 N->printrFull(Msg, CurDAG);
3823 Msg << "\nIn function: " << MF->getName();
3824 } else {
3825 bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other;
3826 unsigned iid =
3827 cast<ConstantSDNode>(N->getOperand(HasInputChain))->getZExtValue();
3828 if (iid < Intrinsic::num_intrinsics)
3829 Msg << "intrinsic %" << Intrinsic::getName((Intrinsic::ID)iid, None);
3830 else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo())
3831 Msg << "target intrinsic %" << TII->getName(iid);
3832 else
3833 Msg << "unknown intrinsic #" << iid;
3834 }
3835 report_fatal_error(Msg.str());
3836}
3837
3838char SelectionDAGISel::ID = 0;