File: | llvm/include/llvm/CodeGen/SelectionDAGNodes.h |
Warning: | line 1114, column 10 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===// | ||||||||||||
2 | // | ||||||||||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||||||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||||||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||||||||
6 | // | ||||||||||||
7 | //===----------------------------------------------------------------------===// | ||||||||||||
8 | // | ||||||||||||
9 | // This implements routines for translating from LLVM IR into SelectionDAG IR. | ||||||||||||
10 | // | ||||||||||||
11 | //===----------------------------------------------------------------------===// | ||||||||||||
12 | |||||||||||||
13 | #include "SelectionDAGBuilder.h" | ||||||||||||
14 | #include "SDNodeDbgValue.h" | ||||||||||||
15 | #include "llvm/ADT/APFloat.h" | ||||||||||||
16 | #include "llvm/ADT/APInt.h" | ||||||||||||
17 | #include "llvm/ADT/BitVector.h" | ||||||||||||
18 | #include "llvm/ADT/None.h" | ||||||||||||
19 | #include "llvm/ADT/Optional.h" | ||||||||||||
20 | #include "llvm/ADT/STLExtras.h" | ||||||||||||
21 | #include "llvm/ADT/SmallPtrSet.h" | ||||||||||||
22 | #include "llvm/ADT/SmallSet.h" | ||||||||||||
23 | #include "llvm/ADT/StringRef.h" | ||||||||||||
24 | #include "llvm/ADT/Triple.h" | ||||||||||||
25 | #include "llvm/ADT/Twine.h" | ||||||||||||
26 | #include "llvm/Analysis/AliasAnalysis.h" | ||||||||||||
27 | #include "llvm/Analysis/BlockFrequencyInfo.h" | ||||||||||||
28 | #include "llvm/Analysis/BranchProbabilityInfo.h" | ||||||||||||
29 | #include "llvm/Analysis/ConstantFolding.h" | ||||||||||||
30 | #include "llvm/Analysis/EHPersonalities.h" | ||||||||||||
31 | #include "llvm/Analysis/Loads.h" | ||||||||||||
32 | #include "llvm/Analysis/MemoryLocation.h" | ||||||||||||
33 | #include "llvm/Analysis/ProfileSummaryInfo.h" | ||||||||||||
34 | #include "llvm/Analysis/TargetLibraryInfo.h" | ||||||||||||
35 | #include "llvm/Analysis/ValueTracking.h" | ||||||||||||
36 | #include "llvm/Analysis/VectorUtils.h" | ||||||||||||
37 | #include "llvm/CodeGen/Analysis.h" | ||||||||||||
38 | #include "llvm/CodeGen/FunctionLoweringInfo.h" | ||||||||||||
39 | #include "llvm/CodeGen/GCMetadata.h" | ||||||||||||
40 | #include "llvm/CodeGen/MachineBasicBlock.h" | ||||||||||||
41 | #include "llvm/CodeGen/MachineFrameInfo.h" | ||||||||||||
42 | #include "llvm/CodeGen/MachineFunction.h" | ||||||||||||
43 | #include "llvm/CodeGen/MachineInstr.h" | ||||||||||||
44 | #include "llvm/CodeGen/MachineInstrBuilder.h" | ||||||||||||
45 | #include "llvm/CodeGen/MachineJumpTableInfo.h" | ||||||||||||
46 | #include "llvm/CodeGen/MachineMemOperand.h" | ||||||||||||
47 | #include "llvm/CodeGen/MachineModuleInfo.h" | ||||||||||||
48 | #include "llvm/CodeGen/MachineOperand.h" | ||||||||||||
49 | #include "llvm/CodeGen/MachineRegisterInfo.h" | ||||||||||||
50 | #include "llvm/CodeGen/RuntimeLibcalls.h" | ||||||||||||
51 | #include "llvm/CodeGen/SelectionDAG.h" | ||||||||||||
52 | #include "llvm/CodeGen/SelectionDAGTargetInfo.h" | ||||||||||||
53 | #include "llvm/CodeGen/StackMaps.h" | ||||||||||||
54 | #include "llvm/CodeGen/SwiftErrorValueTracking.h" | ||||||||||||
55 | #include "llvm/CodeGen/TargetFrameLowering.h" | ||||||||||||
56 | #include "llvm/CodeGen/TargetInstrInfo.h" | ||||||||||||
57 | #include "llvm/CodeGen/TargetOpcodes.h" | ||||||||||||
58 | #include "llvm/CodeGen/TargetRegisterInfo.h" | ||||||||||||
59 | #include "llvm/CodeGen/TargetSubtargetInfo.h" | ||||||||||||
60 | #include "llvm/CodeGen/WinEHFuncInfo.h" | ||||||||||||
61 | #include "llvm/IR/Argument.h" | ||||||||||||
62 | #include "llvm/IR/Attributes.h" | ||||||||||||
63 | #include "llvm/IR/BasicBlock.h" | ||||||||||||
64 | #include "llvm/IR/CFG.h" | ||||||||||||
65 | #include "llvm/IR/CallingConv.h" | ||||||||||||
66 | #include "llvm/IR/Constant.h" | ||||||||||||
67 | #include "llvm/IR/ConstantRange.h" | ||||||||||||
68 | #include "llvm/IR/Constants.h" | ||||||||||||
69 | #include "llvm/IR/DataLayout.h" | ||||||||||||
70 | #include "llvm/IR/DebugInfoMetadata.h" | ||||||||||||
71 | #include "llvm/IR/DerivedTypes.h" | ||||||||||||
72 | #include "llvm/IR/DiagnosticInfo.h" | ||||||||||||
73 | #include "llvm/IR/Function.h" | ||||||||||||
74 | #include "llvm/IR/GetElementPtrTypeIterator.h" | ||||||||||||
75 | #include "llvm/IR/InlineAsm.h" | ||||||||||||
76 | #include "llvm/IR/InstrTypes.h" | ||||||||||||
77 | #include "llvm/IR/Instructions.h" | ||||||||||||
78 | #include "llvm/IR/IntrinsicInst.h" | ||||||||||||
79 | #include "llvm/IR/Intrinsics.h" | ||||||||||||
80 | #include "llvm/IR/IntrinsicsAArch64.h" | ||||||||||||
81 | #include "llvm/IR/IntrinsicsWebAssembly.h" | ||||||||||||
82 | #include "llvm/IR/LLVMContext.h" | ||||||||||||
83 | #include "llvm/IR/Metadata.h" | ||||||||||||
84 | #include "llvm/IR/Module.h" | ||||||||||||
85 | #include "llvm/IR/Operator.h" | ||||||||||||
86 | #include "llvm/IR/PatternMatch.h" | ||||||||||||
87 | #include "llvm/IR/Statepoint.h" | ||||||||||||
88 | #include "llvm/IR/Type.h" | ||||||||||||
89 | #include "llvm/IR/User.h" | ||||||||||||
90 | #include "llvm/IR/Value.h" | ||||||||||||
91 | #include "llvm/MC/MCContext.h" | ||||||||||||
92 | #include "llvm/MC/MCSymbol.h" | ||||||||||||
93 | #include "llvm/Support/AtomicOrdering.h" | ||||||||||||
94 | #include "llvm/Support/Casting.h" | ||||||||||||
95 | #include "llvm/Support/CommandLine.h" | ||||||||||||
96 | #include "llvm/Support/Compiler.h" | ||||||||||||
97 | #include "llvm/Support/Debug.h" | ||||||||||||
98 | #include "llvm/Support/MathExtras.h" | ||||||||||||
99 | #include "llvm/Support/raw_ostream.h" | ||||||||||||
100 | #include "llvm/Target/TargetIntrinsicInfo.h" | ||||||||||||
101 | #include "llvm/Target/TargetMachine.h" | ||||||||||||
102 | #include "llvm/Target/TargetOptions.h" | ||||||||||||
103 | #include "llvm/Transforms/Utils/Local.h" | ||||||||||||
104 | #include <cstddef> | ||||||||||||
105 | #include <cstring> | ||||||||||||
106 | #include <iterator> | ||||||||||||
107 | #include <limits> | ||||||||||||
108 | #include <numeric> | ||||||||||||
109 | #include <tuple> | ||||||||||||
110 | |||||||||||||
111 | using namespace llvm; | ||||||||||||
112 | using namespace PatternMatch; | ||||||||||||
113 | using namespace SwitchCG; | ||||||||||||
114 | |||||||||||||
115 | #define DEBUG_TYPE"isel" "isel" | ||||||||||||
116 | |||||||||||||
117 | /// LimitFloatPrecision - Generate low-precision inline sequences for | ||||||||||||
118 | /// some float libcalls (6, 8 or 12 bits). | ||||||||||||
119 | static unsigned LimitFloatPrecision; | ||||||||||||
120 | |||||||||||||
121 | static cl::opt<bool> | ||||||||||||
122 | InsertAssertAlign("insert-assert-align", cl::init(true), | ||||||||||||
123 | cl::desc("Insert the experimental `assertalign` node."), | ||||||||||||
124 | cl::ReallyHidden); | ||||||||||||
125 | |||||||||||||
126 | static cl::opt<unsigned, true> | ||||||||||||
127 | LimitFPPrecision("limit-float-precision", | ||||||||||||
128 | cl::desc("Generate low-precision inline sequences " | ||||||||||||
129 | "for some float libcalls"), | ||||||||||||
130 | cl::location(LimitFloatPrecision), cl::Hidden, | ||||||||||||
131 | cl::init(0)); | ||||||||||||
132 | |||||||||||||
133 | static cl::opt<unsigned> SwitchPeelThreshold( | ||||||||||||
134 | "switch-peel-threshold", cl::Hidden, cl::init(66), | ||||||||||||
135 | cl::desc("Set the case probability threshold for peeling the case from a " | ||||||||||||
136 | "switch statement. A value greater than 100 will void this " | ||||||||||||
137 | "optimization")); | ||||||||||||
138 | |||||||||||||
139 | // Limit the width of DAG chains. This is important in general to prevent | ||||||||||||
140 | // DAG-based analysis from blowing up. For example, alias analysis and | ||||||||||||
141 | // load clustering may not complete in reasonable time. It is difficult to | ||||||||||||
142 | // recognize and avoid this situation within each individual analysis, and | ||||||||||||
143 | // future analyses are likely to have the same behavior. Limiting DAG width is | ||||||||||||
144 | // the safe approach and will be especially important with global DAGs. | ||||||||||||
145 | // | ||||||||||||
146 | // MaxParallelChains default is arbitrarily high to avoid affecting | ||||||||||||
147 | // optimization, but could be lowered to improve compile time. Any ld-ld-st-st | ||||||||||||
148 | // sequence over this should have been converted to llvm.memcpy by the | ||||||||||||
149 | // frontend. It is easy to induce this behavior with .ll code such as: | ||||||||||||
150 | // %buffer = alloca [4096 x i8] | ||||||||||||
151 | // %data = load [4096 x i8]* %argPtr | ||||||||||||
152 | // store [4096 x i8] %data, [4096 x i8]* %buffer | ||||||||||||
153 | static const unsigned MaxParallelChains = 64; | ||||||||||||
154 | |||||||||||||
155 | static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, | ||||||||||||
156 | const SDValue *Parts, unsigned NumParts, | ||||||||||||
157 | MVT PartVT, EVT ValueVT, const Value *V, | ||||||||||||
158 | Optional<CallingConv::ID> CC); | ||||||||||||
159 | |||||||||||||
160 | /// getCopyFromParts - Create a value that contains the specified legal parts | ||||||||||||
161 | /// combined into the value they represent. If the parts combine to a type | ||||||||||||
162 | /// larger than ValueVT then AssertOp can be used to specify whether the extra | ||||||||||||
163 | /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT | ||||||||||||
164 | /// (ISD::AssertSext). | ||||||||||||
165 | static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, | ||||||||||||
166 | const SDValue *Parts, unsigned NumParts, | ||||||||||||
167 | MVT PartVT, EVT ValueVT, const Value *V, | ||||||||||||
168 | Optional<CallingConv::ID> CC = None, | ||||||||||||
169 | Optional<ISD::NodeType> AssertOp = None) { | ||||||||||||
170 | // Let the target assemble the parts if it wants to | ||||||||||||
171 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
172 | if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts, | ||||||||||||
173 | PartVT, ValueVT, CC)) | ||||||||||||
174 | return Val; | ||||||||||||
175 | |||||||||||||
176 | if (ValueVT.isVector()) | ||||||||||||
177 | return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V, | ||||||||||||
178 | CC); | ||||||||||||
179 | |||||||||||||
180 | assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!" ) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 180, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
181 | SDValue Val = Parts[0]; | ||||||||||||
182 | |||||||||||||
183 | if (NumParts > 1) { | ||||||||||||
184 | // Assemble the value from multiple parts. | ||||||||||||
185 | if (ValueVT.isInteger()) { | ||||||||||||
186 | unsigned PartBits = PartVT.getSizeInBits(); | ||||||||||||
187 | unsigned ValueBits = ValueVT.getSizeInBits(); | ||||||||||||
188 | |||||||||||||
189 | // Assemble the power of 2 part. | ||||||||||||
190 | unsigned RoundParts = | ||||||||||||
191 | (NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts; | ||||||||||||
192 | unsigned RoundBits = PartBits * RoundParts; | ||||||||||||
193 | EVT RoundVT = RoundBits == ValueBits ? | ||||||||||||
194 | ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits); | ||||||||||||
195 | SDValue Lo, Hi; | ||||||||||||
196 | |||||||||||||
197 | EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2); | ||||||||||||
198 | |||||||||||||
199 | if (RoundParts > 2) { | ||||||||||||
200 | Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, | ||||||||||||
201 | PartVT, HalfVT, V); | ||||||||||||
202 | Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, | ||||||||||||
203 | RoundParts / 2, PartVT, HalfVT, V); | ||||||||||||
204 | } else { | ||||||||||||
205 | Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]); | ||||||||||||
206 | Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]); | ||||||||||||
207 | } | ||||||||||||
208 | |||||||||||||
209 | if (DAG.getDataLayout().isBigEndian()) | ||||||||||||
210 | std::swap(Lo, Hi); | ||||||||||||
211 | |||||||||||||
212 | Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi); | ||||||||||||
213 | |||||||||||||
214 | if (RoundParts < NumParts) { | ||||||||||||
215 | // Assemble the trailing non-power-of-2 part. | ||||||||||||
216 | unsigned OddParts = NumParts - RoundParts; | ||||||||||||
217 | EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits); | ||||||||||||
218 | Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT, | ||||||||||||
219 | OddVT, V, CC); | ||||||||||||
220 | |||||||||||||
221 | // Combine the round and odd parts. | ||||||||||||
222 | Lo = Val; | ||||||||||||
223 | if (DAG.getDataLayout().isBigEndian()) | ||||||||||||
224 | std::swap(Lo, Hi); | ||||||||||||
225 | EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); | ||||||||||||
226 | Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi); | ||||||||||||
227 | Hi = | ||||||||||||
228 | DAG.getNode(ISD::SHL, DL, TotalVT, Hi, | ||||||||||||
229 | DAG.getConstant(Lo.getValueSizeInBits(), DL, | ||||||||||||
230 | TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||||||
231 | Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo); | ||||||||||||
232 | Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi); | ||||||||||||
233 | } | ||||||||||||
234 | } else if (PartVT.isFloatingPoint()) { | ||||||||||||
235 | // FP split into multiple FP parts (for ppcf128) | ||||||||||||
236 | assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && "Unexpected split") ? void (0) : __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 237, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
237 | "Unexpected split")(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && "Unexpected split") ? void (0) : __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 237, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
238 | SDValue Lo, Hi; | ||||||||||||
239 | Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]); | ||||||||||||
240 | Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]); | ||||||||||||
241 | if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout())) | ||||||||||||
242 | std::swap(Lo, Hi); | ||||||||||||
243 | Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi); | ||||||||||||
244 | } else { | ||||||||||||
245 | // FP split into integer parts (soft fp) | ||||||||||||
246 | assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&(static_cast <bool> (ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && "Unexpected split" ) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 247, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
247 | !PartVT.isVector() && "Unexpected split")(static_cast <bool> (ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && "Unexpected split" ) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 247, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
248 | EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); | ||||||||||||
249 | Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC); | ||||||||||||
250 | } | ||||||||||||
251 | } | ||||||||||||
252 | |||||||||||||
253 | // There is now one part, held in Val. Correct it to match ValueVT. | ||||||||||||
254 | // PartEVT is the type of the register class that holds the value. | ||||||||||||
255 | // ValueVT is the type of the inline asm operation. | ||||||||||||
256 | EVT PartEVT = Val.getValueType(); | ||||||||||||
257 | |||||||||||||
258 | if (PartEVT == ValueVT) | ||||||||||||
259 | return Val; | ||||||||||||
260 | |||||||||||||
261 | if (PartEVT.isInteger() && ValueVT.isFloatingPoint() && | ||||||||||||
262 | ValueVT.bitsLT(PartEVT)) { | ||||||||||||
263 | // For an FP value in an integer part, we need to truncate to the right | ||||||||||||
264 | // width first. | ||||||||||||
265 | PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); | ||||||||||||
266 | Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val); | ||||||||||||
267 | } | ||||||||||||
268 | |||||||||||||
269 | // Handle types that have the same size. | ||||||||||||
270 | if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits()) | ||||||||||||
271 | return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); | ||||||||||||
272 | |||||||||||||
273 | // Handle types with different sizes. | ||||||||||||
274 | if (PartEVT.isInteger() && ValueVT.isInteger()) { | ||||||||||||
275 | if (ValueVT.bitsLT(PartEVT)) { | ||||||||||||
276 | // For a truncate, see if we have any information to | ||||||||||||
277 | // indicate whether the truncated bits will always be | ||||||||||||
278 | // zero or sign-extension. | ||||||||||||
279 | if (AssertOp.hasValue()) | ||||||||||||
280 | Val = DAG.getNode(*AssertOp, DL, PartEVT, Val, | ||||||||||||
281 | DAG.getValueType(ValueVT)); | ||||||||||||
282 | return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); | ||||||||||||
283 | } | ||||||||||||
284 | return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); | ||||||||||||
285 | } | ||||||||||||
286 | |||||||||||||
287 | if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { | ||||||||||||
288 | // FP_ROUND's are always exact here. | ||||||||||||
289 | if (ValueVT.bitsLT(Val.getValueType())) | ||||||||||||
290 | return DAG.getNode( | ||||||||||||
291 | ISD::FP_ROUND, DL, ValueVT, Val, | ||||||||||||
292 | DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||||||
293 | |||||||||||||
294 | return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val); | ||||||||||||
295 | } | ||||||||||||
296 | |||||||||||||
297 | // Handle MMX to a narrower integer type by bitcasting MMX to integer and | ||||||||||||
298 | // then truncating. | ||||||||||||
299 | if (PartEVT == MVT::x86mmx && ValueVT.isInteger() && | ||||||||||||
300 | ValueVT.bitsLT(PartEVT)) { | ||||||||||||
301 | Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val); | ||||||||||||
302 | return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); | ||||||||||||
303 | } | ||||||||||||
304 | |||||||||||||
305 | report_fatal_error("Unknown mismatch in getCopyFromParts!"); | ||||||||||||
306 | } | ||||||||||||
307 | |||||||||||||
308 | static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, | ||||||||||||
309 | const Twine &ErrMsg) { | ||||||||||||
310 | const Instruction *I = dyn_cast_or_null<Instruction>(V); | ||||||||||||
311 | if (!V) | ||||||||||||
312 | return Ctx.emitError(ErrMsg); | ||||||||||||
313 | |||||||||||||
314 | const char *AsmError = ", possible invalid constraint for vector type"; | ||||||||||||
315 | if (const CallInst *CI = dyn_cast<CallInst>(I)) | ||||||||||||
316 | if (CI->isInlineAsm()) | ||||||||||||
317 | return Ctx.emitError(I, ErrMsg + AsmError); | ||||||||||||
318 | |||||||||||||
319 | return Ctx.emitError(I, ErrMsg); | ||||||||||||
320 | } | ||||||||||||
321 | |||||||||||||
322 | /// getCopyFromPartsVector - Create a value that contains the specified legal | ||||||||||||
323 | /// parts combined into the value they represent. If the parts combine to a | ||||||||||||
324 | /// type larger than ValueVT then AssertOp can be used to specify whether the | ||||||||||||
325 | /// extra bits are known to be zero (ISD::AssertZext) or sign extended from | ||||||||||||
326 | /// ValueVT (ISD::AssertSext). | ||||||||||||
327 | static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, | ||||||||||||
328 | const SDValue *Parts, unsigned NumParts, | ||||||||||||
329 | MVT PartVT, EVT ValueVT, const Value *V, | ||||||||||||
330 | Optional<CallingConv::ID> CallConv) { | ||||||||||||
331 | assert(ValueVT.isVector() && "Not a vector value")(static_cast <bool> (ValueVT.isVector() && "Not a vector value" ) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector value\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 331, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
332 | assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!" ) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 332, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
333 | const bool IsABIRegCopy = CallConv.hasValue(); | ||||||||||||
334 | |||||||||||||
335 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
336 | SDValue Val = Parts[0]; | ||||||||||||
337 | |||||||||||||
338 | // Handle a multi-element vector. | ||||||||||||
339 | if (NumParts > 1) { | ||||||||||||
340 | EVT IntermediateVT; | ||||||||||||
341 | MVT RegisterVT; | ||||||||||||
342 | unsigned NumIntermediates; | ||||||||||||
343 | unsigned NumRegs; | ||||||||||||
344 | |||||||||||||
345 | if (IsABIRegCopy) { | ||||||||||||
346 | NumRegs = TLI.getVectorTypeBreakdownForCallingConv( | ||||||||||||
347 | *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT, | ||||||||||||
348 | NumIntermediates, RegisterVT); | ||||||||||||
349 | } else { | ||||||||||||
350 | NumRegs = | ||||||||||||
351 | TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, | ||||||||||||
352 | NumIntermediates, RegisterVT); | ||||||||||||
353 | } | ||||||||||||
354 | |||||||||||||
355 | assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!" ) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 355, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
356 | NumParts = NumRegs; // Silence a compiler warning. | ||||||||||||
357 | assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!" ) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 357, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
358 | assert(RegisterVT.getSizeInBits() ==(static_cast <bool> (RegisterVT.getSizeInBits() == Parts [0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!" ) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 360, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
359 | Parts[0].getSimpleValueType().getSizeInBits() &&(static_cast <bool> (RegisterVT.getSizeInBits() == Parts [0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!" ) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 360, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
360 | "Part type sizes don't match!")(static_cast <bool> (RegisterVT.getSizeInBits() == Parts [0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!" ) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 360, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
361 | |||||||||||||
362 | // Assemble the parts into intermediate operands. | ||||||||||||
363 | SmallVector<SDValue, 8> Ops(NumIntermediates); | ||||||||||||
364 | if (NumIntermediates == NumParts) { | ||||||||||||
365 | // If the register was not expanded, truncate or copy the value, | ||||||||||||
366 | // as appropriate. | ||||||||||||
367 | for (unsigned i = 0; i != NumParts; ++i) | ||||||||||||
368 | Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, | ||||||||||||
369 | PartVT, IntermediateVT, V, CallConv); | ||||||||||||
370 | } else if (NumParts > 0) { | ||||||||||||
371 | // If the intermediate type was expanded, build the intermediate | ||||||||||||
372 | // operands from the parts. | ||||||||||||
373 | assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!") ? void (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 374, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
374 | "Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!") ? void (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 374, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
375 | unsigned Factor = NumParts / NumIntermediates; | ||||||||||||
376 | for (unsigned i = 0; i != NumIntermediates; ++i) | ||||||||||||
377 | Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, | ||||||||||||
378 | PartVT, IntermediateVT, V, CallConv); | ||||||||||||
379 | } | ||||||||||||
380 | |||||||||||||
381 | // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the | ||||||||||||
382 | // intermediate operands. | ||||||||||||
383 | EVT BuiltVectorTy = | ||||||||||||
384 | IntermediateVT.isVector() | ||||||||||||
385 | ? EVT::getVectorVT( | ||||||||||||
386 | *DAG.getContext(), IntermediateVT.getScalarType(), | ||||||||||||
387 | IntermediateVT.getVectorElementCount() * NumParts) | ||||||||||||
388 | : EVT::getVectorVT(*DAG.getContext(), | ||||||||||||
389 | IntermediateVT.getScalarType(), | ||||||||||||
390 | NumIntermediates); | ||||||||||||
391 | Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS | ||||||||||||
392 | : ISD::BUILD_VECTOR, | ||||||||||||
393 | DL, BuiltVectorTy, Ops); | ||||||||||||
394 | } | ||||||||||||
395 | |||||||||||||
396 | // There is now one part, held in Val. Correct it to match ValueVT. | ||||||||||||
397 | EVT PartEVT = Val.getValueType(); | ||||||||||||
398 | |||||||||||||
399 | if (PartEVT == ValueVT) | ||||||||||||
400 | return Val; | ||||||||||||
401 | |||||||||||||
402 | if (PartEVT.isVector()) { | ||||||||||||
403 | // Vector/Vector bitcast. | ||||||||||||
404 | if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) | ||||||||||||
405 | return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); | ||||||||||||
406 | |||||||||||||
407 | // If the element type of the source/dest vectors are the same, but the | ||||||||||||
408 | // parts vector has more elements than the value vector, then we have a | ||||||||||||
409 | // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the | ||||||||||||
410 | // elements we want. | ||||||||||||
411 | if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) { | ||||||||||||
412 | assert((PartEVT.getVectorElementCount().getKnownMinValue() >(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue () > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount ().isScalable()) && "Cannot narrow, it would be a lossy transformation" ) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 416, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
413 | ValueVT.getVectorElementCount().getKnownMinValue()) &&(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue () > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount ().isScalable()) && "Cannot narrow, it would be a lossy transformation" ) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 416, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
414 | (PartEVT.getVectorElementCount().isScalable() ==(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue () > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount ().isScalable()) && "Cannot narrow, it would be a lossy transformation" ) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 416, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
415 | ValueVT.getVectorElementCount().isScalable()) &&(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue () > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount ().isScalable()) && "Cannot narrow, it would be a lossy transformation" ) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 416, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
416 | "Cannot narrow, it would be a lossy transformation")(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue () > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount ().isScalable()) && "Cannot narrow, it would be a lossy transformation" ) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 416, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
417 | PartEVT = | ||||||||||||
418 | EVT::getVectorVT(*DAG.getContext(), PartEVT.getVectorElementType(), | ||||||||||||
419 | ValueVT.getVectorElementCount()); | ||||||||||||
420 | Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val, | ||||||||||||
421 | DAG.getVectorIdxConstant(0, DL)); | ||||||||||||
422 | if (PartEVT == ValueVT) | ||||||||||||
423 | return Val; | ||||||||||||
424 | } | ||||||||||||
425 | |||||||||||||
426 | // Promoted vector extract | ||||||||||||
427 | return DAG.getAnyExtOrTrunc(Val, DL, ValueVT); | ||||||||||||
428 | } | ||||||||||||
429 | |||||||||||||
430 | // Trivial bitcast if the types are the same size and the destination | ||||||||||||
431 | // vector type is legal. | ||||||||||||
432 | if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() && | ||||||||||||
433 | TLI.isTypeLegal(ValueVT)) | ||||||||||||
434 | return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); | ||||||||||||
435 | |||||||||||||
436 | if (ValueVT.getVectorNumElements() != 1) { | ||||||||||||
437 | // Certain ABIs require that vectors are passed as integers. For vectors | ||||||||||||
438 | // are the same size, this is an obvious bitcast. | ||||||||||||
439 | if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) { | ||||||||||||
440 | return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); | ||||||||||||
441 | } else if (ValueVT.bitsLT(PartEVT)) { | ||||||||||||
442 | const uint64_t ValueSize = ValueVT.getFixedSizeInBits(); | ||||||||||||
443 | EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); | ||||||||||||
444 | // Drop the extra bits. | ||||||||||||
445 | Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val); | ||||||||||||
446 | return DAG.getBitcast(ValueVT, Val); | ||||||||||||
447 | } | ||||||||||||
448 | |||||||||||||
449 | diagnosePossiblyInvalidConstraint( | ||||||||||||
450 | *DAG.getContext(), V, "non-trivial scalar-to-vector conversion"); | ||||||||||||
451 | return DAG.getUNDEF(ValueVT); | ||||||||||||
452 | } | ||||||||||||
453 | |||||||||||||
454 | // Handle cases such as i8 -> <1 x i1> | ||||||||||||
455 | EVT ValueSVT = ValueVT.getVectorElementType(); | ||||||||||||
456 | if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) { | ||||||||||||
457 | if (ValueSVT.getSizeInBits() == PartEVT.getSizeInBits()) | ||||||||||||
458 | Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val); | ||||||||||||
459 | else | ||||||||||||
460 | Val = ValueVT.isFloatingPoint() | ||||||||||||
461 | ? DAG.getFPExtendOrRound(Val, DL, ValueSVT) | ||||||||||||
462 | : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT); | ||||||||||||
463 | } | ||||||||||||
464 | |||||||||||||
465 | return DAG.getBuildVector(ValueVT, DL, Val); | ||||||||||||
466 | } | ||||||||||||
467 | |||||||||||||
468 | static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, | ||||||||||||
469 | SDValue Val, SDValue *Parts, unsigned NumParts, | ||||||||||||
470 | MVT PartVT, const Value *V, | ||||||||||||
471 | Optional<CallingConv::ID> CallConv); | ||||||||||||
472 | |||||||||||||
473 | /// getCopyToParts - Create a series of nodes that contain the specified value | ||||||||||||
474 | /// split into legal parts. If the parts contain more bits than Val, then, for | ||||||||||||
475 | /// integers, ExtendKind can be used to specify how to generate the extra bits. | ||||||||||||
476 | static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, | ||||||||||||
477 | SDValue *Parts, unsigned NumParts, MVT PartVT, | ||||||||||||
478 | const Value *V, | ||||||||||||
479 | Optional<CallingConv::ID> CallConv = None, | ||||||||||||
480 | ISD::NodeType ExtendKind = ISD::ANY_EXTEND) { | ||||||||||||
481 | // Let the target split the parts if it wants to | ||||||||||||
482 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
483 | if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT, | ||||||||||||
484 | CallConv)) | ||||||||||||
485 | return; | ||||||||||||
486 | EVT ValueVT = Val.getValueType(); | ||||||||||||
487 | |||||||||||||
488 | // Handle the vector case separately. | ||||||||||||
489 | if (ValueVT.isVector()) | ||||||||||||
490 | return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V, | ||||||||||||
491 | CallConv); | ||||||||||||
492 | |||||||||||||
493 | unsigned PartBits = PartVT.getSizeInBits(); | ||||||||||||
494 | unsigned OrigNumParts = NumParts; | ||||||||||||
495 | assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal (PartVT) && "Copying to an illegal type!") ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 496, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
496 | "Copying to an illegal type!")(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal (PartVT) && "Copying to an illegal type!") ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 496, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
497 | |||||||||||||
498 | if (NumParts == 0) | ||||||||||||
499 | return; | ||||||||||||
500 | |||||||||||||
501 | assert(!ValueVT.isVector() && "Vector case handled elsewhere")(static_cast <bool> (!ValueVT.isVector() && "Vector case handled elsewhere" ) ? void (0) : __assert_fail ("!ValueVT.isVector() && \"Vector case handled elsewhere\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 501, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
502 | EVT PartEVT = PartVT; | ||||||||||||
503 | if (PartEVT == ValueVT) { | ||||||||||||
504 | assert(NumParts == 1 && "No-op copy with multiple parts!")(static_cast <bool> (NumParts == 1 && "No-op copy with multiple parts!" ) ? void (0) : __assert_fail ("NumParts == 1 && \"No-op copy with multiple parts!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 504, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
505 | Parts[0] = Val; | ||||||||||||
506 | return; | ||||||||||||
507 | } | ||||||||||||
508 | |||||||||||||
509 | if (NumParts * PartBits > ValueVT.getSizeInBits()) { | ||||||||||||
510 | // If the parts cover more bits than the value has, promote the value. | ||||||||||||
511 | if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { | ||||||||||||
512 | assert(NumParts == 1 && "Do not know what to promote to!")(static_cast <bool> (NumParts == 1 && "Do not know what to promote to!" ) ? void (0) : __assert_fail ("NumParts == 1 && \"Do not know what to promote to!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 512, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
513 | Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val); | ||||||||||||
514 | } else { | ||||||||||||
515 | if (ValueVT.isFloatingPoint()) { | ||||||||||||
516 | // FP values need to be bitcast, then extended if they are being put | ||||||||||||
517 | // into a larger container. | ||||||||||||
518 | ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); | ||||||||||||
519 | Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); | ||||||||||||
520 | } | ||||||||||||
521 | assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT ::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!" ) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 523, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
522 | ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT ::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!" ) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 523, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
523 | "Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT ::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!" ) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 523, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
524 | ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); | ||||||||||||
525 | Val = DAG.getNode(ExtendKind, DL, ValueVT, Val); | ||||||||||||
526 | if (PartVT == MVT::x86mmx) | ||||||||||||
527 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); | ||||||||||||
528 | } | ||||||||||||
529 | } else if (PartBits == ValueVT.getSizeInBits()) { | ||||||||||||
530 | // Different types of the same size. | ||||||||||||
531 | assert(NumParts == 1 && PartEVT != ValueVT)(static_cast <bool> (NumParts == 1 && PartEVT != ValueVT) ? void (0) : __assert_fail ("NumParts == 1 && PartEVT != ValueVT" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 531, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
532 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); | ||||||||||||
533 | } else if (NumParts * PartBits < ValueVT.getSizeInBits()) { | ||||||||||||
534 | // If the parts cover less bits than value has, truncate the value. | ||||||||||||
535 | assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT ::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!" ) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 537, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
536 | ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT ::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!" ) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 537, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
537 | "Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT ::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!" ) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 537, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
538 | ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); | ||||||||||||
539 | Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); | ||||||||||||
540 | if (PartVT == MVT::x86mmx) | ||||||||||||
541 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); | ||||||||||||
542 | } | ||||||||||||
543 | |||||||||||||
544 | // The value may have changed - recompute ValueVT. | ||||||||||||
545 | ValueVT = Val.getValueType(); | ||||||||||||
546 | assert(NumParts * PartBits == ValueVT.getSizeInBits() &&(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits () && "Failed to tile the value with PartVT!") ? void (0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 547, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
547 | "Failed to tile the value with PartVT!")(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits () && "Failed to tile the value with PartVT!") ? void (0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 547, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
548 | |||||||||||||
549 | if (NumParts == 1) { | ||||||||||||
550 | if (PartEVT != ValueVT) { | ||||||||||||
551 | diagnosePossiblyInvalidConstraint(*DAG.getContext(), V, | ||||||||||||
552 | "scalar-to-vector conversion failed"); | ||||||||||||
553 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); | ||||||||||||
554 | } | ||||||||||||
555 | |||||||||||||
556 | Parts[0] = Val; | ||||||||||||
557 | return; | ||||||||||||
558 | } | ||||||||||||
559 | |||||||||||||
560 | // Expand the value into multiple parts. | ||||||||||||
561 | if (NumParts & (NumParts - 1)) { | ||||||||||||
562 | // The number of parts is not a power of 2. Split off and copy the tail. | ||||||||||||
563 | assert(PartVT.isInteger() && ValueVT.isInteger() &&(static_cast <bool> (PartVT.isInteger() && ValueVT .isInteger() && "Do not know what to expand to!") ? void (0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 564, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
564 | "Do not know what to expand to!")(static_cast <bool> (PartVT.isInteger() && ValueVT .isInteger() && "Do not know what to expand to!") ? void (0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 564, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
565 | unsigned RoundParts = 1 << Log2_32(NumParts); | ||||||||||||
566 | unsigned RoundBits = RoundParts * PartBits; | ||||||||||||
567 | unsigned OddParts = NumParts - RoundParts; | ||||||||||||
568 | SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val, | ||||||||||||
569 | DAG.getShiftAmountConstant(RoundBits, ValueVT, DL, /*LegalTypes*/false)); | ||||||||||||
570 | |||||||||||||
571 | getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V, | ||||||||||||
572 | CallConv); | ||||||||||||
573 | |||||||||||||
574 | if (DAG.getDataLayout().isBigEndian()) | ||||||||||||
575 | // The odd parts were reversed by getCopyToParts - unreverse them. | ||||||||||||
576 | std::reverse(Parts + RoundParts, Parts + NumParts); | ||||||||||||
577 | |||||||||||||
578 | NumParts = RoundParts; | ||||||||||||
579 | ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); | ||||||||||||
580 | Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); | ||||||||||||
581 | } | ||||||||||||
582 | |||||||||||||
583 | // The number of parts is a power of 2. Repeatedly bisect the value using | ||||||||||||
584 | // EXTRACT_ELEMENT. | ||||||||||||
585 | Parts[0] = DAG.getNode(ISD::BITCAST, DL, | ||||||||||||
586 | EVT::getIntegerVT(*DAG.getContext(), | ||||||||||||
587 | ValueVT.getSizeInBits()), | ||||||||||||
588 | Val); | ||||||||||||
589 | |||||||||||||
590 | for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) { | ||||||||||||
591 | for (unsigned i = 0; i < NumParts; i += StepSize) { | ||||||||||||
592 | unsigned ThisBits = StepSize * PartBits / 2; | ||||||||||||
593 | EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits); | ||||||||||||
594 | SDValue &Part0 = Parts[i]; | ||||||||||||
595 | SDValue &Part1 = Parts[i+StepSize/2]; | ||||||||||||
596 | |||||||||||||
597 | Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, | ||||||||||||
598 | ThisVT, Part0, DAG.getIntPtrConstant(1, DL)); | ||||||||||||
599 | Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, | ||||||||||||
600 | ThisVT, Part0, DAG.getIntPtrConstant(0, DL)); | ||||||||||||
601 | |||||||||||||
602 | if (ThisBits == PartBits && ThisVT != PartVT) { | ||||||||||||
603 | Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0); | ||||||||||||
604 | Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1); | ||||||||||||
605 | } | ||||||||||||
606 | } | ||||||||||||
607 | } | ||||||||||||
608 | |||||||||||||
609 | if (DAG.getDataLayout().isBigEndian()) | ||||||||||||
610 | std::reverse(Parts, Parts + OrigNumParts); | ||||||||||||
611 | } | ||||||||||||
612 | |||||||||||||
613 | static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, | ||||||||||||
614 | const SDLoc &DL, EVT PartVT) { | ||||||||||||
615 | if (!PartVT.isVector()) | ||||||||||||
616 | return SDValue(); | ||||||||||||
617 | |||||||||||||
618 | EVT ValueVT = Val.getValueType(); | ||||||||||||
619 | ElementCount PartNumElts = PartVT.getVectorElementCount(); | ||||||||||||
620 | ElementCount ValueNumElts = ValueVT.getVectorElementCount(); | ||||||||||||
621 | |||||||||||||
622 | // We only support widening vectors with equivalent element types and | ||||||||||||
623 | // fixed/scalable properties. If a target needs to widen a fixed-length type | ||||||||||||
624 | // to a scalable one, it should be possible to use INSERT_SUBVECTOR below. | ||||||||||||
625 | if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) || | ||||||||||||
626 | PartNumElts.isScalable() != ValueNumElts.isScalable() || | ||||||||||||
627 | PartVT.getVectorElementType() != ValueVT.getVectorElementType()) | ||||||||||||
628 | return SDValue(); | ||||||||||||
629 | |||||||||||||
630 | // Widening a scalable vector to another scalable vector is done by inserting | ||||||||||||
631 | // the vector into a larger undef one. | ||||||||||||
632 | if (PartNumElts.isScalable()) | ||||||||||||
633 | return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT), | ||||||||||||
634 | Val, DAG.getVectorIdxConstant(0, DL)); | ||||||||||||
635 | |||||||||||||
636 | EVT ElementVT = PartVT.getVectorElementType(); | ||||||||||||
637 | // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in | ||||||||||||
638 | // undef elements. | ||||||||||||
639 | SmallVector<SDValue, 16> Ops; | ||||||||||||
640 | DAG.ExtractVectorElements(Val, Ops); | ||||||||||||
641 | SDValue EltUndef = DAG.getUNDEF(ElementVT); | ||||||||||||
642 | Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef); | ||||||||||||
643 | |||||||||||||
644 | // FIXME: Use CONCAT for 2x -> 4x. | ||||||||||||
645 | return DAG.getBuildVector(PartVT, DL, Ops); | ||||||||||||
646 | } | ||||||||||||
647 | |||||||||||||
648 | /// getCopyToPartsVector - Create a series of nodes that contain the specified | ||||||||||||
649 | /// value split into legal parts. | ||||||||||||
650 | static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL, | ||||||||||||
651 | SDValue Val, SDValue *Parts, unsigned NumParts, | ||||||||||||
652 | MVT PartVT, const Value *V, | ||||||||||||
653 | Optional<CallingConv::ID> CallConv) { | ||||||||||||
654 | EVT ValueVT = Val.getValueType(); | ||||||||||||
655 | assert(ValueVT.isVector() && "Not a vector")(static_cast <bool> (ValueVT.isVector() && "Not a vector" ) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 655, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
656 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
657 | const bool IsABIRegCopy = CallConv.hasValue(); | ||||||||||||
658 | |||||||||||||
659 | if (NumParts == 1) { | ||||||||||||
660 | EVT PartEVT = PartVT; | ||||||||||||
661 | if (PartEVT == ValueVT) { | ||||||||||||
662 | // Nothing to do. | ||||||||||||
663 | } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) { | ||||||||||||
664 | // Bitconvert vector->vector case. | ||||||||||||
665 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); | ||||||||||||
666 | } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) { | ||||||||||||
667 | Val = Widened; | ||||||||||||
668 | } else if (PartVT.isVector() && | ||||||||||||
669 | PartEVT.getVectorElementType().bitsGE( | ||||||||||||
670 | ValueVT.getVectorElementType()) && | ||||||||||||
671 | PartEVT.getVectorElementCount() == | ||||||||||||
672 | ValueVT.getVectorElementCount()) { | ||||||||||||
673 | |||||||||||||
674 | // Promoted vector extract | ||||||||||||
675 | Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT); | ||||||||||||
676 | } else { | ||||||||||||
677 | if (ValueVT.getVectorElementCount().isScalar()) { | ||||||||||||
678 | Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val, | ||||||||||||
679 | DAG.getVectorIdxConstant(0, DL)); | ||||||||||||
680 | } else { | ||||||||||||
681 | uint64_t ValueSize = ValueVT.getFixedSizeInBits(); | ||||||||||||
682 | assert(PartVT.getFixedSizeInBits() > ValueSize &&(static_cast <bool> (PartVT.getFixedSizeInBits() > ValueSize && "lossy conversion of vector to scalar type") ? void (0) : __assert_fail ("PartVT.getFixedSizeInBits() > ValueSize && \"lossy conversion of vector to scalar type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 683, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
683 | "lossy conversion of vector to scalar type")(static_cast <bool> (PartVT.getFixedSizeInBits() > ValueSize && "lossy conversion of vector to scalar type") ? void (0) : __assert_fail ("PartVT.getFixedSizeInBits() > ValueSize && \"lossy conversion of vector to scalar type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 683, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
684 | EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); | ||||||||||||
685 | Val = DAG.getBitcast(IntermediateType, Val); | ||||||||||||
686 | Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT); | ||||||||||||
687 | } | ||||||||||||
688 | } | ||||||||||||
689 | |||||||||||||
690 | assert(Val.getValueType() == PartVT && "Unexpected vector part value type")(static_cast <bool> (Val.getValueType() == PartVT && "Unexpected vector part value type") ? void (0) : __assert_fail ("Val.getValueType() == PartVT && \"Unexpected vector part value type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 690, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
691 | Parts[0] = Val; | ||||||||||||
692 | return; | ||||||||||||
693 | } | ||||||||||||
694 | |||||||||||||
695 | // Handle a multi-element vector. | ||||||||||||
696 | EVT IntermediateVT; | ||||||||||||
697 | MVT RegisterVT; | ||||||||||||
698 | unsigned NumIntermediates; | ||||||||||||
699 | unsigned NumRegs; | ||||||||||||
700 | if (IsABIRegCopy) { | ||||||||||||
701 | NumRegs = TLI.getVectorTypeBreakdownForCallingConv( | ||||||||||||
702 | *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT, | ||||||||||||
703 | NumIntermediates, RegisterVT); | ||||||||||||
704 | } else { | ||||||||||||
705 | NumRegs = | ||||||||||||
706 | TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, | ||||||||||||
707 | NumIntermediates, RegisterVT); | ||||||||||||
708 | } | ||||||||||||
709 | |||||||||||||
710 | assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!" ) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 710, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
711 | NumParts = NumRegs; // Silence a compiler warning. | ||||||||||||
712 | assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!" ) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 712, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
713 | |||||||||||||
714 | assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&(static_cast <bool> (IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && "Mixing scalable and fixed vectors when copying in parts" ) ? void (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 715, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
715 | "Mixing scalable and fixed vectors when copying in parts")(static_cast <bool> (IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && "Mixing scalable and fixed vectors when copying in parts" ) ? void (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 715, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
716 | |||||||||||||
717 | Optional<ElementCount> DestEltCnt; | ||||||||||||
718 | |||||||||||||
719 | if (IntermediateVT.isVector()) | ||||||||||||
720 | DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates; | ||||||||||||
721 | else | ||||||||||||
722 | DestEltCnt = ElementCount::getFixed(NumIntermediates); | ||||||||||||
723 | |||||||||||||
724 | EVT BuiltVectorTy = EVT::getVectorVT( | ||||||||||||
725 | *DAG.getContext(), IntermediateVT.getScalarType(), DestEltCnt.getValue()); | ||||||||||||
726 | |||||||||||||
727 | if (ValueVT == BuiltVectorTy) { | ||||||||||||
728 | // Nothing to do. | ||||||||||||
729 | } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) { | ||||||||||||
730 | // Bitconvert vector->vector case. | ||||||||||||
731 | Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val); | ||||||||||||
732 | } else { | ||||||||||||
733 | if (BuiltVectorTy.getVectorElementType().bitsGT( | ||||||||||||
734 | ValueVT.getVectorElementType())) { | ||||||||||||
735 | // Integer promotion. | ||||||||||||
736 | ValueVT = EVT::getVectorVT(*DAG.getContext(), | ||||||||||||
737 | BuiltVectorTy.getVectorElementType(), | ||||||||||||
738 | ValueVT.getVectorElementCount()); | ||||||||||||
739 | Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); | ||||||||||||
740 | } | ||||||||||||
741 | |||||||||||||
742 | if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) { | ||||||||||||
743 | Val = Widened; | ||||||||||||
744 | } | ||||||||||||
745 | } | ||||||||||||
746 | |||||||||||||
747 | assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type")(static_cast <bool> (Val.getValueType() == BuiltVectorTy && "Unexpected vector value type") ? void (0) : __assert_fail ("Val.getValueType() == BuiltVectorTy && \"Unexpected vector value type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 747, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
748 | |||||||||||||
749 | // Split the vector into intermediate operands. | ||||||||||||
750 | SmallVector<SDValue, 8> Ops(NumIntermediates); | ||||||||||||
751 | for (unsigned i = 0; i != NumIntermediates; ++i) { | ||||||||||||
752 | if (IntermediateVT.isVector()) { | ||||||||||||
753 | // This does something sensible for scalable vectors - see the | ||||||||||||
754 | // definition of EXTRACT_SUBVECTOR for further details. | ||||||||||||
755 | unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements(); | ||||||||||||
756 | Ops[i] = | ||||||||||||
757 | DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val, | ||||||||||||
758 | DAG.getVectorIdxConstant(i * IntermediateNumElts, DL)); | ||||||||||||
759 | } else { | ||||||||||||
760 | Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val, | ||||||||||||
761 | DAG.getVectorIdxConstant(i, DL)); | ||||||||||||
762 | } | ||||||||||||
763 | } | ||||||||||||
764 | |||||||||||||
765 | // Split the intermediate operands into legal parts. | ||||||||||||
766 | if (NumParts == NumIntermediates) { | ||||||||||||
767 | // If the register was not expanded, promote or copy the value, | ||||||||||||
768 | // as appropriate. | ||||||||||||
769 | for (unsigned i = 0; i != NumParts; ++i) | ||||||||||||
770 | getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv); | ||||||||||||
771 | } else if (NumParts > 0) { | ||||||||||||
772 | // If the intermediate type was expanded, split each the value into | ||||||||||||
773 | // legal parts. | ||||||||||||
774 | assert(NumIntermediates != 0 && "division by zero")(static_cast <bool> (NumIntermediates != 0 && "division by zero" ) ? void (0) : __assert_fail ("NumIntermediates != 0 && \"division by zero\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 774, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
775 | assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!") ? void (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 776, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
776 | "Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!") ? void (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 776, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
777 | unsigned Factor = NumParts / NumIntermediates; | ||||||||||||
778 | for (unsigned i = 0; i != NumIntermediates; ++i) | ||||||||||||
779 | getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V, | ||||||||||||
780 | CallConv); | ||||||||||||
781 | } | ||||||||||||
782 | } | ||||||||||||
783 | |||||||||||||
784 | RegsForValue::RegsForValue(const SmallVector<unsigned, 4> ®s, MVT regvt, | ||||||||||||
785 | EVT valuevt, Optional<CallingConv::ID> CC) | ||||||||||||
786 | : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs), | ||||||||||||
787 | RegCount(1, regs.size()), CallConv(CC) {} | ||||||||||||
788 | |||||||||||||
789 | RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI, | ||||||||||||
790 | const DataLayout &DL, unsigned Reg, Type *Ty, | ||||||||||||
791 | Optional<CallingConv::ID> CC) { | ||||||||||||
792 | ComputeValueVTs(TLI, DL, Ty, ValueVTs); | ||||||||||||
793 | |||||||||||||
794 | CallConv = CC; | ||||||||||||
795 | |||||||||||||
796 | for (EVT ValueVT : ValueVTs) { | ||||||||||||
797 | unsigned NumRegs = | ||||||||||||
798 | isABIMangled() | ||||||||||||
799 | ? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT) | ||||||||||||
800 | : TLI.getNumRegisters(Context, ValueVT); | ||||||||||||
801 | MVT RegisterVT = | ||||||||||||
802 | isABIMangled() | ||||||||||||
803 | ? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT) | ||||||||||||
804 | : TLI.getRegisterType(Context, ValueVT); | ||||||||||||
805 | for (unsigned i = 0; i != NumRegs; ++i) | ||||||||||||
806 | Regs.push_back(Reg + i); | ||||||||||||
807 | RegVTs.push_back(RegisterVT); | ||||||||||||
808 | RegCount.push_back(NumRegs); | ||||||||||||
809 | Reg += NumRegs; | ||||||||||||
810 | } | ||||||||||||
811 | } | ||||||||||||
812 | |||||||||||||
813 | SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, | ||||||||||||
814 | FunctionLoweringInfo &FuncInfo, | ||||||||||||
815 | const SDLoc &dl, SDValue &Chain, | ||||||||||||
816 | SDValue *Flag, const Value *V) const { | ||||||||||||
817 | // A Value with type {} or [0 x %t] needs no registers. | ||||||||||||
818 | if (ValueVTs.empty()) | ||||||||||||
819 | return SDValue(); | ||||||||||||
820 | |||||||||||||
821 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
822 | |||||||||||||
823 | // Assemble the legal parts into the final values. | ||||||||||||
824 | SmallVector<SDValue, 4> Values(ValueVTs.size()); | ||||||||||||
825 | SmallVector<SDValue, 8> Parts; | ||||||||||||
826 | for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { | ||||||||||||
827 | // Copy the legal parts from the registers. | ||||||||||||
828 | EVT ValueVT = ValueVTs[Value]; | ||||||||||||
829 | unsigned NumRegs = RegCount[Value]; | ||||||||||||
830 | MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv( | ||||||||||||
831 | *DAG.getContext(), | ||||||||||||
832 | CallConv.getValue(), RegVTs[Value]) | ||||||||||||
833 | : RegVTs[Value]; | ||||||||||||
834 | |||||||||||||
835 | Parts.resize(NumRegs); | ||||||||||||
836 | for (unsigned i = 0; i != NumRegs; ++i) { | ||||||||||||
837 | SDValue P; | ||||||||||||
838 | if (!Flag) { | ||||||||||||
839 | P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT); | ||||||||||||
840 | } else { | ||||||||||||
841 | P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag); | ||||||||||||
842 | *Flag = P.getValue(2); | ||||||||||||
843 | } | ||||||||||||
844 | |||||||||||||
845 | Chain = P.getValue(1); | ||||||||||||
846 | Parts[i] = P; | ||||||||||||
847 | |||||||||||||
848 | // If the source register was virtual and if we know something about it, | ||||||||||||
849 | // add an assert node. | ||||||||||||
850 | if (!Register::isVirtualRegister(Regs[Part + i]) || | ||||||||||||
851 | !RegisterVT.isInteger()) | ||||||||||||
852 | continue; | ||||||||||||
853 | |||||||||||||
854 | const FunctionLoweringInfo::LiveOutInfo *LOI = | ||||||||||||
855 | FuncInfo.GetLiveOutRegInfo(Regs[Part+i]); | ||||||||||||
856 | if (!LOI) | ||||||||||||
857 | continue; | ||||||||||||
858 | |||||||||||||
859 | unsigned RegSize = RegisterVT.getScalarSizeInBits(); | ||||||||||||
860 | unsigned NumSignBits = LOI->NumSignBits; | ||||||||||||
861 | unsigned NumZeroBits = LOI->Known.countMinLeadingZeros(); | ||||||||||||
862 | |||||||||||||
863 | if (NumZeroBits == RegSize) { | ||||||||||||
864 | // The current value is a zero. | ||||||||||||
865 | // Explicitly express that as it would be easier for | ||||||||||||
866 | // optimizations to kick in. | ||||||||||||
867 | Parts[i] = DAG.getConstant(0, dl, RegisterVT); | ||||||||||||
868 | continue; | ||||||||||||
869 | } | ||||||||||||
870 | |||||||||||||
871 | // FIXME: We capture more information than the dag can represent. For | ||||||||||||
872 | // now, just use the tightest assertzext/assertsext possible. | ||||||||||||
873 | bool isSExt; | ||||||||||||
874 | EVT FromVT(MVT::Other); | ||||||||||||
875 | if (NumZeroBits) { | ||||||||||||
876 | FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits); | ||||||||||||
877 | isSExt = false; | ||||||||||||
878 | } else if (NumSignBits > 1) { | ||||||||||||
879 | FromVT = | ||||||||||||
880 | EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1); | ||||||||||||
881 | isSExt = true; | ||||||||||||
882 | } else { | ||||||||||||
883 | continue; | ||||||||||||
884 | } | ||||||||||||
885 | // Add an assertion node. | ||||||||||||
886 | assert(FromVT != MVT::Other)(static_cast <bool> (FromVT != MVT::Other) ? void (0) : __assert_fail ("FromVT != MVT::Other", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 886, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
887 | Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl, | ||||||||||||
888 | RegisterVT, P, DAG.getValueType(FromVT)); | ||||||||||||
889 | } | ||||||||||||
890 | |||||||||||||
891 | Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs, | ||||||||||||
892 | RegisterVT, ValueVT, V, CallConv); | ||||||||||||
893 | Part += NumRegs; | ||||||||||||
894 | Parts.clear(); | ||||||||||||
895 | } | ||||||||||||
896 | |||||||||||||
897 | return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values); | ||||||||||||
898 | } | ||||||||||||
899 | |||||||||||||
900 | void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, | ||||||||||||
901 | const SDLoc &dl, SDValue &Chain, SDValue *Flag, | ||||||||||||
902 | const Value *V, | ||||||||||||
903 | ISD::NodeType PreferredExtendType) const { | ||||||||||||
904 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
905 | ISD::NodeType ExtendKind = PreferredExtendType; | ||||||||||||
906 | |||||||||||||
907 | // Get the list of the values's legal parts. | ||||||||||||
908 | unsigned NumRegs = Regs.size(); | ||||||||||||
909 | SmallVector<SDValue, 8> Parts(NumRegs); | ||||||||||||
910 | for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { | ||||||||||||
911 | unsigned NumParts = RegCount[Value]; | ||||||||||||
912 | |||||||||||||
913 | MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv( | ||||||||||||
914 | *DAG.getContext(), | ||||||||||||
915 | CallConv.getValue(), RegVTs[Value]) | ||||||||||||
916 | : RegVTs[Value]; | ||||||||||||
917 | |||||||||||||
918 | if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT)) | ||||||||||||
919 | ExtendKind = ISD::ZERO_EXTEND; | ||||||||||||
920 | |||||||||||||
921 | getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part], | ||||||||||||
922 | NumParts, RegisterVT, V, CallConv, ExtendKind); | ||||||||||||
923 | Part += NumParts; | ||||||||||||
924 | } | ||||||||||||
925 | |||||||||||||
926 | // Copy the parts into the registers. | ||||||||||||
927 | SmallVector<SDValue, 8> Chains(NumRegs); | ||||||||||||
928 | for (unsigned i = 0; i != NumRegs; ++i) { | ||||||||||||
929 | SDValue Part; | ||||||||||||
930 | if (!Flag) { | ||||||||||||
931 | Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]); | ||||||||||||
932 | } else { | ||||||||||||
933 | Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag); | ||||||||||||
934 | *Flag = Part.getValue(1); | ||||||||||||
935 | } | ||||||||||||
936 | |||||||||||||
937 | Chains[i] = Part.getValue(0); | ||||||||||||
938 | } | ||||||||||||
939 | |||||||||||||
940 | if (NumRegs == 1 || Flag) | ||||||||||||
941 | // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is | ||||||||||||
942 | // flagged to it. That is the CopyToReg nodes and the user are considered | ||||||||||||
943 | // a single scheduling unit. If we create a TokenFactor and return it as | ||||||||||||
944 | // chain, then the TokenFactor is both a predecessor (operand) of the | ||||||||||||
945 | // user as well as a successor (the TF operands are flagged to the user). | ||||||||||||
946 | // c1, f1 = CopyToReg | ||||||||||||
947 | // c2, f2 = CopyToReg | ||||||||||||
948 | // c3 = TokenFactor c1, c2 | ||||||||||||
949 | // ... | ||||||||||||
950 | // = op c3, ..., f2 | ||||||||||||
951 | Chain = Chains[NumRegs-1]; | ||||||||||||
952 | else | ||||||||||||
953 | Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); | ||||||||||||
954 | } | ||||||||||||
955 | |||||||||||||
956 | void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching, | ||||||||||||
957 | unsigned MatchingIdx, const SDLoc &dl, | ||||||||||||
958 | SelectionDAG &DAG, | ||||||||||||
959 | std::vector<SDValue> &Ops) const { | ||||||||||||
960 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
961 | |||||||||||||
962 | unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size()); | ||||||||||||
963 | if (HasMatching) | ||||||||||||
964 | Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx); | ||||||||||||
965 | else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) { | ||||||||||||
966 | // Put the register class of the virtual registers in the flag word. That | ||||||||||||
967 | // way, later passes can recompute register class constraints for inline | ||||||||||||
968 | // assembly as well as normal instructions. | ||||||||||||
969 | // Don't do this for tied operands that can use the regclass information | ||||||||||||
970 | // from the def. | ||||||||||||
971 | const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); | ||||||||||||
972 | const TargetRegisterClass *RC = MRI.getRegClass(Regs.front()); | ||||||||||||
973 | Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID()); | ||||||||||||
974 | } | ||||||||||||
975 | |||||||||||||
976 | SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32); | ||||||||||||
977 | Ops.push_back(Res); | ||||||||||||
978 | |||||||||||||
979 | if (Code == InlineAsm::Kind_Clobber) { | ||||||||||||
980 | // Clobbers should always have a 1:1 mapping with registers, and may | ||||||||||||
981 | // reference registers that have illegal (e.g. vector) types. Hence, we | ||||||||||||
982 | // shouldn't try to apply any sort of splitting logic to them. | ||||||||||||
983 | assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&(static_cast <bool> (Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?" ) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 984, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
984 | "No 1:1 mapping from clobbers to regs?")(static_cast <bool> (Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?" ) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 984, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
985 | Register SP = TLI.getStackPointerRegisterToSaveRestore(); | ||||||||||||
986 | (void)SP; | ||||||||||||
987 | for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) { | ||||||||||||
988 | Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I])); | ||||||||||||
989 | assert((static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction ().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it." ) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 992, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
990 | (Regs[I] != SP ||(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction ().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it." ) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 992, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
991 | DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction ().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it." ) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 992, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
992 | "If we clobbered the stack pointer, MFI should know about it.")(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction ().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it." ) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 992, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
993 | } | ||||||||||||
994 | return; | ||||||||||||
995 | } | ||||||||||||
996 | |||||||||||||
997 | for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { | ||||||||||||
998 | MVT RegisterVT = RegVTs[Value]; | ||||||||||||
999 | unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value], | ||||||||||||
1000 | RegisterVT); | ||||||||||||
1001 | for (unsigned i = 0; i != NumRegs; ++i) { | ||||||||||||
1002 | assert(Reg < Regs.size() && "Mismatch in # registers expected")(static_cast <bool> (Reg < Regs.size() && "Mismatch in # registers expected" ) ? void (0) : __assert_fail ("Reg < Regs.size() && \"Mismatch in # registers expected\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1002, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
1003 | unsigned TheReg = Regs[Reg++]; | ||||||||||||
1004 | Ops.push_back(DAG.getRegister(TheReg, RegisterVT)); | ||||||||||||
1005 | } | ||||||||||||
1006 | } | ||||||||||||
1007 | } | ||||||||||||
1008 | |||||||||||||
1009 | SmallVector<std::pair<unsigned, TypeSize>, 4> | ||||||||||||
1010 | RegsForValue::getRegsAndSizes() const { | ||||||||||||
1011 | SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec; | ||||||||||||
1012 | unsigned I = 0; | ||||||||||||
1013 | for (auto CountAndVT : zip_first(RegCount, RegVTs)) { | ||||||||||||
1014 | unsigned RegCount = std::get<0>(CountAndVT); | ||||||||||||
1015 | MVT RegisterVT = std::get<1>(CountAndVT); | ||||||||||||
1016 | TypeSize RegisterSize = RegisterVT.getSizeInBits(); | ||||||||||||
1017 | for (unsigned E = I + RegCount; I != E; ++I) | ||||||||||||
1018 | OutVec.push_back(std::make_pair(Regs[I], RegisterSize)); | ||||||||||||
1019 | } | ||||||||||||
1020 | return OutVec; | ||||||||||||
1021 | } | ||||||||||||
1022 | |||||||||||||
1023 | void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa, | ||||||||||||
1024 | const TargetLibraryInfo *li) { | ||||||||||||
1025 | AA = aa; | ||||||||||||
1026 | GFI = gfi; | ||||||||||||
1027 | LibInfo = li; | ||||||||||||
1028 | DL = &DAG.getDataLayout(); | ||||||||||||
1029 | Context = DAG.getContext(); | ||||||||||||
1030 | LPadToCallSiteMap.clear(); | ||||||||||||
1031 | SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout()); | ||||||||||||
1032 | } | ||||||||||||
1033 | |||||||||||||
1034 | void SelectionDAGBuilder::clear() { | ||||||||||||
1035 | NodeMap.clear(); | ||||||||||||
1036 | UnusedArgNodeMap.clear(); | ||||||||||||
1037 | PendingLoads.clear(); | ||||||||||||
1038 | PendingExports.clear(); | ||||||||||||
1039 | PendingConstrainedFP.clear(); | ||||||||||||
1040 | PendingConstrainedFPStrict.clear(); | ||||||||||||
1041 | CurInst = nullptr; | ||||||||||||
1042 | HasTailCall = false; | ||||||||||||
1043 | SDNodeOrder = LowestSDNodeOrder; | ||||||||||||
1044 | StatepointLowering.clear(); | ||||||||||||
1045 | } | ||||||||||||
1046 | |||||||||||||
1047 | void SelectionDAGBuilder::clearDanglingDebugInfo() { | ||||||||||||
1048 | DanglingDebugInfoMap.clear(); | ||||||||||||
1049 | } | ||||||||||||
1050 | |||||||||||||
1051 | // Update DAG root to include dependencies on Pending chains. | ||||||||||||
1052 | SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) { | ||||||||||||
1053 | SDValue Root = DAG.getRoot(); | ||||||||||||
1054 | |||||||||||||
1055 | if (Pending.empty()) | ||||||||||||
1056 | return Root; | ||||||||||||
1057 | |||||||||||||
1058 | // Add current root to PendingChains, unless we already indirectly | ||||||||||||
1059 | // depend on it. | ||||||||||||
1060 | if (Root.getOpcode() != ISD::EntryToken) { | ||||||||||||
1061 | unsigned i = 0, e = Pending.size(); | ||||||||||||
1062 | for (; i != e; ++i) { | ||||||||||||
1063 | assert(Pending[i].getNode()->getNumOperands() > 1)(static_cast <bool> (Pending[i].getNode()->getNumOperands () > 1) ? void (0) : __assert_fail ("Pending[i].getNode()->getNumOperands() > 1" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1063, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
1064 | if (Pending[i].getNode()->getOperand(0) == Root) | ||||||||||||
1065 | break; // Don't add the root if we already indirectly depend on it. | ||||||||||||
1066 | } | ||||||||||||
1067 | |||||||||||||
1068 | if (i == e) | ||||||||||||
1069 | Pending.push_back(Root); | ||||||||||||
1070 | } | ||||||||||||
1071 | |||||||||||||
1072 | if (Pending.size() == 1) | ||||||||||||
1073 | Root = Pending[0]; | ||||||||||||
1074 | else | ||||||||||||
1075 | Root = DAG.getTokenFactor(getCurSDLoc(), Pending); | ||||||||||||
1076 | |||||||||||||
1077 | DAG.setRoot(Root); | ||||||||||||
1078 | Pending.clear(); | ||||||||||||
1079 | return Root; | ||||||||||||
1080 | } | ||||||||||||
1081 | |||||||||||||
1082 | SDValue SelectionDAGBuilder::getMemoryRoot() { | ||||||||||||
1083 | return updateRoot(PendingLoads); | ||||||||||||
1084 | } | ||||||||||||
1085 | |||||||||||||
1086 | SDValue SelectionDAGBuilder::getRoot() { | ||||||||||||
1087 | // Chain up all pending constrained intrinsics together with all | ||||||||||||
1088 | // pending loads, by simply appending them to PendingLoads and | ||||||||||||
1089 | // then calling getMemoryRoot(). | ||||||||||||
1090 | PendingLoads.reserve(PendingLoads.size() + | ||||||||||||
1091 | PendingConstrainedFP.size() + | ||||||||||||
1092 | PendingConstrainedFPStrict.size()); | ||||||||||||
1093 | PendingLoads.append(PendingConstrainedFP.begin(), | ||||||||||||
1094 | PendingConstrainedFP.end()); | ||||||||||||
1095 | PendingLoads.append(PendingConstrainedFPStrict.begin(), | ||||||||||||
1096 | PendingConstrainedFPStrict.end()); | ||||||||||||
1097 | PendingConstrainedFP.clear(); | ||||||||||||
1098 | PendingConstrainedFPStrict.clear(); | ||||||||||||
1099 | return getMemoryRoot(); | ||||||||||||
1100 | } | ||||||||||||
1101 | |||||||||||||
1102 | SDValue SelectionDAGBuilder::getControlRoot() { | ||||||||||||
1103 | // We need to emit pending fpexcept.strict constrained intrinsics, | ||||||||||||
1104 | // so append them to the PendingExports list. | ||||||||||||
1105 | PendingExports.append(PendingConstrainedFPStrict.begin(), | ||||||||||||
1106 | PendingConstrainedFPStrict.end()); | ||||||||||||
1107 | PendingConstrainedFPStrict.clear(); | ||||||||||||
1108 | return updateRoot(PendingExports); | ||||||||||||
1109 | } | ||||||||||||
1110 | |||||||||||||
1111 | void SelectionDAGBuilder::visit(const Instruction &I) { | ||||||||||||
1112 | // Set up outgoing PHI node register values before emitting the terminator. | ||||||||||||
1113 | if (I.isTerminator()) { | ||||||||||||
1114 | HandlePHINodesInSuccessorBlocks(I.getParent()); | ||||||||||||
1115 | } | ||||||||||||
1116 | |||||||||||||
1117 | // Increase the SDNodeOrder if dealing with a non-debug instruction. | ||||||||||||
1118 | if (!isa<DbgInfoIntrinsic>(I)) | ||||||||||||
1119 | ++SDNodeOrder; | ||||||||||||
1120 | |||||||||||||
1121 | CurInst = &I; | ||||||||||||
1122 | |||||||||||||
1123 | visit(I.getOpcode(), I); | ||||||||||||
1124 | |||||||||||||
1125 | if (!I.isTerminator() && !HasTailCall && | ||||||||||||
1126 | !isa<GCStatepointInst>(I)) // statepoints handle their exports internally | ||||||||||||
1127 | CopyToExportRegsIfNeeded(&I); | ||||||||||||
1128 | |||||||||||||
1129 | CurInst = nullptr; | ||||||||||||
1130 | } | ||||||||||||
1131 | |||||||||||||
1132 | void SelectionDAGBuilder::visitPHI(const PHINode &) { | ||||||||||||
1133 | llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit PHI nodes!" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1133); | ||||||||||||
1134 | } | ||||||||||||
1135 | |||||||||||||
1136 | void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) { | ||||||||||||
1137 | // Note: this doesn't use InstVisitor, because it has to work with | ||||||||||||
1138 | // ConstantExpr's in addition to instructions. | ||||||||||||
1139 | switch (Opcode) { | ||||||||||||
1140 | default: llvm_unreachable("Unknown instruction type encountered!")::llvm::llvm_unreachable_internal("Unknown instruction type encountered!" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1140); | ||||||||||||
1141 | // Build the switch statement using the Instruction.def file. | ||||||||||||
1142 | #define HANDLE_INST(NUM, OPCODE, CLASS) \ | ||||||||||||
1143 | case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break; | ||||||||||||
1144 | #include "llvm/IR/Instruction.def" | ||||||||||||
1145 | } | ||||||||||||
1146 | } | ||||||||||||
1147 | |||||||||||||
1148 | void SelectionDAGBuilder::addDanglingDebugInfo(const DbgValueInst *DI, | ||||||||||||
1149 | DebugLoc DL, unsigned Order) { | ||||||||||||
1150 | // We treat variadic dbg_values differently at this stage. | ||||||||||||
1151 | if (DI->hasArgList()) { | ||||||||||||
1152 | // For variadic dbg_values we will now insert an undef. | ||||||||||||
1153 | // FIXME: We can potentially recover these! | ||||||||||||
1154 | SmallVector<SDDbgOperand, 2> Locs; | ||||||||||||
1155 | for (const Value *V : DI->getValues()) { | ||||||||||||
1156 | auto Undef = UndefValue::get(V->getType()); | ||||||||||||
1157 | Locs.push_back(SDDbgOperand::fromConst(Undef)); | ||||||||||||
1158 | } | ||||||||||||
1159 | SDDbgValue *SDV = DAG.getDbgValueList( | ||||||||||||
1160 | DI->getVariable(), DI->getExpression(), Locs, {}, | ||||||||||||
1161 | /*IsIndirect=*/false, DL, Order, /*IsVariadic=*/true); | ||||||||||||
1162 | DAG.AddDbgValue(SDV, /*isParameter=*/false); | ||||||||||||
1163 | } else { | ||||||||||||
1164 | // TODO: Dangling debug info will eventually either be resolved or produce | ||||||||||||
1165 | // an Undef DBG_VALUE. However in the resolution case, a gap may appear | ||||||||||||
1166 | // between the original dbg.value location and its resolved DBG_VALUE, | ||||||||||||
1167 | // which we should ideally fill with an extra Undef DBG_VALUE. | ||||||||||||
1168 | assert(DI->getNumVariableLocationOps() == 1 &&(static_cast <bool> (DI->getNumVariableLocationOps() == 1 && "DbgValueInst without an ArgList should have a single location " "operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1170, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
1169 | "DbgValueInst without an ArgList should have a single location "(static_cast <bool> (DI->getNumVariableLocationOps() == 1 && "DbgValueInst without an ArgList should have a single location " "operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1170, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
1170 | "operand.")(static_cast <bool> (DI->getNumVariableLocationOps() == 1 && "DbgValueInst without an ArgList should have a single location " "operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1170, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
1171 | DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, DL, Order); | ||||||||||||
1172 | } | ||||||||||||
1173 | } | ||||||||||||
1174 | |||||||||||||
1175 | void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable, | ||||||||||||
1176 | const DIExpression *Expr) { | ||||||||||||
1177 | auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) { | ||||||||||||
1178 | const DbgValueInst *DI = DDI.getDI(); | ||||||||||||
1179 | DIVariable *DanglingVariable = DI->getVariable(); | ||||||||||||
1180 | DIExpression *DanglingExpr = DI->getExpression(); | ||||||||||||
1181 | if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) { | ||||||||||||
1182 | LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping dangling debug info for " << *DI << "\n"; } } while (false); | ||||||||||||
1183 | return true; | ||||||||||||
1184 | } | ||||||||||||
1185 | return false; | ||||||||||||
1186 | }; | ||||||||||||
1187 | |||||||||||||
1188 | for (auto &DDIMI : DanglingDebugInfoMap) { | ||||||||||||
1189 | DanglingDebugInfoVector &DDIV = DDIMI.second; | ||||||||||||
1190 | |||||||||||||
1191 | // If debug info is to be dropped, run it through final checks to see | ||||||||||||
1192 | // whether it can be salvaged. | ||||||||||||
1193 | for (auto &DDI : DDIV) | ||||||||||||
1194 | if (isMatchingDbgValue(DDI)) | ||||||||||||
1195 | salvageUnresolvedDbgValue(DDI); | ||||||||||||
1196 | |||||||||||||
1197 | erase_if(DDIV, isMatchingDbgValue); | ||||||||||||
1198 | } | ||||||||||||
1199 | } | ||||||||||||
1200 | |||||||||||||
1201 | // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V, | ||||||||||||
1202 | // generate the debug data structures now that we've seen its definition. | ||||||||||||
1203 | void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V, | ||||||||||||
1204 | SDValue Val) { | ||||||||||||
1205 | auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V); | ||||||||||||
1206 | if (DanglingDbgInfoIt == DanglingDebugInfoMap.end()) | ||||||||||||
1207 | return; | ||||||||||||
1208 | |||||||||||||
1209 | DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second; | ||||||||||||
1210 | for (auto &DDI : DDIV) { | ||||||||||||
1211 | const DbgValueInst *DI = DDI.getDI(); | ||||||||||||
1212 | assert(!DI->hasArgList() && "Not implemented for variadic dbg_values")(static_cast <bool> (!DI->hasArgList() && "Not implemented for variadic dbg_values" ) ? void (0) : __assert_fail ("!DI->hasArgList() && \"Not implemented for variadic dbg_values\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1212, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
1213 | assert(DI && "Ill-formed DanglingDebugInfo")(static_cast <bool> (DI && "Ill-formed DanglingDebugInfo" ) ? void (0) : __assert_fail ("DI && \"Ill-formed DanglingDebugInfo\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1213, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
1214 | DebugLoc dl = DDI.getdl(); | ||||||||||||
1215 | unsigned ValSDNodeOrder = Val.getNode()->getIROrder(); | ||||||||||||
1216 | unsigned DbgSDNodeOrder = DDI.getSDNodeOrder(); | ||||||||||||
1217 | DILocalVariable *Variable = DI->getVariable(); | ||||||||||||
1218 | DIExpression *Expr = DI->getExpression(); | ||||||||||||
1219 | assert(Variable->isValidLocationForIntrinsic(dl) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic (dl) && "Expected inlined-at fields to agree") ? void (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1220, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
1220 | "Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic (dl) && "Expected inlined-at fields to agree") ? void (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1220, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
1221 | SDDbgValue *SDV; | ||||||||||||
1222 | if (Val.getNode()) { | ||||||||||||
1223 | // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a | ||||||||||||
1224 | // FuncArgumentDbgValue (it would be hoisted to the function entry, and if | ||||||||||||
1225 | // we couldn't resolve it directly when examining the DbgValue intrinsic | ||||||||||||
1226 | // in the first place we should not be more successful here). Unless we | ||||||||||||
1227 | // have some test case that prove this to be correct we should avoid | ||||||||||||
1228 | // calling EmitFuncArgumentDbgValue here. | ||||||||||||
1229 | if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) { | ||||||||||||
1230 | LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Resolve dangling debug info [order=" << DbgSDNodeOrder << "] for:\n " << *DI << "\n"; } } while (false) | ||||||||||||
1231 | << DbgSDNodeOrder << "] for:\n " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Resolve dangling debug info [order=" << DbgSDNodeOrder << "] for:\n " << *DI << "\n"; } } while (false); | ||||||||||||
1232 | LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " By mapping to:\n "; Val.dump (); } } while (false); | ||||||||||||
1233 | // Increase the SDNodeOrder for the DbgValue here to make sure it is | ||||||||||||
1234 | // inserted after the definition of Val when emitting the instructions | ||||||||||||
1235 | // after ISel. An alternative could be to teach | ||||||||||||
1236 | // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly. | ||||||||||||
1237 | LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() << "changing SDNodeOrder from " << DbgSDNodeOrder << " to " << ValSDNodeOrder << "\n"; } } while (false ) | ||||||||||||
1238 | << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() << "changing SDNodeOrder from " << DbgSDNodeOrder << " to " << ValSDNodeOrder << "\n"; } } while (false ) | ||||||||||||
1239 | << ValSDNodeOrder << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() << "changing SDNodeOrder from " << DbgSDNodeOrder << " to " << ValSDNodeOrder << "\n"; } } while (false ); | ||||||||||||
1240 | SDV = getDbgValue(Val, Variable, Expr, dl, | ||||||||||||
1241 | std::max(DbgSDNodeOrder, ValSDNodeOrder)); | ||||||||||||
1242 | DAG.AddDbgValue(SDV, false); | ||||||||||||
1243 | } else | ||||||||||||
1244 | LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Resolved dangling debug info for " << *DI << "in EmitFuncArgumentDbgValue\n"; } } while (false) | ||||||||||||
1245 | << "in EmitFuncArgumentDbgValue\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Resolved dangling debug info for " << *DI << "in EmitFuncArgumentDbgValue\n"; } } while (false); | ||||||||||||
1246 | } else { | ||||||||||||
1247 | LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug info for " << *DI << "\n"; } } while (false); | ||||||||||||
1248 | auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType()); | ||||||||||||
1249 | auto SDV = | ||||||||||||
1250 | DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder); | ||||||||||||
1251 | DAG.AddDbgValue(SDV, false); | ||||||||||||
1252 | } | ||||||||||||
1253 | } | ||||||||||||
1254 | DDIV.clear(); | ||||||||||||
1255 | } | ||||||||||||
1256 | |||||||||||||
1257 | void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) { | ||||||||||||
1258 | // TODO: For the variadic implementation, instead of only checking the fail | ||||||||||||
1259 | // state of `handleDebugValue`, we need know specifically which values were | ||||||||||||
1260 | // invalid, so that we attempt to salvage only those values when processing | ||||||||||||
1261 | // a DIArgList. | ||||||||||||
1262 | assert(!DDI.getDI()->hasArgList() &&(static_cast <bool> (!DDI.getDI()->hasArgList() && "Not implemented for variadic dbg_values") ? void (0) : __assert_fail ("!DDI.getDI()->hasArgList() && \"Not implemented for variadic dbg_values\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1263, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
1263 | "Not implemented for variadic dbg_values")(static_cast <bool> (!DDI.getDI()->hasArgList() && "Not implemented for variadic dbg_values") ? void (0) : __assert_fail ("!DDI.getDI()->hasArgList() && \"Not implemented for variadic dbg_values\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1263, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
1264 | Value *V = DDI.getDI()->getValue(0); | ||||||||||||
1265 | DILocalVariable *Var = DDI.getDI()->getVariable(); | ||||||||||||
1266 | DIExpression *Expr = DDI.getDI()->getExpression(); | ||||||||||||
1267 | DebugLoc DL = DDI.getdl(); | ||||||||||||
1268 | DebugLoc InstDL = DDI.getDI()->getDebugLoc(); | ||||||||||||
1269 | unsigned SDOrder = DDI.getSDNodeOrder(); | ||||||||||||
1270 | // Currently we consider only dbg.value intrinsics -- we tell the salvager | ||||||||||||
1271 | // that DW_OP_stack_value is desired. | ||||||||||||
1272 | assert(isa<DbgValueInst>(DDI.getDI()))(static_cast <bool> (isa<DbgValueInst>(DDI.getDI( ))) ? void (0) : __assert_fail ("isa<DbgValueInst>(DDI.getDI())" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1272, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
1273 | bool StackValue = true; | ||||||||||||
1274 | |||||||||||||
1275 | // Can this Value can be encoded without any further work? | ||||||||||||
1276 | if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder, /*IsVariadic=*/false)) | ||||||||||||
1277 | return; | ||||||||||||
1278 | |||||||||||||
1279 | // Attempt to salvage back through as many instructions as possible. Bail if | ||||||||||||
1280 | // a non-instruction is seen, such as a constant expression or global | ||||||||||||
1281 | // variable. FIXME: Further work could recover those too. | ||||||||||||
1282 | while (isa<Instruction>(V)) { | ||||||||||||
1283 | Instruction &VAsInst = *cast<Instruction>(V); | ||||||||||||
1284 | // Temporary "0", awaiting real implementation. | ||||||||||||
1285 | SmallVector<uint64_t, 16> Ops; | ||||||||||||
1286 | SmallVector<Value *, 4> AdditionalValues; | ||||||||||||
1287 | V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops, | ||||||||||||
1288 | AdditionalValues); | ||||||||||||
1289 | // If we cannot salvage any further, and haven't yet found a suitable debug | ||||||||||||
1290 | // expression, bail out. | ||||||||||||
1291 | if (!V) | ||||||||||||
1292 | break; | ||||||||||||
1293 | |||||||||||||
1294 | // TODO: If AdditionalValues isn't empty, then the salvage can only be | ||||||||||||
1295 | // represented with a DBG_VALUE_LIST, so we give up. When we have support | ||||||||||||
1296 | // here for variadic dbg_values, remove that condition. | ||||||||||||
1297 | if (!AdditionalValues.empty()) | ||||||||||||
1298 | break; | ||||||||||||
1299 | |||||||||||||
1300 | // New value and expr now represent this debuginfo. | ||||||||||||
1301 | Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue); | ||||||||||||
1302 | |||||||||||||
1303 | // Some kind of simplification occurred: check whether the operand of the | ||||||||||||
1304 | // salvaged debug expression can be encoded in this DAG. | ||||||||||||
1305 | if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder, | ||||||||||||
1306 | /*IsVariadic=*/false)) { | ||||||||||||
1307 | LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Salvaged debug location info for:\n " << DDI.getDI() << "\nBy stripping back to:\n " << V; } } while (false) | ||||||||||||
1308 | << DDI.getDI() << "\nBy stripping back to:\n " << V)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Salvaged debug location info for:\n " << DDI.getDI() << "\nBy stripping back to:\n " << V; } } while (false); | ||||||||||||
1309 | return; | ||||||||||||
1310 | } | ||||||||||||
1311 | } | ||||||||||||
1312 | |||||||||||||
1313 | // This was the final opportunity to salvage this debug information, and it | ||||||||||||
1314 | // couldn't be done. Place an undef DBG_VALUE at this location to terminate | ||||||||||||
1315 | // any earlier variable location. | ||||||||||||
1316 | auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType()); | ||||||||||||
1317 | auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder); | ||||||||||||
1318 | DAG.AddDbgValue(SDV, false); | ||||||||||||
1319 | |||||||||||||
1320 | LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << DDI.getDI()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug value info for:\n " << DDI.getDI() << "\n"; } } while (false) | ||||||||||||
1321 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug value info for:\n " << DDI.getDI() << "\n"; } } while (false); | ||||||||||||
1322 | LLVM_DEBUG(dbgs() << " Last seen at:\n " << *DDI.getDI()->getOperand(0)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " Last seen at:\n " << * DDI.getDI()->getOperand(0) << "\n"; } } while (false ) | ||||||||||||
1323 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " Last seen at:\n " << * DDI.getDI()->getOperand(0) << "\n"; } } while (false ); | ||||||||||||
1324 | } | ||||||||||||
1325 | |||||||||||||
1326 | bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values, | ||||||||||||
1327 | DILocalVariable *Var, | ||||||||||||
1328 | DIExpression *Expr, DebugLoc dl, | ||||||||||||
1329 | DebugLoc InstDL, unsigned Order, | ||||||||||||
1330 | bool IsVariadic) { | ||||||||||||
1331 | if (Values.empty()) | ||||||||||||
1332 | return true; | ||||||||||||
1333 | SmallVector<SDDbgOperand> LocationOps; | ||||||||||||
1334 | SmallVector<SDNode *> Dependencies; | ||||||||||||
1335 | for (const Value *V : Values) { | ||||||||||||
1336 | // Constant value. | ||||||||||||
1337 | if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) || | ||||||||||||
1338 | isa<ConstantPointerNull>(V)) { | ||||||||||||
1339 | LocationOps.emplace_back(SDDbgOperand::fromConst(V)); | ||||||||||||
1340 | continue; | ||||||||||||
1341 | } | ||||||||||||
1342 | |||||||||||||
1343 | // If the Value is a frame index, we can create a FrameIndex debug value | ||||||||||||
1344 | // without relying on the DAG at all. | ||||||||||||
1345 | if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) { | ||||||||||||
1346 | auto SI = FuncInfo.StaticAllocaMap.find(AI); | ||||||||||||
1347 | if (SI != FuncInfo.StaticAllocaMap.end()) { | ||||||||||||
1348 | LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second)); | ||||||||||||
1349 | continue; | ||||||||||||
1350 | } | ||||||||||||
1351 | } | ||||||||||||
1352 | |||||||||||||
1353 | // Do not use getValue() in here; we don't want to generate code at | ||||||||||||
1354 | // this point if it hasn't been done yet. | ||||||||||||
1355 | SDValue N = NodeMap[V]; | ||||||||||||
1356 | if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map. | ||||||||||||
1357 | N = UnusedArgNodeMap[V]; | ||||||||||||
1358 | if (N.getNode()) { | ||||||||||||
1359 | // Only emit func arg dbg value for non-variadic dbg.values for now. | ||||||||||||
1360 | if (!IsVariadic && EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N)) | ||||||||||||
1361 | return true; | ||||||||||||
1362 | if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) { | ||||||||||||
1363 | // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can | ||||||||||||
1364 | // describe stack slot locations. | ||||||||||||
1365 | // | ||||||||||||
1366 | // Consider "int x = 0; int *px = &x;". There are two kinds of | ||||||||||||
1367 | // interesting debug values here after optimization: | ||||||||||||
1368 | // | ||||||||||||
1369 | // dbg.value(i32* %px, !"int *px", !DIExpression()), and | ||||||||||||
1370 | // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref)) | ||||||||||||
1371 | // | ||||||||||||
1372 | // Both describe the direct values of their associated variables. | ||||||||||||
1373 | Dependencies.push_back(N.getNode()); | ||||||||||||
1374 | LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex())); | ||||||||||||
1375 | continue; | ||||||||||||
1376 | } | ||||||||||||
1377 | LocationOps.emplace_back( | ||||||||||||
1378 | SDDbgOperand::fromNode(N.getNode(), N.getResNo())); | ||||||||||||
1379 | continue; | ||||||||||||
1380 | } | ||||||||||||
1381 | |||||||||||||
1382 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
1383 | // Special rules apply for the first dbg.values of parameter variables in a | ||||||||||||
1384 | // function. Identify them by the fact they reference Argument Values, that | ||||||||||||
1385 | // they're parameters, and they are parameters of the current function. We | ||||||||||||
1386 | // need to let them dangle until they get an SDNode. | ||||||||||||
1387 | bool IsParamOfFunc = | ||||||||||||
1388 | isa<Argument>(V) && Var->isParameter() && !InstDL.getInlinedAt(); | ||||||||||||
1389 | if (IsParamOfFunc) | ||||||||||||
1390 | return false; | ||||||||||||
1391 | |||||||||||||
1392 | // The value is not used in this block yet (or it would have an SDNode). | ||||||||||||
1393 | // We still want the value to appear for the user if possible -- if it has | ||||||||||||
1394 | // an associated VReg, we can refer to that instead. | ||||||||||||
1395 | auto VMI = FuncInfo.ValueMap.find(V); | ||||||||||||
1396 | if (VMI != FuncInfo.ValueMap.end()) { | ||||||||||||
1397 | unsigned Reg = VMI->second; | ||||||||||||
1398 | // If this is a PHI node, it may be split up into several MI PHI nodes | ||||||||||||
1399 | // (in FunctionLoweringInfo::set). | ||||||||||||
1400 | RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, | ||||||||||||
1401 | V->getType(), None); | ||||||||||||
1402 | if (RFV.occupiesMultipleRegs()) { | ||||||||||||
1403 | // FIXME: We could potentially support variadic dbg_values here. | ||||||||||||
1404 | if (IsVariadic) | ||||||||||||
1405 | return false; | ||||||||||||
1406 | unsigned Offset = 0; | ||||||||||||
1407 | unsigned BitsToDescribe = 0; | ||||||||||||
1408 | if (auto VarSize = Var->getSizeInBits()) | ||||||||||||
1409 | BitsToDescribe = *VarSize; | ||||||||||||
1410 | if (auto Fragment = Expr->getFragmentInfo()) | ||||||||||||
1411 | BitsToDescribe = Fragment->SizeInBits; | ||||||||||||
1412 | for (auto RegAndSize : RFV.getRegsAndSizes()) { | ||||||||||||
1413 | // Bail out if all bits are described already. | ||||||||||||
1414 | if (Offset >= BitsToDescribe) | ||||||||||||
1415 | break; | ||||||||||||
1416 | // TODO: handle scalable vectors. | ||||||||||||
1417 | unsigned RegisterSize = RegAndSize.second; | ||||||||||||
1418 | unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe) | ||||||||||||
1419 | ? BitsToDescribe - Offset | ||||||||||||
1420 | : RegisterSize; | ||||||||||||
1421 | auto FragmentExpr = DIExpression::createFragmentExpression( | ||||||||||||
1422 | Expr, Offset, FragmentSize); | ||||||||||||
1423 | if (!FragmentExpr) | ||||||||||||
1424 | continue; | ||||||||||||
1425 | SDDbgValue *SDV = DAG.getVRegDbgValue( | ||||||||||||
1426 | Var, *FragmentExpr, RegAndSize.first, false, dl, SDNodeOrder); | ||||||||||||
1427 | DAG.AddDbgValue(SDV, false); | ||||||||||||
1428 | Offset += RegisterSize; | ||||||||||||
1429 | } | ||||||||||||
1430 | return true; | ||||||||||||
1431 | } | ||||||||||||
1432 | // We can use simple vreg locations for variadic dbg_values as well. | ||||||||||||
1433 | LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg)); | ||||||||||||
1434 | continue; | ||||||||||||
1435 | } | ||||||||||||
1436 | // We failed to create a SDDbgOperand for V. | ||||||||||||
1437 | return false; | ||||||||||||
1438 | } | ||||||||||||
1439 | |||||||||||||
1440 | // We have created a SDDbgOperand for each Value in Values. | ||||||||||||
1441 | // Should use Order instead of SDNodeOrder? | ||||||||||||
1442 | assert(!LocationOps.empty())(static_cast <bool> (!LocationOps.empty()) ? void (0) : __assert_fail ("!LocationOps.empty()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1442, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
1443 | SDDbgValue *SDV = | ||||||||||||
1444 | DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies, | ||||||||||||
1445 | /*IsIndirect=*/false, dl, SDNodeOrder, IsVariadic); | ||||||||||||
1446 | DAG.AddDbgValue(SDV, /*isParameter=*/false); | ||||||||||||
1447 | return true; | ||||||||||||
1448 | } | ||||||||||||
1449 | |||||||||||||
1450 | void SelectionDAGBuilder::resolveOrClearDbgInfo() { | ||||||||||||
1451 | // Try to fixup any remaining dangling debug info -- and drop it if we can't. | ||||||||||||
1452 | for (auto &Pair : DanglingDebugInfoMap) | ||||||||||||
1453 | for (auto &DDI : Pair.second) | ||||||||||||
1454 | salvageUnresolvedDbgValue(DDI); | ||||||||||||
1455 | clearDanglingDebugInfo(); | ||||||||||||
1456 | } | ||||||||||||
1457 | |||||||||||||
1458 | /// getCopyFromRegs - If there was virtual register allocated for the value V | ||||||||||||
1459 | /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise. | ||||||||||||
1460 | SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) { | ||||||||||||
1461 | DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V); | ||||||||||||
1462 | SDValue Result; | ||||||||||||
1463 | |||||||||||||
1464 | if (It != FuncInfo.ValueMap.end()) { | ||||||||||||
1465 | Register InReg = It->second; | ||||||||||||
1466 | |||||||||||||
1467 | RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), | ||||||||||||
1468 | DAG.getDataLayout(), InReg, Ty, | ||||||||||||
1469 | None); // This is not an ABI copy. | ||||||||||||
1470 | SDValue Chain = DAG.getEntryNode(); | ||||||||||||
1471 | Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, | ||||||||||||
1472 | V); | ||||||||||||
1473 | resolveDanglingDebugInfo(V, Result); | ||||||||||||
1474 | } | ||||||||||||
1475 | |||||||||||||
1476 | return Result; | ||||||||||||
1477 | } | ||||||||||||
1478 | |||||||||||||
1479 | /// getValue - Return an SDValue for the given Value. | ||||||||||||
1480 | SDValue SelectionDAGBuilder::getValue(const Value *V) { | ||||||||||||
1481 | // If we already have an SDValue for this value, use it. It's important | ||||||||||||
1482 | // to do this first, so that we don't create a CopyFromReg if we already | ||||||||||||
1483 | // have a regular SDValue. | ||||||||||||
1484 | SDValue &N = NodeMap[V]; | ||||||||||||
1485 | if (N.getNode()) return N; | ||||||||||||
1486 | |||||||||||||
1487 | // If there's a virtual register allocated and initialized for this | ||||||||||||
1488 | // value, use it. | ||||||||||||
1489 | if (SDValue copyFromReg = getCopyFromRegs(V, V->getType())) | ||||||||||||
1490 | return copyFromReg; | ||||||||||||
1491 | |||||||||||||
1492 | // Otherwise create a new SDValue and remember it. | ||||||||||||
1493 | SDValue Val = getValueImpl(V); | ||||||||||||
1494 | NodeMap[V] = Val; | ||||||||||||
1495 | resolveDanglingDebugInfo(V, Val); | ||||||||||||
1496 | return Val; | ||||||||||||
1497 | } | ||||||||||||
1498 | |||||||||||||
1499 | /// getNonRegisterValue - Return an SDValue for the given Value, but | ||||||||||||
1500 | /// don't look in FuncInfo.ValueMap for a virtual register. | ||||||||||||
1501 | SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) { | ||||||||||||
1502 | // If we already have an SDValue for this value, use it. | ||||||||||||
1503 | SDValue &N = NodeMap[V]; | ||||||||||||
1504 | if (N.getNode()) { | ||||||||||||
1505 | if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) { | ||||||||||||
1506 | // Remove the debug location from the node as the node is about to be used | ||||||||||||
1507 | // in a location which may differ from the original debug location. This | ||||||||||||
1508 | // is relevant to Constant and ConstantFP nodes because they can appear | ||||||||||||
1509 | // as constant expressions inside PHI nodes. | ||||||||||||
1510 | N->setDebugLoc(DebugLoc()); | ||||||||||||
1511 | } | ||||||||||||
1512 | return N; | ||||||||||||
1513 | } | ||||||||||||
1514 | |||||||||||||
1515 | // Otherwise create a new SDValue and remember it. | ||||||||||||
1516 | SDValue Val = getValueImpl(V); | ||||||||||||
1517 | NodeMap[V] = Val; | ||||||||||||
1518 | resolveDanglingDebugInfo(V, Val); | ||||||||||||
1519 | return Val; | ||||||||||||
1520 | } | ||||||||||||
1521 | |||||||||||||
1522 | /// getValueImpl - Helper function for getValue and getNonRegisterValue. | ||||||||||||
1523 | /// Create an SDValue for the given value. | ||||||||||||
1524 | SDValue SelectionDAGBuilder::getValueImpl(const Value *V) { | ||||||||||||
1525 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
1526 | |||||||||||||
1527 | if (const Constant *C = dyn_cast<Constant>(V)) { | ||||||||||||
1528 | EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true); | ||||||||||||
1529 | |||||||||||||
1530 | if (const ConstantInt *CI = dyn_cast<ConstantInt>(C)) | ||||||||||||
1531 | return DAG.getConstant(*CI, getCurSDLoc(), VT); | ||||||||||||
1532 | |||||||||||||
1533 | if (const GlobalValue *GV = dyn_cast<GlobalValue>(C)) | ||||||||||||
1534 | return DAG.getGlobalAddress(GV, getCurSDLoc(), VT); | ||||||||||||
1535 | |||||||||||||
1536 | if (isa<ConstantPointerNull>(C)) { | ||||||||||||
1537 | unsigned AS = V->getType()->getPointerAddressSpace(); | ||||||||||||
1538 | return DAG.getConstant(0, getCurSDLoc(), | ||||||||||||
1539 | TLI.getPointerTy(DAG.getDataLayout(), AS)); | ||||||||||||
1540 | } | ||||||||||||
1541 | |||||||||||||
1542 | if (match(C, m_VScale(DAG.getDataLayout()))) | ||||||||||||
1543 | return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1)); | ||||||||||||
1544 | |||||||||||||
1545 | if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) | ||||||||||||
1546 | return DAG.getConstantFP(*CFP, getCurSDLoc(), VT); | ||||||||||||
1547 | |||||||||||||
1548 | if (isa<UndefValue>(C) && !V->getType()->isAggregateType()) | ||||||||||||
1549 | return DAG.getUNDEF(VT); | ||||||||||||
1550 | |||||||||||||
1551 | if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { | ||||||||||||
1552 | visit(CE->getOpcode(), *CE); | ||||||||||||
1553 | SDValue N1 = NodeMap[V]; | ||||||||||||
1554 | assert(N1.getNode() && "visit didn't populate the NodeMap!")(static_cast <bool> (N1.getNode() && "visit didn't populate the NodeMap!" ) ? void (0) : __assert_fail ("N1.getNode() && \"visit didn't populate the NodeMap!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1554, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
1555 | return N1; | ||||||||||||
1556 | } | ||||||||||||
1557 | |||||||||||||
1558 | if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) { | ||||||||||||
1559 | SmallVector<SDValue, 4> Constants; | ||||||||||||
1560 | for (const Use &U : C->operands()) { | ||||||||||||
1561 | SDNode *Val = getValue(U).getNode(); | ||||||||||||
1562 | // If the operand is an empty aggregate, there are no values. | ||||||||||||
1563 | if (!Val) continue; | ||||||||||||
1564 | // Add each leaf value from the operand to the Constants list | ||||||||||||
1565 | // to form a flattened list of all the values. | ||||||||||||
1566 | for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) | ||||||||||||
1567 | Constants.push_back(SDValue(Val, i)); | ||||||||||||
1568 | } | ||||||||||||
1569 | |||||||||||||
1570 | return DAG.getMergeValues(Constants, getCurSDLoc()); | ||||||||||||
1571 | } | ||||||||||||
1572 | |||||||||||||
1573 | if (const ConstantDataSequential *CDS = | ||||||||||||
1574 | dyn_cast<ConstantDataSequential>(C)) { | ||||||||||||
1575 | SmallVector<SDValue, 4> Ops; | ||||||||||||
1576 | for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) { | ||||||||||||
1577 | SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode(); | ||||||||||||
1578 | // Add each leaf value from the operand to the Constants list | ||||||||||||
1579 | // to form a flattened list of all the values. | ||||||||||||
1580 | for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) | ||||||||||||
1581 | Ops.push_back(SDValue(Val, i)); | ||||||||||||
1582 | } | ||||||||||||
1583 | |||||||||||||
1584 | if (isa<ArrayType>(CDS->getType())) | ||||||||||||
1585 | return DAG.getMergeValues(Ops, getCurSDLoc()); | ||||||||||||
1586 | return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops); | ||||||||||||
1587 | } | ||||||||||||
1588 | |||||||||||||
1589 | if (C->getType()->isStructTy() || C->getType()->isArrayTy()) { | ||||||||||||
1590 | assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&(static_cast <bool> ((isa<ConstantAggregateZero>( C) || isa<UndefValue>(C)) && "Unknown struct or array constant!" ) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1591, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
1591 | "Unknown struct or array constant!")(static_cast <bool> ((isa<ConstantAggregateZero>( C) || isa<UndefValue>(C)) && "Unknown struct or array constant!" ) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1591, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
1592 | |||||||||||||
1593 | SmallVector<EVT, 4> ValueVTs; | ||||||||||||
1594 | ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs); | ||||||||||||
1595 | unsigned NumElts = ValueVTs.size(); | ||||||||||||
1596 | if (NumElts == 0) | ||||||||||||
1597 | return SDValue(); // empty struct | ||||||||||||
1598 | SmallVector<SDValue, 4> Constants(NumElts); | ||||||||||||
1599 | for (unsigned i = 0; i != NumElts; ++i) { | ||||||||||||
1600 | EVT EltVT = ValueVTs[i]; | ||||||||||||
1601 | if (isa<UndefValue>(C)) | ||||||||||||
1602 | Constants[i] = DAG.getUNDEF(EltVT); | ||||||||||||
1603 | else if (EltVT.isFloatingPoint()) | ||||||||||||
1604 | Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT); | ||||||||||||
1605 | else | ||||||||||||
1606 | Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT); | ||||||||||||
1607 | } | ||||||||||||
1608 | |||||||||||||
1609 | return DAG.getMergeValues(Constants, getCurSDLoc()); | ||||||||||||
1610 | } | ||||||||||||
1611 | |||||||||||||
1612 | if (const BlockAddress *BA = dyn_cast<BlockAddress>(C)) | ||||||||||||
1613 | return DAG.getBlockAddress(BA, VT); | ||||||||||||
1614 | |||||||||||||
1615 | if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C)) | ||||||||||||
1616 | return getValue(Equiv->getGlobalValue()); | ||||||||||||
1617 | |||||||||||||
1618 | VectorType *VecTy = cast<VectorType>(V->getType()); | ||||||||||||
1619 | |||||||||||||
1620 | // Now that we know the number and type of the elements, get that number of | ||||||||||||
1621 | // elements into the Ops array based on what kind of constant it is. | ||||||||||||
1622 | if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) { | ||||||||||||
1623 | SmallVector<SDValue, 16> Ops; | ||||||||||||
1624 | unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements(); | ||||||||||||
1625 | for (unsigned i = 0; i != NumElements; ++i) | ||||||||||||
1626 | Ops.push_back(getValue(CV->getOperand(i))); | ||||||||||||
1627 | |||||||||||||
1628 | return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops); | ||||||||||||
1629 | } else if (isa<ConstantAggregateZero>(C)) { | ||||||||||||
1630 | EVT EltVT = | ||||||||||||
1631 | TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType()); | ||||||||||||
1632 | |||||||||||||
1633 | SDValue Op; | ||||||||||||
1634 | if (EltVT.isFloatingPoint()) | ||||||||||||
1635 | Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT); | ||||||||||||
1636 | else | ||||||||||||
1637 | Op = DAG.getConstant(0, getCurSDLoc(), EltVT); | ||||||||||||
1638 | |||||||||||||
1639 | if (isa<ScalableVectorType>(VecTy)) | ||||||||||||
1640 | return NodeMap[V] = DAG.getSplatVector(VT, getCurSDLoc(), Op); | ||||||||||||
1641 | else { | ||||||||||||
1642 | SmallVector<SDValue, 16> Ops; | ||||||||||||
1643 | Ops.assign(cast<FixedVectorType>(VecTy)->getNumElements(), Op); | ||||||||||||
1644 | return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops); | ||||||||||||
1645 | } | ||||||||||||
1646 | } | ||||||||||||
1647 | llvm_unreachable("Unknown vector constant")::llvm::llvm_unreachable_internal("Unknown vector constant", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1647); | ||||||||||||
1648 | } | ||||||||||||
1649 | |||||||||||||
1650 | // If this is a static alloca, generate it as the frameindex instead of | ||||||||||||
1651 | // computation. | ||||||||||||
1652 | if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) { | ||||||||||||
1653 | DenseMap<const AllocaInst*, int>::iterator SI = | ||||||||||||
1654 | FuncInfo.StaticAllocaMap.find(AI); | ||||||||||||
1655 | if (SI != FuncInfo.StaticAllocaMap.end()) | ||||||||||||
1656 | return DAG.getFrameIndex(SI->second, | ||||||||||||
1657 | TLI.getFrameIndexTy(DAG.getDataLayout())); | ||||||||||||
1658 | } | ||||||||||||
1659 | |||||||||||||
1660 | // If this is an instruction which fast-isel has deferred, select it now. | ||||||||||||
1661 | if (const Instruction *Inst = dyn_cast<Instruction>(V)) { | ||||||||||||
1662 | unsigned InReg = FuncInfo.InitializeRegForValue(Inst); | ||||||||||||
1663 | |||||||||||||
1664 | RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg, | ||||||||||||
1665 | Inst->getType(), None); | ||||||||||||
1666 | SDValue Chain = DAG.getEntryNode(); | ||||||||||||
1667 | return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V); | ||||||||||||
1668 | } | ||||||||||||
1669 | |||||||||||||
1670 | if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V)) { | ||||||||||||
1671 | return DAG.getMDNode(cast<MDNode>(MD->getMetadata())); | ||||||||||||
1672 | } | ||||||||||||
1673 | llvm_unreachable("Can't get register for value!")::llvm::llvm_unreachable_internal("Can't get register for value!" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1673); | ||||||||||||
1674 | } | ||||||||||||
1675 | |||||||||||||
1676 | void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) { | ||||||||||||
1677 | auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); | ||||||||||||
1678 | bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX; | ||||||||||||
1679 | bool IsCoreCLR = Pers == EHPersonality::CoreCLR; | ||||||||||||
1680 | bool IsSEH = isAsynchronousEHPersonality(Pers); | ||||||||||||
1681 | MachineBasicBlock *CatchPadMBB = FuncInfo.MBB; | ||||||||||||
1682 | if (!IsSEH) | ||||||||||||
1683 | CatchPadMBB->setIsEHScopeEntry(); | ||||||||||||
1684 | // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues. | ||||||||||||
1685 | if (IsMSVCCXX || IsCoreCLR) | ||||||||||||
1686 | CatchPadMBB->setIsEHFuncletEntry(); | ||||||||||||
1687 | } | ||||||||||||
1688 | |||||||||||||
1689 | void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) { | ||||||||||||
1690 | // Update machine-CFG edge. | ||||||||||||
1691 | MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()]; | ||||||||||||
1692 | FuncInfo.MBB->addSuccessor(TargetMBB); | ||||||||||||
1693 | TargetMBB->setIsEHCatchretTarget(true); | ||||||||||||
1694 | DAG.getMachineFunction().setHasEHCatchret(true); | ||||||||||||
1695 | |||||||||||||
1696 | auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); | ||||||||||||
1697 | bool IsSEH = isAsynchronousEHPersonality(Pers); | ||||||||||||
1698 | if (IsSEH) { | ||||||||||||
1699 | // If this is not a fall-through branch or optimizations are switched off, | ||||||||||||
1700 | // emit the branch. | ||||||||||||
1701 | if (TargetMBB != NextBlock(FuncInfo.MBB) || | ||||||||||||
1702 | TM.getOptLevel() == CodeGenOpt::None) | ||||||||||||
1703 | DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, | ||||||||||||
1704 | getControlRoot(), DAG.getBasicBlock(TargetMBB))); | ||||||||||||
1705 | return; | ||||||||||||
1706 | } | ||||||||||||
1707 | |||||||||||||
1708 | // Figure out the funclet membership for the catchret's successor. | ||||||||||||
1709 | // This will be used by the FuncletLayout pass to determine how to order the | ||||||||||||
1710 | // BB's. | ||||||||||||
1711 | // A 'catchret' returns to the outer scope's color. | ||||||||||||
1712 | Value *ParentPad = I.getCatchSwitchParentPad(); | ||||||||||||
1713 | const BasicBlock *SuccessorColor; | ||||||||||||
1714 | if (isa<ConstantTokenNone>(ParentPad)) | ||||||||||||
1715 | SuccessorColor = &FuncInfo.Fn->getEntryBlock(); | ||||||||||||
1716 | else | ||||||||||||
1717 | SuccessorColor = cast<Instruction>(ParentPad)->getParent(); | ||||||||||||
1718 | assert(SuccessorColor && "No parent funclet for catchret!")(static_cast <bool> (SuccessorColor && "No parent funclet for catchret!" ) ? void (0) : __assert_fail ("SuccessorColor && \"No parent funclet for catchret!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1718, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
1719 | MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor]; | ||||||||||||
1720 | assert(SuccessorColorMBB && "No MBB for SuccessorColor!")(static_cast <bool> (SuccessorColorMBB && "No MBB for SuccessorColor!" ) ? void (0) : __assert_fail ("SuccessorColorMBB && \"No MBB for SuccessorColor!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1720, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
1721 | |||||||||||||
1722 | // Create the terminator node. | ||||||||||||
1723 | SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other, | ||||||||||||
1724 | getControlRoot(), DAG.getBasicBlock(TargetMBB), | ||||||||||||
1725 | DAG.getBasicBlock(SuccessorColorMBB)); | ||||||||||||
1726 | DAG.setRoot(Ret); | ||||||||||||
1727 | } | ||||||||||||
1728 | |||||||||||||
1729 | void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) { | ||||||||||||
1730 | // Don't emit any special code for the cleanuppad instruction. It just marks | ||||||||||||
1731 | // the start of an EH scope/funclet. | ||||||||||||
1732 | FuncInfo.MBB->setIsEHScopeEntry(); | ||||||||||||
1733 | auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); | ||||||||||||
1734 | if (Pers != EHPersonality::Wasm_CXX) { | ||||||||||||
1735 | FuncInfo.MBB->setIsEHFuncletEntry(); | ||||||||||||
1736 | FuncInfo.MBB->setIsCleanupFuncletEntry(); | ||||||||||||
1737 | } | ||||||||||||
1738 | } | ||||||||||||
1739 | |||||||||||||
1740 | // In wasm EH, even though a catchpad may not catch an exception if a tag does | ||||||||||||
1741 | // not match, it is OK to add only the first unwind destination catchpad to the | ||||||||||||
1742 | // successors, because there will be at least one invoke instruction within the | ||||||||||||
1743 | // catch scope that points to the next unwind destination, if one exists, so | ||||||||||||
1744 | // CFGSort cannot mess up with BB sorting order. | ||||||||||||
1745 | // (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic | ||||||||||||
1746 | // call within them, and catchpads only consisting of 'catch (...)' have a | ||||||||||||
1747 | // '__cxa_end_catch' call within them, both of which generate invokes in case | ||||||||||||
1748 | // the next unwind destination exists, i.e., the next unwind destination is not | ||||||||||||
1749 | // the caller.) | ||||||||||||
1750 | // | ||||||||||||
1751 | // Having at most one EH pad successor is also simpler and helps later | ||||||||||||
1752 | // transformations. | ||||||||||||
1753 | // | ||||||||||||
1754 | // For example, | ||||||||||||
1755 | // current: | ||||||||||||
1756 | // invoke void @foo to ... unwind label %catch.dispatch | ||||||||||||
1757 | // catch.dispatch: | ||||||||||||
1758 | // %0 = catchswitch within ... [label %catch.start] unwind label %next | ||||||||||||
1759 | // catch.start: | ||||||||||||
1760 | // ... | ||||||||||||
1761 | // ... in this BB or some other child BB dominated by this BB there will be an | ||||||||||||
1762 | // invoke that points to 'next' BB as an unwind destination | ||||||||||||
1763 | // | ||||||||||||
1764 | // next: ; We don't need to add this to 'current' BB's successor | ||||||||||||
1765 | // ... | ||||||||||||
1766 | static void findWasmUnwindDestinations( | ||||||||||||
1767 | FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, | ||||||||||||
1768 | BranchProbability Prob, | ||||||||||||
1769 | SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>> | ||||||||||||
1770 | &UnwindDests) { | ||||||||||||
1771 | while (EHPadBB) { | ||||||||||||
1772 | const Instruction *Pad = EHPadBB->getFirstNonPHI(); | ||||||||||||
1773 | if (isa<CleanupPadInst>(Pad)) { | ||||||||||||
1774 | // Stop on cleanup pads. | ||||||||||||
1775 | UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); | ||||||||||||
1776 | UnwindDests.back().first->setIsEHScopeEntry(); | ||||||||||||
1777 | break; | ||||||||||||
1778 | } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) { | ||||||||||||
1779 | // Add the catchpad handlers to the possible destinations. We don't | ||||||||||||
1780 | // continue to the unwind destination of the catchswitch for wasm. | ||||||||||||
1781 | for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) { | ||||||||||||
1782 | UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob); | ||||||||||||
1783 | UnwindDests.back().first->setIsEHScopeEntry(); | ||||||||||||
1784 | } | ||||||||||||
1785 | break; | ||||||||||||
1786 | } else { | ||||||||||||
1787 | continue; | ||||||||||||
1788 | } | ||||||||||||
1789 | } | ||||||||||||
1790 | } | ||||||||||||
1791 | |||||||||||||
1792 | /// When an invoke or a cleanupret unwinds to the next EH pad, there are | ||||||||||||
1793 | /// many places it could ultimately go. In the IR, we have a single unwind | ||||||||||||
1794 | /// destination, but in the machine CFG, we enumerate all the possible blocks. | ||||||||||||
1795 | /// This function skips over imaginary basic blocks that hold catchswitch | ||||||||||||
1796 | /// instructions, and finds all the "real" machine | ||||||||||||
1797 | /// basic block destinations. As those destinations may not be successors of | ||||||||||||
1798 | /// EHPadBB, here we also calculate the edge probability to those destinations. | ||||||||||||
1799 | /// The passed-in Prob is the edge probability to EHPadBB. | ||||||||||||
1800 | static void findUnwindDestinations( | ||||||||||||
1801 | FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, | ||||||||||||
1802 | BranchProbability Prob, | ||||||||||||
1803 | SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>> | ||||||||||||
1804 | &UnwindDests) { | ||||||||||||
1805 | EHPersonality Personality = | ||||||||||||
1806 | classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); | ||||||||||||
1807 | bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX; | ||||||||||||
1808 | bool IsCoreCLR = Personality == EHPersonality::CoreCLR; | ||||||||||||
1809 | bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX; | ||||||||||||
1810 | bool IsSEH = isAsynchronousEHPersonality(Personality); | ||||||||||||
1811 | |||||||||||||
1812 | if (IsWasmCXX) { | ||||||||||||
1813 | findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests); | ||||||||||||
1814 | assert(UnwindDests.size() <= 1 &&(static_cast <bool> (UnwindDests.size() <= 1 && "There should be at most one unwind destination for wasm") ? void (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1815, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
1815 | "There should be at most one unwind destination for wasm")(static_cast <bool> (UnwindDests.size() <= 1 && "There should be at most one unwind destination for wasm") ? void (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1815, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
1816 | return; | ||||||||||||
1817 | } | ||||||||||||
1818 | |||||||||||||
1819 | while (EHPadBB) { | ||||||||||||
1820 | const Instruction *Pad = EHPadBB->getFirstNonPHI(); | ||||||||||||
1821 | BasicBlock *NewEHPadBB = nullptr; | ||||||||||||
1822 | if (isa<LandingPadInst>(Pad)) { | ||||||||||||
1823 | // Stop on landingpads. They are not funclets. | ||||||||||||
1824 | UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); | ||||||||||||
1825 | break; | ||||||||||||
1826 | } else if (isa<CleanupPadInst>(Pad)) { | ||||||||||||
1827 | // Stop on cleanup pads. Cleanups are always funclet entries for all known | ||||||||||||
1828 | // personalities. | ||||||||||||
1829 | UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); | ||||||||||||
1830 | UnwindDests.back().first->setIsEHScopeEntry(); | ||||||||||||
1831 | UnwindDests.back().first->setIsEHFuncletEntry(); | ||||||||||||
1832 | break; | ||||||||||||
1833 | } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) { | ||||||||||||
1834 | // Add the catchpad handlers to the possible destinations. | ||||||||||||
1835 | for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) { | ||||||||||||
1836 | UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob); | ||||||||||||
1837 | // For MSVC++ and the CLR, catchblocks are funclets and need prologues. | ||||||||||||
1838 | if (IsMSVCCXX || IsCoreCLR) | ||||||||||||
1839 | UnwindDests.back().first->setIsEHFuncletEntry(); | ||||||||||||
1840 | if (!IsSEH) | ||||||||||||
1841 | UnwindDests.back().first->setIsEHScopeEntry(); | ||||||||||||
1842 | } | ||||||||||||
1843 | NewEHPadBB = CatchSwitch->getUnwindDest(); | ||||||||||||
1844 | } else { | ||||||||||||
1845 | continue; | ||||||||||||
1846 | } | ||||||||||||
1847 | |||||||||||||
1848 | BranchProbabilityInfo *BPI = FuncInfo.BPI; | ||||||||||||
1849 | if (BPI && NewEHPadBB) | ||||||||||||
1850 | Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB); | ||||||||||||
1851 | EHPadBB = NewEHPadBB; | ||||||||||||
1852 | } | ||||||||||||
1853 | } | ||||||||||||
1854 | |||||||||||||
1855 | void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) { | ||||||||||||
1856 | // Update successor info. | ||||||||||||
1857 | SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests; | ||||||||||||
1858 | auto UnwindDest = I.getUnwindDest(); | ||||||||||||
1859 | BranchProbabilityInfo *BPI = FuncInfo.BPI; | ||||||||||||
1860 | BranchProbability UnwindDestProb = | ||||||||||||
1861 | (BPI && UnwindDest) | ||||||||||||
1862 | ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest) | ||||||||||||
1863 | : BranchProbability::getZero(); | ||||||||||||
1864 | findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests); | ||||||||||||
1865 | for (auto &UnwindDest : UnwindDests) { | ||||||||||||
1866 | UnwindDest.first->setIsEHPad(); | ||||||||||||
1867 | addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second); | ||||||||||||
1868 | } | ||||||||||||
1869 | FuncInfo.MBB->normalizeSuccProbs(); | ||||||||||||
1870 | |||||||||||||
1871 | // Create the terminator node. | ||||||||||||
1872 | SDValue Ret = | ||||||||||||
1873 | DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot()); | ||||||||||||
1874 | DAG.setRoot(Ret); | ||||||||||||
1875 | } | ||||||||||||
1876 | |||||||||||||
1877 | void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) { | ||||||||||||
1878 | report_fatal_error("visitCatchSwitch not yet implemented!"); | ||||||||||||
1879 | } | ||||||||||||
1880 | |||||||||||||
1881 | void SelectionDAGBuilder::visitRet(const ReturnInst &I) { | ||||||||||||
1882 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
1883 | auto &DL = DAG.getDataLayout(); | ||||||||||||
1884 | SDValue Chain = getControlRoot(); | ||||||||||||
1885 | SmallVector<ISD::OutputArg, 8> Outs; | ||||||||||||
1886 | SmallVector<SDValue, 8> OutVals; | ||||||||||||
1887 | |||||||||||||
1888 | // Calls to @llvm.experimental.deoptimize don't generate a return value, so | ||||||||||||
1889 | // lower | ||||||||||||
1890 | // | ||||||||||||
1891 | // %val = call <ty> @llvm.experimental.deoptimize() | ||||||||||||
1892 | // ret <ty> %val | ||||||||||||
1893 | // | ||||||||||||
1894 | // differently. | ||||||||||||
1895 | if (I.getParent()->getTerminatingDeoptimizeCall()) { | ||||||||||||
1896 | LowerDeoptimizingReturn(); | ||||||||||||
1897 | return; | ||||||||||||
1898 | } | ||||||||||||
1899 | |||||||||||||
1900 | if (!FuncInfo.CanLowerReturn) { | ||||||||||||
1901 | unsigned DemoteReg = FuncInfo.DemoteRegister; | ||||||||||||
1902 | const Function *F = I.getParent()->getParent(); | ||||||||||||
1903 | |||||||||||||
1904 | // Emit a store of the return value through the virtual register. | ||||||||||||
1905 | // Leave Outs empty so that LowerReturn won't try to load return | ||||||||||||
1906 | // registers the usual way. | ||||||||||||
1907 | SmallVector<EVT, 1> PtrValueVTs; | ||||||||||||
1908 | ComputeValueVTs(TLI, DL, | ||||||||||||
1909 | F->getReturnType()->getPointerTo( | ||||||||||||
1910 | DAG.getDataLayout().getAllocaAddrSpace()), | ||||||||||||
1911 | PtrValueVTs); | ||||||||||||
1912 | |||||||||||||
1913 | SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), | ||||||||||||
1914 | DemoteReg, PtrValueVTs[0]); | ||||||||||||
1915 | SDValue RetOp = getValue(I.getOperand(0)); | ||||||||||||
1916 | |||||||||||||
1917 | SmallVector<EVT, 4> ValueVTs, MemVTs; | ||||||||||||
1918 | SmallVector<uint64_t, 4> Offsets; | ||||||||||||
1919 | ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs, | ||||||||||||
1920 | &Offsets); | ||||||||||||
1921 | unsigned NumValues = ValueVTs.size(); | ||||||||||||
1922 | |||||||||||||
1923 | SmallVector<SDValue, 4> Chains(NumValues); | ||||||||||||
1924 | Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType()); | ||||||||||||
1925 | for (unsigned i = 0; i != NumValues; ++i) { | ||||||||||||
1926 | // An aggregate return value cannot wrap around the address space, so | ||||||||||||
1927 | // offsets to its parts don't wrap either. | ||||||||||||
1928 | SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, | ||||||||||||
1929 | TypeSize::Fixed(Offsets[i])); | ||||||||||||
1930 | |||||||||||||
1931 | SDValue Val = RetOp.getValue(RetOp.getResNo() + i); | ||||||||||||
1932 | if (MemVTs[i] != ValueVTs[i]) | ||||||||||||
1933 | Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]); | ||||||||||||
1934 | Chains[i] = DAG.getStore( | ||||||||||||
1935 | Chain, getCurSDLoc(), Val, | ||||||||||||
1936 | // FIXME: better loc info would be nice. | ||||||||||||
1937 | Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), | ||||||||||||
1938 | commonAlignment(BaseAlign, Offsets[i])); | ||||||||||||
1939 | } | ||||||||||||
1940 | |||||||||||||
1941 | Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), | ||||||||||||
1942 | MVT::Other, Chains); | ||||||||||||
1943 | } else if (I.getNumOperands() != 0) { | ||||||||||||
1944 | SmallVector<EVT, 4> ValueVTs; | ||||||||||||
1945 | ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs); | ||||||||||||
1946 | unsigned NumValues = ValueVTs.size(); | ||||||||||||
1947 | if (NumValues) { | ||||||||||||
1948 | SDValue RetOp = getValue(I.getOperand(0)); | ||||||||||||
1949 | |||||||||||||
1950 | const Function *F = I.getParent()->getParent(); | ||||||||||||
1951 | |||||||||||||
1952 | bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters( | ||||||||||||
1953 | I.getOperand(0)->getType(), F->getCallingConv(), | ||||||||||||
1954 | /*IsVarArg*/ false, DL); | ||||||||||||
1955 | |||||||||||||
1956 | ISD::NodeType ExtendKind = ISD::ANY_EXTEND; | ||||||||||||
1957 | if (F->getAttributes().hasRetAttr(Attribute::SExt)) | ||||||||||||
1958 | ExtendKind = ISD::SIGN_EXTEND; | ||||||||||||
1959 | else if (F->getAttributes().hasRetAttr(Attribute::ZExt)) | ||||||||||||
1960 | ExtendKind = ISD::ZERO_EXTEND; | ||||||||||||
1961 | |||||||||||||
1962 | LLVMContext &Context = F->getContext(); | ||||||||||||
1963 | bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg); | ||||||||||||
1964 | |||||||||||||
1965 | for (unsigned j = 0; j != NumValues; ++j) { | ||||||||||||
1966 | EVT VT = ValueVTs[j]; | ||||||||||||
1967 | |||||||||||||
1968 | if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) | ||||||||||||
1969 | VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind); | ||||||||||||
1970 | |||||||||||||
1971 | CallingConv::ID CC = F->getCallingConv(); | ||||||||||||
1972 | |||||||||||||
1973 | unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT); | ||||||||||||
1974 | MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT); | ||||||||||||
1975 | SmallVector<SDValue, 4> Parts(NumParts); | ||||||||||||
1976 | getCopyToParts(DAG, getCurSDLoc(), | ||||||||||||
1977 | SDValue(RetOp.getNode(), RetOp.getResNo() + j), | ||||||||||||
1978 | &Parts[0], NumParts, PartVT, &I, CC, ExtendKind); | ||||||||||||
1979 | |||||||||||||
1980 | // 'inreg' on function refers to return value | ||||||||||||
1981 | ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); | ||||||||||||
1982 | if (RetInReg) | ||||||||||||
1983 | Flags.setInReg(); | ||||||||||||
1984 | |||||||||||||
1985 | if (I.getOperand(0)->getType()->isPointerTy()) { | ||||||||||||
1986 | Flags.setPointer(); | ||||||||||||
1987 | Flags.setPointerAddrSpace( | ||||||||||||
1988 | cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace()); | ||||||||||||
1989 | } | ||||||||||||
1990 | |||||||||||||
1991 | if (NeedsRegBlock) { | ||||||||||||
1992 | Flags.setInConsecutiveRegs(); | ||||||||||||
1993 | if (j == NumValues - 1) | ||||||||||||
1994 | Flags.setInConsecutiveRegsLast(); | ||||||||||||
1995 | } | ||||||||||||
1996 | |||||||||||||
1997 | // Propagate extension type if any | ||||||||||||
1998 | if (ExtendKind == ISD::SIGN_EXTEND) | ||||||||||||
1999 | Flags.setSExt(); | ||||||||||||
2000 | else if (ExtendKind == ISD::ZERO_EXTEND) | ||||||||||||
2001 | Flags.setZExt(); | ||||||||||||
2002 | |||||||||||||
2003 | for (unsigned i = 0; i < NumParts; ++i) { | ||||||||||||
2004 | Outs.push_back(ISD::OutputArg(Flags, | ||||||||||||
2005 | Parts[i].getValueType().getSimpleVT(), | ||||||||||||
2006 | VT, /*isfixed=*/true, 0, 0)); | ||||||||||||
2007 | OutVals.push_back(Parts[i]); | ||||||||||||
2008 | } | ||||||||||||
2009 | } | ||||||||||||
2010 | } | ||||||||||||
2011 | } | ||||||||||||
2012 | |||||||||||||
2013 | // Push in swifterror virtual register as the last element of Outs. This makes | ||||||||||||
2014 | // sure swifterror virtual register will be returned in the swifterror | ||||||||||||
2015 | // physical register. | ||||||||||||
2016 | const Function *F = I.getParent()->getParent(); | ||||||||||||
2017 | if (TLI.supportSwiftError() && | ||||||||||||
2018 | F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) { | ||||||||||||
2019 | assert(SwiftError.getFunctionArg() && "Need a swift error argument")(static_cast <bool> (SwiftError.getFunctionArg() && "Need a swift error argument") ? void (0) : __assert_fail ("SwiftError.getFunctionArg() && \"Need a swift error argument\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2019, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
2020 | ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); | ||||||||||||
2021 | Flags.setSwiftError(); | ||||||||||||
2022 | Outs.push_back(ISD::OutputArg( | ||||||||||||
2023 | Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)), | ||||||||||||
2024 | /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0)); | ||||||||||||
2025 | // Create SDNode for the swifterror virtual register. | ||||||||||||
2026 | OutVals.push_back( | ||||||||||||
2027 | DAG.getRegister(SwiftError.getOrCreateVRegUseAt( | ||||||||||||
2028 | &I, FuncInfo.MBB, SwiftError.getFunctionArg()), | ||||||||||||
2029 | EVT(TLI.getPointerTy(DL)))); | ||||||||||||
2030 | } | ||||||||||||
2031 | |||||||||||||
2032 | bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg(); | ||||||||||||
2033 | CallingConv::ID CallConv = | ||||||||||||
2034 | DAG.getMachineFunction().getFunction().getCallingConv(); | ||||||||||||
2035 | Chain = DAG.getTargetLoweringInfo().LowerReturn( | ||||||||||||
2036 | Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG); | ||||||||||||
2037 | |||||||||||||
2038 | // Verify that the target's LowerReturn behaved as expected. | ||||||||||||
2039 | assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&(static_cast <bool> (Chain.getNode() && Chain.getValueType () == MVT::Other && "LowerReturn didn't return a valid chain!" ) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2040, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
2040 | "LowerReturn didn't return a valid chain!")(static_cast <bool> (Chain.getNode() && Chain.getValueType () == MVT::Other && "LowerReturn didn't return a valid chain!" ) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2040, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
2041 | |||||||||||||
2042 | // Update the DAG with the new chain value resulting from return lowering. | ||||||||||||
2043 | DAG.setRoot(Chain); | ||||||||||||
2044 | } | ||||||||||||
2045 | |||||||||||||
2046 | /// CopyToExportRegsIfNeeded - If the given value has virtual registers | ||||||||||||
2047 | /// created for it, emit nodes to copy the value into the virtual | ||||||||||||
2048 | /// registers. | ||||||||||||
2049 | void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) { | ||||||||||||
2050 | // Skip empty types | ||||||||||||
2051 | if (V->getType()->isEmptyTy()) | ||||||||||||
2052 | return; | ||||||||||||
2053 | |||||||||||||
2054 | DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V); | ||||||||||||
2055 | if (VMI != FuncInfo.ValueMap.end()) { | ||||||||||||
2056 | assert(!V->use_empty() && "Unused value assigned virtual registers!")(static_cast <bool> (!V->use_empty() && "Unused value assigned virtual registers!" ) ? void (0) : __assert_fail ("!V->use_empty() && \"Unused value assigned virtual registers!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2056, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
2057 | CopyValueToVirtualRegister(V, VMI->second); | ||||||||||||
2058 | } | ||||||||||||
2059 | } | ||||||||||||
2060 | |||||||||||||
2061 | /// ExportFromCurrentBlock - If this condition isn't known to be exported from | ||||||||||||
2062 | /// the current basic block, add it to ValueMap now so that we'll get a | ||||||||||||
2063 | /// CopyTo/FromReg. | ||||||||||||
2064 | void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) { | ||||||||||||
2065 | // No need to export constants. | ||||||||||||
2066 | if (!isa<Instruction>(V) && !isa<Argument>(V)) return; | ||||||||||||
2067 | |||||||||||||
2068 | // Already exported? | ||||||||||||
2069 | if (FuncInfo.isExportedInst(V)) return; | ||||||||||||
2070 | |||||||||||||
2071 | unsigned Reg = FuncInfo.InitializeRegForValue(V); | ||||||||||||
2072 | CopyValueToVirtualRegister(V, Reg); | ||||||||||||
2073 | } | ||||||||||||
2074 | |||||||||||||
2075 | bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V, | ||||||||||||
2076 | const BasicBlock *FromBB) { | ||||||||||||
2077 | // The operands of the setcc have to be in this block. We don't know | ||||||||||||
2078 | // how to export them from some other block. | ||||||||||||
2079 | if (const Instruction *VI = dyn_cast<Instruction>(V)) { | ||||||||||||
2080 | // Can export from current BB. | ||||||||||||
2081 | if (VI->getParent() == FromBB) | ||||||||||||
2082 | return true; | ||||||||||||
2083 | |||||||||||||
2084 | // Is already exported, noop. | ||||||||||||
2085 | return FuncInfo.isExportedInst(V); | ||||||||||||
2086 | } | ||||||||||||
2087 | |||||||||||||
2088 | // If this is an argument, we can export it if the BB is the entry block or | ||||||||||||
2089 | // if it is already exported. | ||||||||||||
2090 | if (isa<Argument>(V)) { | ||||||||||||
2091 | if (FromBB->isEntryBlock()) | ||||||||||||
2092 | return true; | ||||||||||||
2093 | |||||||||||||
2094 | // Otherwise, can only export this if it is already exported. | ||||||||||||
2095 | return FuncInfo.isExportedInst(V); | ||||||||||||
2096 | } | ||||||||||||
2097 | |||||||||||||
2098 | // Otherwise, constants can always be exported. | ||||||||||||
2099 | return true; | ||||||||||||
2100 | } | ||||||||||||
2101 | |||||||||||||
2102 | /// Return branch probability calculated by BranchProbabilityInfo for IR blocks. | ||||||||||||
2103 | BranchProbability | ||||||||||||
2104 | SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src, | ||||||||||||
2105 | const MachineBasicBlock *Dst) const { | ||||||||||||
2106 | BranchProbabilityInfo *BPI = FuncInfo.BPI; | ||||||||||||
2107 | const BasicBlock *SrcBB = Src->getBasicBlock(); | ||||||||||||
2108 | const BasicBlock *DstBB = Dst->getBasicBlock(); | ||||||||||||
2109 | if (!BPI) { | ||||||||||||
2110 | // If BPI is not available, set the default probability as 1 / N, where N is | ||||||||||||
2111 | // the number of successors. | ||||||||||||
2112 | auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1); | ||||||||||||
2113 | return BranchProbability(1, SuccSize); | ||||||||||||
2114 | } | ||||||||||||
2115 | return BPI->getEdgeProbability(SrcBB, DstBB); | ||||||||||||
2116 | } | ||||||||||||
2117 | |||||||||||||
2118 | void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src, | ||||||||||||
2119 | MachineBasicBlock *Dst, | ||||||||||||
2120 | BranchProbability Prob) { | ||||||||||||
2121 | if (!FuncInfo.BPI) | ||||||||||||
2122 | Src->addSuccessorWithoutProb(Dst); | ||||||||||||
2123 | else { | ||||||||||||
2124 | if (Prob.isUnknown()) | ||||||||||||
2125 | Prob = getEdgeProbability(Src, Dst); | ||||||||||||
2126 | Src->addSuccessor(Dst, Prob); | ||||||||||||
2127 | } | ||||||||||||
2128 | } | ||||||||||||
2129 | |||||||||||||
2130 | static bool InBlock(const Value *V, const BasicBlock *BB) { | ||||||||||||
2131 | if (const Instruction *I = dyn_cast<Instruction>(V)) | ||||||||||||
2132 | return I->getParent() == BB; | ||||||||||||
2133 | return true; | ||||||||||||
2134 | } | ||||||||||||
2135 | |||||||||||||
2136 | /// EmitBranchForMergedCondition - Helper method for FindMergedConditions. | ||||||||||||
2137 | /// This function emits a branch and is used at the leaves of an OR or an | ||||||||||||
2138 | /// AND operator tree. | ||||||||||||
2139 | void | ||||||||||||
2140 | SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, | ||||||||||||
2141 | MachineBasicBlock *TBB, | ||||||||||||
2142 | MachineBasicBlock *FBB, | ||||||||||||
2143 | MachineBasicBlock *CurBB, | ||||||||||||
2144 | MachineBasicBlock *SwitchBB, | ||||||||||||
2145 | BranchProbability TProb, | ||||||||||||
2146 | BranchProbability FProb, | ||||||||||||
2147 | bool InvertCond) { | ||||||||||||
2148 | const BasicBlock *BB = CurBB->getBasicBlock(); | ||||||||||||
2149 | |||||||||||||
2150 | // If the leaf of the tree is a comparison, merge the condition into | ||||||||||||
2151 | // the caseblock. | ||||||||||||
2152 | if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) { | ||||||||||||
2153 | // The operands of the cmp have to be in this block. We don't know | ||||||||||||
2154 | // how to export them from some other block. If this is the first block | ||||||||||||
2155 | // of the sequence, no exporting is needed. | ||||||||||||
2156 | if (CurBB == SwitchBB || | ||||||||||||
2157 | (isExportableFromCurrentBlock(BOp->getOperand(0), BB) && | ||||||||||||
2158 | isExportableFromCurrentBlock(BOp->getOperand(1), BB))) { | ||||||||||||
2159 | ISD::CondCode Condition; | ||||||||||||
2160 | if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) { | ||||||||||||
2161 | ICmpInst::Predicate Pred = | ||||||||||||
2162 | InvertCond ? IC->getInversePredicate() : IC->getPredicate(); | ||||||||||||
2163 | Condition = getICmpCondCode(Pred); | ||||||||||||
2164 | } else { | ||||||||||||
2165 | const FCmpInst *FC = cast<FCmpInst>(Cond); | ||||||||||||
2166 | FCmpInst::Predicate Pred = | ||||||||||||
2167 | InvertCond ? FC->getInversePredicate() : FC->getPredicate(); | ||||||||||||
2168 | Condition = getFCmpCondCode(Pred); | ||||||||||||
2169 | if (TM.Options.NoNaNsFPMath) | ||||||||||||
2170 | Condition = getFCmpCodeWithoutNaN(Condition); | ||||||||||||
2171 | } | ||||||||||||
2172 | |||||||||||||
2173 | CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr, | ||||||||||||
2174 | TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb); | ||||||||||||
2175 | SL->SwitchCases.push_back(CB); | ||||||||||||
2176 | return; | ||||||||||||
2177 | } | ||||||||||||
2178 | } | ||||||||||||
2179 | |||||||||||||
2180 | // Create a CaseBlock record representing this branch. | ||||||||||||
2181 | ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ; | ||||||||||||
2182 | CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()), | ||||||||||||
2183 | nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb); | ||||||||||||
2184 | SL->SwitchCases.push_back(CB); | ||||||||||||
2185 | } | ||||||||||||
2186 | |||||||||||||
2187 | void SelectionDAGBuilder::FindMergedConditions(const Value *Cond, | ||||||||||||
2188 | MachineBasicBlock *TBB, | ||||||||||||
2189 | MachineBasicBlock *FBB, | ||||||||||||
2190 | MachineBasicBlock *CurBB, | ||||||||||||
2191 | MachineBasicBlock *SwitchBB, | ||||||||||||
2192 | Instruction::BinaryOps Opc, | ||||||||||||
2193 | BranchProbability TProb, | ||||||||||||
2194 | BranchProbability FProb, | ||||||||||||
2195 | bool InvertCond) { | ||||||||||||
2196 | // Skip over not part of the tree and remember to invert op and operands at | ||||||||||||
2197 | // next level. | ||||||||||||
2198 | Value *NotCond; | ||||||||||||
2199 | if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) && | ||||||||||||
2200 | InBlock(NotCond, CurBB->getBasicBlock())) { | ||||||||||||
2201 | FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb, | ||||||||||||
2202 | !InvertCond); | ||||||||||||
2203 | return; | ||||||||||||
2204 | } | ||||||||||||
2205 | |||||||||||||
2206 | const Instruction *BOp = dyn_cast<Instruction>(Cond); | ||||||||||||
2207 | const Value *BOpOp0, *BOpOp1; | ||||||||||||
2208 | // Compute the effective opcode for Cond, taking into account whether it needs | ||||||||||||
2209 | // to be inverted, e.g. | ||||||||||||
2210 | // and (not (or A, B)), C | ||||||||||||
2211 | // gets lowered as | ||||||||||||
2212 | // and (and (not A, not B), C) | ||||||||||||
2213 | Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0; | ||||||||||||
2214 | if (BOp) { | ||||||||||||
2215 | BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1))) | ||||||||||||
2216 | ? Instruction::And | ||||||||||||
2217 | : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1))) | ||||||||||||
2218 | ? Instruction::Or | ||||||||||||
2219 | : (Instruction::BinaryOps)0); | ||||||||||||
2220 | if (InvertCond) { | ||||||||||||
2221 | if (BOpc == Instruction::And) | ||||||||||||
2222 | BOpc = Instruction::Or; | ||||||||||||
2223 | else if (BOpc == Instruction::Or) | ||||||||||||
2224 | BOpc = Instruction::And; | ||||||||||||
2225 | } | ||||||||||||
2226 | } | ||||||||||||
2227 | |||||||||||||
2228 | // If this node is not part of the or/and tree, emit it as a branch. | ||||||||||||
2229 | // Note that all nodes in the tree should have same opcode. | ||||||||||||
2230 | bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse(); | ||||||||||||
2231 | if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() || | ||||||||||||
2232 | !InBlock(BOpOp0, CurBB->getBasicBlock()) || | ||||||||||||
2233 | !InBlock(BOpOp1, CurBB->getBasicBlock())) { | ||||||||||||
2234 | EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, | ||||||||||||
2235 | TProb, FProb, InvertCond); | ||||||||||||
2236 | return; | ||||||||||||
2237 | } | ||||||||||||
2238 | |||||||||||||
2239 | // Create TmpBB after CurBB. | ||||||||||||
2240 | MachineFunction::iterator BBI(CurBB); | ||||||||||||
2241 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
2242 | MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock()); | ||||||||||||
2243 | CurBB->getParent()->insert(++BBI, TmpBB); | ||||||||||||
2244 | |||||||||||||
2245 | if (Opc == Instruction::Or) { | ||||||||||||
2246 | // Codegen X | Y as: | ||||||||||||
2247 | // BB1: | ||||||||||||
2248 | // jmp_if_X TBB | ||||||||||||
2249 | // jmp TmpBB | ||||||||||||
2250 | // TmpBB: | ||||||||||||
2251 | // jmp_if_Y TBB | ||||||||||||
2252 | // jmp FBB | ||||||||||||
2253 | // | ||||||||||||
2254 | |||||||||||||
2255 | // We have flexibility in setting Prob for BB1 and Prob for TmpBB. | ||||||||||||
2256 | // The requirement is that | ||||||||||||
2257 | // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB) | ||||||||||||
2258 | // = TrueProb for original BB. | ||||||||||||
2259 | // Assuming the original probabilities are A and B, one choice is to set | ||||||||||||
2260 | // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to | ||||||||||||
2261 | // A/(1+B) and 2B/(1+B). This choice assumes that | ||||||||||||
2262 | // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB. | ||||||||||||
2263 | // Another choice is to assume TrueProb for BB1 equals to TrueProb for | ||||||||||||
2264 | // TmpBB, but the math is more complicated. | ||||||||||||
2265 | |||||||||||||
2266 | auto NewTrueProb = TProb / 2; | ||||||||||||
2267 | auto NewFalseProb = TProb / 2 + FProb; | ||||||||||||
2268 | // Emit the LHS condition. | ||||||||||||
2269 | FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb, | ||||||||||||
2270 | NewFalseProb, InvertCond); | ||||||||||||
2271 | |||||||||||||
2272 | // Normalize A/2 and B to get A/(1+B) and 2B/(1+B). | ||||||||||||
2273 | SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb}; | ||||||||||||
2274 | BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end()); | ||||||||||||
2275 | // Emit the RHS condition into TmpBB. | ||||||||||||
2276 | FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0], | ||||||||||||
2277 | Probs[1], InvertCond); | ||||||||||||
2278 | } else { | ||||||||||||
2279 | assert(Opc == Instruction::And && "Unknown merge op!")(static_cast <bool> (Opc == Instruction::And && "Unknown merge op!") ? void (0) : __assert_fail ("Opc == Instruction::And && \"Unknown merge op!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2279, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
2280 | // Codegen X & Y as: | ||||||||||||
2281 | // BB1: | ||||||||||||
2282 | // jmp_if_X TmpBB | ||||||||||||
2283 | // jmp FBB | ||||||||||||
2284 | // TmpBB: | ||||||||||||
2285 | // jmp_if_Y TBB | ||||||||||||
2286 | // jmp FBB | ||||||||||||
2287 | // | ||||||||||||
2288 | // This requires creation of TmpBB after CurBB. | ||||||||||||
2289 | |||||||||||||
2290 | // We have flexibility in setting Prob for BB1 and Prob for TmpBB. | ||||||||||||
2291 | // The requirement is that | ||||||||||||
2292 | // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB) | ||||||||||||
2293 | // = FalseProb for original BB. | ||||||||||||
2294 | // Assuming the original probabilities are A and B, one choice is to set | ||||||||||||
2295 | // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to | ||||||||||||
2296 | // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 == | ||||||||||||
2297 | // TrueProb for BB1 * FalseProb for TmpBB. | ||||||||||||
2298 | |||||||||||||
2299 | auto NewTrueProb = TProb + FProb / 2; | ||||||||||||
2300 | auto NewFalseProb = FProb / 2; | ||||||||||||
2301 | // Emit the LHS condition. | ||||||||||||
2302 | FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb, | ||||||||||||
2303 | NewFalseProb, InvertCond); | ||||||||||||
2304 | |||||||||||||
2305 | // Normalize A and B/2 to get 2A/(1+A) and B/(1+A). | ||||||||||||
2306 | SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2}; | ||||||||||||
2307 | BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end()); | ||||||||||||
2308 | // Emit the RHS condition into TmpBB. | ||||||||||||
2309 | FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0], | ||||||||||||
2310 | Probs[1], InvertCond); | ||||||||||||
2311 | } | ||||||||||||
2312 | } | ||||||||||||
2313 | |||||||||||||
2314 | /// If the set of cases should be emitted as a series of branches, return true. | ||||||||||||
2315 | /// If we should emit this as a bunch of and/or'd together conditions, return | ||||||||||||
2316 | /// false. | ||||||||||||
2317 | bool | ||||||||||||
2318 | SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) { | ||||||||||||
2319 | if (Cases.size() != 2) return true; | ||||||||||||
2320 | |||||||||||||
2321 | // If this is two comparisons of the same values or'd or and'd together, they | ||||||||||||
2322 | // will get folded into a single comparison, so don't emit two blocks. | ||||||||||||
2323 | if ((Cases[0].CmpLHS == Cases[1].CmpLHS && | ||||||||||||
2324 | Cases[0].CmpRHS == Cases[1].CmpRHS) || | ||||||||||||
2325 | (Cases[0].CmpRHS == Cases[1].CmpLHS && | ||||||||||||
2326 | Cases[0].CmpLHS == Cases[1].CmpRHS)) { | ||||||||||||
2327 | return false; | ||||||||||||
2328 | } | ||||||||||||
2329 | |||||||||||||
2330 | // Handle: (X != null) | (Y != null) --> (X|Y) != 0 | ||||||||||||
2331 | // Handle: (X == null) & (Y == null) --> (X|Y) == 0 | ||||||||||||
2332 | if (Cases[0].CmpRHS == Cases[1].CmpRHS && | ||||||||||||
2333 | Cases[0].CC == Cases[1].CC && | ||||||||||||
2334 | isa<Constant>(Cases[0].CmpRHS) && | ||||||||||||
2335 | cast<Constant>(Cases[0].CmpRHS)->isNullValue()) { | ||||||||||||
2336 | if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB) | ||||||||||||
2337 | return false; | ||||||||||||
2338 | if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB) | ||||||||||||
2339 | return false; | ||||||||||||
2340 | } | ||||||||||||
2341 | |||||||||||||
2342 | return true; | ||||||||||||
2343 | } | ||||||||||||
2344 | |||||||||||||
2345 | void SelectionDAGBuilder::visitBr(const BranchInst &I) { | ||||||||||||
2346 | MachineBasicBlock *BrMBB = FuncInfo.MBB; | ||||||||||||
2347 | |||||||||||||
2348 | // Update machine-CFG edges. | ||||||||||||
2349 | MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)]; | ||||||||||||
2350 | |||||||||||||
2351 | if (I.isUnconditional()) { | ||||||||||||
2352 | // Update machine-CFG edges. | ||||||||||||
2353 | BrMBB->addSuccessor(Succ0MBB); | ||||||||||||
2354 | |||||||||||||
2355 | // If this is not a fall-through branch or optimizations are switched off, | ||||||||||||
2356 | // emit the branch. | ||||||||||||
2357 | if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None) | ||||||||||||
2358 | DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), | ||||||||||||
2359 | MVT::Other, getControlRoot(), | ||||||||||||
2360 | DAG.getBasicBlock(Succ0MBB))); | ||||||||||||
2361 | |||||||||||||
2362 | return; | ||||||||||||
2363 | } | ||||||||||||
2364 | |||||||||||||
2365 | // If this condition is one of the special cases we handle, do special stuff | ||||||||||||
2366 | // now. | ||||||||||||
2367 | const Value *CondVal = I.getCondition(); | ||||||||||||
2368 | MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)]; | ||||||||||||
2369 | |||||||||||||
2370 | // If this is a series of conditions that are or'd or and'd together, emit | ||||||||||||
2371 | // this as a sequence of branches instead of setcc's with and/or operations. | ||||||||||||
2372 | // As long as jumps are not expensive (exceptions for multi-use logic ops, | ||||||||||||
2373 | // unpredictable branches, and vector extracts because those jumps are likely | ||||||||||||
2374 | // expensive for any target), this should improve performance. | ||||||||||||
2375 | // For example, instead of something like: | ||||||||||||
2376 | // cmp A, B | ||||||||||||
2377 | // C = seteq | ||||||||||||
2378 | // cmp D, E | ||||||||||||
2379 | // F = setle | ||||||||||||
2380 | // or C, F | ||||||||||||
2381 | // jnz foo | ||||||||||||
2382 | // Emit: | ||||||||||||
2383 | // cmp A, B | ||||||||||||
2384 | // je foo | ||||||||||||
2385 | // cmp D, E | ||||||||||||
2386 | // jle foo | ||||||||||||
2387 | const Instruction *BOp = dyn_cast<Instruction>(CondVal); | ||||||||||||
2388 | if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp && | ||||||||||||
2389 | BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) { | ||||||||||||
2390 | Value *Vec; | ||||||||||||
2391 | const Value *BOp0, *BOp1; | ||||||||||||
2392 | Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0; | ||||||||||||
2393 | if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1)))) | ||||||||||||
2394 | Opcode = Instruction::And; | ||||||||||||
2395 | else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1)))) | ||||||||||||
2396 | Opcode = Instruction::Or; | ||||||||||||
2397 | |||||||||||||
2398 | if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) && | ||||||||||||
2399 | match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) { | ||||||||||||
2400 | FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode, | ||||||||||||
2401 | getEdgeProbability(BrMBB, Succ0MBB), | ||||||||||||
2402 | getEdgeProbability(BrMBB, Succ1MBB), | ||||||||||||
2403 | /*InvertCond=*/false); | ||||||||||||
2404 | // If the compares in later blocks need to use values not currently | ||||||||||||
2405 | // exported from this block, export them now. This block should always | ||||||||||||
2406 | // be the first entry. | ||||||||||||
2407 | assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!")(static_cast <bool> (SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!") ? void (0) : __assert_fail ("SL->SwitchCases[0].ThisBB == BrMBB && \"Unexpected lowering!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2407, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
2408 | |||||||||||||
2409 | // Allow some cases to be rejected. | ||||||||||||
2410 | if (ShouldEmitAsBranches(SL->SwitchCases)) { | ||||||||||||
2411 | for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) { | ||||||||||||
2412 | ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS); | ||||||||||||
2413 | ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS); | ||||||||||||
2414 | } | ||||||||||||
2415 | |||||||||||||
2416 | // Emit the branch for this block. | ||||||||||||
2417 | visitSwitchCase(SL->SwitchCases[0], BrMBB); | ||||||||||||
2418 | SL->SwitchCases.erase(SL->SwitchCases.begin()); | ||||||||||||
2419 | return; | ||||||||||||
2420 | } | ||||||||||||
2421 | |||||||||||||
2422 | // Okay, we decided not to do this, remove any inserted MBB's and clear | ||||||||||||
2423 | // SwitchCases. | ||||||||||||
2424 | for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) | ||||||||||||
2425 | FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB); | ||||||||||||
2426 | |||||||||||||
2427 | SL->SwitchCases.clear(); | ||||||||||||
2428 | } | ||||||||||||
2429 | } | ||||||||||||
2430 | |||||||||||||
2431 | // Create a CaseBlock record representing this branch. | ||||||||||||
2432 | CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()), | ||||||||||||
2433 | nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc()); | ||||||||||||
2434 | |||||||||||||
2435 | // Use visitSwitchCase to actually insert the fast branch sequence for this | ||||||||||||
2436 | // cond branch. | ||||||||||||
2437 | visitSwitchCase(CB, BrMBB); | ||||||||||||
2438 | } | ||||||||||||
2439 | |||||||||||||
2440 | /// visitSwitchCase - Emits the necessary code to represent a single node in | ||||||||||||
2441 | /// the binary search tree resulting from lowering a switch instruction. | ||||||||||||
2442 | void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, | ||||||||||||
2443 | MachineBasicBlock *SwitchBB) { | ||||||||||||
2444 | SDValue Cond; | ||||||||||||
2445 | SDValue CondLHS = getValue(CB.CmpLHS); | ||||||||||||
2446 | SDLoc dl = CB.DL; | ||||||||||||
2447 | |||||||||||||
2448 | if (CB.CC == ISD::SETTRUE) { | ||||||||||||
2449 | // Branch or fall through to TrueBB. | ||||||||||||
2450 | addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb); | ||||||||||||
2451 | SwitchBB->normalizeSuccProbs(); | ||||||||||||
2452 | if (CB.TrueBB != NextBlock(SwitchBB)) { | ||||||||||||
2453 | DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(), | ||||||||||||
2454 | DAG.getBasicBlock(CB.TrueBB))); | ||||||||||||
2455 | } | ||||||||||||
2456 | return; | ||||||||||||
2457 | } | ||||||||||||
2458 | |||||||||||||
2459 | auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
2460 | EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType()); | ||||||||||||
2461 | |||||||||||||
2462 | // Build the setcc now. | ||||||||||||
2463 | if (!CB.CmpMHS) { | ||||||||||||
2464 | // Fold "(X == true)" to X and "(X == false)" to !X to | ||||||||||||
2465 | // handle common cases produced by branch lowering. | ||||||||||||
2466 | if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) && | ||||||||||||
2467 | CB.CC == ISD::SETEQ) | ||||||||||||
2468 | Cond = CondLHS; | ||||||||||||
2469 | else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) && | ||||||||||||
2470 | CB.CC == ISD::SETEQ) { | ||||||||||||
2471 | SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType()); | ||||||||||||
2472 | Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True); | ||||||||||||
2473 | } else { | ||||||||||||
2474 | SDValue CondRHS = getValue(CB.CmpRHS); | ||||||||||||
2475 | |||||||||||||
2476 | // If a pointer's DAG type is larger than its memory type then the DAG | ||||||||||||
2477 | // values are zero-extended. This breaks signed comparisons so truncate | ||||||||||||
2478 | // back to the underlying type before doing the compare. | ||||||||||||
2479 | if (CondLHS.getValueType() != MemVT) { | ||||||||||||
2480 | CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT); | ||||||||||||
2481 | CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT); | ||||||||||||
2482 | } | ||||||||||||
2483 | Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC); | ||||||||||||
2484 | } | ||||||||||||
2485 | } else { | ||||||||||||
2486 | assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now")(static_cast <bool> (CB.CC == ISD::SETLE && "Can handle only LE ranges now" ) ? void (0) : __assert_fail ("CB.CC == ISD::SETLE && \"Can handle only LE ranges now\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2486, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
2487 | |||||||||||||
2488 | const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue(); | ||||||||||||
2489 | const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue(); | ||||||||||||
2490 | |||||||||||||
2491 | SDValue CmpOp = getValue(CB.CmpMHS); | ||||||||||||
2492 | EVT VT = CmpOp.getValueType(); | ||||||||||||
2493 | |||||||||||||
2494 | if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) { | ||||||||||||
2495 | Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT), | ||||||||||||
2496 | ISD::SETLE); | ||||||||||||
2497 | } else { | ||||||||||||
2498 | SDValue SUB = DAG.getNode(ISD::SUB, dl, | ||||||||||||
2499 | VT, CmpOp, DAG.getConstant(Low, dl, VT)); | ||||||||||||
2500 | Cond = DAG.getSetCC(dl, MVT::i1, SUB, | ||||||||||||
2501 | DAG.getConstant(High-Low, dl, VT), ISD::SETULE); | ||||||||||||
2502 | } | ||||||||||||
2503 | } | ||||||||||||
2504 | |||||||||||||
2505 | // Update successor info | ||||||||||||
2506 | addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb); | ||||||||||||
2507 | // TrueBB and FalseBB are always different unless the incoming IR is | ||||||||||||
2508 | // degenerate. This only happens when running llc on weird IR. | ||||||||||||
2509 | if (CB.TrueBB != CB.FalseBB) | ||||||||||||
2510 | addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb); | ||||||||||||
2511 | SwitchBB->normalizeSuccProbs(); | ||||||||||||
2512 | |||||||||||||
2513 | // If the lhs block is the next block, invert the condition so that we can | ||||||||||||
2514 | // fall through to the lhs instead of the rhs block. | ||||||||||||
2515 | if (CB.TrueBB == NextBlock(SwitchBB)) { | ||||||||||||
2516 | std::swap(CB.TrueBB, CB.FalseBB); | ||||||||||||
2517 | SDValue True = DAG.getConstant(1, dl, Cond.getValueType()); | ||||||||||||
2518 | Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True); | ||||||||||||
2519 | } | ||||||||||||
2520 | |||||||||||||
2521 | SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, | ||||||||||||
2522 | MVT::Other, getControlRoot(), Cond, | ||||||||||||
2523 | DAG.getBasicBlock(CB.TrueBB)); | ||||||||||||
2524 | |||||||||||||
2525 | // Insert the false branch. Do this even if it's a fall through branch, | ||||||||||||
2526 | // this makes it easier to do DAG optimizations which require inverting | ||||||||||||
2527 | // the branch condition. | ||||||||||||
2528 | BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, | ||||||||||||
2529 | DAG.getBasicBlock(CB.FalseBB)); | ||||||||||||
2530 | |||||||||||||
2531 | DAG.setRoot(BrCond); | ||||||||||||
2532 | } | ||||||||||||
2533 | |||||||||||||
2534 | /// visitJumpTable - Emit JumpTable node in the current MBB | ||||||||||||
2535 | void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) { | ||||||||||||
2536 | // Emit the code for the jump table | ||||||||||||
2537 | assert(JT.Reg != -1U && "Should lower JT Header first!")(static_cast <bool> (JT.Reg != -1U && "Should lower JT Header first!" ) ? void (0) : __assert_fail ("JT.Reg != -1U && \"Should lower JT Header first!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2537, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
2538 | EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); | ||||||||||||
2539 | SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(), | ||||||||||||
2540 | JT.Reg, PTy); | ||||||||||||
2541 | SDValue Table = DAG.getJumpTable(JT.JTI, PTy); | ||||||||||||
2542 | SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(), | ||||||||||||
2543 | MVT::Other, Index.getValue(1), | ||||||||||||
2544 | Table, Index); | ||||||||||||
2545 | DAG.setRoot(BrJumpTable); | ||||||||||||
2546 | } | ||||||||||||
2547 | |||||||||||||
2548 | /// visitJumpTableHeader - This function emits necessary code to produce index | ||||||||||||
2549 | /// in the JumpTable from switch case. | ||||||||||||
2550 | void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT, | ||||||||||||
2551 | JumpTableHeader &JTH, | ||||||||||||
2552 | MachineBasicBlock *SwitchBB) { | ||||||||||||
2553 | SDLoc dl = getCurSDLoc(); | ||||||||||||
2554 | |||||||||||||
2555 | // Subtract the lowest switch case value from the value being switched on. | ||||||||||||
2556 | SDValue SwitchOp = getValue(JTH.SValue); | ||||||||||||
2557 | EVT VT = SwitchOp.getValueType(); | ||||||||||||
2558 | SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp, | ||||||||||||
2559 | DAG.getConstant(JTH.First, dl, VT)); | ||||||||||||
2560 | |||||||||||||
2561 | // The SDNode we just created, which holds the value being switched on minus | ||||||||||||
2562 | // the smallest case value, needs to be copied to a virtual register so it | ||||||||||||
2563 | // can be used as an index into the jump table in a subsequent basic block. | ||||||||||||
2564 | // This value may be smaller or larger than the target's pointer type, and | ||||||||||||
2565 | // therefore require extension or truncating. | ||||||||||||
2566 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
2567 | SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||||||
2568 | |||||||||||||
2569 | unsigned JumpTableReg = | ||||||||||||
2570 | FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout())); | ||||||||||||
2571 | SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, | ||||||||||||
2572 | JumpTableReg, SwitchOp); | ||||||||||||
2573 | JT.Reg = JumpTableReg; | ||||||||||||
2574 | |||||||||||||
2575 | if (!JTH.OmitRangeCheck) { | ||||||||||||
2576 | // Emit the range check for the jump table, and branch to the default block | ||||||||||||
2577 | // for the switch statement if the value being switched on exceeds the | ||||||||||||
2578 | // largest case in the switch. | ||||||||||||
2579 | SDValue CMP = DAG.getSetCC( | ||||||||||||
2580 | dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), | ||||||||||||
2581 | Sub.getValueType()), | ||||||||||||
2582 | Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT); | ||||||||||||
2583 | |||||||||||||
2584 | SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, | ||||||||||||
2585 | MVT::Other, CopyTo, CMP, | ||||||||||||
2586 | DAG.getBasicBlock(JT.Default)); | ||||||||||||
2587 | |||||||||||||
2588 | // Avoid emitting unnecessary branches to the next block. | ||||||||||||
2589 | if (JT.MBB != NextBlock(SwitchBB)) | ||||||||||||
2590 | BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, | ||||||||||||
2591 | DAG.getBasicBlock(JT.MBB)); | ||||||||||||
2592 | |||||||||||||
2593 | DAG.setRoot(BrCond); | ||||||||||||
2594 | } else { | ||||||||||||
2595 | // Avoid emitting unnecessary branches to the next block. | ||||||||||||
2596 | if (JT.MBB != NextBlock(SwitchBB)) | ||||||||||||
2597 | DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo, | ||||||||||||
2598 | DAG.getBasicBlock(JT.MBB))); | ||||||||||||
2599 | else | ||||||||||||
2600 | DAG.setRoot(CopyTo); | ||||||||||||
2601 | } | ||||||||||||
2602 | } | ||||||||||||
2603 | |||||||||||||
2604 | /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global | ||||||||||||
2605 | /// variable if there exists one. | ||||||||||||
2606 | static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, | ||||||||||||
2607 | SDValue &Chain) { | ||||||||||||
2608 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
2609 | EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); | ||||||||||||
2610 | EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout()); | ||||||||||||
2611 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
2612 | Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent()); | ||||||||||||
2613 | MachineSDNode *Node = | ||||||||||||
2614 | DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain); | ||||||||||||
2615 | if (Global) { | ||||||||||||
2616 | MachinePointerInfo MPInfo(Global); | ||||||||||||
2617 | auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant | | ||||||||||||
2618 | MachineMemOperand::MODereferenceable; | ||||||||||||
2619 | MachineMemOperand *MemRef = MF.getMachineMemOperand( | ||||||||||||
2620 | MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy)); | ||||||||||||
2621 | DAG.setNodeMemRefs(Node, {MemRef}); | ||||||||||||
2622 | } | ||||||||||||
2623 | if (PtrTy != PtrMemTy) | ||||||||||||
2624 | return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy); | ||||||||||||
2625 | return SDValue(Node, 0); | ||||||||||||
2626 | } | ||||||||||||
2627 | |||||||||||||
2628 | /// Codegen a new tail for a stack protector check ParentMBB which has had its | ||||||||||||
2629 | /// tail spliced into a stack protector check success bb. | ||||||||||||
2630 | /// | ||||||||||||
2631 | /// For a high level explanation of how this fits into the stack protector | ||||||||||||
2632 | /// generation see the comment on the declaration of class | ||||||||||||
2633 | /// StackProtectorDescriptor. | ||||||||||||
2634 | void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD, | ||||||||||||
2635 | MachineBasicBlock *ParentBB) { | ||||||||||||
2636 | |||||||||||||
2637 | // First create the loads to the guard/stack slot for the comparison. | ||||||||||||
2638 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
2639 | EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); | ||||||||||||
2640 | EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout()); | ||||||||||||
2641 | |||||||||||||
2642 | MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo(); | ||||||||||||
2643 | int FI = MFI.getStackProtectorIndex(); | ||||||||||||
2644 | |||||||||||||
2645 | SDValue Guard; | ||||||||||||
2646 | SDLoc dl = getCurSDLoc(); | ||||||||||||
2647 | SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy); | ||||||||||||
2648 | const Module &M = *ParentBB->getParent()->getFunction().getParent(); | ||||||||||||
2649 | Align Align = DL->getPrefTypeAlign(Type::getInt8PtrTy(M.getContext())); | ||||||||||||
2650 | |||||||||||||
2651 | // Generate code to load the content of the guard slot. | ||||||||||||
2652 | SDValue GuardVal = DAG.getLoad( | ||||||||||||
2653 | PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr, | ||||||||||||
2654 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align, | ||||||||||||
2655 | MachineMemOperand::MOVolatile); | ||||||||||||
2656 | |||||||||||||
2657 | if (TLI.useStackGuardXorFP()) | ||||||||||||
2658 | GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl); | ||||||||||||
2659 | |||||||||||||
2660 | // Retrieve guard check function, nullptr if instrumentation is inlined. | ||||||||||||
2661 | if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) { | ||||||||||||
2662 | // The target provides a guard check function to validate the guard value. | ||||||||||||
2663 | // Generate a call to that function with the content of the guard slot as | ||||||||||||
2664 | // argument. | ||||||||||||
2665 | FunctionType *FnTy = GuardCheckFn->getFunctionType(); | ||||||||||||
2666 | assert(FnTy->getNumParams() == 1 && "Invalid function signature")(static_cast <bool> (FnTy->getNumParams() == 1 && "Invalid function signature") ? void (0) : __assert_fail ("FnTy->getNumParams() == 1 && \"Invalid function signature\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2666, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
2667 | |||||||||||||
2668 | TargetLowering::ArgListTy Args; | ||||||||||||
2669 | TargetLowering::ArgListEntry Entry; | ||||||||||||
2670 | Entry.Node = GuardVal; | ||||||||||||
2671 | Entry.Ty = FnTy->getParamType(0); | ||||||||||||
2672 | if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg)) | ||||||||||||
2673 | Entry.IsInReg = true; | ||||||||||||
2674 | Args.push_back(Entry); | ||||||||||||
2675 | |||||||||||||
2676 | TargetLowering::CallLoweringInfo CLI(DAG); | ||||||||||||
2677 | CLI.setDebugLoc(getCurSDLoc()) | ||||||||||||
2678 | .setChain(DAG.getEntryNode()) | ||||||||||||
2679 | .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(), | ||||||||||||
2680 | getValue(GuardCheckFn), std::move(Args)); | ||||||||||||
2681 | |||||||||||||
2682 | std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); | ||||||||||||
2683 | DAG.setRoot(Result.second); | ||||||||||||
2684 | return; | ||||||||||||
2685 | } | ||||||||||||
2686 | |||||||||||||
2687 | // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD. | ||||||||||||
2688 | // Otherwise, emit a volatile load to retrieve the stack guard value. | ||||||||||||
2689 | SDValue Chain = DAG.getEntryNode(); | ||||||||||||
2690 | if (TLI.useLoadStackGuardNode()) { | ||||||||||||
2691 | Guard = getLoadStackGuard(DAG, dl, Chain); | ||||||||||||
2692 | } else { | ||||||||||||
2693 | const Value *IRGuard = TLI.getSDagStackGuard(M); | ||||||||||||
2694 | SDValue GuardPtr = getValue(IRGuard); | ||||||||||||
2695 | |||||||||||||
2696 | Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr, | ||||||||||||
2697 | MachinePointerInfo(IRGuard, 0), Align, | ||||||||||||
2698 | MachineMemOperand::MOVolatile); | ||||||||||||
2699 | } | ||||||||||||
2700 | |||||||||||||
2701 | // Perform the comparison via a getsetcc. | ||||||||||||
2702 | SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(), | ||||||||||||
2703 | *DAG.getContext(), | ||||||||||||
2704 | Guard.getValueType()), | ||||||||||||
2705 | Guard, GuardVal, ISD::SETNE); | ||||||||||||
2706 | |||||||||||||
2707 | // If the guard/stackslot do not equal, branch to failure MBB. | ||||||||||||
2708 | SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, | ||||||||||||
2709 | MVT::Other, GuardVal.getOperand(0), | ||||||||||||
2710 | Cmp, DAG.getBasicBlock(SPD.getFailureMBB())); | ||||||||||||
2711 | // Otherwise branch to success MBB. | ||||||||||||
2712 | SDValue Br = DAG.getNode(ISD::BR, dl, | ||||||||||||
2713 | MVT::Other, BrCond, | ||||||||||||
2714 | DAG.getBasicBlock(SPD.getSuccessMBB())); | ||||||||||||
2715 | |||||||||||||
2716 | DAG.setRoot(Br); | ||||||||||||
2717 | } | ||||||||||||
2718 | |||||||||||||
2719 | /// Codegen the failure basic block for a stack protector check. | ||||||||||||
2720 | /// | ||||||||||||
2721 | /// A failure stack protector machine basic block consists simply of a call to | ||||||||||||
2722 | /// __stack_chk_fail(). | ||||||||||||
2723 | /// | ||||||||||||
2724 | /// For a high level explanation of how this fits into the stack protector | ||||||||||||
2725 | /// generation see the comment on the declaration of class | ||||||||||||
2726 | /// StackProtectorDescriptor. | ||||||||||||
2727 | void | ||||||||||||
2728 | SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) { | ||||||||||||
2729 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
2730 | TargetLowering::MakeLibCallOptions CallOptions; | ||||||||||||
2731 | CallOptions.setDiscardResult(true); | ||||||||||||
2732 | SDValue Chain = | ||||||||||||
2733 | TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid, | ||||||||||||
2734 | None, CallOptions, getCurSDLoc()).second; | ||||||||||||
2735 | // On PS4, the "return address" must still be within the calling function, | ||||||||||||
2736 | // even if it's at the very end, so emit an explicit TRAP here. | ||||||||||||
2737 | // Passing 'true' for doesNotReturn above won't generate the trap for us. | ||||||||||||
2738 | if (TM.getTargetTriple().isPS4CPU()) | ||||||||||||
2739 | Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain); | ||||||||||||
2740 | // WebAssembly needs an unreachable instruction after a non-returning call, | ||||||||||||
2741 | // because the function return type can be different from __stack_chk_fail's | ||||||||||||
2742 | // return type (void). | ||||||||||||
2743 | if (TM.getTargetTriple().isWasm()) | ||||||||||||
2744 | Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain); | ||||||||||||
2745 | |||||||||||||
2746 | DAG.setRoot(Chain); | ||||||||||||
2747 | } | ||||||||||||
2748 | |||||||||||||
2749 | /// visitBitTestHeader - This function emits necessary code to produce value | ||||||||||||
2750 | /// suitable for "bit tests" | ||||||||||||
2751 | void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, | ||||||||||||
2752 | MachineBasicBlock *SwitchBB) { | ||||||||||||
2753 | SDLoc dl = getCurSDLoc(); | ||||||||||||
2754 | |||||||||||||
2755 | // Subtract the minimum value. | ||||||||||||
2756 | SDValue SwitchOp = getValue(B.SValue); | ||||||||||||
2757 | EVT VT = SwitchOp.getValueType(); | ||||||||||||
2758 | SDValue RangeSub = | ||||||||||||
2759 | DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT)); | ||||||||||||
2760 | |||||||||||||
2761 | // Determine the type of the test operands. | ||||||||||||
2762 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
2763 | bool UsePtrType = false; | ||||||||||||
2764 | if (!TLI.isTypeLegal(VT)) { | ||||||||||||
2765 | UsePtrType = true; | ||||||||||||
2766 | } else { | ||||||||||||
2767 | for (unsigned i = 0, e = B.Cases.size(); i != e; ++i) | ||||||||||||
2768 | if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) { | ||||||||||||
2769 | // Switch table case range are encoded into series of masks. | ||||||||||||
2770 | // Just use pointer type, it's guaranteed to fit. | ||||||||||||
2771 | UsePtrType = true; | ||||||||||||
2772 | break; | ||||||||||||
2773 | } | ||||||||||||
2774 | } | ||||||||||||
2775 | SDValue Sub = RangeSub; | ||||||||||||
2776 | if (UsePtrType) { | ||||||||||||
2777 | VT = TLI.getPointerTy(DAG.getDataLayout()); | ||||||||||||
2778 | Sub = DAG.getZExtOrTrunc(Sub, dl, VT); | ||||||||||||
2779 | } | ||||||||||||
2780 | |||||||||||||
2781 | B.RegVT = VT.getSimpleVT(); | ||||||||||||
2782 | B.Reg = FuncInfo.CreateReg(B.RegVT); | ||||||||||||
2783 | SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub); | ||||||||||||
2784 | |||||||||||||
2785 | MachineBasicBlock* MBB = B.Cases[0].ThisBB; | ||||||||||||
2786 | |||||||||||||
2787 | if (!B.OmitRangeCheck) | ||||||||||||
2788 | addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb); | ||||||||||||
2789 | addSuccessorWithProb(SwitchBB, MBB, B.Prob); | ||||||||||||
2790 | SwitchBB->normalizeSuccProbs(); | ||||||||||||
2791 | |||||||||||||
2792 | SDValue Root = CopyTo; | ||||||||||||
2793 | if (!B.OmitRangeCheck) { | ||||||||||||
2794 | // Conditional branch to the default block. | ||||||||||||
2795 | SDValue RangeCmp = DAG.getSetCC(dl, | ||||||||||||
2796 | TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), | ||||||||||||
2797 | RangeSub.getValueType()), | ||||||||||||
2798 | RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()), | ||||||||||||
2799 | ISD::SETUGT); | ||||||||||||
2800 | |||||||||||||
2801 | Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp, | ||||||||||||
2802 | DAG.getBasicBlock(B.Default)); | ||||||||||||
2803 | } | ||||||||||||
2804 | |||||||||||||
2805 | // Avoid emitting unnecessary branches to the next block. | ||||||||||||
2806 | if (MBB != NextBlock(SwitchBB)) | ||||||||||||
2807 | Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB)); | ||||||||||||
2808 | |||||||||||||
2809 | DAG.setRoot(Root); | ||||||||||||
2810 | } | ||||||||||||
2811 | |||||||||||||
2812 | /// visitBitTestCase - this function produces one "bit test" | ||||||||||||
2813 | void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB, | ||||||||||||
2814 | MachineBasicBlock* NextMBB, | ||||||||||||
2815 | BranchProbability BranchProbToNext, | ||||||||||||
2816 | unsigned Reg, | ||||||||||||
2817 | BitTestCase &B, | ||||||||||||
2818 | MachineBasicBlock *SwitchBB) { | ||||||||||||
2819 | SDLoc dl = getCurSDLoc(); | ||||||||||||
2820 | MVT VT = BB.RegVT; | ||||||||||||
2821 | SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT); | ||||||||||||
2822 | SDValue Cmp; | ||||||||||||
2823 | unsigned PopCount = countPopulation(B.Mask); | ||||||||||||
2824 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
2825 | if (PopCount == 1) { | ||||||||||||
2826 | // Testing for a single bit; just compare the shift count with what it | ||||||||||||
2827 | // would need to be to shift a 1 bit in that position. | ||||||||||||
2828 | Cmp = DAG.getSetCC( | ||||||||||||
2829 | dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), | ||||||||||||
2830 | ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT), | ||||||||||||
2831 | ISD::SETEQ); | ||||||||||||
2832 | } else if (PopCount == BB.Range) { | ||||||||||||
2833 | // There is only one zero bit in the range, test for it directly. | ||||||||||||
2834 | Cmp = DAG.getSetCC( | ||||||||||||
2835 | dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), | ||||||||||||
2836 | ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT), | ||||||||||||
2837 | ISD::SETNE); | ||||||||||||
2838 | } else { | ||||||||||||
2839 | // Make desired shift | ||||||||||||
2840 | SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT, | ||||||||||||
2841 | DAG.getConstant(1, dl, VT), ShiftOp); | ||||||||||||
2842 | |||||||||||||
2843 | // Emit bit tests and jumps | ||||||||||||
2844 | SDValue AndOp = DAG.getNode(ISD::AND, dl, | ||||||||||||
2845 | VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT)); | ||||||||||||
2846 | Cmp = DAG.getSetCC( | ||||||||||||
2847 | dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), | ||||||||||||
2848 | AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE); | ||||||||||||
2849 | } | ||||||||||||
2850 | |||||||||||||
2851 | // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb. | ||||||||||||
2852 | addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb); | ||||||||||||
2853 | // The branch probability from SwitchBB to NextMBB is BranchProbToNext. | ||||||||||||
2854 | addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext); | ||||||||||||
2855 | // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is | ||||||||||||
2856 | // one as they are relative probabilities (and thus work more like weights), | ||||||||||||
2857 | // and hence we need to normalize them to let the sum of them become one. | ||||||||||||
2858 | SwitchBB->normalizeSuccProbs(); | ||||||||||||
2859 | |||||||||||||
2860 | SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl, | ||||||||||||
2861 | MVT::Other, getControlRoot(), | ||||||||||||
2862 | Cmp, DAG.getBasicBlock(B.TargetBB)); | ||||||||||||
2863 | |||||||||||||
2864 | // Avoid emitting unnecessary branches to the next block. | ||||||||||||
2865 | if (NextMBB != NextBlock(SwitchBB)) | ||||||||||||
2866 | BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd, | ||||||||||||
2867 | DAG.getBasicBlock(NextMBB)); | ||||||||||||
2868 | |||||||||||||
2869 | DAG.setRoot(BrAnd); | ||||||||||||
2870 | } | ||||||||||||
2871 | |||||||||||||
2872 | void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { | ||||||||||||
2873 | MachineBasicBlock *InvokeMBB = FuncInfo.MBB; | ||||||||||||
2874 | |||||||||||||
2875 | // Retrieve successors. Look through artificial IR level blocks like | ||||||||||||
2876 | // catchswitch for successors. | ||||||||||||
2877 | MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)]; | ||||||||||||
2878 | const BasicBlock *EHPadBB = I.getSuccessor(1); | ||||||||||||
2879 | |||||||||||||
2880 | // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't | ||||||||||||
2881 | // have to do anything here to lower funclet bundles. | ||||||||||||
2882 | assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live , LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext ::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2887, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
2883 | {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live , LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext ::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2887, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
2884 | LLVMContext::OB_gc_live, LLVMContext::OB_funclet,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live , LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext ::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2887, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
2885 | LLVMContext::OB_cfguardtarget,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live , LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext ::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2887, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
2886 | LLVMContext::OB_clang_arc_attachedcall}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live , LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext ::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2887, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
2887 | "Cannot lower invokes with arbitrary operand bundles yet!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live , LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext ::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2887, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
2888 | |||||||||||||
2889 | const Value *Callee(I.getCalledOperand()); | ||||||||||||
2890 | const Function *Fn = dyn_cast<Function>(Callee); | ||||||||||||
2891 | if (isa<InlineAsm>(Callee)) | ||||||||||||
2892 | visitInlineAsm(I, EHPadBB); | ||||||||||||
2893 | else if (Fn && Fn->isIntrinsic()) { | ||||||||||||
2894 | switch (Fn->getIntrinsicID()) { | ||||||||||||
2895 | default: | ||||||||||||
2896 | llvm_unreachable("Cannot invoke this intrinsic")::llvm::llvm_unreachable_internal("Cannot invoke this intrinsic" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2896); | ||||||||||||
2897 | case Intrinsic::donothing: | ||||||||||||
2898 | // Ignore invokes to @llvm.donothing: jump directly to the next BB. | ||||||||||||
2899 | case Intrinsic::seh_try_begin: | ||||||||||||
2900 | case Intrinsic::seh_scope_begin: | ||||||||||||
2901 | case Intrinsic::seh_try_end: | ||||||||||||
2902 | case Intrinsic::seh_scope_end: | ||||||||||||
2903 | break; | ||||||||||||
2904 | case Intrinsic::experimental_patchpoint_void: | ||||||||||||
2905 | case Intrinsic::experimental_patchpoint_i64: | ||||||||||||
2906 | visitPatchpoint(I, EHPadBB); | ||||||||||||
2907 | break; | ||||||||||||
2908 | case Intrinsic::experimental_gc_statepoint: | ||||||||||||
2909 | LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB); | ||||||||||||
2910 | break; | ||||||||||||
2911 | case Intrinsic::wasm_rethrow: { | ||||||||||||
2912 | // This is usually done in visitTargetIntrinsic, but this intrinsic is | ||||||||||||
2913 | // special because it can be invoked, so we manually lower it to a DAG | ||||||||||||
2914 | // node here. | ||||||||||||
2915 | SmallVector<SDValue, 8> Ops; | ||||||||||||
2916 | Ops.push_back(getRoot()); // inchain | ||||||||||||
2917 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
2918 | Ops.push_back( | ||||||||||||
2919 | DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(), | ||||||||||||
2920 | TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||||||
2921 | SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain | ||||||||||||
2922 | DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops)); | ||||||||||||
2923 | break; | ||||||||||||
2924 | } | ||||||||||||
2925 | } | ||||||||||||
2926 | } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) { | ||||||||||||
2927 | // Currently we do not lower any intrinsic calls with deopt operand bundles. | ||||||||||||
2928 | // Eventually we will support lowering the @llvm.experimental.deoptimize | ||||||||||||
2929 | // intrinsic, and right now there are no plans to support other intrinsics | ||||||||||||
2930 | // with deopt state. | ||||||||||||
2931 | LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB); | ||||||||||||
2932 | } else { | ||||||||||||
2933 | LowerCallTo(I, getValue(Callee), false, false, EHPadBB); | ||||||||||||
2934 | } | ||||||||||||
2935 | |||||||||||||
2936 | // If the value of the invoke is used outside of its defining block, make it | ||||||||||||
2937 | // available as a virtual register. | ||||||||||||
2938 | // We already took care of the exported value for the statepoint instruction | ||||||||||||
2939 | // during call to the LowerStatepoint. | ||||||||||||
2940 | if (!isa<GCStatepointInst>(I)) { | ||||||||||||
2941 | CopyToExportRegsIfNeeded(&I); | ||||||||||||
2942 | } | ||||||||||||
2943 | |||||||||||||
2944 | SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests; | ||||||||||||
2945 | BranchProbabilityInfo *BPI = FuncInfo.BPI; | ||||||||||||
2946 | BranchProbability EHPadBBProb = | ||||||||||||
2947 | BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB) | ||||||||||||
2948 | : BranchProbability::getZero(); | ||||||||||||
2949 | findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests); | ||||||||||||
2950 | |||||||||||||
2951 | // Update successor info. | ||||||||||||
2952 | addSuccessorWithProb(InvokeMBB, Return); | ||||||||||||
2953 | for (auto &UnwindDest : UnwindDests) { | ||||||||||||
2954 | UnwindDest.first->setIsEHPad(); | ||||||||||||
2955 | addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second); | ||||||||||||
2956 | } | ||||||||||||
2957 | InvokeMBB->normalizeSuccProbs(); | ||||||||||||
2958 | |||||||||||||
2959 | // Drop into normal successor. | ||||||||||||
2960 | DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(), | ||||||||||||
2961 | DAG.getBasicBlock(Return))); | ||||||||||||
2962 | } | ||||||||||||
2963 | |||||||||||||
2964 | void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { | ||||||||||||
2965 | MachineBasicBlock *CallBrMBB = FuncInfo.MBB; | ||||||||||||
2966 | |||||||||||||
2967 | // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't | ||||||||||||
2968 | // have to do anything here to lower funclet bundles. | ||||||||||||
2969 | assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2971, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
2970 | {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2971, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
2971 | "Cannot lower callbrs with arbitrary operand bundles yet!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2971, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
2972 | |||||||||||||
2973 | assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr")(static_cast <bool> (I.isInlineAsm() && "Only know how to handle inlineasm callbr" ) ? void (0) : __assert_fail ("I.isInlineAsm() && \"Only know how to handle inlineasm callbr\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2973, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
2974 | visitInlineAsm(I); | ||||||||||||
2975 | CopyToExportRegsIfNeeded(&I); | ||||||||||||
2976 | |||||||||||||
2977 | // Retrieve successors. | ||||||||||||
2978 | MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()]; | ||||||||||||
2979 | |||||||||||||
2980 | // Update successor info. | ||||||||||||
2981 | addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne()); | ||||||||||||
2982 | for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) { | ||||||||||||
2983 | MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)]; | ||||||||||||
2984 | addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero()); | ||||||||||||
2985 | Target->setIsInlineAsmBrIndirectTarget(); | ||||||||||||
2986 | } | ||||||||||||
2987 | CallBrMBB->normalizeSuccProbs(); | ||||||||||||
2988 | |||||||||||||
2989 | // Drop into default successor. | ||||||||||||
2990 | DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), | ||||||||||||
2991 | MVT::Other, getControlRoot(), | ||||||||||||
2992 | DAG.getBasicBlock(Return))); | ||||||||||||
2993 | } | ||||||||||||
2994 | |||||||||||||
2995 | void SelectionDAGBuilder::visitResume(const ResumeInst &RI) { | ||||||||||||
2996 | llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit resume instructions!" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 2996); | ||||||||||||
2997 | } | ||||||||||||
2998 | |||||||||||||
2999 | void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) { | ||||||||||||
3000 | assert(FuncInfo.MBB->isEHPad() &&(static_cast <bool> (FuncInfo.MBB->isEHPad() && "Call to landingpad not in landing pad!") ? void (0) : __assert_fail ("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 3001, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
3001 | "Call to landingpad not in landing pad!")(static_cast <bool> (FuncInfo.MBB->isEHPad() && "Call to landingpad not in landing pad!") ? void (0) : __assert_fail ("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 3001, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
3002 | |||||||||||||
3003 | // If there aren't registers to copy the values into (e.g., during SjLj | ||||||||||||
3004 | // exceptions), then don't bother to create these DAG nodes. | ||||||||||||
3005 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
3006 | const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn(); | ||||||||||||
3007 | if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 && | ||||||||||||
3008 | TLI.getExceptionSelectorRegister(PersonalityFn) == 0) | ||||||||||||
3009 | return; | ||||||||||||
3010 | |||||||||||||
3011 | // If landingpad's return type is token type, we don't create DAG nodes | ||||||||||||
3012 | // for its exception pointer and selector value. The extraction of exception | ||||||||||||
3013 | // pointer or selector value from token type landingpads is not currently | ||||||||||||
3014 | // supported. | ||||||||||||
3015 | if (LP.getType()->isTokenTy()) | ||||||||||||
3016 | return; | ||||||||||||
3017 | |||||||||||||
3018 | SmallVector<EVT, 2> ValueVTs; | ||||||||||||
3019 | SDLoc dl = getCurSDLoc(); | ||||||||||||
3020 | ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs); | ||||||||||||
3021 | assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported")(static_cast <bool> (ValueVTs.size() == 2 && "Only two-valued landingpads are supported" ) ? void (0) : __assert_fail ("ValueVTs.size() == 2 && \"Only two-valued landingpads are supported\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 3021, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
3022 | |||||||||||||
3023 | // Get the two live-in registers as SDValues. The physregs have already been | ||||||||||||
3024 | // copied into virtual registers. | ||||||||||||
3025 | SDValue Ops[2]; | ||||||||||||
3026 | if (FuncInfo.ExceptionPointerVirtReg) { | ||||||||||||
3027 | Ops[0] = DAG.getZExtOrTrunc( | ||||||||||||
3028 | DAG.getCopyFromReg(DAG.getEntryNode(), dl, | ||||||||||||
3029 | FuncInfo.ExceptionPointerVirtReg, | ||||||||||||
3030 | TLI.getPointerTy(DAG.getDataLayout())), | ||||||||||||
3031 | dl, ValueVTs[0]); | ||||||||||||
3032 | } else { | ||||||||||||
3033 | Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||||||
3034 | } | ||||||||||||
3035 | Ops[1] = DAG.getZExtOrTrunc( | ||||||||||||
3036 | DAG.getCopyFromReg(DAG.getEntryNode(), dl, | ||||||||||||
3037 | FuncInfo.ExceptionSelectorVirtReg, | ||||||||||||
3038 | TLI.getPointerTy(DAG.getDataLayout())), | ||||||||||||
3039 | dl, ValueVTs[1]); | ||||||||||||
3040 | |||||||||||||
3041 | // Merge into one. | ||||||||||||
3042 | SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl, | ||||||||||||
3043 | DAG.getVTList(ValueVTs), Ops); | ||||||||||||
3044 | setValue(&LP, Res); | ||||||||||||
3045 | } | ||||||||||||
3046 | |||||||||||||
3047 | void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First, | ||||||||||||
3048 | MachineBasicBlock *Last) { | ||||||||||||
3049 | // Update JTCases. | ||||||||||||
3050 | for (unsigned i = 0, e = SL->JTCases.size(); i != e; ++i) | ||||||||||||
3051 | if (SL->JTCases[i].first.HeaderBB == First) | ||||||||||||
3052 | SL->JTCases[i].first.HeaderBB = Last; | ||||||||||||
3053 | |||||||||||||
3054 | // Update BitTestCases. | ||||||||||||
3055 | for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i) | ||||||||||||
3056 | if (SL->BitTestCases[i].Parent == First) | ||||||||||||
3057 | SL->BitTestCases[i].Parent = Last; | ||||||||||||
3058 | } | ||||||||||||
3059 | |||||||||||||
3060 | void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) { | ||||||||||||
3061 | MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB; | ||||||||||||
3062 | |||||||||||||
3063 | // Update machine-CFG edges with unique successors. | ||||||||||||
3064 | SmallSet<BasicBlock*, 32> Done; | ||||||||||||
3065 | for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) { | ||||||||||||
3066 | BasicBlock *BB = I.getSuccessor(i); | ||||||||||||
3067 | bool Inserted = Done.insert(BB).second; | ||||||||||||
3068 | if (!Inserted) | ||||||||||||
3069 | continue; | ||||||||||||
3070 | |||||||||||||
3071 | MachineBasicBlock *Succ = FuncInfo.MBBMap[BB]; | ||||||||||||
3072 | addSuccessorWithProb(IndirectBrMBB, Succ); | ||||||||||||
3073 | } | ||||||||||||
3074 | IndirectBrMBB->normalizeSuccProbs(); | ||||||||||||
3075 | |||||||||||||
3076 | DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(), | ||||||||||||
3077 | MVT::Other, getControlRoot(), | ||||||||||||
3078 | getValue(I.getAddress()))); | ||||||||||||
3079 | } | ||||||||||||
3080 | |||||||||||||
3081 | void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) { | ||||||||||||
3082 | if (!DAG.getTarget().Options.TrapUnreachable) | ||||||||||||
3083 | return; | ||||||||||||
3084 | |||||||||||||
3085 | // We may be able to ignore unreachable behind a noreturn call. | ||||||||||||
3086 | if (DAG.getTarget().Options.NoTrapAfterNoreturn) { | ||||||||||||
3087 | const BasicBlock &BB = *I.getParent(); | ||||||||||||
3088 | if (&I != &BB.front()) { | ||||||||||||
3089 | BasicBlock::const_iterator PredI = | ||||||||||||
3090 | std::prev(BasicBlock::const_iterator(&I)); | ||||||||||||
3091 | if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) { | ||||||||||||
3092 | if (Call->doesNotReturn()) | ||||||||||||
3093 | return; | ||||||||||||
3094 | } | ||||||||||||
3095 | } | ||||||||||||
3096 | } | ||||||||||||
3097 | |||||||||||||
3098 | DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot())); | ||||||||||||
3099 | } | ||||||||||||
3100 | |||||||||||||
3101 | void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) { | ||||||||||||
3102 | SDNodeFlags Flags; | ||||||||||||
3103 | |||||||||||||
3104 | SDValue Op = getValue(I.getOperand(0)); | ||||||||||||
3105 | SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(), | ||||||||||||
3106 | Op, Flags); | ||||||||||||
3107 | setValue(&I, UnNodeValue); | ||||||||||||
3108 | } | ||||||||||||
3109 | |||||||||||||
3110 | void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) { | ||||||||||||
3111 | SDNodeFlags Flags; | ||||||||||||
3112 | if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) { | ||||||||||||
3113 | Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap()); | ||||||||||||
3114 | Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap()); | ||||||||||||
3115 | } | ||||||||||||
3116 | if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) | ||||||||||||
3117 | Flags.setExact(ExactOp->isExact()); | ||||||||||||
3118 | if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) | ||||||||||||
3119 | Flags.copyFMF(*FPOp); | ||||||||||||
3120 | |||||||||||||
3121 | SDValue Op1 = getValue(I.getOperand(0)); | ||||||||||||
3122 | SDValue Op2 = getValue(I.getOperand(1)); | ||||||||||||
3123 | SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), | ||||||||||||
3124 | Op1, Op2, Flags); | ||||||||||||
3125 | setValue(&I, BinNodeValue); | ||||||||||||
3126 | } | ||||||||||||
3127 | |||||||||||||
3128 | void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) { | ||||||||||||
3129 | SDValue Op1 = getValue(I.getOperand(0)); | ||||||||||||
3130 | SDValue Op2 = getValue(I.getOperand(1)); | ||||||||||||
3131 | |||||||||||||
3132 | EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy( | ||||||||||||
3133 | Op1.getValueType(), DAG.getDataLayout()); | ||||||||||||
3134 | |||||||||||||
3135 | // Coerce the shift amount to the right type if we can. | ||||||||||||
3136 | if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) { | ||||||||||||
3137 | unsigned ShiftSize = ShiftTy.getSizeInBits(); | ||||||||||||
3138 | unsigned Op2Size = Op2.getValueSizeInBits(); | ||||||||||||
3139 | SDLoc DL = getCurSDLoc(); | ||||||||||||
3140 | |||||||||||||
3141 | // If the operand is smaller than the shift count type, promote it. | ||||||||||||
3142 | if (ShiftSize > Op2Size) | ||||||||||||
3143 | Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2); | ||||||||||||
3144 | |||||||||||||
3145 | // If the operand is larger than the shift count type but the shift | ||||||||||||
3146 | // count type has enough bits to represent any shift value, truncate | ||||||||||||
3147 | // it now. This is a common case and it exposes the truncate to | ||||||||||||
3148 | // optimization early. | ||||||||||||
3149 | else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits())) | ||||||||||||
3150 | Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2); | ||||||||||||
3151 | // Otherwise we'll need to temporarily settle for some other convenient | ||||||||||||
3152 | // type. Type legalization will make adjustments once the shiftee is split. | ||||||||||||
3153 | else | ||||||||||||
3154 | Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32); | ||||||||||||
3155 | } | ||||||||||||
3156 | |||||||||||||
3157 | bool nuw = false; | ||||||||||||
3158 | bool nsw = false; | ||||||||||||
3159 | bool exact = false; | ||||||||||||
3160 | |||||||||||||
3161 | if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) { | ||||||||||||
3162 | |||||||||||||
3163 | if (const OverflowingBinaryOperator *OFBinOp = | ||||||||||||
3164 | dyn_cast<const OverflowingBinaryOperator>(&I)) { | ||||||||||||
3165 | nuw = OFBinOp->hasNoUnsignedWrap(); | ||||||||||||
3166 | nsw = OFBinOp->hasNoSignedWrap(); | ||||||||||||
3167 | } | ||||||||||||
3168 | if (const PossiblyExactOperator *ExactOp = | ||||||||||||
3169 | dyn_cast<const PossiblyExactOperator>(&I)) | ||||||||||||
3170 | exact = ExactOp->isExact(); | ||||||||||||
3171 | } | ||||||||||||
3172 | SDNodeFlags Flags; | ||||||||||||
3173 | Flags.setExact(exact); | ||||||||||||
3174 | Flags.setNoSignedWrap(nsw); | ||||||||||||
3175 | Flags.setNoUnsignedWrap(nuw); | ||||||||||||
3176 | SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2, | ||||||||||||
3177 | Flags); | ||||||||||||
3178 | setValue(&I, Res); | ||||||||||||
3179 | } | ||||||||||||
3180 | |||||||||||||
3181 | void SelectionDAGBuilder::visitSDiv(const User &I) { | ||||||||||||
3182 | SDValue Op1 = getValue(I.getOperand(0)); | ||||||||||||
3183 | SDValue Op2 = getValue(I.getOperand(1)); | ||||||||||||
3184 | |||||||||||||
3185 | SDNodeFlags Flags; | ||||||||||||
3186 | Flags.setExact(isa<PossiblyExactOperator>(&I) && | ||||||||||||
3187 | cast<PossiblyExactOperator>(&I)->isExact()); | ||||||||||||
3188 | setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1, | ||||||||||||
3189 | Op2, Flags)); | ||||||||||||
3190 | } | ||||||||||||
3191 | |||||||||||||
3192 | void SelectionDAGBuilder::visitICmp(const User &I) { | ||||||||||||
3193 | ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE; | ||||||||||||
3194 | if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I)) | ||||||||||||
3195 | predicate = IC->getPredicate(); | ||||||||||||
3196 | else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I)) | ||||||||||||
3197 | predicate = ICmpInst::Predicate(IC->getPredicate()); | ||||||||||||
3198 | SDValue Op1 = getValue(I.getOperand(0)); | ||||||||||||
3199 | SDValue Op2 = getValue(I.getOperand(1)); | ||||||||||||
3200 | ISD::CondCode Opcode = getICmpCondCode(predicate); | ||||||||||||
3201 | |||||||||||||
3202 | auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
3203 | EVT MemVT = | ||||||||||||
3204 | TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType()); | ||||||||||||
3205 | |||||||||||||
3206 | // If a pointer's DAG type is larger than its memory type then the DAG values | ||||||||||||
3207 | // are zero-extended. This breaks signed comparisons so truncate back to the | ||||||||||||
3208 | // underlying type before doing the compare. | ||||||||||||
3209 | if (Op1.getValueType() != MemVT) { | ||||||||||||
3210 | Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT); | ||||||||||||
3211 | Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT); | ||||||||||||
3212 | } | ||||||||||||
3213 | |||||||||||||
3214 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||||||
3215 | I.getType()); | ||||||||||||
3216 | setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode)); | ||||||||||||
3217 | } | ||||||||||||
3218 | |||||||||||||
3219 | void SelectionDAGBuilder::visitFCmp(const User &I) { | ||||||||||||
3220 | FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE; | ||||||||||||
3221 | if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I)) | ||||||||||||
3222 | predicate = FC->getPredicate(); | ||||||||||||
3223 | else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I)) | ||||||||||||
3224 | predicate = FCmpInst::Predicate(FC->getPredicate()); | ||||||||||||
3225 | SDValue Op1 = getValue(I.getOperand(0)); | ||||||||||||
3226 | SDValue Op2 = getValue(I.getOperand(1)); | ||||||||||||
3227 | |||||||||||||
3228 | ISD::CondCode Condition = getFCmpCondCode(predicate); | ||||||||||||
3229 | auto *FPMO = cast<FPMathOperator>(&I); | ||||||||||||
3230 | if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath) | ||||||||||||
3231 | Condition = getFCmpCodeWithoutNaN(Condition); | ||||||||||||
3232 | |||||||||||||
3233 | SDNodeFlags Flags; | ||||||||||||
3234 | Flags.copyFMF(*FPMO); | ||||||||||||
3235 | SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); | ||||||||||||
3236 | |||||||||||||
3237 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||||||
3238 | I.getType()); | ||||||||||||
3239 | setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition)); | ||||||||||||
3240 | } | ||||||||||||
3241 | |||||||||||||
3242 | // Check if the condition of the select has one use or two users that are both | ||||||||||||
3243 | // selects with the same condition. | ||||||||||||
3244 | static bool hasOnlySelectUsers(const Value *Cond) { | ||||||||||||
3245 | return llvm::all_of(Cond->users(), [](const Value *V) { | ||||||||||||
3246 | return isa<SelectInst>(V); | ||||||||||||
3247 | }); | ||||||||||||
3248 | } | ||||||||||||
3249 | |||||||||||||
3250 | void SelectionDAGBuilder::visitSelect(const User &I) { | ||||||||||||
3251 | SmallVector<EVT, 4> ValueVTs; | ||||||||||||
3252 | ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(), | ||||||||||||
3253 | ValueVTs); | ||||||||||||
3254 | unsigned NumValues = ValueVTs.size(); | ||||||||||||
3255 | if (NumValues == 0) return; | ||||||||||||
3256 | |||||||||||||
3257 | SmallVector<SDValue, 4> Values(NumValues); | ||||||||||||
3258 | SDValue Cond = getValue(I.getOperand(0)); | ||||||||||||
3259 | SDValue LHSVal = getValue(I.getOperand(1)); | ||||||||||||
3260 | SDValue RHSVal = getValue(I.getOperand(2)); | ||||||||||||
3261 | SmallVector<SDValue, 1> BaseOps(1, Cond); | ||||||||||||
3262 | ISD::NodeType OpCode = | ||||||||||||
3263 | Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT; | ||||||||||||
3264 | |||||||||||||
3265 | bool IsUnaryAbs = false; | ||||||||||||
3266 | bool Negate = false; | ||||||||||||
3267 | |||||||||||||
3268 | SDNodeFlags Flags; | ||||||||||||
3269 | if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) | ||||||||||||
3270 | Flags.copyFMF(*FPOp); | ||||||||||||
3271 | |||||||||||||
3272 | // Min/max matching is only viable if all output VTs are the same. | ||||||||||||
3273 | if (is_splat(ValueVTs)) { | ||||||||||||
3274 | EVT VT = ValueVTs[0]; | ||||||||||||
3275 | LLVMContext &Ctx = *DAG.getContext(); | ||||||||||||
3276 | auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
3277 | |||||||||||||
3278 | // We care about the legality of the operation after it has been type | ||||||||||||
3279 | // legalized. | ||||||||||||
3280 | while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal) | ||||||||||||
3281 | VT = TLI.getTypeToTransformTo(Ctx, VT); | ||||||||||||
3282 | |||||||||||||
3283 | // If the vselect is legal, assume we want to leave this as a vector setcc + | ||||||||||||
3284 | // vselect. Otherwise, if this is going to be scalarized, we want to see if | ||||||||||||
3285 | // min/max is legal on the scalar type. | ||||||||||||
3286 | bool UseScalarMinMax = VT.isVector() && | ||||||||||||
3287 | !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT); | ||||||||||||
3288 | |||||||||||||
3289 | Value *LHS, *RHS; | ||||||||||||
3290 | auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS); | ||||||||||||
3291 | ISD::NodeType Opc = ISD::DELETED_NODE; | ||||||||||||
3292 | switch (SPR.Flavor) { | ||||||||||||
3293 | case SPF_UMAX: Opc = ISD::UMAX; break; | ||||||||||||
3294 | case SPF_UMIN: Opc = ISD::UMIN; break; | ||||||||||||
3295 | case SPF_SMAX: Opc = ISD::SMAX; break; | ||||||||||||
3296 | case SPF_SMIN: Opc = ISD::SMIN; break; | ||||||||||||
3297 | case SPF_FMINNUM: | ||||||||||||
3298 | switch (SPR.NaNBehavior) { | ||||||||||||
3299 | case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 3299); | ||||||||||||
3300 | case SPNB_RETURNS_NAN: Opc = ISD::FMINIMUM; break; | ||||||||||||
3301 | case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break; | ||||||||||||
3302 | case SPNB_RETURNS_ANY: { | ||||||||||||
3303 | if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT)) | ||||||||||||
3304 | Opc = ISD::FMINNUM; | ||||||||||||
3305 | else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT)) | ||||||||||||
3306 | Opc = ISD::FMINIMUM; | ||||||||||||
3307 | else if (UseScalarMinMax) | ||||||||||||
3308 | Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ? | ||||||||||||
3309 | ISD::FMINNUM : ISD::FMINIMUM; | ||||||||||||
3310 | break; | ||||||||||||
3311 | } | ||||||||||||
3312 | } | ||||||||||||
3313 | break; | ||||||||||||
3314 | case SPF_FMAXNUM: | ||||||||||||
3315 | switch (SPR.NaNBehavior) { | ||||||||||||
3316 | case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 3316); | ||||||||||||
3317 | case SPNB_RETURNS_NAN: Opc = ISD::FMAXIMUM; break; | ||||||||||||
3318 | case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break; | ||||||||||||
3319 | case SPNB_RETURNS_ANY: | ||||||||||||
3320 | |||||||||||||
3321 | if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT)) | ||||||||||||
3322 | Opc = ISD::FMAXNUM; | ||||||||||||
3323 | else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT)) | ||||||||||||
3324 | Opc = ISD::FMAXIMUM; | ||||||||||||
3325 | else if (UseScalarMinMax) | ||||||||||||
3326 | Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ? | ||||||||||||
3327 | ISD::FMAXNUM : ISD::FMAXIMUM; | ||||||||||||
3328 | break; | ||||||||||||
3329 | } | ||||||||||||
3330 | break; | ||||||||||||
3331 | case SPF_NABS: | ||||||||||||
3332 | Negate = true; | ||||||||||||
3333 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||||||||
3334 | case SPF_ABS: | ||||||||||||
3335 | IsUnaryAbs = true; | ||||||||||||
3336 | Opc = ISD::ABS; | ||||||||||||
3337 | break; | ||||||||||||
3338 | default: break; | ||||||||||||
3339 | } | ||||||||||||
3340 | |||||||||||||
3341 | if (!IsUnaryAbs && Opc != ISD::DELETED_NODE && | ||||||||||||
3342 | (TLI.isOperationLegalOrCustom(Opc, VT) || | ||||||||||||
3343 | (UseScalarMinMax && | ||||||||||||
3344 | TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) && | ||||||||||||
3345 | // If the underlying comparison instruction is used by any other | ||||||||||||
3346 | // instruction, the consumed instructions won't be destroyed, so it is | ||||||||||||
3347 | // not profitable to convert to a min/max. | ||||||||||||
3348 | hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) { | ||||||||||||
3349 | OpCode = Opc; | ||||||||||||
3350 | LHSVal = getValue(LHS); | ||||||||||||
3351 | RHSVal = getValue(RHS); | ||||||||||||
3352 | BaseOps.clear(); | ||||||||||||
3353 | } | ||||||||||||
3354 | |||||||||||||
3355 | if (IsUnaryAbs) { | ||||||||||||
3356 | OpCode = Opc; | ||||||||||||
3357 | LHSVal = getValue(LHS); | ||||||||||||
3358 | BaseOps.clear(); | ||||||||||||
3359 | } | ||||||||||||
3360 | } | ||||||||||||
3361 | |||||||||||||
3362 | if (IsUnaryAbs) { | ||||||||||||
3363 | for (unsigned i = 0; i != NumValues; ++i) { | ||||||||||||
3364 | SDLoc dl = getCurSDLoc(); | ||||||||||||
3365 | EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i); | ||||||||||||
3366 | Values[i] = | ||||||||||||
3367 | DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i)); | ||||||||||||
3368 | if (Negate) | ||||||||||||
3369 | Values[i] = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), | ||||||||||||
3370 | Values[i]); | ||||||||||||
3371 | } | ||||||||||||
3372 | } else { | ||||||||||||
3373 | for (unsigned i = 0; i != NumValues; ++i) { | ||||||||||||
3374 | SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end()); | ||||||||||||
3375 | Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i)); | ||||||||||||
3376 | Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i)); | ||||||||||||
3377 | Values[i] = DAG.getNode( | ||||||||||||
3378 | OpCode, getCurSDLoc(), | ||||||||||||
3379 | LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags); | ||||||||||||
3380 | } | ||||||||||||
3381 | } | ||||||||||||
3382 | |||||||||||||
3383 | setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), | ||||||||||||
3384 | DAG.getVTList(ValueVTs), Values)); | ||||||||||||
3385 | } | ||||||||||||
3386 | |||||||||||||
3387 | void SelectionDAGBuilder::visitTrunc(const User &I) { | ||||||||||||
3388 | // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest). | ||||||||||||
3389 | SDValue N = getValue(I.getOperand(0)); | ||||||||||||
3390 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||||||
3391 | I.getType()); | ||||||||||||
3392 | setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N)); | ||||||||||||
3393 | } | ||||||||||||
3394 | |||||||||||||
3395 | void SelectionDAGBuilder::visitZExt(const User &I) { | ||||||||||||
3396 | // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest). | ||||||||||||
3397 | // ZExt also can't be a cast to bool for same reason. So, nothing much to do | ||||||||||||
3398 | SDValue N = getValue(I.getOperand(0)); | ||||||||||||
3399 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||||||
3400 | I.getType()); | ||||||||||||
3401 | setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N)); | ||||||||||||
3402 | } | ||||||||||||
3403 | |||||||||||||
3404 | void SelectionDAGBuilder::visitSExt(const User &I) { | ||||||||||||
3405 | // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest). | ||||||||||||
3406 | // SExt also can't be a cast to bool for same reason. So, nothing much to do | ||||||||||||
3407 | SDValue N = getValue(I.getOperand(0)); | ||||||||||||
3408 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||||||
3409 | I.getType()); | ||||||||||||
3410 | setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N)); | ||||||||||||
3411 | } | ||||||||||||
3412 | |||||||||||||
3413 | void SelectionDAGBuilder::visitFPTrunc(const User &I) { | ||||||||||||
3414 | // FPTrunc is never a no-op cast, no need to check | ||||||||||||
3415 | SDValue N = getValue(I.getOperand(0)); | ||||||||||||
3416 | SDLoc dl = getCurSDLoc(); | ||||||||||||
3417 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
3418 | EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
3419 | setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N, | ||||||||||||
3420 | DAG.getTargetConstant( | ||||||||||||
3421 | 0, dl, TLI.getPointerTy(DAG.getDataLayout())))); | ||||||||||||
3422 | } | ||||||||||||
3423 | |||||||||||||
3424 | void SelectionDAGBuilder::visitFPExt(const User &I) { | ||||||||||||
3425 | // FPExt is never a no-op cast, no need to check | ||||||||||||
3426 | SDValue N = getValue(I.getOperand(0)); | ||||||||||||
3427 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||||||
3428 | I.getType()); | ||||||||||||
3429 | setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N)); | ||||||||||||
3430 | } | ||||||||||||
3431 | |||||||||||||
3432 | void SelectionDAGBuilder::visitFPToUI(const User &I) { | ||||||||||||
3433 | // FPToUI is never a no-op cast, no need to check | ||||||||||||
3434 | SDValue N = getValue(I.getOperand(0)); | ||||||||||||
3435 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||||||
3436 | I.getType()); | ||||||||||||
3437 | setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N)); | ||||||||||||
3438 | } | ||||||||||||
3439 | |||||||||||||
3440 | void SelectionDAGBuilder::visitFPToSI(const User &I) { | ||||||||||||
3441 | // FPToSI is never a no-op cast, no need to check | ||||||||||||
3442 | SDValue N = getValue(I.getOperand(0)); | ||||||||||||
3443 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||||||
3444 | I.getType()); | ||||||||||||
3445 | setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N)); | ||||||||||||
3446 | } | ||||||||||||
3447 | |||||||||||||
3448 | void SelectionDAGBuilder::visitUIToFP(const User &I) { | ||||||||||||
3449 | // UIToFP is never a no-op cast, no need to check | ||||||||||||
3450 | SDValue N = getValue(I.getOperand(0)); | ||||||||||||
3451 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||||||
3452 | I.getType()); | ||||||||||||
3453 | setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N)); | ||||||||||||
3454 | } | ||||||||||||
3455 | |||||||||||||
3456 | void SelectionDAGBuilder::visitSIToFP(const User &I) { | ||||||||||||
3457 | // SIToFP is never a no-op cast, no need to check | ||||||||||||
3458 | SDValue N = getValue(I.getOperand(0)); | ||||||||||||
3459 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||||||
3460 | I.getType()); | ||||||||||||
3461 | setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N)); | ||||||||||||
3462 | } | ||||||||||||
3463 | |||||||||||||
3464 | void SelectionDAGBuilder::visitPtrToInt(const User &I) { | ||||||||||||
3465 | // What to do depends on the size of the integer and the size of the pointer. | ||||||||||||
3466 | // We can either truncate, zero extend, or no-op, accordingly. | ||||||||||||
3467 | SDValue N = getValue(I.getOperand(0)); | ||||||||||||
3468 | auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
3469 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||||||
3470 | I.getType()); | ||||||||||||
3471 | EVT PtrMemVT = | ||||||||||||
3472 | TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType()); | ||||||||||||
3473 | N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT); | ||||||||||||
3474 | N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT); | ||||||||||||
3475 | setValue(&I, N); | ||||||||||||
3476 | } | ||||||||||||
3477 | |||||||||||||
3478 | void SelectionDAGBuilder::visitIntToPtr(const User &I) { | ||||||||||||
3479 | // What to do depends on the size of the integer and the size of the pointer. | ||||||||||||
3480 | // We can either truncate, zero extend, or no-op, accordingly. | ||||||||||||
3481 | SDValue N = getValue(I.getOperand(0)); | ||||||||||||
3482 | auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
3483 | EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
3484 | EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
3485 | N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT); | ||||||||||||
3486 | N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT); | ||||||||||||
3487 | setValue(&I, N); | ||||||||||||
3488 | } | ||||||||||||
3489 | |||||||||||||
3490 | void SelectionDAGBuilder::visitBitCast(const User &I) { | ||||||||||||
3491 | SDValue N = getValue(I.getOperand(0)); | ||||||||||||
3492 | SDLoc dl = getCurSDLoc(); | ||||||||||||
3493 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||||||
3494 | I.getType()); | ||||||||||||
3495 | |||||||||||||
3496 | // BitCast assures us that source and destination are the same size so this is | ||||||||||||
3497 | // either a BITCAST or a no-op. | ||||||||||||
3498 | if (DestVT != N.getValueType()) | ||||||||||||
3499 | setValue(&I, DAG.getNode(ISD::BITCAST, dl, | ||||||||||||
3500 | DestVT, N)); // convert types. | ||||||||||||
3501 | // Check if the original LLVM IR Operand was a ConstantInt, because getValue() | ||||||||||||
3502 | // might fold any kind of constant expression to an integer constant and that | ||||||||||||
3503 | // is not what we are looking for. Only recognize a bitcast of a genuine | ||||||||||||
3504 | // constant integer as an opaque constant. | ||||||||||||
3505 | else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0))) | ||||||||||||
3506 | setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false, | ||||||||||||
3507 | /*isOpaque*/true)); | ||||||||||||
3508 | else | ||||||||||||
3509 | setValue(&I, N); // noop cast. | ||||||||||||
3510 | } | ||||||||||||
3511 | |||||||||||||
3512 | void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) { | ||||||||||||
3513 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
3514 | const Value *SV = I.getOperand(0); | ||||||||||||
3515 | SDValue N = getValue(SV); | ||||||||||||
3516 | EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
3517 | |||||||||||||
3518 | unsigned SrcAS = SV->getType()->getPointerAddressSpace(); | ||||||||||||
3519 | unsigned DestAS = I.getType()->getPointerAddressSpace(); | ||||||||||||
3520 | |||||||||||||
3521 | if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS)) | ||||||||||||
3522 | N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS); | ||||||||||||
3523 | |||||||||||||
3524 | setValue(&I, N); | ||||||||||||
3525 | } | ||||||||||||
3526 | |||||||||||||
3527 | void SelectionDAGBuilder::visitInsertElement(const User &I) { | ||||||||||||
3528 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
3529 | SDValue InVec = getValue(I.getOperand(0)); | ||||||||||||
3530 | SDValue InVal = getValue(I.getOperand(1)); | ||||||||||||
3531 | SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(), | ||||||||||||
3532 | TLI.getVectorIdxTy(DAG.getDataLayout())); | ||||||||||||
3533 | setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(), | ||||||||||||
3534 | TLI.getValueType(DAG.getDataLayout(), I.getType()), | ||||||||||||
3535 | InVec, InVal, InIdx)); | ||||||||||||
3536 | } | ||||||||||||
3537 | |||||||||||||
3538 | void SelectionDAGBuilder::visitExtractElement(const User &I) { | ||||||||||||
3539 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
3540 | SDValue InVec = getValue(I.getOperand(0)); | ||||||||||||
3541 | SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(), | ||||||||||||
3542 | TLI.getVectorIdxTy(DAG.getDataLayout())); | ||||||||||||
3543 | setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(), | ||||||||||||
3544 | TLI.getValueType(DAG.getDataLayout(), I.getType()), | ||||||||||||
3545 | InVec, InIdx)); | ||||||||||||
3546 | } | ||||||||||||
3547 | |||||||||||||
3548 | void SelectionDAGBuilder::visitShuffleVector(const User &I) { | ||||||||||||
3549 | SDValue Src1 = getValue(I.getOperand(0)); | ||||||||||||
3550 | SDValue Src2 = getValue(I.getOperand(1)); | ||||||||||||
3551 | ArrayRef<int> Mask; | ||||||||||||
3552 | if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I)) | ||||||||||||
3553 | Mask = SVI->getShuffleMask(); | ||||||||||||
3554 | else | ||||||||||||
3555 | Mask = cast<ConstantExpr>(I).getShuffleMask(); | ||||||||||||
3556 | SDLoc DL = getCurSDLoc(); | ||||||||||||
3557 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
3558 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
3559 | EVT SrcVT = Src1.getValueType(); | ||||||||||||
3560 | |||||||||||||
3561 | if (all_of(Mask, [](int Elem) { return Elem == 0; }) && | ||||||||||||
3562 | VT.isScalableVector()) { | ||||||||||||
3563 | // Canonical splat form of first element of first input vector. | ||||||||||||
3564 | SDValue FirstElt = | ||||||||||||
3565 | DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1, | ||||||||||||
3566 | DAG.getVectorIdxConstant(0, DL)); | ||||||||||||
3567 | setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt)); | ||||||||||||
3568 | return; | ||||||||||||
3569 | } | ||||||||||||
3570 | |||||||||||||
3571 | // For now, we only handle splats for scalable vectors. | ||||||||||||
3572 | // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation | ||||||||||||
3573 | // for targets that support a SPLAT_VECTOR for non-scalable vector types. | ||||||||||||
3574 | assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle")(static_cast <bool> (!VT.isScalableVector() && "Unsupported scalable vector shuffle" ) ? void (0) : __assert_fail ("!VT.isScalableVector() && \"Unsupported scalable vector shuffle\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 3574, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
3575 | |||||||||||||
3576 | unsigned SrcNumElts = SrcVT.getVectorNumElements(); | ||||||||||||
3577 | unsigned MaskNumElts = Mask.size(); | ||||||||||||
3578 | |||||||||||||
3579 | if (SrcNumElts == MaskNumElts) { | ||||||||||||
3580 | setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask)); | ||||||||||||
3581 | return; | ||||||||||||
3582 | } | ||||||||||||
3583 | |||||||||||||
3584 | // Normalize the shuffle vector since mask and vector length don't match. | ||||||||||||
3585 | if (SrcNumElts < MaskNumElts) { | ||||||||||||
3586 | // Mask is longer than the source vectors. We can use concatenate vector to | ||||||||||||
3587 | // make the mask and vectors lengths match. | ||||||||||||
3588 | |||||||||||||
3589 | if (MaskNumElts % SrcNumElts == 0) { | ||||||||||||
3590 | // Mask length is a multiple of the source vector length. | ||||||||||||
3591 | // Check if the shuffle is some kind of concatenation of the input | ||||||||||||
3592 | // vectors. | ||||||||||||
3593 | unsigned NumConcat = MaskNumElts / SrcNumElts; | ||||||||||||
3594 | bool IsConcat = true; | ||||||||||||
3595 | SmallVector<int, 8> ConcatSrcs(NumConcat, -1); | ||||||||||||
3596 | for (unsigned i = 0; i != MaskNumElts; ++i) { | ||||||||||||
3597 | int Idx = Mask[i]; | ||||||||||||
3598 | if (Idx < 0) | ||||||||||||
3599 | continue; | ||||||||||||
3600 | // Ensure the indices in each SrcVT sized piece are sequential and that | ||||||||||||
3601 | // the same source is used for the whole piece. | ||||||||||||
3602 | if ((Idx % SrcNumElts != (i % SrcNumElts)) || | ||||||||||||
3603 | (ConcatSrcs[i / SrcNumElts] >= 0 && | ||||||||||||
3604 | ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) { | ||||||||||||
3605 | IsConcat = false; | ||||||||||||
3606 | break; | ||||||||||||
3607 | } | ||||||||||||
3608 | // Remember which source this index came from. | ||||||||||||
3609 | ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts; | ||||||||||||
3610 | } | ||||||||||||
3611 | |||||||||||||
3612 | // The shuffle is concatenating multiple vectors together. Just emit | ||||||||||||
3613 | // a CONCAT_VECTORS operation. | ||||||||||||
3614 | if (IsConcat) { | ||||||||||||
3615 | SmallVector<SDValue, 8> ConcatOps; | ||||||||||||
3616 | for (auto Src : ConcatSrcs) { | ||||||||||||
3617 | if (Src < 0) | ||||||||||||
3618 | ConcatOps.push_back(DAG.getUNDEF(SrcVT)); | ||||||||||||
3619 | else if (Src == 0) | ||||||||||||
3620 | ConcatOps.push_back(Src1); | ||||||||||||
3621 | else | ||||||||||||
3622 | ConcatOps.push_back(Src2); | ||||||||||||
3623 | } | ||||||||||||
3624 | setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps)); | ||||||||||||
3625 | return; | ||||||||||||
3626 | } | ||||||||||||
3627 | } | ||||||||||||
3628 | |||||||||||||
3629 | unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts); | ||||||||||||
3630 | unsigned NumConcat = PaddedMaskNumElts / SrcNumElts; | ||||||||||||
3631 | EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), | ||||||||||||
3632 | PaddedMaskNumElts); | ||||||||||||
3633 | |||||||||||||
3634 | // Pad both vectors with undefs to make them the same length as the mask. | ||||||||||||
3635 | SDValue UndefVal = DAG.getUNDEF(SrcVT); | ||||||||||||
3636 | |||||||||||||
3637 | SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal); | ||||||||||||
3638 | SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal); | ||||||||||||
3639 | MOps1[0] = Src1; | ||||||||||||
3640 | MOps2[0] = Src2; | ||||||||||||
3641 | |||||||||||||
3642 | Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1); | ||||||||||||
3643 | Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2); | ||||||||||||
3644 | |||||||||||||
3645 | // Readjust mask for new input vector length. | ||||||||||||
3646 | SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1); | ||||||||||||
3647 | for (unsigned i = 0; i != MaskNumElts; ++i) { | ||||||||||||
3648 | int Idx = Mask[i]; | ||||||||||||
3649 | if (Idx >= (int)SrcNumElts) | ||||||||||||
3650 | Idx -= SrcNumElts - PaddedMaskNumElts; | ||||||||||||
3651 | MappedOps[i] = Idx; | ||||||||||||
3652 | } | ||||||||||||
3653 | |||||||||||||
3654 | SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps); | ||||||||||||
3655 | |||||||||||||
3656 | // If the concatenated vector was padded, extract a subvector with the | ||||||||||||
3657 | // correct number of elements. | ||||||||||||
3658 | if (MaskNumElts != PaddedMaskNumElts) | ||||||||||||
3659 | Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result, | ||||||||||||
3660 | DAG.getVectorIdxConstant(0, DL)); | ||||||||||||
3661 | |||||||||||||
3662 | setValue(&I, Result); | ||||||||||||
3663 | return; | ||||||||||||
3664 | } | ||||||||||||
3665 | |||||||||||||
3666 | if (SrcNumElts > MaskNumElts) { | ||||||||||||
3667 | // Analyze the access pattern of the vector to see if we can extract | ||||||||||||
3668 | // two subvectors and do the shuffle. | ||||||||||||
3669 | int StartIdx[2] = { -1, -1 }; // StartIdx to extract from | ||||||||||||
3670 | bool CanExtract = true; | ||||||||||||
3671 | for (int Idx : Mask) { | ||||||||||||
3672 | unsigned Input = 0; | ||||||||||||
3673 | if (Idx < 0) | ||||||||||||
3674 | continue; | ||||||||||||
3675 | |||||||||||||
3676 | if (Idx >= (int)SrcNumElts) { | ||||||||||||
3677 | Input = 1; | ||||||||||||
3678 | Idx -= SrcNumElts; | ||||||||||||
3679 | } | ||||||||||||
3680 | |||||||||||||
3681 | // If all the indices come from the same MaskNumElts sized portion of | ||||||||||||
3682 | // the sources we can use extract. Also make sure the extract wouldn't | ||||||||||||
3683 | // extract past the end of the source. | ||||||||||||
3684 | int NewStartIdx = alignDown(Idx, MaskNumElts); | ||||||||||||
3685 | if (NewStartIdx + MaskNumElts > SrcNumElts || | ||||||||||||
3686 | (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx)) | ||||||||||||
3687 | CanExtract = false; | ||||||||||||
3688 | // Make sure we always update StartIdx as we use it to track if all | ||||||||||||
3689 | // elements are undef. | ||||||||||||
3690 | StartIdx[Input] = NewStartIdx; | ||||||||||||
3691 | } | ||||||||||||
3692 | |||||||||||||
3693 | if (StartIdx[0] < 0 && StartIdx[1] < 0) { | ||||||||||||
3694 | setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used. | ||||||||||||
3695 | return; | ||||||||||||
3696 | } | ||||||||||||
3697 | if (CanExtract) { | ||||||||||||
3698 | // Extract appropriate subvector and generate a vector shuffle | ||||||||||||
3699 | for (unsigned Input = 0; Input < 2; ++Input) { | ||||||||||||
3700 | SDValue &Src = Input == 0 ? Src1 : Src2; | ||||||||||||
3701 | if (StartIdx[Input] < 0) | ||||||||||||
3702 | Src = DAG.getUNDEF(VT); | ||||||||||||
3703 | else { | ||||||||||||
3704 | Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src, | ||||||||||||
3705 | DAG.getVectorIdxConstant(StartIdx[Input], DL)); | ||||||||||||
3706 | } | ||||||||||||
3707 | } | ||||||||||||
3708 | |||||||||||||
3709 | // Calculate new mask. | ||||||||||||
3710 | SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end()); | ||||||||||||
3711 | for (int &Idx : MappedOps) { | ||||||||||||
3712 | if (Idx >= (int)SrcNumElts) | ||||||||||||
3713 | Idx -= SrcNumElts + StartIdx[1] - MaskNumElts; | ||||||||||||
3714 | else if (Idx >= 0) | ||||||||||||
3715 | Idx -= StartIdx[0]; | ||||||||||||
3716 | } | ||||||||||||
3717 | |||||||||||||
3718 | setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps)); | ||||||||||||
3719 | return; | ||||||||||||
3720 | } | ||||||||||||
3721 | } | ||||||||||||
3722 | |||||||||||||
3723 | // We can't use either concat vectors or extract subvectors so fall back to | ||||||||||||
3724 | // replacing the shuffle with extract and build vector. | ||||||||||||
3725 | // to insert and build vector. | ||||||||||||
3726 | EVT EltVT = VT.getVectorElementType(); | ||||||||||||
3727 | SmallVector<SDValue,8> Ops; | ||||||||||||
3728 | for (int Idx : Mask) { | ||||||||||||
3729 | SDValue Res; | ||||||||||||
3730 | |||||||||||||
3731 | if (Idx < 0) { | ||||||||||||
3732 | Res = DAG.getUNDEF(EltVT); | ||||||||||||
3733 | } else { | ||||||||||||
3734 | SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2; | ||||||||||||
3735 | if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts; | ||||||||||||
3736 | |||||||||||||
3737 | Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src, | ||||||||||||
3738 | DAG.getVectorIdxConstant(Idx, DL)); | ||||||||||||
3739 | } | ||||||||||||
3740 | |||||||||||||
3741 | Ops.push_back(Res); | ||||||||||||
3742 | } | ||||||||||||
3743 | |||||||||||||
3744 | setValue(&I, DAG.getBuildVector(VT, DL, Ops)); | ||||||||||||
3745 | } | ||||||||||||
3746 | |||||||||||||
3747 | void SelectionDAGBuilder::visitInsertValue(const User &I) { | ||||||||||||
3748 | ArrayRef<unsigned> Indices; | ||||||||||||
3749 | if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I)) | ||||||||||||
3750 | Indices = IV->getIndices(); | ||||||||||||
3751 | else | ||||||||||||
3752 | Indices = cast<ConstantExpr>(&I)->getIndices(); | ||||||||||||
3753 | |||||||||||||
3754 | const Value *Op0 = I.getOperand(0); | ||||||||||||
3755 | const Value *Op1 = I.getOperand(1); | ||||||||||||
3756 | Type *AggTy = I.getType(); | ||||||||||||
3757 | Type *ValTy = Op1->getType(); | ||||||||||||
3758 | bool IntoUndef = isa<UndefValue>(Op0); | ||||||||||||
3759 | bool FromUndef = isa<UndefValue>(Op1); | ||||||||||||
3760 | |||||||||||||
3761 | unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices); | ||||||||||||
3762 | |||||||||||||
3763 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
3764 | SmallVector<EVT, 4> AggValueVTs; | ||||||||||||
3765 | ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs); | ||||||||||||
3766 | SmallVector<EVT, 4> ValValueVTs; | ||||||||||||
3767 | ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs); | ||||||||||||
3768 | |||||||||||||
3769 | unsigned NumAggValues = AggValueVTs.size(); | ||||||||||||
3770 | unsigned NumValValues = ValValueVTs.size(); | ||||||||||||
3771 | SmallVector<SDValue, 4> Values(NumAggValues); | ||||||||||||
3772 | |||||||||||||
3773 | // Ignore an insertvalue that produces an empty object | ||||||||||||
3774 | if (!NumAggValues) { | ||||||||||||
3775 | setValue(&I, DAG.getUNDEF(MVT(MVT::Other))); | ||||||||||||
3776 | return; | ||||||||||||
3777 | } | ||||||||||||
3778 | |||||||||||||
3779 | SDValue Agg = getValue(Op0); | ||||||||||||
3780 | unsigned i = 0; | ||||||||||||
3781 | // Copy the beginning value(s) from the original aggregate. | ||||||||||||
3782 | for (; i != LinearIndex; ++i) | ||||||||||||
3783 | Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : | ||||||||||||
3784 | SDValue(Agg.getNode(), Agg.getResNo() + i); | ||||||||||||
3785 | // Copy values from the inserted value(s). | ||||||||||||
3786 | if (NumValValues) { | ||||||||||||
3787 | SDValue Val = getValue(Op1); | ||||||||||||
3788 | for (; i != LinearIndex + NumValValues; ++i) | ||||||||||||
3789 | Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) : | ||||||||||||
3790 | SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex); | ||||||||||||
3791 | } | ||||||||||||
3792 | // Copy remaining value(s) from the original aggregate. | ||||||||||||
3793 | for (; i != NumAggValues; ++i) | ||||||||||||
3794 | Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : | ||||||||||||
3795 | SDValue(Agg.getNode(), Agg.getResNo() + i); | ||||||||||||
3796 | |||||||||||||
3797 | setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), | ||||||||||||
3798 | DAG.getVTList(AggValueVTs), Values)); | ||||||||||||
3799 | } | ||||||||||||
3800 | |||||||||||||
3801 | void SelectionDAGBuilder::visitExtractValue(const User &I) { | ||||||||||||
3802 | ArrayRef<unsigned> Indices; | ||||||||||||
3803 | if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I)) | ||||||||||||
3804 | Indices = EV->getIndices(); | ||||||||||||
3805 | else | ||||||||||||
3806 | Indices = cast<ConstantExpr>(&I)->getIndices(); | ||||||||||||
3807 | |||||||||||||
3808 | const Value *Op0 = I.getOperand(0); | ||||||||||||
3809 | Type *AggTy = Op0->getType(); | ||||||||||||
3810 | Type *ValTy = I.getType(); | ||||||||||||
3811 | bool OutOfUndef = isa<UndefValue>(Op0); | ||||||||||||
3812 | |||||||||||||
3813 | unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices); | ||||||||||||
3814 | |||||||||||||
3815 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
3816 | SmallVector<EVT, 4> ValValueVTs; | ||||||||||||
3817 | ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs); | ||||||||||||
3818 | |||||||||||||
3819 | unsigned NumValValues = ValValueVTs.size(); | ||||||||||||
3820 | |||||||||||||
3821 | // Ignore a extractvalue that produces an empty object | ||||||||||||
3822 | if (!NumValValues) { | ||||||||||||
3823 | setValue(&I, DAG.getUNDEF(MVT(MVT::Other))); | ||||||||||||
3824 | return; | ||||||||||||
3825 | } | ||||||||||||
3826 | |||||||||||||
3827 | SmallVector<SDValue, 4> Values(NumValValues); | ||||||||||||
3828 | |||||||||||||
3829 | SDValue Agg = getValue(Op0); | ||||||||||||
3830 | // Copy out the selected value(s). | ||||||||||||
3831 | for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i) | ||||||||||||
3832 | Values[i - LinearIndex] = | ||||||||||||
3833 | OutOfUndef ? | ||||||||||||
3834 | DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) : | ||||||||||||
3835 | SDValue(Agg.getNode(), Agg.getResNo() + i); | ||||||||||||
3836 | |||||||||||||
3837 | setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), | ||||||||||||
3838 | DAG.getVTList(ValValueVTs), Values)); | ||||||||||||
3839 | } | ||||||||||||
3840 | |||||||||||||
3841 | void SelectionDAGBuilder::visitGetElementPtr(const User &I) { | ||||||||||||
3842 | Value *Op0 = I.getOperand(0); | ||||||||||||
3843 | // Note that the pointer operand may be a vector of pointers. Take the scalar | ||||||||||||
3844 | // element which holds a pointer. | ||||||||||||
3845 | unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace(); | ||||||||||||
3846 | SDValue N = getValue(Op0); | ||||||||||||
3847 | SDLoc dl = getCurSDLoc(); | ||||||||||||
3848 | auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
3849 | |||||||||||||
3850 | // Normalize Vector GEP - all scalar operands should be converted to the | ||||||||||||
3851 | // splat vector. | ||||||||||||
3852 | bool IsVectorGEP = I.getType()->isVectorTy(); | ||||||||||||
3853 | ElementCount VectorElementCount = | ||||||||||||
3854 | IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount() | ||||||||||||
3855 | : ElementCount::getFixed(0); | ||||||||||||
3856 | |||||||||||||
3857 | if (IsVectorGEP && !N.getValueType().isVector()) { | ||||||||||||
3858 | LLVMContext &Context = *DAG.getContext(); | ||||||||||||
3859 | EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount); | ||||||||||||
3860 | if (VectorElementCount.isScalable()) | ||||||||||||
3861 | N = DAG.getSplatVector(VT, dl, N); | ||||||||||||
3862 | else | ||||||||||||
3863 | N = DAG.getSplatBuildVector(VT, dl, N); | ||||||||||||
3864 | } | ||||||||||||
3865 | |||||||||||||
3866 | for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I); | ||||||||||||
3867 | GTI != E; ++GTI) { | ||||||||||||
3868 | const Value *Idx = GTI.getOperand(); | ||||||||||||
3869 | if (StructType *StTy = GTI.getStructTypeOrNull()) { | ||||||||||||
3870 | unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue(); | ||||||||||||
3871 | if (Field) { | ||||||||||||
3872 | // N = N + Offset | ||||||||||||
3873 | uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field); | ||||||||||||
3874 | |||||||||||||
3875 | // In an inbounds GEP with an offset that is nonnegative even when | ||||||||||||
3876 | // interpreted as signed, assume there is no unsigned overflow. | ||||||||||||
3877 | SDNodeFlags Flags; | ||||||||||||
3878 | if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds()) | ||||||||||||
3879 | Flags.setNoUnsignedWrap(true); | ||||||||||||
3880 | |||||||||||||
3881 | N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, | ||||||||||||
3882 | DAG.getConstant(Offset, dl, N.getValueType()), Flags); | ||||||||||||
3883 | } | ||||||||||||
3884 | } else { | ||||||||||||
3885 | // IdxSize is the width of the arithmetic according to IR semantics. | ||||||||||||
3886 | // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth | ||||||||||||
3887 | // (and fix up the result later). | ||||||||||||
3888 | unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS); | ||||||||||||
3889 | MVT IdxTy = MVT::getIntegerVT(IdxSize); | ||||||||||||
3890 | TypeSize ElementSize = DL->getTypeAllocSize(GTI.getIndexedType()); | ||||||||||||
3891 | // We intentionally mask away the high bits here; ElementSize may not | ||||||||||||
3892 | // fit in IdxTy. | ||||||||||||
3893 | APInt ElementMul(IdxSize, ElementSize.getKnownMinSize()); | ||||||||||||
3894 | bool ElementScalable = ElementSize.isScalable(); | ||||||||||||
3895 | |||||||||||||
3896 | // If this is a scalar constant or a splat vector of constants, | ||||||||||||
3897 | // handle it quickly. | ||||||||||||
3898 | const auto *C = dyn_cast<Constant>(Idx); | ||||||||||||
3899 | if (C && isa<VectorType>(C->getType())) | ||||||||||||
3900 | C = C->getSplatValue(); | ||||||||||||
3901 | |||||||||||||
3902 | const auto *CI = dyn_cast_or_null<ConstantInt>(C); | ||||||||||||
3903 | if (CI && CI->isZero()) | ||||||||||||
3904 | continue; | ||||||||||||
3905 | if (CI && !ElementScalable) { | ||||||||||||
3906 | APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize); | ||||||||||||
3907 | LLVMContext &Context = *DAG.getContext(); | ||||||||||||
3908 | SDValue OffsVal; | ||||||||||||
3909 | if (IsVectorGEP) | ||||||||||||
3910 | OffsVal = DAG.getConstant( | ||||||||||||
3911 | Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount)); | ||||||||||||
3912 | else | ||||||||||||
3913 | OffsVal = DAG.getConstant(Offs, dl, IdxTy); | ||||||||||||
3914 | |||||||||||||
3915 | // In an inbounds GEP with an offset that is nonnegative even when | ||||||||||||
3916 | // interpreted as signed, assume there is no unsigned overflow. | ||||||||||||
3917 | SDNodeFlags Flags; | ||||||||||||
3918 | if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds()) | ||||||||||||
3919 | Flags.setNoUnsignedWrap(true); | ||||||||||||
3920 | |||||||||||||
3921 | OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType()); | ||||||||||||
3922 | |||||||||||||
3923 | N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags); | ||||||||||||
3924 | continue; | ||||||||||||
3925 | } | ||||||||||||
3926 | |||||||||||||
3927 | // N = N + Idx * ElementMul; | ||||||||||||
3928 | SDValue IdxN = getValue(Idx); | ||||||||||||
3929 | |||||||||||||
3930 | if (!IdxN.getValueType().isVector() && IsVectorGEP) { | ||||||||||||
3931 | EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), | ||||||||||||
3932 | VectorElementCount); | ||||||||||||
3933 | if (VectorElementCount.isScalable()) | ||||||||||||
3934 | IdxN = DAG.getSplatVector(VT, dl, IdxN); | ||||||||||||
3935 | else | ||||||||||||
3936 | IdxN = DAG.getSplatBuildVector(VT, dl, IdxN); | ||||||||||||
3937 | } | ||||||||||||
3938 | |||||||||||||
3939 | // If the index is smaller or larger than intptr_t, truncate or extend | ||||||||||||
3940 | // it. | ||||||||||||
3941 | IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType()); | ||||||||||||
3942 | |||||||||||||
3943 | if (ElementScalable) { | ||||||||||||
3944 | EVT VScaleTy = N.getValueType().getScalarType(); | ||||||||||||
3945 | SDValue VScale = DAG.getNode( | ||||||||||||
3946 | ISD::VSCALE, dl, VScaleTy, | ||||||||||||
3947 | DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy)); | ||||||||||||
3948 | if (IsVectorGEP) | ||||||||||||
3949 | VScale = DAG.getSplatVector(N.getValueType(), dl, VScale); | ||||||||||||
3950 | IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale); | ||||||||||||
3951 | } else { | ||||||||||||
3952 | // If this is a multiply by a power of two, turn it into a shl | ||||||||||||
3953 | // immediately. This is a very common case. | ||||||||||||
3954 | if (ElementMul != 1) { | ||||||||||||
3955 | if (ElementMul.isPowerOf2()) { | ||||||||||||
3956 | unsigned Amt = ElementMul.logBase2(); | ||||||||||||
3957 | IdxN = DAG.getNode(ISD::SHL, dl, | ||||||||||||
3958 | N.getValueType(), IdxN, | ||||||||||||
3959 | DAG.getConstant(Amt, dl, IdxN.getValueType())); | ||||||||||||
3960 | } else { | ||||||||||||
3961 | SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl, | ||||||||||||
3962 | IdxN.getValueType()); | ||||||||||||
3963 | IdxN = DAG.getNode(ISD::MUL, dl, | ||||||||||||
3964 | N.getValueType(), IdxN, Scale); | ||||||||||||
3965 | } | ||||||||||||
3966 | } | ||||||||||||
3967 | } | ||||||||||||
3968 | |||||||||||||
3969 | N = DAG.getNode(ISD::ADD, dl, | ||||||||||||
3970 | N.getValueType(), N, IdxN); | ||||||||||||
3971 | } | ||||||||||||
3972 | } | ||||||||||||
3973 | |||||||||||||
3974 | MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS); | ||||||||||||
3975 | MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS); | ||||||||||||
3976 | if (IsVectorGEP) { | ||||||||||||
3977 | PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount); | ||||||||||||
3978 | PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount); | ||||||||||||
3979 | } | ||||||||||||
3980 | |||||||||||||
3981 | if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds()) | ||||||||||||
3982 | N = DAG.getPtrExtendInReg(N, dl, PtrMemTy); | ||||||||||||
3983 | |||||||||||||
3984 | setValue(&I, N); | ||||||||||||
3985 | } | ||||||||||||
3986 | |||||||||||||
3987 | void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) { | ||||||||||||
3988 | // If this is a fixed sized alloca in the entry block of the function, | ||||||||||||
3989 | // allocate it statically on the stack. | ||||||||||||
3990 | if (FuncInfo.StaticAllocaMap.count(&I)) | ||||||||||||
3991 | return; // getValue will auto-populate this. | ||||||||||||
3992 | |||||||||||||
3993 | SDLoc dl = getCurSDLoc(); | ||||||||||||
3994 | Type *Ty = I.getAllocatedType(); | ||||||||||||
3995 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
3996 | auto &DL = DAG.getDataLayout(); | ||||||||||||
3997 | uint64_t TySize = DL.getTypeAllocSize(Ty); | ||||||||||||
3998 | MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign()); | ||||||||||||
3999 | |||||||||||||
4000 | SDValue AllocSize = getValue(I.getArraySize()); | ||||||||||||
4001 | |||||||||||||
4002 | EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace()); | ||||||||||||
4003 | if (AllocSize.getValueType() != IntPtr) | ||||||||||||
4004 | AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr); | ||||||||||||
4005 | |||||||||||||
4006 | AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, | ||||||||||||
4007 | AllocSize, | ||||||||||||
4008 | DAG.getConstant(TySize, dl, IntPtr)); | ||||||||||||
4009 | |||||||||||||
4010 | // Handle alignment. If the requested alignment is less than or equal to | ||||||||||||
4011 | // the stack alignment, ignore it. If the size is greater than or equal to | ||||||||||||
4012 | // the stack alignment, we note this in the DYNAMIC_STACKALLOC node. | ||||||||||||
4013 | Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign(); | ||||||||||||
4014 | if (*Alignment <= StackAlign) | ||||||||||||
4015 | Alignment = None; | ||||||||||||
4016 | |||||||||||||
4017 | const uint64_t StackAlignMask = StackAlign.value() - 1U; | ||||||||||||
4018 | // Round the size of the allocation up to the stack alignment size | ||||||||||||
4019 | // by add SA-1 to the size. This doesn't overflow because we're computing | ||||||||||||
4020 | // an address inside an alloca. | ||||||||||||
4021 | SDNodeFlags Flags; | ||||||||||||
4022 | Flags.setNoUnsignedWrap(true); | ||||||||||||
4023 | AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize, | ||||||||||||
4024 | DAG.getConstant(StackAlignMask, dl, IntPtr), Flags); | ||||||||||||
4025 | |||||||||||||
4026 | // Mask out the low bits for alignment purposes. | ||||||||||||
4027 | AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize, | ||||||||||||
4028 | DAG.getConstant(~StackAlignMask, dl, IntPtr)); | ||||||||||||
4029 | |||||||||||||
4030 | SDValue Ops[] = { | ||||||||||||
4031 | getRoot(), AllocSize, | ||||||||||||
4032 | DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)}; | ||||||||||||
4033 | SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other); | ||||||||||||
4034 | SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops); | ||||||||||||
4035 | setValue(&I, DSA); | ||||||||||||
4036 | DAG.setRoot(DSA.getValue(1)); | ||||||||||||
4037 | |||||||||||||
4038 | assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects())(static_cast <bool> (FuncInfo.MF->getFrameInfo().hasVarSizedObjects ()) ? void (0) : __assert_fail ("FuncInfo.MF->getFrameInfo().hasVarSizedObjects()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4038, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
4039 | } | ||||||||||||
4040 | |||||||||||||
4041 | void SelectionDAGBuilder::visitLoad(const LoadInst &I) { | ||||||||||||
4042 | if (I.isAtomic()) | ||||||||||||
4043 | return visitAtomicLoad(I); | ||||||||||||
4044 | |||||||||||||
4045 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
4046 | const Value *SV = I.getOperand(0); | ||||||||||||
4047 | if (TLI.supportSwiftError()) { | ||||||||||||
4048 | // Swifterror values can come from either a function parameter with | ||||||||||||
4049 | // swifterror attribute or an alloca with swifterror attribute. | ||||||||||||
4050 | if (const Argument *Arg = dyn_cast<Argument>(SV)) { | ||||||||||||
4051 | if (Arg->hasSwiftErrorAttr()) | ||||||||||||
4052 | return visitLoadFromSwiftError(I); | ||||||||||||
4053 | } | ||||||||||||
4054 | |||||||||||||
4055 | if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) { | ||||||||||||
4056 | if (Alloca->isSwiftError()) | ||||||||||||
4057 | return visitLoadFromSwiftError(I); | ||||||||||||
4058 | } | ||||||||||||
4059 | } | ||||||||||||
4060 | |||||||||||||
4061 | SDValue Ptr = getValue(SV); | ||||||||||||
4062 | |||||||||||||
4063 | Type *Ty = I.getType(); | ||||||||||||
4064 | Align Alignment = I.getAlign(); | ||||||||||||
4065 | |||||||||||||
4066 | AAMDNodes AAInfo; | ||||||||||||
4067 | I.getAAMetadata(AAInfo); | ||||||||||||
4068 | const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); | ||||||||||||
4069 | |||||||||||||
4070 | SmallVector<EVT, 4> ValueVTs, MemVTs; | ||||||||||||
4071 | SmallVector<uint64_t, 4> Offsets; | ||||||||||||
4072 | ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets); | ||||||||||||
4073 | unsigned NumValues = ValueVTs.size(); | ||||||||||||
4074 | if (NumValues == 0) | ||||||||||||
4075 | return; | ||||||||||||
4076 | |||||||||||||
4077 | bool isVolatile = I.isVolatile(); | ||||||||||||
4078 | |||||||||||||
4079 | SDValue Root; | ||||||||||||
4080 | bool ConstantMemory = false; | ||||||||||||
4081 | if (isVolatile) | ||||||||||||
4082 | // Serialize volatile loads with other side effects. | ||||||||||||
4083 | Root = getRoot(); | ||||||||||||
4084 | else if (NumValues > MaxParallelChains) | ||||||||||||
4085 | Root = getMemoryRoot(); | ||||||||||||
4086 | else if (AA && | ||||||||||||
4087 | AA->pointsToConstantMemory(MemoryLocation( | ||||||||||||
4088 | SV, | ||||||||||||
4089 | LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), | ||||||||||||
4090 | AAInfo))) { | ||||||||||||
4091 | // Do not serialize (non-volatile) loads of constant memory with anything. | ||||||||||||
4092 | Root = DAG.getEntryNode(); | ||||||||||||
4093 | ConstantMemory = true; | ||||||||||||
4094 | } else { | ||||||||||||
4095 | // Do not serialize non-volatile loads against each other. | ||||||||||||
4096 | Root = DAG.getRoot(); | ||||||||||||
4097 | } | ||||||||||||
4098 | |||||||||||||
4099 | SDLoc dl = getCurSDLoc(); | ||||||||||||
4100 | |||||||||||||
4101 | if (isVolatile) | ||||||||||||
4102 | Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG); | ||||||||||||
4103 | |||||||||||||
4104 | // An aggregate load cannot wrap around the address space, so offsets to its | ||||||||||||
4105 | // parts don't wrap either. | ||||||||||||
4106 | SDNodeFlags Flags; | ||||||||||||
4107 | Flags.setNoUnsignedWrap(true); | ||||||||||||
4108 | |||||||||||||
4109 | SmallVector<SDValue, 4> Values(NumValues); | ||||||||||||
4110 | SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues)); | ||||||||||||
4111 | EVT PtrVT = Ptr.getValueType(); | ||||||||||||
4112 | |||||||||||||
4113 | MachineMemOperand::Flags MMOFlags | ||||||||||||
4114 | = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout()); | ||||||||||||
4115 | |||||||||||||
4116 | unsigned ChainI = 0; | ||||||||||||
4117 | for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { | ||||||||||||
4118 | // Serializing loads here may result in excessive register pressure, and | ||||||||||||
4119 | // TokenFactor places arbitrary choke points on the scheduler. SD scheduling | ||||||||||||
4120 | // could recover a bit by hoisting nodes upward in the chain by recognizing | ||||||||||||
4121 | // they are side-effect free or do not alias. The optimizer should really | ||||||||||||
4122 | // avoid this case by converting large object/array copies to llvm.memcpy | ||||||||||||
4123 | // (MaxParallelChains should always remain as failsafe). | ||||||||||||
4124 | if (ChainI == MaxParallelChains) { | ||||||||||||
4125 | assert(PendingLoads.empty() && "PendingLoads must be serialized first")(static_cast <bool> (PendingLoads.empty() && "PendingLoads must be serialized first" ) ? void (0) : __assert_fail ("PendingLoads.empty() && \"PendingLoads must be serialized first\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4125, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
4126 | SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, | ||||||||||||
4127 | makeArrayRef(Chains.data(), ChainI)); | ||||||||||||
4128 | Root = Chain; | ||||||||||||
4129 | ChainI = 0; | ||||||||||||
4130 | } | ||||||||||||
4131 | SDValue A = DAG.getNode(ISD::ADD, dl, | ||||||||||||
4132 | PtrVT, Ptr, | ||||||||||||
4133 | DAG.getConstant(Offsets[i], dl, PtrVT), | ||||||||||||
4134 | Flags); | ||||||||||||
4135 | |||||||||||||
4136 | SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, | ||||||||||||
4137 | MachinePointerInfo(SV, Offsets[i]), Alignment, | ||||||||||||
4138 | MMOFlags, AAInfo, Ranges); | ||||||||||||
4139 | Chains[ChainI] = L.getValue(1); | ||||||||||||
4140 | |||||||||||||
4141 | if (MemVTs[i] != ValueVTs[i]) | ||||||||||||
4142 | L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]); | ||||||||||||
4143 | |||||||||||||
4144 | Values[i] = L; | ||||||||||||
4145 | } | ||||||||||||
4146 | |||||||||||||
4147 | if (!ConstantMemory) { | ||||||||||||
4148 | SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, | ||||||||||||
4149 | makeArrayRef(Chains.data(), ChainI)); | ||||||||||||
4150 | if (isVolatile) | ||||||||||||
4151 | DAG.setRoot(Chain); | ||||||||||||
4152 | else | ||||||||||||
4153 | PendingLoads.push_back(Chain); | ||||||||||||
4154 | } | ||||||||||||
4155 | |||||||||||||
4156 | setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl, | ||||||||||||
4157 | DAG.getVTList(ValueVTs), Values)); | ||||||||||||
4158 | } | ||||||||||||
4159 | |||||||||||||
4160 | void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) { | ||||||||||||
4161 | assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError () && "call visitStoreToSwiftError when backend supports swifterror" ) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4162, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
4162 | "call visitStoreToSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError () && "call visitStoreToSwiftError when backend supports swifterror" ) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4162, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
4163 | |||||||||||||
4164 | SmallVector<EVT, 4> ValueVTs; | ||||||||||||
4165 | SmallVector<uint64_t, 4> Offsets; | ||||||||||||
4166 | const Value *SrcV = I.getOperand(0); | ||||||||||||
4167 | ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), | ||||||||||||
4168 | SrcV->getType(), ValueVTs, &Offsets); | ||||||||||||
4169 | assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets [0] == 0 && "expect a single EVT for swifterror") ? void (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4170, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
4170 | "expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets [0] == 0 && "expect a single EVT for swifterror") ? void (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4170, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
4171 | |||||||||||||
4172 | SDValue Src = getValue(SrcV); | ||||||||||||
4173 | // Create a virtual register, then update the virtual register. | ||||||||||||
4174 | Register VReg = | ||||||||||||
4175 | SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand()); | ||||||||||||
4176 | // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue | ||||||||||||
4177 | // Chain can be getRoot or getControlRoot. | ||||||||||||
4178 | SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg, | ||||||||||||
4179 | SDValue(Src.getNode(), Src.getResNo())); | ||||||||||||
4180 | DAG.setRoot(CopyNode); | ||||||||||||
4181 | } | ||||||||||||
4182 | |||||||||||||
4183 | void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) { | ||||||||||||
4184 | assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError () && "call visitLoadFromSwiftError when backend supports swifterror" ) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4185, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
4185 | "call visitLoadFromSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError () && "call visitLoadFromSwiftError when backend supports swifterror" ) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4185, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
4186 | |||||||||||||
4187 | assert(!I.isVolatile() &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata (LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext ::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error" ) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4190, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
4188 | !I.hasMetadata(LLVMContext::MD_nontemporal) &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata (LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext ::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error" ) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4190, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
4189 | !I.hasMetadata(LLVMContext::MD_invariant_load) &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata (LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext ::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error" ) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4190, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
4190 | "Support volatile, non temporal, invariant for load_from_swift_error")(static_cast <bool> (!I.isVolatile() && !I.hasMetadata (LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext ::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error" ) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4190, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
4191 | |||||||||||||
4192 | const Value *SV = I.getOperand(0); | ||||||||||||
4193 | Type *Ty = I.getType(); | ||||||||||||
4194 | AAMDNodes AAInfo; | ||||||||||||
4195 | I.getAAMetadata(AAInfo); | ||||||||||||
4196 | assert((static_cast <bool> ((!AA || !AA->pointsToConstantMemory (MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout( ).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory" ) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4201, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
4197 | (!AA ||(static_cast <bool> ((!AA || !AA->pointsToConstantMemory (MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout( ).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory" ) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4201, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
4198 | !AA->pointsToConstantMemory(MemoryLocation((static_cast <bool> ((!AA || !AA->pointsToConstantMemory (MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout( ).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory" ) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4201, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
4199 | SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),(static_cast <bool> ((!AA || !AA->pointsToConstantMemory (MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout( ).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory" ) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4201, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
4200 | AAInfo))) &&(static_cast <bool> ((!AA || !AA->pointsToConstantMemory (MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout( ).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory" ) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4201, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
4201 | "load_from_swift_error should not be constant memory")(static_cast <bool> ((!AA || !AA->pointsToConstantMemory (MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout( ).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory" ) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4201, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
4202 | |||||||||||||
4203 | SmallVector<EVT, 4> ValueVTs; | ||||||||||||
4204 | SmallVector<uint64_t, 4> Offsets; | ||||||||||||
4205 | ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty, | ||||||||||||
4206 | ValueVTs, &Offsets); | ||||||||||||
4207 | assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets [0] == 0 && "expect a single EVT for swifterror") ? void (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4208, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
4208 | "expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets [0] == 0 && "expect a single EVT for swifterror") ? void (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4208, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
4209 | |||||||||||||
4210 | // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT | ||||||||||||
4211 | SDValue L = DAG.getCopyFromReg( | ||||||||||||
4212 | getRoot(), getCurSDLoc(), | ||||||||||||
4213 | SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]); | ||||||||||||
4214 | |||||||||||||
4215 | setValue(&I, L); | ||||||||||||
4216 | } | ||||||||||||
4217 | |||||||||||||
4218 | void SelectionDAGBuilder::visitStore(const StoreInst &I) { | ||||||||||||
4219 | if (I.isAtomic()) | ||||||||||||
4220 | return visitAtomicStore(I); | ||||||||||||
4221 | |||||||||||||
4222 | const Value *SrcV = I.getOperand(0); | ||||||||||||
4223 | const Value *PtrV = I.getOperand(1); | ||||||||||||
4224 | |||||||||||||
4225 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
4226 | if (TLI.supportSwiftError()) { | ||||||||||||
4227 | // Swifterror values can come from either a function parameter with | ||||||||||||
4228 | // swifterror attribute or an alloca with swifterror attribute. | ||||||||||||
4229 | if (const Argument *Arg = dyn_cast<Argument>(PtrV)) { | ||||||||||||
4230 | if (Arg->hasSwiftErrorAttr()) | ||||||||||||
4231 | return visitStoreToSwiftError(I); | ||||||||||||
4232 | } | ||||||||||||
4233 | |||||||||||||
4234 | if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) { | ||||||||||||
4235 | if (Alloca->isSwiftError()) | ||||||||||||
4236 | return visitStoreToSwiftError(I); | ||||||||||||
4237 | } | ||||||||||||
4238 | } | ||||||||||||
4239 | |||||||||||||
4240 | SmallVector<EVT, 4> ValueVTs, MemVTs; | ||||||||||||
4241 | SmallVector<uint64_t, 4> Offsets; | ||||||||||||
4242 | ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), | ||||||||||||
4243 | SrcV->getType(), ValueVTs, &MemVTs, &Offsets); | ||||||||||||
4244 | unsigned NumValues = ValueVTs.size(); | ||||||||||||
4245 | if (NumValues == 0) | ||||||||||||
4246 | return; | ||||||||||||
4247 | |||||||||||||
4248 | // Get the lowered operands. Note that we do this after | ||||||||||||
4249 | // checking if NumResults is zero, because with zero results | ||||||||||||
4250 | // the operands won't have values in the map. | ||||||||||||
4251 | SDValue Src = getValue(SrcV); | ||||||||||||
4252 | SDValue Ptr = getValue(PtrV); | ||||||||||||
4253 | |||||||||||||
4254 | SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot(); | ||||||||||||
4255 | SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues)); | ||||||||||||
4256 | SDLoc dl = getCurSDLoc(); | ||||||||||||
4257 | Align Alignment = I.getAlign(); | ||||||||||||
4258 | AAMDNodes AAInfo; | ||||||||||||
4259 | I.getAAMetadata(AAInfo); | ||||||||||||
4260 | |||||||||||||
4261 | auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout()); | ||||||||||||
4262 | |||||||||||||
4263 | // An aggregate load cannot wrap around the address space, so offsets to its | ||||||||||||
4264 | // parts don't wrap either. | ||||||||||||
4265 | SDNodeFlags Flags; | ||||||||||||
4266 | Flags.setNoUnsignedWrap(true); | ||||||||||||
4267 | |||||||||||||
4268 | unsigned ChainI = 0; | ||||||||||||
4269 | for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { | ||||||||||||
4270 | // See visitLoad comments. | ||||||||||||
4271 | if (ChainI == MaxParallelChains) { | ||||||||||||
4272 | SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, | ||||||||||||
4273 | makeArrayRef(Chains.data(), ChainI)); | ||||||||||||
4274 | Root = Chain; | ||||||||||||
4275 | ChainI = 0; | ||||||||||||
4276 | } | ||||||||||||
4277 | SDValue Add = | ||||||||||||
4278 | DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags); | ||||||||||||
4279 | SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i); | ||||||||||||
4280 | if (MemVTs[i] != ValueVTs[i]) | ||||||||||||
4281 | Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]); | ||||||||||||
4282 | SDValue St = | ||||||||||||
4283 | DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]), | ||||||||||||
4284 | Alignment, MMOFlags, AAInfo); | ||||||||||||
4285 | Chains[ChainI] = St; | ||||||||||||
4286 | } | ||||||||||||
4287 | |||||||||||||
4288 | SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, | ||||||||||||
4289 | makeArrayRef(Chains.data(), ChainI)); | ||||||||||||
4290 | DAG.setRoot(StoreNode); | ||||||||||||
4291 | } | ||||||||||||
4292 | |||||||||||||
4293 | void SelectionDAGBuilder::visitMaskedStore(const CallInst &I, | ||||||||||||
4294 | bool IsCompressing) { | ||||||||||||
4295 | SDLoc sdl = getCurSDLoc(); | ||||||||||||
4296 | |||||||||||||
4297 | auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, | ||||||||||||
4298 | MaybeAlign &Alignment) { | ||||||||||||
4299 | // llvm.masked.store.*(Src0, Ptr, alignment, Mask) | ||||||||||||
4300 | Src0 = I.getArgOperand(0); | ||||||||||||
4301 | Ptr = I.getArgOperand(1); | ||||||||||||
4302 | Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue(); | ||||||||||||
4303 | Mask = I.getArgOperand(3); | ||||||||||||
4304 | }; | ||||||||||||
4305 | auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, | ||||||||||||
4306 | MaybeAlign &Alignment) { | ||||||||||||
4307 | // llvm.masked.compressstore.*(Src0, Ptr, Mask) | ||||||||||||
4308 | Src0 = I.getArgOperand(0); | ||||||||||||
4309 | Ptr = I.getArgOperand(1); | ||||||||||||
4310 | Mask = I.getArgOperand(2); | ||||||||||||
4311 | Alignment = None; | ||||||||||||
4312 | }; | ||||||||||||
4313 | |||||||||||||
4314 | Value *PtrOperand, *MaskOperand, *Src0Operand; | ||||||||||||
4315 | MaybeAlign Alignment; | ||||||||||||
4316 | if (IsCompressing) | ||||||||||||
4317 | getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment); | ||||||||||||
4318 | else | ||||||||||||
4319 | getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment); | ||||||||||||
4320 | |||||||||||||
4321 | SDValue Ptr = getValue(PtrOperand); | ||||||||||||
4322 | SDValue Src0 = getValue(Src0Operand); | ||||||||||||
4323 | SDValue Mask = getValue(MaskOperand); | ||||||||||||
4324 | SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); | ||||||||||||
4325 | |||||||||||||
4326 | EVT VT = Src0.getValueType(); | ||||||||||||
4327 | if (!Alignment) | ||||||||||||
4328 | Alignment = DAG.getEVTAlign(VT); | ||||||||||||
4329 | |||||||||||||
4330 | AAMDNodes AAInfo; | ||||||||||||
4331 | I.getAAMetadata(AAInfo); | ||||||||||||
4332 | |||||||||||||
4333 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||||||
4334 | MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore, | ||||||||||||
4335 | // TODO: Make MachineMemOperands aware of scalable | ||||||||||||
4336 | // vectors. | ||||||||||||
4337 | VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo); | ||||||||||||
4338 | SDValue StoreNode = | ||||||||||||
4339 | DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO, | ||||||||||||
4340 | ISD::UNINDEXED, false /* Truncating */, IsCompressing); | ||||||||||||
4341 | DAG.setRoot(StoreNode); | ||||||||||||
4342 | setValue(&I, StoreNode); | ||||||||||||
4343 | } | ||||||||||||
4344 | |||||||||||||
4345 | // Get a uniform base for the Gather/Scatter intrinsic. | ||||||||||||
4346 | // The first argument of the Gather/Scatter intrinsic is a vector of pointers. | ||||||||||||
4347 | // We try to represent it as a base pointer + vector of indices. | ||||||||||||
4348 | // Usually, the vector of pointers comes from a 'getelementptr' instruction. | ||||||||||||
4349 | // The first operand of the GEP may be a single pointer or a vector of pointers | ||||||||||||
4350 | // Example: | ||||||||||||
4351 | // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind | ||||||||||||
4352 | // or | ||||||||||||
4353 | // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind | ||||||||||||
4354 | // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, .. | ||||||||||||
4355 | // | ||||||||||||
4356 | // When the first GEP operand is a single pointer - it is the uniform base we | ||||||||||||
4357 | // are looking for. If first operand of the GEP is a splat vector - we | ||||||||||||
4358 | // extract the splat value and use it as a uniform base. | ||||||||||||
4359 | // In all other cases the function returns 'false'. | ||||||||||||
4360 | static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, | ||||||||||||
4361 | ISD::MemIndexType &IndexType, SDValue &Scale, | ||||||||||||
4362 | SelectionDAGBuilder *SDB, const BasicBlock *CurBB) { | ||||||||||||
4363 | SelectionDAG& DAG = SDB->DAG; | ||||||||||||
4364 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
4365 | const DataLayout &DL = DAG.getDataLayout(); | ||||||||||||
4366 | |||||||||||||
4367 | assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type")(static_cast <bool> (Ptr->getType()->isVectorTy() && "Uexpected pointer type") ? void (0) : __assert_fail ("Ptr->getType()->isVectorTy() && \"Uexpected pointer type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4367, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
4368 | |||||||||||||
4369 | // Handle splat constant pointer. | ||||||||||||
4370 | if (auto *C = dyn_cast<Constant>(Ptr)) { | ||||||||||||
4371 | C = C->getSplatValue(); | ||||||||||||
4372 | if (!C) | ||||||||||||
4373 | return false; | ||||||||||||
4374 | |||||||||||||
4375 | Base = SDB->getValue(C); | ||||||||||||
4376 | |||||||||||||
4377 | ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount(); | ||||||||||||
4378 | EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts); | ||||||||||||
4379 | Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT); | ||||||||||||
4380 | IndexType = ISD::SIGNED_SCALED; | ||||||||||||
4381 | Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL)); | ||||||||||||
4382 | return true; | ||||||||||||
4383 | } | ||||||||||||
4384 | |||||||||||||
4385 | const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr); | ||||||||||||
4386 | if (!GEP || GEP->getParent() != CurBB) | ||||||||||||
4387 | return false; | ||||||||||||
4388 | |||||||||||||
4389 | if (GEP->getNumOperands() != 2) | ||||||||||||
4390 | return false; | ||||||||||||
4391 | |||||||||||||
4392 | const Value *BasePtr = GEP->getPointerOperand(); | ||||||||||||
4393 | const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1); | ||||||||||||
4394 | |||||||||||||
4395 | // Make sure the base is scalar and the index is a vector. | ||||||||||||
4396 | if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy()) | ||||||||||||
4397 | return false; | ||||||||||||
4398 | |||||||||||||
4399 | Base = SDB->getValue(BasePtr); | ||||||||||||
4400 | Index = SDB->getValue(IndexVal); | ||||||||||||
4401 | IndexType = ISD::SIGNED_SCALED; | ||||||||||||
4402 | Scale = DAG.getTargetConstant( | ||||||||||||
4403 | DL.getTypeAllocSize(GEP->getResultElementType()), | ||||||||||||
4404 | SDB->getCurSDLoc(), TLI.getPointerTy(DL)); | ||||||||||||
4405 | return true; | ||||||||||||
4406 | } | ||||||||||||
4407 | |||||||||||||
4408 | void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) { | ||||||||||||
4409 | SDLoc sdl = getCurSDLoc(); | ||||||||||||
4410 | |||||||||||||
4411 | // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask) | ||||||||||||
4412 | const Value *Ptr = I.getArgOperand(1); | ||||||||||||
4413 | SDValue Src0 = getValue(I.getArgOperand(0)); | ||||||||||||
4414 | SDValue Mask = getValue(I.getArgOperand(3)); | ||||||||||||
4415 | EVT VT = Src0.getValueType(); | ||||||||||||
4416 | Align Alignment = cast<ConstantInt>(I.getArgOperand(2)) | ||||||||||||
4417 | ->getMaybeAlignValue() | ||||||||||||
4418 | .getValueOr(DAG.getEVTAlign(VT.getScalarType())); | ||||||||||||
4419 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
4420 | |||||||||||||
4421 | AAMDNodes AAInfo; | ||||||||||||
4422 | I.getAAMetadata(AAInfo); | ||||||||||||
4423 | |||||||||||||
4424 | SDValue Base; | ||||||||||||
4425 | SDValue Index; | ||||||||||||
4426 | ISD::MemIndexType IndexType; | ||||||||||||
4427 | SDValue Scale; | ||||||||||||
4428 | bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this, | ||||||||||||
4429 | I.getParent()); | ||||||||||||
4430 | |||||||||||||
4431 | unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace(); | ||||||||||||
4432 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||||||
4433 | MachinePointerInfo(AS), MachineMemOperand::MOStore, | ||||||||||||
4434 | // TODO: Make MachineMemOperands aware of scalable | ||||||||||||
4435 | // vectors. | ||||||||||||
4436 | MemoryLocation::UnknownSize, Alignment, AAInfo); | ||||||||||||
4437 | if (!UniformBase) { | ||||||||||||
4438 | Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||||||
4439 | Index = getValue(Ptr); | ||||||||||||
4440 | IndexType = ISD::SIGNED_UNSCALED; | ||||||||||||
4441 | Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||||||
4442 | } | ||||||||||||
4443 | |||||||||||||
4444 | EVT IdxVT = Index.getValueType(); | ||||||||||||
4445 | EVT EltTy = IdxVT.getVectorElementType(); | ||||||||||||
4446 | if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) { | ||||||||||||
4447 | EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy); | ||||||||||||
4448 | Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index); | ||||||||||||
4449 | } | ||||||||||||
4450 | |||||||||||||
4451 | SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale }; | ||||||||||||
4452 | SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl, | ||||||||||||
4453 | Ops, MMO, IndexType, false); | ||||||||||||
4454 | DAG.setRoot(Scatter); | ||||||||||||
4455 | setValue(&I, Scatter); | ||||||||||||
4456 | } | ||||||||||||
4457 | |||||||||||||
4458 | void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) { | ||||||||||||
4459 | SDLoc sdl = getCurSDLoc(); | ||||||||||||
4460 | |||||||||||||
4461 | auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, | ||||||||||||
4462 | MaybeAlign &Alignment) { | ||||||||||||
4463 | // @llvm.masked.load.*(Ptr, alignment, Mask, Src0) | ||||||||||||
4464 | Ptr = I.getArgOperand(0); | ||||||||||||
4465 | Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue(); | ||||||||||||
4466 | Mask = I.getArgOperand(2); | ||||||||||||
4467 | Src0 = I.getArgOperand(3); | ||||||||||||
4468 | }; | ||||||||||||
4469 | auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, | ||||||||||||
4470 | MaybeAlign &Alignment) { | ||||||||||||
4471 | // @llvm.masked.expandload.*(Ptr, Mask, Src0) | ||||||||||||
4472 | Ptr = I.getArgOperand(0); | ||||||||||||
4473 | Alignment = None; | ||||||||||||
4474 | Mask = I.getArgOperand(1); | ||||||||||||
4475 | Src0 = I.getArgOperand(2); | ||||||||||||
4476 | }; | ||||||||||||
4477 | |||||||||||||
4478 | Value *PtrOperand, *MaskOperand, *Src0Operand; | ||||||||||||
4479 | MaybeAlign Alignment; | ||||||||||||
4480 | if (IsExpanding) | ||||||||||||
4481 | getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment); | ||||||||||||
4482 | else | ||||||||||||
4483 | getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment); | ||||||||||||
4484 | |||||||||||||
4485 | SDValue Ptr = getValue(PtrOperand); | ||||||||||||
4486 | SDValue Src0 = getValue(Src0Operand); | ||||||||||||
4487 | SDValue Mask = getValue(MaskOperand); | ||||||||||||
4488 | SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); | ||||||||||||
4489 | |||||||||||||
4490 | EVT VT = Src0.getValueType(); | ||||||||||||
4491 | if (!Alignment) | ||||||||||||
4492 | Alignment = DAG.getEVTAlign(VT); | ||||||||||||
4493 | |||||||||||||
4494 | AAMDNodes AAInfo; | ||||||||||||
4495 | I.getAAMetadata(AAInfo); | ||||||||||||
4496 | const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); | ||||||||||||
4497 | |||||||||||||
4498 | // Do not serialize masked loads of constant memory with anything. | ||||||||||||
4499 | MemoryLocation ML; | ||||||||||||
4500 | if (VT.isScalableVector()) | ||||||||||||
4501 | ML = MemoryLocation::getAfter(PtrOperand); | ||||||||||||
4502 | else | ||||||||||||
4503 | ML = MemoryLocation(PtrOperand, LocationSize::precise( | ||||||||||||
4504 | DAG.getDataLayout().getTypeStoreSize(I.getType())), | ||||||||||||
4505 | AAInfo); | ||||||||||||
4506 | bool AddToChain = !AA || !AA->pointsToConstantMemory(ML); | ||||||||||||
4507 | |||||||||||||
4508 | SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); | ||||||||||||
4509 | |||||||||||||
4510 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||||||
4511 | MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad, | ||||||||||||
4512 | // TODO: Make MachineMemOperands aware of scalable | ||||||||||||
4513 | // vectors. | ||||||||||||
4514 | VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo, Ranges); | ||||||||||||
4515 | |||||||||||||
4516 | SDValue Load = | ||||||||||||
4517 | DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO, | ||||||||||||
4518 | ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding); | ||||||||||||
4519 | if (AddToChain) | ||||||||||||
4520 | PendingLoads.push_back(Load.getValue(1)); | ||||||||||||
4521 | setValue(&I, Load); | ||||||||||||
4522 | } | ||||||||||||
4523 | |||||||||||||
4524 | void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) { | ||||||||||||
4525 | SDLoc sdl = getCurSDLoc(); | ||||||||||||
4526 | |||||||||||||
4527 | // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0) | ||||||||||||
4528 | const Value *Ptr = I.getArgOperand(0); | ||||||||||||
4529 | SDValue Src0 = getValue(I.getArgOperand(3)); | ||||||||||||
4530 | SDValue Mask = getValue(I.getArgOperand(2)); | ||||||||||||
4531 | |||||||||||||
4532 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
4533 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
4534 | Align Alignment = cast<ConstantInt>(I.getArgOperand(1)) | ||||||||||||
4535 | ->getMaybeAlignValue() | ||||||||||||
4536 | .getValueOr(DAG.getEVTAlign(VT.getScalarType())); | ||||||||||||
4537 | |||||||||||||
4538 | AAMDNodes AAInfo; | ||||||||||||
4539 | I.getAAMetadata(AAInfo); | ||||||||||||
4540 | const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); | ||||||||||||
4541 | |||||||||||||
4542 | SDValue Root = DAG.getRoot(); | ||||||||||||
4543 | SDValue Base; | ||||||||||||
4544 | SDValue Index; | ||||||||||||
4545 | ISD::MemIndexType IndexType; | ||||||||||||
4546 | SDValue Scale; | ||||||||||||
4547 | bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this, | ||||||||||||
4548 | I.getParent()); | ||||||||||||
4549 | unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace(); | ||||||||||||
4550 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||||||
4551 | MachinePointerInfo(AS), MachineMemOperand::MOLoad, | ||||||||||||
4552 | // TODO: Make MachineMemOperands aware of scalable | ||||||||||||
4553 | // vectors. | ||||||||||||
4554 | MemoryLocation::UnknownSize, Alignment, AAInfo, Ranges); | ||||||||||||
4555 | |||||||||||||
4556 | if (!UniformBase) { | ||||||||||||
4557 | Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||||||
4558 | Index = getValue(Ptr); | ||||||||||||
4559 | IndexType = ISD::SIGNED_UNSCALED; | ||||||||||||
4560 | Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||||||
4561 | } | ||||||||||||
4562 | |||||||||||||
4563 | EVT IdxVT = Index.getValueType(); | ||||||||||||
4564 | EVT EltTy = IdxVT.getVectorElementType(); | ||||||||||||
4565 | if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) { | ||||||||||||
4566 | EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy); | ||||||||||||
4567 | Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index); | ||||||||||||
4568 | } | ||||||||||||
4569 | |||||||||||||
4570 | SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale }; | ||||||||||||
4571 | SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl, | ||||||||||||
4572 | Ops, MMO, IndexType, ISD::NON_EXTLOAD); | ||||||||||||
4573 | |||||||||||||
4574 | PendingLoads.push_back(Gather.getValue(1)); | ||||||||||||
4575 | setValue(&I, Gather); | ||||||||||||
4576 | } | ||||||||||||
4577 | |||||||||||||
4578 | void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { | ||||||||||||
4579 | SDLoc dl = getCurSDLoc(); | ||||||||||||
4580 | AtomicOrdering SuccessOrdering = I.getSuccessOrdering(); | ||||||||||||
4581 | AtomicOrdering FailureOrdering = I.getFailureOrdering(); | ||||||||||||
4582 | SyncScope::ID SSID = I.getSyncScopeID(); | ||||||||||||
4583 | |||||||||||||
4584 | SDValue InChain = getRoot(); | ||||||||||||
4585 | |||||||||||||
4586 | MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType(); | ||||||||||||
4587 | SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other); | ||||||||||||
4588 | |||||||||||||
4589 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
4590 | auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout()); | ||||||||||||
4591 | |||||||||||||
4592 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
4593 | MachineMemOperand *MMO = MF.getMachineMemOperand( | ||||||||||||
4594 | MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), | ||||||||||||
4595 | DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering, | ||||||||||||
4596 | FailureOrdering); | ||||||||||||
4597 | |||||||||||||
4598 | SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, | ||||||||||||
4599 | dl, MemVT, VTs, InChain, | ||||||||||||
4600 | getValue(I.getPointerOperand()), | ||||||||||||
4601 | getValue(I.getCompareOperand()), | ||||||||||||
4602 | getValue(I.getNewValOperand()), MMO); | ||||||||||||
4603 | |||||||||||||
4604 | SDValue OutChain = L.getValue(2); | ||||||||||||
4605 | |||||||||||||
4606 | setValue(&I, L); | ||||||||||||
4607 | DAG.setRoot(OutChain); | ||||||||||||
4608 | } | ||||||||||||
4609 | |||||||||||||
4610 | void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { | ||||||||||||
4611 | SDLoc dl = getCurSDLoc(); | ||||||||||||
4612 | ISD::NodeType NT; | ||||||||||||
4613 | switch (I.getOperation()) { | ||||||||||||
4614 | default: llvm_unreachable("Unknown atomicrmw operation")::llvm::llvm_unreachable_internal("Unknown atomicrmw operation" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4614); | ||||||||||||
4615 | case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break; | ||||||||||||
4616 | case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break; | ||||||||||||
4617 | case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break; | ||||||||||||
4618 | case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break; | ||||||||||||
4619 | case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break; | ||||||||||||
4620 | case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break; | ||||||||||||
4621 | case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break; | ||||||||||||
4622 | case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break; | ||||||||||||
4623 | case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break; | ||||||||||||
4624 | case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break; | ||||||||||||
4625 | case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break; | ||||||||||||
4626 | case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break; | ||||||||||||
4627 | case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break; | ||||||||||||
4628 | } | ||||||||||||
4629 | AtomicOrdering Ordering = I.getOrdering(); | ||||||||||||
4630 | SyncScope::ID SSID = I.getSyncScopeID(); | ||||||||||||
4631 | |||||||||||||
4632 | SDValue InChain = getRoot(); | ||||||||||||
4633 | |||||||||||||
4634 | auto MemVT = getValue(I.getValOperand()).getSimpleValueType(); | ||||||||||||
4635 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
4636 | auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout()); | ||||||||||||
4637 | |||||||||||||
4638 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
4639 | MachineMemOperand *MMO = MF.getMachineMemOperand( | ||||||||||||
4640 | MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), | ||||||||||||
4641 | DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering); | ||||||||||||
4642 | |||||||||||||
4643 | SDValue L = | ||||||||||||
4644 | DAG.getAtomic(NT, dl, MemVT, InChain, | ||||||||||||
4645 | getValue(I.getPointerOperand()), getValue(I.getValOperand()), | ||||||||||||
4646 | MMO); | ||||||||||||
4647 | |||||||||||||
4648 | SDValue OutChain = L.getValue(1); | ||||||||||||
4649 | |||||||||||||
4650 | setValue(&I, L); | ||||||||||||
4651 | DAG.setRoot(OutChain); | ||||||||||||
4652 | } | ||||||||||||
4653 | |||||||||||||
4654 | void SelectionDAGBuilder::visitFence(const FenceInst &I) { | ||||||||||||
4655 | SDLoc dl = getCurSDLoc(); | ||||||||||||
4656 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
4657 | SDValue Ops[3]; | ||||||||||||
4658 | Ops[0] = getRoot(); | ||||||||||||
4659 | Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl, | ||||||||||||
4660 | TLI.getFenceOperandTy(DAG.getDataLayout())); | ||||||||||||
4661 | Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl, | ||||||||||||
4662 | TLI.getFenceOperandTy(DAG.getDataLayout())); | ||||||||||||
4663 | DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops)); | ||||||||||||
4664 | } | ||||||||||||
4665 | |||||||||||||
4666 | void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { | ||||||||||||
4667 | SDLoc dl = getCurSDLoc(); | ||||||||||||
4668 | AtomicOrdering Order = I.getOrdering(); | ||||||||||||
4669 | SyncScope::ID SSID = I.getSyncScopeID(); | ||||||||||||
4670 | |||||||||||||
4671 | SDValue InChain = getRoot(); | ||||||||||||
4672 | |||||||||||||
4673 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
4674 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
4675 | EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
4676 | |||||||||||||
4677 | if (!TLI.supportsUnalignedAtomics() && | ||||||||||||
4678 | I.getAlignment() < MemVT.getSizeInBits() / 8) | ||||||||||||
4679 | report_fatal_error("Cannot generate unaligned atomic load"); | ||||||||||||
4680 | |||||||||||||
4681 | auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout()); | ||||||||||||
4682 | |||||||||||||
4683 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||||||
4684 | MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), | ||||||||||||
4685 | I.getAlign(), AAMDNodes(), nullptr, SSID, Order); | ||||||||||||
4686 | |||||||||||||
4687 | InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG); | ||||||||||||
4688 | |||||||||||||
4689 | SDValue Ptr = getValue(I.getPointerOperand()); | ||||||||||||
4690 | |||||||||||||
4691 | if (TLI.lowerAtomicLoadAsLoadSDNode(I)) { | ||||||||||||
4692 | // TODO: Once this is better exercised by tests, it should be merged with | ||||||||||||
4693 | // the normal path for loads to prevent future divergence. | ||||||||||||
4694 | SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO); | ||||||||||||
4695 | if (MemVT != VT) | ||||||||||||
4696 | L = DAG.getPtrExtOrTrunc(L, dl, VT); | ||||||||||||
4697 | |||||||||||||
4698 | setValue(&I, L); | ||||||||||||
4699 | SDValue OutChain = L.getValue(1); | ||||||||||||
4700 | if (!I.isUnordered()) | ||||||||||||
4701 | DAG.setRoot(OutChain); | ||||||||||||
4702 | else | ||||||||||||
4703 | PendingLoads.push_back(OutChain); | ||||||||||||
4704 | return; | ||||||||||||
4705 | } | ||||||||||||
4706 | |||||||||||||
4707 | SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain, | ||||||||||||
4708 | Ptr, MMO); | ||||||||||||
4709 | |||||||||||||
4710 | SDValue OutChain = L.getValue(1); | ||||||||||||
4711 | if (MemVT != VT) | ||||||||||||
4712 | L = DAG.getPtrExtOrTrunc(L, dl, VT); | ||||||||||||
4713 | |||||||||||||
4714 | setValue(&I, L); | ||||||||||||
4715 | DAG.setRoot(OutChain); | ||||||||||||
4716 | } | ||||||||||||
4717 | |||||||||||||
4718 | void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { | ||||||||||||
4719 | SDLoc dl = getCurSDLoc(); | ||||||||||||
4720 | |||||||||||||
4721 | AtomicOrdering Ordering = I.getOrdering(); | ||||||||||||
4722 | SyncScope::ID SSID = I.getSyncScopeID(); | ||||||||||||
4723 | |||||||||||||
4724 | SDValue InChain = getRoot(); | ||||||||||||
4725 | |||||||||||||
4726 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
4727 | EVT MemVT = | ||||||||||||
4728 | TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType()); | ||||||||||||
4729 | |||||||||||||
4730 | if (I.getAlignment() < MemVT.getSizeInBits() / 8) | ||||||||||||
4731 | report_fatal_error("Cannot generate unaligned atomic store"); | ||||||||||||
4732 | |||||||||||||
4733 | auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout()); | ||||||||||||
4734 | |||||||||||||
4735 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
4736 | MachineMemOperand *MMO = MF.getMachineMemOperand( | ||||||||||||
4737 | MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), | ||||||||||||
4738 | I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering); | ||||||||||||
4739 | |||||||||||||
4740 | SDValue Val = getValue(I.getValueOperand()); | ||||||||||||
4741 | if (Val.getValueType() != MemVT) | ||||||||||||
4742 | Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT); | ||||||||||||
4743 | SDValue Ptr = getValue(I.getPointerOperand()); | ||||||||||||
4744 | |||||||||||||
4745 | if (TLI.lowerAtomicStoreAsStoreSDNode(I)) { | ||||||||||||
4746 | // TODO: Once this is better exercised by tests, it should be merged with | ||||||||||||
4747 | // the normal path for stores to prevent future divergence. | ||||||||||||
4748 | SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO); | ||||||||||||
4749 | DAG.setRoot(S); | ||||||||||||
4750 | return; | ||||||||||||
4751 | } | ||||||||||||
4752 | SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, | ||||||||||||
4753 | Ptr, Val, MMO); | ||||||||||||
4754 | |||||||||||||
4755 | |||||||||||||
4756 | DAG.setRoot(OutChain); | ||||||||||||
4757 | } | ||||||||||||
4758 | |||||||||||||
4759 | /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC | ||||||||||||
4760 | /// node. | ||||||||||||
4761 | void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, | ||||||||||||
4762 | unsigned Intrinsic) { | ||||||||||||
4763 | // Ignore the callsite's attributes. A specific call site may be marked with | ||||||||||||
4764 | // readnone, but the lowering code will expect the chain based on the | ||||||||||||
4765 | // definition. | ||||||||||||
4766 | const Function *F = I.getCalledFunction(); | ||||||||||||
4767 | bool HasChain = !F->doesNotAccessMemory(); | ||||||||||||
4768 | bool OnlyLoad = HasChain && F->onlyReadsMemory(); | ||||||||||||
4769 | |||||||||||||
4770 | // Build the operand list. | ||||||||||||
4771 | SmallVector<SDValue, 8> Ops; | ||||||||||||
4772 | if (HasChain) { // If this intrinsic has side-effects, chainify it. | ||||||||||||
4773 | if (OnlyLoad) { | ||||||||||||
4774 | // We don't need to serialize loads against other loads. | ||||||||||||
4775 | Ops.push_back(DAG.getRoot()); | ||||||||||||
4776 | } else { | ||||||||||||
4777 | Ops.push_back(getRoot()); | ||||||||||||
4778 | } | ||||||||||||
4779 | } | ||||||||||||
4780 | |||||||||||||
4781 | // Info is set by getTgtMemInstrinsic | ||||||||||||
4782 | TargetLowering::IntrinsicInfo Info; | ||||||||||||
4783 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
4784 | bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, | ||||||||||||
4785 | DAG.getMachineFunction(), | ||||||||||||
4786 | Intrinsic); | ||||||||||||
4787 | |||||||||||||
4788 | // Add the intrinsic ID as an integer operand if it's not a target intrinsic. | ||||||||||||
4789 | if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID || | ||||||||||||
4790 | Info.opc == ISD::INTRINSIC_W_CHAIN) | ||||||||||||
4791 | Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(), | ||||||||||||
4792 | TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||||||
4793 | |||||||||||||
4794 | // Add all operands of the call to the operand list. | ||||||||||||
4795 | for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) { | ||||||||||||
4796 | const Value *Arg = I.getArgOperand(i); | ||||||||||||
4797 | if (!I.paramHasAttr(i, Attribute::ImmArg)) { | ||||||||||||
4798 | Ops.push_back(getValue(Arg)); | ||||||||||||
4799 | continue; | ||||||||||||
4800 | } | ||||||||||||
4801 | |||||||||||||
4802 | // Use TargetConstant instead of a regular constant for immarg. | ||||||||||||
4803 | EVT VT = TLI.getValueType(*DL, Arg->getType(), true); | ||||||||||||
4804 | if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) { | ||||||||||||
4805 | assert(CI->getBitWidth() <= 64 &&(static_cast <bool> (CI->getBitWidth() <= 64 && "large intrinsic immediates not handled") ? void (0) : __assert_fail ("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4806, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
4806 | "large intrinsic immediates not handled")(static_cast <bool> (CI->getBitWidth() <= 64 && "large intrinsic immediates not handled") ? void (0) : __assert_fail ("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 4806, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
4807 | Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT)); | ||||||||||||
4808 | } else { | ||||||||||||
4809 | Ops.push_back( | ||||||||||||
4810 | DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT)); | ||||||||||||
4811 | } | ||||||||||||
4812 | } | ||||||||||||
4813 | |||||||||||||
4814 | SmallVector<EVT, 4> ValueVTs; | ||||||||||||
4815 | ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs); | ||||||||||||
4816 | |||||||||||||
4817 | if (HasChain) | ||||||||||||
4818 | ValueVTs.push_back(MVT::Other); | ||||||||||||
4819 | |||||||||||||
4820 | SDVTList VTs = DAG.getVTList(ValueVTs); | ||||||||||||
4821 | |||||||||||||
4822 | // Propagate fast-math-flags from IR to node(s). | ||||||||||||
4823 | SDNodeFlags Flags; | ||||||||||||
4824 | if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) | ||||||||||||
4825 | Flags.copyFMF(*FPMO); | ||||||||||||
4826 | SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); | ||||||||||||
4827 | |||||||||||||
4828 | // Create the node. | ||||||||||||
4829 | SDValue Result; | ||||||||||||
4830 | if (IsTgtIntrinsic) { | ||||||||||||
4831 | // This is target intrinsic that touches memory | ||||||||||||
4832 | AAMDNodes AAInfo; | ||||||||||||
4833 | I.getAAMetadata(AAInfo); | ||||||||||||
4834 | Result = | ||||||||||||
4835 | DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT, | ||||||||||||
4836 | MachinePointerInfo(Info.ptrVal, Info.offset), | ||||||||||||
4837 | Info.align, Info.flags, Info.size, AAInfo); | ||||||||||||
4838 | } else if (!HasChain) { | ||||||||||||
4839 | Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops); | ||||||||||||
4840 | } else if (!I.getType()->isVoidTy()) { | ||||||||||||
4841 | Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops); | ||||||||||||
4842 | } else { | ||||||||||||
4843 | Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops); | ||||||||||||
4844 | } | ||||||||||||
4845 | |||||||||||||
4846 | if (HasChain) { | ||||||||||||
4847 | SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1); | ||||||||||||
4848 | if (OnlyLoad) | ||||||||||||
4849 | PendingLoads.push_back(Chain); | ||||||||||||
4850 | else | ||||||||||||
4851 | DAG.setRoot(Chain); | ||||||||||||
4852 | } | ||||||||||||
4853 | |||||||||||||
4854 | if (!I.getType()->isVoidTy()) { | ||||||||||||
4855 | if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) { | ||||||||||||
4856 | EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy); | ||||||||||||
4857 | Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result); | ||||||||||||
4858 | } else | ||||||||||||
4859 | Result = lowerRangeToAssertZExt(DAG, I, Result); | ||||||||||||
4860 | |||||||||||||
4861 | MaybeAlign Alignment = I.getRetAlign(); | ||||||||||||
4862 | if (!Alignment) | ||||||||||||
4863 | Alignment = F->getAttributes().getRetAlignment(); | ||||||||||||
4864 | // Insert `assertalign` node if there's an alignment. | ||||||||||||
4865 | if (InsertAssertAlign && Alignment) { | ||||||||||||
4866 | Result = | ||||||||||||
4867 | DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne()); | ||||||||||||
4868 | } | ||||||||||||
4869 | |||||||||||||
4870 | setValue(&I, Result); | ||||||||||||
4871 | } | ||||||||||||
4872 | } | ||||||||||||
4873 | |||||||||||||
4874 | /// GetSignificand - Get the significand and build it into a floating-point | ||||||||||||
4875 | /// number with exponent of 1: | ||||||||||||
4876 | /// | ||||||||||||
4877 | /// Op = (Op & 0x007fffff) | 0x3f800000; | ||||||||||||
4878 | /// | ||||||||||||
4879 | /// where Op is the hexadecimal representation of floating point value. | ||||||||||||
4880 | static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) { | ||||||||||||
4881 | SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, | ||||||||||||
4882 | DAG.getConstant(0x007fffff, dl, MVT::i32)); | ||||||||||||
4883 | SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1, | ||||||||||||
4884 | DAG.getConstant(0x3f800000, dl, MVT::i32)); | ||||||||||||
4885 | return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2); | ||||||||||||
4886 | } | ||||||||||||
4887 | |||||||||||||
4888 | /// GetExponent - Get the exponent: | ||||||||||||
4889 | /// | ||||||||||||
4890 | /// (float)(int)(((Op & 0x7f800000) >> 23) - 127); | ||||||||||||
4891 | /// | ||||||||||||
4892 | /// where Op is the hexadecimal representation of floating point value. | ||||||||||||
4893 | static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, | ||||||||||||
4894 | const TargetLowering &TLI, const SDLoc &dl) { | ||||||||||||
4895 | SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, | ||||||||||||
4896 | DAG.getConstant(0x7f800000, dl, MVT::i32)); | ||||||||||||
4897 | SDValue t1 = DAG.getNode( | ||||||||||||
4898 | ISD::SRL, dl, MVT::i32, t0, | ||||||||||||
4899 | DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||||||
4900 | SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1, | ||||||||||||
4901 | DAG.getConstant(127, dl, MVT::i32)); | ||||||||||||
4902 | return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2); | ||||||||||||
4903 | } | ||||||||||||
4904 | |||||||||||||
4905 | /// getF32Constant - Get 32-bit floating point constant. | ||||||||||||
4906 | static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, | ||||||||||||
4907 | const SDLoc &dl) { | ||||||||||||
4908 | return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl, | ||||||||||||
4909 | MVT::f32); | ||||||||||||
4910 | } | ||||||||||||
4911 | |||||||||||||
4912 | static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, | ||||||||||||
4913 | SelectionDAG &DAG) { | ||||||||||||
4914 | // TODO: What fast-math-flags should be set on the floating-point nodes? | ||||||||||||
4915 | |||||||||||||
4916 | // IntegerPartOfX = ((int32_t)(t0); | ||||||||||||
4917 | SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); | ||||||||||||
4918 | |||||||||||||
4919 | // FractionalPartOfX = t0 - (float)IntegerPartOfX; | ||||||||||||
4920 | SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); | ||||||||||||
4921 | SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); | ||||||||||||
4922 | |||||||||||||
4923 | // IntegerPartOfX <<= 23; | ||||||||||||
4924 | IntegerPartOfX = DAG.getNode( | ||||||||||||
4925 | ISD::SHL, dl, MVT::i32, IntegerPartOfX, | ||||||||||||
4926 | DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy( | ||||||||||||
4927 | DAG.getDataLayout()))); | ||||||||||||
4928 | |||||||||||||
4929 | SDValue TwoToFractionalPartOfX; | ||||||||||||
4930 | if (LimitFloatPrecision <= 6) { | ||||||||||||
4931 | // For floating-point precision of 6: | ||||||||||||
4932 | // | ||||||||||||
4933 | // TwoToFractionalPartOfX = | ||||||||||||
4934 | // 0.997535578f + | ||||||||||||
4935 | // (0.735607626f + 0.252464424f * x) * x; | ||||||||||||
4936 | // | ||||||||||||
4937 | // error 0.0144103317, which is 6 bits | ||||||||||||
4938 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||||||
4939 | getF32Constant(DAG, 0x3e814304, dl)); | ||||||||||||
4940 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, | ||||||||||||
4941 | getF32Constant(DAG, 0x3f3c50c8, dl)); | ||||||||||||
4942 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||||||
4943 | TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, | ||||||||||||
4944 | getF32Constant(DAG, 0x3f7f5e7e, dl)); | ||||||||||||
4945 | } else if (LimitFloatPrecision <= 12) { | ||||||||||||
4946 | // For floating-point precision of 12: | ||||||||||||
4947 | // | ||||||||||||
4948 | // TwoToFractionalPartOfX = | ||||||||||||
4949 | // 0.999892986f + | ||||||||||||
4950 | // (0.696457318f + | ||||||||||||
4951 | // (0.224338339f + 0.792043434e-1f * x) * x) * x; | ||||||||||||
4952 | // | ||||||||||||
4953 | // error 0.000107046256, which is 13 to 14 bits | ||||||||||||
4954 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||||||
4955 | getF32Constant(DAG, 0x3da235e3, dl)); | ||||||||||||
4956 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, | ||||||||||||
4957 | getF32Constant(DAG, 0x3e65b8f3, dl)); | ||||||||||||
4958 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||||||
4959 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, | ||||||||||||
4960 | getF32Constant(DAG, 0x3f324b07, dl)); | ||||||||||||
4961 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); | ||||||||||||
4962 | TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, | ||||||||||||
4963 | getF32Constant(DAG, 0x3f7ff8fd, dl)); | ||||||||||||
4964 | } else { // LimitFloatPrecision <= 18 | ||||||||||||
4965 | // For floating-point precision of 18: | ||||||||||||
4966 | // | ||||||||||||
4967 | // TwoToFractionalPartOfX = | ||||||||||||
4968 | // 0.999999982f + | ||||||||||||
4969 | // (0.693148872f + | ||||||||||||
4970 | // (0.240227044f + | ||||||||||||
4971 | // (0.554906021e-1f + | ||||||||||||
4972 | // (0.961591928e-2f + | ||||||||||||
4973 | // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; | ||||||||||||
4974 | // error 2.47208000*10^(-7), which is better than 18 bits | ||||||||||||
4975 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||||||
4976 | getF32Constant(DAG, 0x3924b03e, dl)); | ||||||||||||
4977 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, | ||||||||||||
4978 | getF32Constant(DAG, 0x3ab24b87, dl)); | ||||||||||||
4979 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||||||
4980 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, | ||||||||||||
4981 | getF32Constant(DAG, 0x3c1d8c17, dl)); | ||||||||||||
4982 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); | ||||||||||||
4983 | SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, | ||||||||||||
4984 | getF32Constant(DAG, 0x3d634a1d, dl)); | ||||||||||||
4985 | SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); | ||||||||||||
4986 | SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, | ||||||||||||
4987 | getF32Constant(DAG, 0x3e75fe14, dl)); | ||||||||||||
4988 | SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); | ||||||||||||
4989 | SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, | ||||||||||||
4990 | getF32Constant(DAG, 0x3f317234, dl)); | ||||||||||||
4991 | SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); | ||||||||||||
4992 | TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, | ||||||||||||
4993 | getF32Constant(DAG, 0x3f800000, dl)); | ||||||||||||
4994 | } | ||||||||||||
4995 | |||||||||||||
4996 | // Add the exponent into the result in integer domain. | ||||||||||||
4997 | SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX); | ||||||||||||
4998 | return DAG.getNode(ISD::BITCAST, dl, MVT::f32, | ||||||||||||
4999 | DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX)); | ||||||||||||
5000 | } | ||||||||||||
5001 | |||||||||||||
5002 | /// expandExp - Lower an exp intrinsic. Handles the special sequences for | ||||||||||||
5003 | /// limited-precision mode. | ||||||||||||
5004 | static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, | ||||||||||||
5005 | const TargetLowering &TLI, SDNodeFlags Flags) { | ||||||||||||
5006 | if (Op.getValueType() == MVT::f32 && | ||||||||||||
5007 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { | ||||||||||||
5008 | |||||||||||||
5009 | // Put the exponent in the right bit position for later addition to the | ||||||||||||
5010 | // final result: | ||||||||||||
5011 | // | ||||||||||||
5012 | // t0 = Op * log2(e) | ||||||||||||
5013 | |||||||||||||
5014 | // TODO: What fast-math-flags should be set here? | ||||||||||||
5015 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op, | ||||||||||||
5016 | DAG.getConstantFP(numbers::log2ef, dl, MVT::f32)); | ||||||||||||
5017 | return getLimitedPrecisionExp2(t0, dl, DAG); | ||||||||||||
5018 | } | ||||||||||||
5019 | |||||||||||||
5020 | // No special expansion. | ||||||||||||
5021 | return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags); | ||||||||||||
5022 | } | ||||||||||||
5023 | |||||||||||||
5024 | /// expandLog - Lower a log intrinsic. Handles the special sequences for | ||||||||||||
5025 | /// limited-precision mode. | ||||||||||||
5026 | static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, | ||||||||||||
5027 | const TargetLowering &TLI, SDNodeFlags Flags) { | ||||||||||||
5028 | // TODO: What fast-math-flags should be set on the floating-point nodes? | ||||||||||||
5029 | |||||||||||||
5030 | if (Op.getValueType() == MVT::f32 && | ||||||||||||
5031 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { | ||||||||||||
5032 | SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); | ||||||||||||
5033 | |||||||||||||
5034 | // Scale the exponent by log(2). | ||||||||||||
5035 | SDValue Exp = GetExponent(DAG, Op1, TLI, dl); | ||||||||||||
5036 | SDValue LogOfExponent = | ||||||||||||
5037 | DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, | ||||||||||||
5038 | DAG.getConstantFP(numbers::ln2f, dl, MVT::f32)); | ||||||||||||
5039 | |||||||||||||
5040 | // Get the significand and build it into a floating-point number with | ||||||||||||
5041 | // exponent of 1. | ||||||||||||
5042 | SDValue X = GetSignificand(DAG, Op1, dl); | ||||||||||||
5043 | |||||||||||||
5044 | SDValue LogOfMantissa; | ||||||||||||
5045 | if (LimitFloatPrecision <= 6) { | ||||||||||||
5046 | // For floating-point precision of 6: | ||||||||||||
5047 | // | ||||||||||||
5048 | // LogofMantissa = | ||||||||||||
5049 | // -1.1609546f + | ||||||||||||
5050 | // (1.4034025f - 0.23903021f * x) * x; | ||||||||||||
5051 | // | ||||||||||||
5052 | // error 0.0034276066, which is better than 8 bits | ||||||||||||
5053 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||||||
5054 | getF32Constant(DAG, 0xbe74c456, dl)); | ||||||||||||
5055 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, | ||||||||||||
5056 | getF32Constant(DAG, 0x3fb3a2b1, dl)); | ||||||||||||
5057 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||||||
5058 | LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, | ||||||||||||
5059 | getF32Constant(DAG, 0x3f949a29, dl)); | ||||||||||||
5060 | } else if (LimitFloatPrecision <= 12) { | ||||||||||||
5061 | // For floating-point precision of 12: | ||||||||||||
5062 | // | ||||||||||||
5063 | // LogOfMantissa = | ||||||||||||
5064 | // -1.7417939f + | ||||||||||||
5065 | // (2.8212026f + | ||||||||||||
5066 | // (-1.4699568f + | ||||||||||||
5067 | // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x; | ||||||||||||
5068 | // | ||||||||||||
5069 | // error 0.000061011436, which is 14 bits | ||||||||||||
5070 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||||||
5071 | getF32Constant(DAG, 0xbd67b6d6, dl)); | ||||||||||||
5072 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, | ||||||||||||
5073 | getF32Constant(DAG, 0x3ee4f4b8, dl)); | ||||||||||||
5074 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||||||
5075 | SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, | ||||||||||||
5076 | getF32Constant(DAG, 0x3fbc278b, dl)); | ||||||||||||
5077 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||||||
5078 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, | ||||||||||||
5079 | getF32Constant(DAG, 0x40348e95, dl)); | ||||||||||||
5080 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); | ||||||||||||
5081 | LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, | ||||||||||||
5082 | getF32Constant(DAG, 0x3fdef31a, dl)); | ||||||||||||
5083 | } else { // LimitFloatPrecision <= 18 | ||||||||||||
5084 | // For floating-point precision of 18: | ||||||||||||
5085 | // | ||||||||||||
5086 | // LogOfMantissa = | ||||||||||||
5087 | // -2.1072184f + | ||||||||||||
5088 | // (4.2372794f + | ||||||||||||
5089 | // (-3.7029485f + | ||||||||||||
5090 | // (2.2781945f + | ||||||||||||
5091 | // (-0.87823314f + | ||||||||||||
5092 | // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x; | ||||||||||||
5093 | // | ||||||||||||
5094 | // error 0.0000023660568, which is better than 18 bits | ||||||||||||
5095 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||||||
5096 | getF32Constant(DAG, 0xbc91e5ac, dl)); | ||||||||||||
5097 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, | ||||||||||||
5098 | getF32Constant(DAG, 0x3e4350aa, dl)); | ||||||||||||
5099 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||||||
5100 | SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, | ||||||||||||
5101 | getF32Constant(DAG, 0x3f60d3e3, dl)); | ||||||||||||
5102 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||||||
5103 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, | ||||||||||||
5104 | getF32Constant(DAG, 0x4011cdf0, dl)); | ||||||||||||
5105 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); | ||||||||||||
5106 | SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, | ||||||||||||
5107 | getF32Constant(DAG, 0x406cfd1c, dl)); | ||||||||||||
5108 | SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); | ||||||||||||
5109 | SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, | ||||||||||||
5110 | getF32Constant(DAG, 0x408797cb, dl)); | ||||||||||||
5111 | SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); | ||||||||||||
5112 | LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, | ||||||||||||
5113 | getF32Constant(DAG, 0x4006dcab, dl)); | ||||||||||||
5114 | } | ||||||||||||
5115 | |||||||||||||
5116 | return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); | ||||||||||||
5117 | } | ||||||||||||
5118 | |||||||||||||
5119 | // No special expansion. | ||||||||||||
5120 | return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags); | ||||||||||||
5121 | } | ||||||||||||
5122 | |||||||||||||
5123 | /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for | ||||||||||||
5124 | /// limited-precision mode. | ||||||||||||
5125 | static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, | ||||||||||||
5126 | const TargetLowering &TLI, SDNodeFlags Flags) { | ||||||||||||
5127 | // TODO: What fast-math-flags should be set on the floating-point nodes? | ||||||||||||
5128 | |||||||||||||
5129 | if (Op.getValueType() == MVT::f32 && | ||||||||||||
5130 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { | ||||||||||||
5131 | SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); | ||||||||||||
5132 | |||||||||||||
5133 | // Get the exponent. | ||||||||||||
5134 | SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl); | ||||||||||||
5135 | |||||||||||||
5136 | // Get the significand and build it into a floating-point number with | ||||||||||||
5137 | // exponent of 1. | ||||||||||||
5138 | SDValue X = GetSignificand(DAG, Op1, dl); | ||||||||||||
5139 | |||||||||||||
5140 | // Different possible minimax approximations of significand in | ||||||||||||
5141 | // floating-point for various degrees of accuracy over [1,2]. | ||||||||||||
5142 | SDValue Log2ofMantissa; | ||||||||||||
5143 | if (LimitFloatPrecision <= 6) { | ||||||||||||
5144 | // For floating-point precision of 6: | ||||||||||||
5145 | // | ||||||||||||
5146 | // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x; | ||||||||||||
5147 | // | ||||||||||||
5148 | // error 0.0049451742, which is more than 7 bits | ||||||||||||
5149 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||||||
5150 | getF32Constant(DAG, 0xbeb08fe0, dl)); | ||||||||||||
5151 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, | ||||||||||||
5152 | getF32Constant(DAG, 0x40019463, dl)); | ||||||||||||
5153 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||||||
5154 | Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, | ||||||||||||
5155 | getF32Constant(DAG, 0x3fd6633d, dl)); | ||||||||||||
5156 | } else if (LimitFloatPrecision <= 12) { | ||||||||||||
5157 | // For floating-point precision of 12: | ||||||||||||
5158 | // | ||||||||||||
5159 | // Log2ofMantissa = | ||||||||||||
5160 | // -2.51285454f + | ||||||||||||
5161 | // (4.07009056f + | ||||||||||||
5162 | // (-2.12067489f + | ||||||||||||
5163 | // (.645142248f - 0.816157886e-1f * x) * x) * x) * x; | ||||||||||||
5164 | // | ||||||||||||
5165 | // error 0.0000876136000, which is better than 13 bits | ||||||||||||
5166 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||||||
5167 | getF32Constant(DAG, 0xbda7262e, dl)); | ||||||||||||
5168 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, | ||||||||||||
5169 | getF32Constant(DAG, 0x3f25280b, dl)); | ||||||||||||
5170 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||||||
5171 | SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, | ||||||||||||
5172 | getF32Constant(DAG, 0x4007b923, dl)); | ||||||||||||
5173 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||||||
5174 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, | ||||||||||||
5175 | getF32Constant(DAG, 0x40823e2f, dl)); | ||||||||||||
5176 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); | ||||||||||||
5177 | Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, | ||||||||||||
5178 | getF32Constant(DAG, 0x4020d29c, dl)); | ||||||||||||
5179 | } else { // LimitFloatPrecision <= 18 | ||||||||||||
5180 | // For floating-point precision of 18: | ||||||||||||
5181 | // | ||||||||||||
5182 | // Log2ofMantissa = | ||||||||||||
5183 | // -3.0400495f + | ||||||||||||
5184 | // (6.1129976f + | ||||||||||||
5185 | // (-5.3420409f + | ||||||||||||
5186 | // (3.2865683f + | ||||||||||||
5187 | // (-1.2669343f + | ||||||||||||
5188 | // (0.27515199f - | ||||||||||||
5189 | // 0.25691327e-1f * x) * x) * x) * x) * x) * x; | ||||||||||||
5190 | // | ||||||||||||
5191 | // error 0.0000018516, which is better than 18 bits | ||||||||||||
5192 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||||||
5193 | getF32Constant(DAG, 0xbcd2769e, dl)); | ||||||||||||
5194 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, | ||||||||||||
5195 | getF32Constant(DAG, 0x3e8ce0b9, dl)); | ||||||||||||
5196 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||||||
5197 | SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, | ||||||||||||
5198 | getF32Constant(DAG, 0x3fa22ae7, dl)); | ||||||||||||
5199 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||||||
5200 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, | ||||||||||||
5201 | getF32Constant(DAG, 0x40525723, dl)); | ||||||||||||
5202 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); | ||||||||||||
5203 | SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, | ||||||||||||
5204 | getF32Constant(DAG, 0x40aaf200, dl)); | ||||||||||||
5205 | SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); | ||||||||||||
5206 | SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, | ||||||||||||
5207 | getF32Constant(DAG, 0x40c39dad, dl)); | ||||||||||||
5208 | SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); | ||||||||||||
5209 | Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, | ||||||||||||
5210 | getF32Constant(DAG, 0x4042902c, dl)); | ||||||||||||
5211 | } | ||||||||||||
5212 | |||||||||||||
5213 | return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); | ||||||||||||
5214 | } | ||||||||||||
5215 | |||||||||||||
5216 | // No special expansion. | ||||||||||||
5217 | return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags); | ||||||||||||
5218 | } | ||||||||||||
5219 | |||||||||||||
5220 | /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for | ||||||||||||
5221 | /// limited-precision mode. | ||||||||||||
5222 | static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, | ||||||||||||
5223 | const TargetLowering &TLI, SDNodeFlags Flags) { | ||||||||||||
5224 | // TODO: What fast-math-flags should be set on the floating-point nodes? | ||||||||||||
5225 | |||||||||||||
5226 | if (Op.getValueType() == MVT::f32 && | ||||||||||||
5227 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { | ||||||||||||
5228 | SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); | ||||||||||||
5229 | |||||||||||||
5230 | // Scale the exponent by log10(2) [0.30102999f]. | ||||||||||||
5231 | SDValue Exp = GetExponent(DAG, Op1, TLI, dl); | ||||||||||||
5232 | SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, | ||||||||||||
5233 | getF32Constant(DAG, 0x3e9a209a, dl)); | ||||||||||||
5234 | |||||||||||||
5235 | // Get the significand and build it into a floating-point number with | ||||||||||||
5236 | // exponent of 1. | ||||||||||||
5237 | SDValue X = GetSignificand(DAG, Op1, dl); | ||||||||||||
5238 | |||||||||||||
5239 | SDValue Log10ofMantissa; | ||||||||||||
5240 | if (LimitFloatPrecision <= 6) { | ||||||||||||
5241 | // For floating-point precision of 6: | ||||||||||||
5242 | // | ||||||||||||
5243 | // Log10ofMantissa = | ||||||||||||
5244 | // -0.50419619f + | ||||||||||||
5245 | // (0.60948995f - 0.10380950f * x) * x; | ||||||||||||
5246 | // | ||||||||||||
5247 | // error 0.0014886165, which is 6 bits | ||||||||||||
5248 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||||||
5249 | getF32Constant(DAG, 0xbdd49a13, dl)); | ||||||||||||
5250 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, | ||||||||||||
5251 | getF32Constant(DAG, 0x3f1c0789, dl)); | ||||||||||||
5252 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||||||
5253 | Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, | ||||||||||||
5254 | getF32Constant(DAG, 0x3f011300, dl)); | ||||||||||||
5255 | } else if (LimitFloatPrecision <= 12) { | ||||||||||||
5256 | // For floating-point precision of 12: | ||||||||||||
5257 | // | ||||||||||||
5258 | // Log10ofMantissa = | ||||||||||||
5259 | // -0.64831180f + | ||||||||||||
5260 | // (0.91751397f + | ||||||||||||
5261 | // (-0.31664806f + 0.47637168e-1f * x) * x) * x; | ||||||||||||
5262 | // | ||||||||||||
5263 | // error 0.00019228036, which is better than 12 bits | ||||||||||||
5264 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||||||
5265 | getF32Constant(DAG, 0x3d431f31, dl)); | ||||||||||||
5266 | SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, | ||||||||||||
5267 | getF32Constant(DAG, 0x3ea21fb2, dl)); | ||||||||||||
5268 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||||||
5269 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, | ||||||||||||
5270 | getF32Constant(DAG, 0x3f6ae232, dl)); | ||||||||||||
5271 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||||||
5272 | Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, | ||||||||||||
5273 | getF32Constant(DAG, 0x3f25f7c3, dl)); | ||||||||||||
5274 | } else { // LimitFloatPrecision <= 18 | ||||||||||||
5275 | // For floating-point precision of 18: | ||||||||||||
5276 | // | ||||||||||||
5277 | // Log10ofMantissa = | ||||||||||||
5278 | // -0.84299375f + | ||||||||||||
5279 | // (1.5327582f + | ||||||||||||
5280 | // (-1.0688956f + | ||||||||||||
5281 | // (0.49102474f + | ||||||||||||
5282 | // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x; | ||||||||||||
5283 | // | ||||||||||||
5284 | // error 0.0000037995730, which is better than 18 bits | ||||||||||||
5285 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||||||
5286 | getF32Constant(DAG, 0x3c5d51ce, dl)); | ||||||||||||
5287 | SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, | ||||||||||||
5288 | getF32Constant(DAG, 0x3e00685a, dl)); | ||||||||||||
5289 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||||||
5290 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, | ||||||||||||
5291 | getF32Constant(DAG, 0x3efb6798, dl)); | ||||||||||||
5292 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||||||
5293 | SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, | ||||||||||||
5294 | getF32Constant(DAG, 0x3f88d192, dl)); | ||||||||||||
5295 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); | ||||||||||||
5296 | SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, | ||||||||||||
5297 | getF32Constant(DAG, 0x3fc4316c, dl)); | ||||||||||||
5298 | SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); | ||||||||||||
5299 | Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8, | ||||||||||||
5300 | getF32Constant(DAG, 0x3f57ce70, dl)); | ||||||||||||
5301 | } | ||||||||||||
5302 | |||||||||||||
5303 | return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); | ||||||||||||
5304 | } | ||||||||||||
5305 | |||||||||||||
5306 | // No special expansion. | ||||||||||||
5307 | return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags); | ||||||||||||
5308 | } | ||||||||||||
5309 | |||||||||||||
5310 | /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for | ||||||||||||
5311 | /// limited-precision mode. | ||||||||||||
5312 | static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, | ||||||||||||
5313 | const TargetLowering &TLI, SDNodeFlags Flags) { | ||||||||||||
5314 | if (Op.getValueType() == MVT::f32 && | ||||||||||||
5315 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) | ||||||||||||
5316 | return getLimitedPrecisionExp2(Op, dl, DAG); | ||||||||||||
5317 | |||||||||||||
5318 | // No special expansion. | ||||||||||||
5319 | return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags); | ||||||||||||
5320 | } | ||||||||||||
5321 | |||||||||||||
5322 | /// visitPow - Lower a pow intrinsic. Handles the special sequences for | ||||||||||||
5323 | /// limited-precision mode with x == 10.0f. | ||||||||||||
5324 | static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, | ||||||||||||
5325 | SelectionDAG &DAG, const TargetLowering &TLI, | ||||||||||||
5326 | SDNodeFlags Flags) { | ||||||||||||
5327 | bool IsExp10 = false; | ||||||||||||
5328 | if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 && | ||||||||||||
5329 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { | ||||||||||||
5330 | if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) { | ||||||||||||
5331 | APFloat Ten(10.0f); | ||||||||||||
5332 | IsExp10 = LHSC->isExactlyValue(Ten); | ||||||||||||
5333 | } | ||||||||||||
5334 | } | ||||||||||||
5335 | |||||||||||||
5336 | // TODO: What fast-math-flags should be set on the FMUL node? | ||||||||||||
5337 | if (IsExp10) { | ||||||||||||
5338 | // Put the exponent in the right bit position for later addition to the | ||||||||||||
5339 | // final result: | ||||||||||||
5340 | // | ||||||||||||
5341 | // #define LOG2OF10 3.3219281f | ||||||||||||
5342 | // t0 = Op * LOG2OF10; | ||||||||||||
5343 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS, | ||||||||||||
5344 | getF32Constant(DAG, 0x40549a78, dl)); | ||||||||||||
5345 | return getLimitedPrecisionExp2(t0, dl, DAG); | ||||||||||||
5346 | } | ||||||||||||
5347 | |||||||||||||
5348 | // No special expansion. | ||||||||||||
5349 | return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags); | ||||||||||||
5350 | } | ||||||||||||
5351 | |||||||||||||
5352 | /// ExpandPowI - Expand a llvm.powi intrinsic. | ||||||||||||
5353 | static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, | ||||||||||||
5354 | SelectionDAG &DAG) { | ||||||||||||
5355 | // If RHS is a constant, we can expand this out to a multiplication tree, | ||||||||||||
5356 | // otherwise we end up lowering to a call to __powidf2 (for example). When | ||||||||||||
5357 | // optimizing for size, we only want to do this if the expansion would produce | ||||||||||||
5358 | // a small number of multiplies, otherwise we do the full expansion. | ||||||||||||
5359 | if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { | ||||||||||||
5360 | // Get the exponent as a positive value. | ||||||||||||
5361 | unsigned Val = RHSC->getSExtValue(); | ||||||||||||
5362 | if ((int)Val < 0) Val = -Val; | ||||||||||||
5363 | |||||||||||||
5364 | // powi(x, 0) -> 1.0 | ||||||||||||
5365 | if (Val == 0) | ||||||||||||
5366 | return DAG.getConstantFP(1.0, DL, LHS.getValueType()); | ||||||||||||
5367 | |||||||||||||
5368 | bool OptForSize = DAG.shouldOptForSize(); | ||||||||||||
5369 | if (!OptForSize || | ||||||||||||
5370 | // If optimizing for size, don't insert too many multiplies. | ||||||||||||
5371 | // This inserts up to 5 multiplies. | ||||||||||||
5372 | countPopulation(Val) + Log2_32(Val) < 7) { | ||||||||||||
5373 | // We use the simple binary decomposition method to generate the multiply | ||||||||||||
5374 | // sequence. There are more optimal ways to do this (for example, | ||||||||||||
5375 | // powi(x,15) generates one more multiply than it should), but this has | ||||||||||||
5376 | // the benefit of being both really simple and much better than a libcall. | ||||||||||||
5377 | SDValue Res; // Logically starts equal to 1.0 | ||||||||||||
5378 | SDValue CurSquare = LHS; | ||||||||||||
5379 | // TODO: Intrinsics should have fast-math-flags that propagate to these | ||||||||||||
5380 | // nodes. | ||||||||||||
5381 | while (Val) { | ||||||||||||
5382 | if (Val & 1) { | ||||||||||||
5383 | if (Res.getNode()) | ||||||||||||
5384 | Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare); | ||||||||||||
5385 | else | ||||||||||||
5386 | Res = CurSquare; // 1.0*CurSquare. | ||||||||||||
5387 | } | ||||||||||||
5388 | |||||||||||||
5389 | CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(), | ||||||||||||
5390 | CurSquare, CurSquare); | ||||||||||||
5391 | Val >>= 1; | ||||||||||||
5392 | } | ||||||||||||
5393 | |||||||||||||
5394 | // If the original was negative, invert the result, producing 1/(x*x*x). | ||||||||||||
5395 | if (RHSC->getSExtValue() < 0) | ||||||||||||
5396 | Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(), | ||||||||||||
5397 | DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res); | ||||||||||||
5398 | return Res; | ||||||||||||
5399 | } | ||||||||||||
5400 | } | ||||||||||||
5401 | |||||||||||||
5402 | // Otherwise, expand to a libcall. | ||||||||||||
5403 | return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS); | ||||||||||||
5404 | } | ||||||||||||
5405 | |||||||||||||
5406 | static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, | ||||||||||||
5407 | SDValue LHS, SDValue RHS, SDValue Scale, | ||||||||||||
5408 | SelectionDAG &DAG, const TargetLowering &TLI) { | ||||||||||||
5409 | EVT VT = LHS.getValueType(); | ||||||||||||
5410 | bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT; | ||||||||||||
5411 | bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT; | ||||||||||||
5412 | LLVMContext &Ctx = *DAG.getContext(); | ||||||||||||
5413 | |||||||||||||
5414 | // If the type is legal but the operation isn't, this node might survive all | ||||||||||||
5415 | // the way to operation legalization. If we end up there and we do not have | ||||||||||||
5416 | // the ability to widen the type (if VT*2 is not legal), we cannot expand the | ||||||||||||
5417 | // node. | ||||||||||||
5418 | |||||||||||||
5419 | // Coax the legalizer into expanding the node during type legalization instead | ||||||||||||
5420 | // by bumping the size by one bit. This will force it to Promote, enabling the | ||||||||||||
5421 | // early expansion and avoiding the need to expand later. | ||||||||||||
5422 | |||||||||||||
5423 | // We don't have to do this if Scale is 0; that can always be expanded, unless | ||||||||||||
5424 | // it's a saturating signed operation. Those can experience true integer | ||||||||||||
5425 | // division overflow, a case which we must avoid. | ||||||||||||
5426 | |||||||||||||
5427 | // FIXME: We wouldn't have to do this (or any of the early | ||||||||||||
5428 | // expansion/promotion) if it was possible to expand a libcall of an | ||||||||||||
5429 | // illegal type during operation legalization. But it's not, so things | ||||||||||||
5430 | // get a bit hacky. | ||||||||||||
5431 | unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue(); | ||||||||||||
5432 | if ((ScaleInt > 0 || (Saturating && Signed)) && | ||||||||||||
5433 | (TLI.isTypeLegal(VT) || | ||||||||||||
5434 | (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) { | ||||||||||||
5435 | TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction( | ||||||||||||
5436 | Opcode, VT, ScaleInt); | ||||||||||||
5437 | if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) { | ||||||||||||
5438 | EVT PromVT; | ||||||||||||
5439 | if (VT.isScalarInteger()) | ||||||||||||
5440 | PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1); | ||||||||||||
5441 | else if (VT.isVector()) { | ||||||||||||
5442 | PromVT = VT.getVectorElementType(); | ||||||||||||
5443 | PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1); | ||||||||||||
5444 | PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount()); | ||||||||||||
5445 | } else | ||||||||||||
5446 | llvm_unreachable("Wrong VT for DIVFIX?")::llvm::llvm_unreachable_internal("Wrong VT for DIVFIX?", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 5446); | ||||||||||||
5447 | if (Signed) { | ||||||||||||
5448 | LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT); | ||||||||||||
5449 | RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT); | ||||||||||||
5450 | } else { | ||||||||||||
5451 | LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT); | ||||||||||||
5452 | RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT); | ||||||||||||
5453 | } | ||||||||||||
5454 | EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout()); | ||||||||||||
5455 | // For saturating operations, we need to shift up the LHS to get the | ||||||||||||
5456 | // proper saturation width, and then shift down again afterwards. | ||||||||||||
5457 | if (Saturating) | ||||||||||||
5458 | LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS, | ||||||||||||
5459 | DAG.getConstant(1, DL, ShiftTy)); | ||||||||||||
5460 | SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale); | ||||||||||||
5461 | if (Saturating) | ||||||||||||
5462 | Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res, | ||||||||||||
5463 | DAG.getConstant(1, DL, ShiftTy)); | ||||||||||||
5464 | return DAG.getZExtOrTrunc(Res, DL, VT); | ||||||||||||
5465 | } | ||||||||||||
5466 | } | ||||||||||||
5467 | |||||||||||||
5468 | return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale); | ||||||||||||
5469 | } | ||||||||||||
5470 | |||||||||||||
5471 | // getUnderlyingArgRegs - Find underlying registers used for a truncated, | ||||||||||||
5472 | // bitcasted, or split argument. Returns a list of <Register, size in bits> | ||||||||||||
5473 | static void | ||||||||||||
5474 | getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs, | ||||||||||||
5475 | const SDValue &N) { | ||||||||||||
5476 | switch (N.getOpcode()) { | ||||||||||||
5477 | case ISD::CopyFromReg: { | ||||||||||||
5478 | SDValue Op = N.getOperand(1); | ||||||||||||
5479 | Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(), | ||||||||||||
5480 | Op.getValueType().getSizeInBits()); | ||||||||||||
5481 | return; | ||||||||||||
5482 | } | ||||||||||||
5483 | case ISD::BITCAST: | ||||||||||||
5484 | case ISD::AssertZext: | ||||||||||||
5485 | case ISD::AssertSext: | ||||||||||||
5486 | case ISD::TRUNCATE: | ||||||||||||
5487 | getUnderlyingArgRegs(Regs, N.getOperand(0)); | ||||||||||||
5488 | return; | ||||||||||||
5489 | case ISD::BUILD_PAIR: | ||||||||||||
5490 | case ISD::BUILD_VECTOR: | ||||||||||||
5491 | case ISD::CONCAT_VECTORS: | ||||||||||||
5492 | for (SDValue Op : N->op_values()) | ||||||||||||
5493 | getUnderlyingArgRegs(Regs, Op); | ||||||||||||
5494 | return; | ||||||||||||
5495 | default: | ||||||||||||
5496 | return; | ||||||||||||
5497 | } | ||||||||||||
5498 | } | ||||||||||||
5499 | |||||||||||||
5500 | /// If the DbgValueInst is a dbg_value of a function argument, create the | ||||||||||||
5501 | /// corresponding DBG_VALUE machine instruction for it now. At the end of | ||||||||||||
5502 | /// instruction selection, they will be inserted to the entry BB. | ||||||||||||
5503 | /// We don't currently support this for variadic dbg_values, as they shouldn't | ||||||||||||
5504 | /// appear for function arguments or in the prologue. | ||||||||||||
5505 | bool SelectionDAGBuilder::EmitFuncArgumentDbgValue( | ||||||||||||
5506 | const Value *V, DILocalVariable *Variable, DIExpression *Expr, | ||||||||||||
5507 | DILocation *DL, bool IsDbgDeclare, const SDValue &N) { | ||||||||||||
5508 | const Argument *Arg = dyn_cast<Argument>(V); | ||||||||||||
5509 | if (!Arg) | ||||||||||||
5510 | return false; | ||||||||||||
5511 | |||||||||||||
5512 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
5513 | const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); | ||||||||||||
5514 | |||||||||||||
5515 | // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind | ||||||||||||
5516 | // we've been asked to pursue. | ||||||||||||
5517 | auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr, | ||||||||||||
5518 | bool Indirect) { | ||||||||||||
5519 | if (Reg.isVirtual() && MF.useDebugInstrRef()) { | ||||||||||||
5520 | // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF | ||||||||||||
5521 | // pointing at the VReg, which will be patched up later. | ||||||||||||
5522 | auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF); | ||||||||||||
5523 | auto MIB = BuildMI(MF, DL, Inst); | ||||||||||||
5524 | MIB.addReg(Reg, RegState::Debug); | ||||||||||||
5525 | MIB.addImm(0); | ||||||||||||
5526 | MIB.addMetadata(Variable); | ||||||||||||
5527 | auto *NewDIExpr = FragExpr; | ||||||||||||
5528 | // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into | ||||||||||||
5529 | // the DIExpression. | ||||||||||||
5530 | if (Indirect) | ||||||||||||
5531 | NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore); | ||||||||||||
5532 | MIB.addMetadata(NewDIExpr); | ||||||||||||
5533 | return MIB; | ||||||||||||
5534 | } else { | ||||||||||||
5535 | // Create a completely standard DBG_VALUE. | ||||||||||||
5536 | auto &Inst = TII->get(TargetOpcode::DBG_VALUE); | ||||||||||||
5537 | return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr); | ||||||||||||
5538 | } | ||||||||||||
5539 | }; | ||||||||||||
5540 | |||||||||||||
5541 | if (!IsDbgDeclare) { | ||||||||||||
5542 | // ArgDbgValues are hoisted to the beginning of the entry block. So we | ||||||||||||
5543 | // should only emit as ArgDbgValue if the dbg.value intrinsic is found in | ||||||||||||
5544 | // the entry block. | ||||||||||||
5545 | bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front(); | ||||||||||||
5546 | if (!IsInEntryBlock) | ||||||||||||
5547 | return false; | ||||||||||||
5548 | |||||||||||||
5549 | // ArgDbgValues are hoisted to the beginning of the entry block. So we | ||||||||||||
5550 | // should only emit as ArgDbgValue if the dbg.value intrinsic describes a | ||||||||||||
5551 | // variable that also is a param. | ||||||||||||
5552 | // | ||||||||||||
5553 | // Although, if we are at the top of the entry block already, we can still | ||||||||||||
5554 | // emit using ArgDbgValue. This might catch some situations when the | ||||||||||||
5555 | // dbg.value refers to an argument that isn't used in the entry block, so | ||||||||||||
5556 | // any CopyToReg node would be optimized out and the only way to express | ||||||||||||
5557 | // this DBG_VALUE is by using the physical reg (or FI) as done in this | ||||||||||||
5558 | // method. ArgDbgValues are hoisted to the beginning of the entry block. So | ||||||||||||
5559 | // we should only emit as ArgDbgValue if the Variable is an argument to the | ||||||||||||
5560 | // current function, and the dbg.value intrinsic is found in the entry | ||||||||||||
5561 | // block. | ||||||||||||
5562 | bool VariableIsFunctionInputArg = Variable->isParameter() && | ||||||||||||
5563 | !DL->getInlinedAt(); | ||||||||||||
5564 | bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder; | ||||||||||||
5565 | if (!IsInPrologue && !VariableIsFunctionInputArg) | ||||||||||||
5566 | return false; | ||||||||||||
5567 | |||||||||||||
5568 | // Here we assume that a function argument on IR level only can be used to | ||||||||||||
5569 | // describe one input parameter on source level. If we for example have | ||||||||||||
5570 | // source code like this | ||||||||||||
5571 | // | ||||||||||||
5572 | // struct A { long x, y; }; | ||||||||||||
5573 | // void foo(struct A a, long b) { | ||||||||||||
5574 | // ... | ||||||||||||
5575 | // b = a.x; | ||||||||||||
5576 | // ... | ||||||||||||
5577 | // } | ||||||||||||
5578 | // | ||||||||||||
5579 | // and IR like this | ||||||||||||
5580 | // | ||||||||||||
5581 | // define void @foo(i32 %a1, i32 %a2, i32 %b) { | ||||||||||||
5582 | // entry: | ||||||||||||
5583 | // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment | ||||||||||||
5584 | // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment | ||||||||||||
5585 | // call void @llvm.dbg.value(metadata i32 %b, "b", | ||||||||||||
5586 | // ... | ||||||||||||
5587 | // call void @llvm.dbg.value(metadata i32 %a1, "b" | ||||||||||||
5588 | // ... | ||||||||||||
5589 | // | ||||||||||||
5590 | // then the last dbg.value is describing a parameter "b" using a value that | ||||||||||||
5591 | // is an argument. But since we already has used %a1 to describe a parameter | ||||||||||||
5592 | // we should not handle that last dbg.value here (that would result in an | ||||||||||||
5593 | // incorrect hoisting of the DBG_VALUE to the function entry). | ||||||||||||
5594 | // Notice that we allow one dbg.value per IR level argument, to accommodate | ||||||||||||
5595 | // for the situation with fragments above. | ||||||||||||
5596 | if (VariableIsFunctionInputArg) { | ||||||||||||
5597 | unsigned ArgNo = Arg->getArgNo(); | ||||||||||||
5598 | if (ArgNo >= FuncInfo.DescribedArgs.size()) | ||||||||||||
5599 | FuncInfo.DescribedArgs.resize(ArgNo + 1, false); | ||||||||||||
5600 | else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo)) | ||||||||||||
5601 | return false; | ||||||||||||
5602 | FuncInfo.DescribedArgs.set(ArgNo); | ||||||||||||
5603 | } | ||||||||||||
5604 | } | ||||||||||||
5605 | |||||||||||||
5606 | bool IsIndirect = false; | ||||||||||||
5607 | Optional<MachineOperand> Op; | ||||||||||||
5608 | // Some arguments' frame index is recorded during argument lowering. | ||||||||||||
5609 | int FI = FuncInfo.getArgumentFrameIndex(Arg); | ||||||||||||
5610 | if (FI != std::numeric_limits<int>::max()) | ||||||||||||
5611 | Op = MachineOperand::CreateFI(FI); | ||||||||||||
5612 | |||||||||||||
5613 | SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes; | ||||||||||||
5614 | if (!Op && N.getNode()) { | ||||||||||||
5615 | getUnderlyingArgRegs(ArgRegsAndSizes, N); | ||||||||||||
5616 | Register Reg; | ||||||||||||
5617 | if (ArgRegsAndSizes.size() == 1) | ||||||||||||
5618 | Reg = ArgRegsAndSizes.front().first; | ||||||||||||
5619 | |||||||||||||
5620 | if (Reg && Reg.isVirtual()) { | ||||||||||||
5621 | MachineRegisterInfo &RegInfo = MF.getRegInfo(); | ||||||||||||
5622 | Register PR = RegInfo.getLiveInPhysReg(Reg); | ||||||||||||
5623 | if (PR) | ||||||||||||
5624 | Reg = PR; | ||||||||||||
5625 | } | ||||||||||||
5626 | if (Reg) { | ||||||||||||
5627 | Op = MachineOperand::CreateReg(Reg, false); | ||||||||||||
5628 | IsIndirect = IsDbgDeclare; | ||||||||||||
5629 | } | ||||||||||||
5630 | } | ||||||||||||
5631 | |||||||||||||
5632 | if (!Op && N.getNode()) { | ||||||||||||
5633 | // Check if frame index is available. | ||||||||||||
5634 | SDValue LCandidate = peekThroughBitcasts(N); | ||||||||||||
5635 | if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode())) | ||||||||||||
5636 | if (FrameIndexSDNode *FINode = | ||||||||||||
5637 | dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) | ||||||||||||
5638 | Op = MachineOperand::CreateFI(FINode->getIndex()); | ||||||||||||
5639 | } | ||||||||||||
5640 | |||||||||||||
5641 | if (!Op) { | ||||||||||||
5642 | // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg | ||||||||||||
5643 | auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>> | ||||||||||||
5644 | SplitRegs) { | ||||||||||||
5645 | unsigned Offset = 0; | ||||||||||||
5646 | for (auto RegAndSize : SplitRegs) { | ||||||||||||
5647 | // If the expression is already a fragment, the current register | ||||||||||||
5648 | // offset+size might extend beyond the fragment. In this case, only | ||||||||||||
5649 | // the register bits that are inside the fragment are relevant. | ||||||||||||
5650 | int RegFragmentSizeInBits = RegAndSize.second; | ||||||||||||
5651 | if (auto ExprFragmentInfo = Expr->getFragmentInfo()) { | ||||||||||||
5652 | uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits; | ||||||||||||
5653 | // The register is entirely outside the expression fragment, | ||||||||||||
5654 | // so is irrelevant for debug info. | ||||||||||||
5655 | if (Offset >= ExprFragmentSizeInBits) | ||||||||||||
5656 | break; | ||||||||||||
5657 | // The register is partially outside the expression fragment, only | ||||||||||||
5658 | // the low bits within the fragment are relevant for debug info. | ||||||||||||
5659 | if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) { | ||||||||||||
5660 | RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset; | ||||||||||||
5661 | } | ||||||||||||
5662 | } | ||||||||||||
5663 | |||||||||||||
5664 | auto FragmentExpr = DIExpression::createFragmentExpression( | ||||||||||||
5665 | Expr, Offset, RegFragmentSizeInBits); | ||||||||||||
5666 | Offset += RegAndSize.second; | ||||||||||||
5667 | // If a valid fragment expression cannot be created, the variable's | ||||||||||||
5668 | // correct value cannot be determined and so it is set as Undef. | ||||||||||||
5669 | if (!FragmentExpr) { | ||||||||||||
5670 | SDDbgValue *SDV = DAG.getConstantDbgValue( | ||||||||||||
5671 | Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder); | ||||||||||||
5672 | DAG.AddDbgValue(SDV, false); | ||||||||||||
5673 | continue; | ||||||||||||
5674 | } | ||||||||||||
5675 | MachineInstr *NewMI = | ||||||||||||
5676 | MakeVRegDbgValue(RegAndSize.first, *FragmentExpr, IsDbgDeclare); | ||||||||||||
5677 | FuncInfo.ArgDbgValues.push_back(NewMI); | ||||||||||||
5678 | } | ||||||||||||
5679 | }; | ||||||||||||
5680 | |||||||||||||
5681 | // Check if ValueMap has reg number. | ||||||||||||
5682 | DenseMap<const Value *, Register>::const_iterator | ||||||||||||
5683 | VMI = FuncInfo.ValueMap.find(V); | ||||||||||||
5684 | if (VMI != FuncInfo.ValueMap.end()) { | ||||||||||||
5685 | const auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
5686 | RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second, | ||||||||||||
5687 | V->getType(), None); | ||||||||||||
5688 | if (RFV.occupiesMultipleRegs()) { | ||||||||||||
5689 | splitMultiRegDbgValue(RFV.getRegsAndSizes()); | ||||||||||||
5690 | return true; | ||||||||||||
5691 | } | ||||||||||||
5692 | |||||||||||||
5693 | Op = MachineOperand::CreateReg(VMI->second, false); | ||||||||||||
5694 | IsIndirect = IsDbgDeclare; | ||||||||||||
5695 | } else if (ArgRegsAndSizes.size() > 1) { | ||||||||||||
5696 | // This was split due to the calling convention, and no virtual register | ||||||||||||
5697 | // mapping exists for the value. | ||||||||||||
5698 | splitMultiRegDbgValue(ArgRegsAndSizes); | ||||||||||||
5699 | return true; | ||||||||||||
5700 | } | ||||||||||||
5701 | } | ||||||||||||
5702 | |||||||||||||
5703 | if (!Op) | ||||||||||||
5704 | return false; | ||||||||||||
5705 | |||||||||||||
5706 | assert(Variable->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic (DL) && "Expected inlined-at fields to agree") ? void (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 5707, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
5707 | "Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic (DL) && "Expected inlined-at fields to agree") ? void (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 5707, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
5708 | MachineInstr *NewMI = nullptr; | ||||||||||||
5709 | |||||||||||||
5710 | if (Op->isReg()) | ||||||||||||
5711 | NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect); | ||||||||||||
5712 | else | ||||||||||||
5713 | NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op, | ||||||||||||
5714 | Variable, Expr); | ||||||||||||
5715 | |||||||||||||
5716 | FuncInfo.ArgDbgValues.push_back(NewMI); | ||||||||||||
5717 | return true; | ||||||||||||
5718 | } | ||||||||||||
5719 | |||||||||||||
5720 | /// Return the appropriate SDDbgValue based on N. | ||||||||||||
5721 | SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N, | ||||||||||||
5722 | DILocalVariable *Variable, | ||||||||||||
5723 | DIExpression *Expr, | ||||||||||||
5724 | const DebugLoc &dl, | ||||||||||||
5725 | unsigned DbgSDNodeOrder) { | ||||||||||||
5726 | if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) { | ||||||||||||
5727 | // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe | ||||||||||||
5728 | // stack slot locations. | ||||||||||||
5729 | // | ||||||||||||
5730 | // Consider "int x = 0; int *px = &x;". There are two kinds of interesting | ||||||||||||
5731 | // debug values here after optimization: | ||||||||||||
5732 | // | ||||||||||||
5733 | // dbg.value(i32* %px, !"int *px", !DIExpression()), and | ||||||||||||
5734 | // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref)) | ||||||||||||
5735 | // | ||||||||||||
5736 | // Both describe the direct values of their associated variables. | ||||||||||||
5737 | return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(), | ||||||||||||
5738 | /*IsIndirect*/ false, dl, DbgSDNodeOrder); | ||||||||||||
5739 | } | ||||||||||||
5740 | return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(), | ||||||||||||
5741 | /*IsIndirect*/ false, dl, DbgSDNodeOrder); | ||||||||||||
5742 | } | ||||||||||||
5743 | |||||||||||||
5744 | static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) { | ||||||||||||
5745 | switch (Intrinsic) { | ||||||||||||
5746 | case Intrinsic::smul_fix: | ||||||||||||
5747 | return ISD::SMULFIX; | ||||||||||||
5748 | case Intrinsic::umul_fix: | ||||||||||||
5749 | return ISD::UMULFIX; | ||||||||||||
5750 | case Intrinsic::smul_fix_sat: | ||||||||||||
5751 | return ISD::SMULFIXSAT; | ||||||||||||
5752 | case Intrinsic::umul_fix_sat: | ||||||||||||
5753 | return ISD::UMULFIXSAT; | ||||||||||||
5754 | case Intrinsic::sdiv_fix: | ||||||||||||
5755 | return ISD::SDIVFIX; | ||||||||||||
5756 | case Intrinsic::udiv_fix: | ||||||||||||
5757 | return ISD::UDIVFIX; | ||||||||||||
5758 | case Intrinsic::sdiv_fix_sat: | ||||||||||||
5759 | return ISD::SDIVFIXSAT; | ||||||||||||
5760 | case Intrinsic::udiv_fix_sat: | ||||||||||||
5761 | return ISD::UDIVFIXSAT; | ||||||||||||
5762 | default: | ||||||||||||
5763 | llvm_unreachable("Unhandled fixed point intrinsic")::llvm::llvm_unreachable_internal("Unhandled fixed point intrinsic" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 5763); | ||||||||||||
5764 | } | ||||||||||||
5765 | } | ||||||||||||
5766 | |||||||||||||
5767 | void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I, | ||||||||||||
5768 | const char *FunctionName) { | ||||||||||||
5769 | assert(FunctionName && "FunctionName must not be nullptr")(static_cast <bool> (FunctionName && "FunctionName must not be nullptr" ) ? void (0) : __assert_fail ("FunctionName && \"FunctionName must not be nullptr\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 5769, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
5770 | SDValue Callee = DAG.getExternalSymbol( | ||||||||||||
5771 | FunctionName, | ||||||||||||
5772 | DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())); | ||||||||||||
5773 | LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall()); | ||||||||||||
5774 | } | ||||||||||||
5775 | |||||||||||||
5776 | /// Given a @llvm.call.preallocated.setup, return the corresponding | ||||||||||||
5777 | /// preallocated call. | ||||||||||||
5778 | static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) { | ||||||||||||
5779 | assert(cast<CallBase>(PreallocatedSetup)(static_cast <bool> (cast<CallBase>(PreallocatedSetup ) ->getCalledFunction() ->getIntrinsicID() == Intrinsic ::call_preallocated_setup && "expected call_preallocated_setup Value" ) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 5782, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
5780 | ->getCalledFunction()(static_cast <bool> (cast<CallBase>(PreallocatedSetup ) ->getCalledFunction() ->getIntrinsicID() == Intrinsic ::call_preallocated_setup && "expected call_preallocated_setup Value" ) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 5782, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
5781 | ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&(static_cast <bool> (cast<CallBase>(PreallocatedSetup ) ->getCalledFunction() ->getIntrinsicID() == Intrinsic ::call_preallocated_setup && "expected call_preallocated_setup Value" ) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 5782, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
5782 | "expected call_preallocated_setup Value")(static_cast <bool> (cast<CallBase>(PreallocatedSetup ) ->getCalledFunction() ->getIntrinsicID() == Intrinsic ::call_preallocated_setup && "expected call_preallocated_setup Value" ) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 5782, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
5783 | for (auto *U : PreallocatedSetup->users()) { | ||||||||||||
5784 | auto *UseCall = cast<CallBase>(U); | ||||||||||||
5785 | const Function *Fn = UseCall->getCalledFunction(); | ||||||||||||
5786 | if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) { | ||||||||||||
5787 | return UseCall; | ||||||||||||
5788 | } | ||||||||||||
5789 | } | ||||||||||||
5790 | llvm_unreachable("expected corresponding call to preallocated setup/arg")::llvm::llvm_unreachable_internal("expected corresponding call to preallocated setup/arg" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 5790); | ||||||||||||
5791 | } | ||||||||||||
5792 | |||||||||||||
5793 | /// Lower the call to the specified intrinsic function. | ||||||||||||
5794 | void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, | ||||||||||||
5795 | unsigned Intrinsic) { | ||||||||||||
5796 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
5797 | SDLoc sdl = getCurSDLoc(); | ||||||||||||
5798 | DebugLoc dl = getCurDebugLoc(); | ||||||||||||
5799 | SDValue Res; | ||||||||||||
5800 | |||||||||||||
5801 | SDNodeFlags Flags; | ||||||||||||
5802 | if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) | ||||||||||||
5803 | Flags.copyFMF(*FPOp); | ||||||||||||
5804 | |||||||||||||
5805 | switch (Intrinsic) { | ||||||||||||
5806 | default: | ||||||||||||
5807 | // By default, turn this into a target intrinsic node. | ||||||||||||
5808 | visitTargetIntrinsic(I, Intrinsic); | ||||||||||||
5809 | return; | ||||||||||||
5810 | case Intrinsic::vscale: { | ||||||||||||
5811 | match(&I, m_VScale(DAG.getDataLayout())); | ||||||||||||
5812 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
5813 | setValue(&I, | ||||||||||||
5814 | DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1))); | ||||||||||||
5815 | return; | ||||||||||||
5816 | } | ||||||||||||
5817 | case Intrinsic::vastart: visitVAStart(I); return; | ||||||||||||
5818 | case Intrinsic::vaend: visitVAEnd(I); return; | ||||||||||||
5819 | case Intrinsic::vacopy: visitVACopy(I); return; | ||||||||||||
5820 | case Intrinsic::returnaddress: | ||||||||||||
5821 | setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, | ||||||||||||
5822 | TLI.getPointerTy(DAG.getDataLayout()), | ||||||||||||
5823 | getValue(I.getArgOperand(0)))); | ||||||||||||
5824 | return; | ||||||||||||
5825 | case Intrinsic::addressofreturnaddress: | ||||||||||||
5826 | setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl, | ||||||||||||
5827 | TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||||||
5828 | return; | ||||||||||||
5829 | case Intrinsic::sponentry: | ||||||||||||
5830 | setValue(&I, DAG.getNode(ISD::SPONENTRY, sdl, | ||||||||||||
5831 | TLI.getFrameIndexTy(DAG.getDataLayout()))); | ||||||||||||
5832 | return; | ||||||||||||
5833 | case Intrinsic::frameaddress: | ||||||||||||
5834 | setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, | ||||||||||||
5835 | TLI.getFrameIndexTy(DAG.getDataLayout()), | ||||||||||||
5836 | getValue(I.getArgOperand(0)))); | ||||||||||||
5837 | return; | ||||||||||||
5838 | case Intrinsic::read_volatile_register: | ||||||||||||
5839 | case Intrinsic::read_register: { | ||||||||||||
5840 | Value *Reg = I.getArgOperand(0); | ||||||||||||
5841 | SDValue Chain = getRoot(); | ||||||||||||
5842 | SDValue RegName = | ||||||||||||
5843 | DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata())); | ||||||||||||
5844 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
5845 | Res = DAG.getNode(ISD::READ_REGISTER, sdl, | ||||||||||||
5846 | DAG.getVTList(VT, MVT::Other), Chain, RegName); | ||||||||||||
5847 | setValue(&I, Res); | ||||||||||||
5848 | DAG.setRoot(Res.getValue(1)); | ||||||||||||
5849 | return; | ||||||||||||
5850 | } | ||||||||||||
5851 | case Intrinsic::write_register: { | ||||||||||||
5852 | Value *Reg = I.getArgOperand(0); | ||||||||||||
5853 | Value *RegValue = I.getArgOperand(1); | ||||||||||||
5854 | SDValue Chain = getRoot(); | ||||||||||||
5855 | SDValue RegName = | ||||||||||||
5856 | DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata())); | ||||||||||||
5857 | DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain, | ||||||||||||
5858 | RegName, getValue(RegValue))); | ||||||||||||
5859 | return; | ||||||||||||
5860 | } | ||||||||||||
5861 | case Intrinsic::memcpy: { | ||||||||||||
5862 | const auto &MCI = cast<MemCpyInst>(I); | ||||||||||||
5863 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
5864 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
5865 | SDValue Op3 = getValue(I.getArgOperand(2)); | ||||||||||||
5866 | // @llvm.memcpy defines 0 and 1 to both mean no alignment. | ||||||||||||
5867 | Align DstAlign = MCI.getDestAlign().valueOrOne(); | ||||||||||||
5868 | Align SrcAlign = MCI.getSourceAlign().valueOrOne(); | ||||||||||||
5869 | Align Alignment = commonAlignment(DstAlign, SrcAlign); | ||||||||||||
5870 | bool isVol = MCI.isVolatile(); | ||||||||||||
5871 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||||||
5872 | // FIXME: Support passing different dest/src alignments to the memcpy DAG | ||||||||||||
5873 | // node. | ||||||||||||
5874 | SDValue Root = isVol ? getRoot() : getMemoryRoot(); | ||||||||||||
5875 | AAMDNodes AAInfo; | ||||||||||||
5876 | I.getAAMetadata(AAInfo); | ||||||||||||
5877 | SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol, | ||||||||||||
5878 | /* AlwaysInline */ false, isTC, | ||||||||||||
5879 | MachinePointerInfo(I.getArgOperand(0)), | ||||||||||||
5880 | MachinePointerInfo(I.getArgOperand(1)), AAInfo); | ||||||||||||
5881 | updateDAGForMaybeTailCall(MC); | ||||||||||||
5882 | return; | ||||||||||||
5883 | } | ||||||||||||
5884 | case Intrinsic::memcpy_inline: { | ||||||||||||
5885 | const auto &MCI = cast<MemCpyInlineInst>(I); | ||||||||||||
5886 | SDValue Dst = getValue(I.getArgOperand(0)); | ||||||||||||
5887 | SDValue Src = getValue(I.getArgOperand(1)); | ||||||||||||
5888 | SDValue Size = getValue(I.getArgOperand(2)); | ||||||||||||
5889 | assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size")(static_cast <bool> (isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size") ? void (0) : __assert_fail ("isa<ConstantSDNode>(Size) && \"memcpy_inline needs constant size\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 5889, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
5890 | // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment. | ||||||||||||
5891 | Align DstAlign = MCI.getDestAlign().valueOrOne(); | ||||||||||||
5892 | Align SrcAlign = MCI.getSourceAlign().valueOrOne(); | ||||||||||||
5893 | Align Alignment = commonAlignment(DstAlign, SrcAlign); | ||||||||||||
5894 | bool isVol = MCI.isVolatile(); | ||||||||||||
5895 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||||||
5896 | // FIXME: Support passing different dest/src alignments to the memcpy DAG | ||||||||||||
5897 | // node. | ||||||||||||
5898 | AAMDNodes AAInfo; | ||||||||||||
5899 | I.getAAMetadata(AAInfo); | ||||||||||||
5900 | SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Alignment, isVol, | ||||||||||||
5901 | /* AlwaysInline */ true, isTC, | ||||||||||||
5902 | MachinePointerInfo(I.getArgOperand(0)), | ||||||||||||
5903 | MachinePointerInfo(I.getArgOperand(1)), AAInfo); | ||||||||||||
5904 | updateDAGForMaybeTailCall(MC); | ||||||||||||
5905 | return; | ||||||||||||
5906 | } | ||||||||||||
5907 | case Intrinsic::memset: { | ||||||||||||
5908 | const auto &MSI = cast<MemSetInst>(I); | ||||||||||||
5909 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
5910 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
5911 | SDValue Op3 = getValue(I.getArgOperand(2)); | ||||||||||||
5912 | // @llvm.memset defines 0 and 1 to both mean no alignment. | ||||||||||||
5913 | Align Alignment = MSI.getDestAlign().valueOrOne(); | ||||||||||||
5914 | bool isVol = MSI.isVolatile(); | ||||||||||||
5915 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||||||
5916 | SDValue Root = isVol ? getRoot() : getMemoryRoot(); | ||||||||||||
5917 | AAMDNodes AAInfo; | ||||||||||||
5918 | I.getAAMetadata(AAInfo); | ||||||||||||
5919 | SDValue MS = DAG.getMemset(Root, sdl, Op1, Op2, Op3, Alignment, isVol, isTC, | ||||||||||||
5920 | MachinePointerInfo(I.getArgOperand(0)), AAInfo); | ||||||||||||
5921 | updateDAGForMaybeTailCall(MS); | ||||||||||||
5922 | return; | ||||||||||||
5923 | } | ||||||||||||
5924 | case Intrinsic::memmove: { | ||||||||||||
5925 | const auto &MMI = cast<MemMoveInst>(I); | ||||||||||||
5926 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
5927 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
5928 | SDValue Op3 = getValue(I.getArgOperand(2)); | ||||||||||||
5929 | // @llvm.memmove defines 0 and 1 to both mean no alignment. | ||||||||||||
5930 | Align DstAlign = MMI.getDestAlign().valueOrOne(); | ||||||||||||
5931 | Align SrcAlign = MMI.getSourceAlign().valueOrOne(); | ||||||||||||
5932 | Align Alignment = commonAlignment(DstAlign, SrcAlign); | ||||||||||||
5933 | bool isVol = MMI.isVolatile(); | ||||||||||||
5934 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||||||
5935 | // FIXME: Support passing different dest/src alignments to the memmove DAG | ||||||||||||
5936 | // node. | ||||||||||||
5937 | SDValue Root = isVol ? getRoot() : getMemoryRoot(); | ||||||||||||
5938 | AAMDNodes AAInfo; | ||||||||||||
5939 | I.getAAMetadata(AAInfo); | ||||||||||||
5940 | SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol, | ||||||||||||
5941 | isTC, MachinePointerInfo(I.getArgOperand(0)), | ||||||||||||
5942 | MachinePointerInfo(I.getArgOperand(1)), AAInfo); | ||||||||||||
5943 | updateDAGForMaybeTailCall(MM); | ||||||||||||
5944 | return; | ||||||||||||
5945 | } | ||||||||||||
5946 | case Intrinsic::memcpy_element_unordered_atomic: { | ||||||||||||
5947 | const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I); | ||||||||||||
5948 | SDValue Dst = getValue(MI.getRawDest()); | ||||||||||||
5949 | SDValue Src = getValue(MI.getRawSource()); | ||||||||||||
5950 | SDValue Length = getValue(MI.getLength()); | ||||||||||||
5951 | |||||||||||||
5952 | unsigned DstAlign = MI.getDestAlignment(); | ||||||||||||
5953 | unsigned SrcAlign = MI.getSourceAlignment(); | ||||||||||||
5954 | Type *LengthTy = MI.getLength()->getType(); | ||||||||||||
5955 | unsigned ElemSz = MI.getElementSizeInBytes(); | ||||||||||||
5956 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||||||
5957 | SDValue MC = DAG.getAtomicMemcpy(getRoot(), sdl, Dst, DstAlign, Src, | ||||||||||||
5958 | SrcAlign, Length, LengthTy, ElemSz, isTC, | ||||||||||||
5959 | MachinePointerInfo(MI.getRawDest()), | ||||||||||||
5960 | MachinePointerInfo(MI.getRawSource())); | ||||||||||||
5961 | updateDAGForMaybeTailCall(MC); | ||||||||||||
5962 | return; | ||||||||||||
5963 | } | ||||||||||||
5964 | case Intrinsic::memmove_element_unordered_atomic: { | ||||||||||||
5965 | auto &MI = cast<AtomicMemMoveInst>(I); | ||||||||||||
5966 | SDValue Dst = getValue(MI.getRawDest()); | ||||||||||||
5967 | SDValue Src = getValue(MI.getRawSource()); | ||||||||||||
5968 | SDValue Length = getValue(MI.getLength()); | ||||||||||||
5969 | |||||||||||||
5970 | unsigned DstAlign = MI.getDestAlignment(); | ||||||||||||
5971 | unsigned SrcAlign = MI.getSourceAlignment(); | ||||||||||||
5972 | Type *LengthTy = MI.getLength()->getType(); | ||||||||||||
5973 | unsigned ElemSz = MI.getElementSizeInBytes(); | ||||||||||||
5974 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||||||
5975 | SDValue MC = DAG.getAtomicMemmove(getRoot(), sdl, Dst, DstAlign, Src, | ||||||||||||
5976 | SrcAlign, Length, LengthTy, ElemSz, isTC, | ||||||||||||
5977 | MachinePointerInfo(MI.getRawDest()), | ||||||||||||
5978 | MachinePointerInfo(MI.getRawSource())); | ||||||||||||
5979 | updateDAGForMaybeTailCall(MC); | ||||||||||||
5980 | return; | ||||||||||||
5981 | } | ||||||||||||
5982 | case Intrinsic::memset_element_unordered_atomic: { | ||||||||||||
5983 | auto &MI = cast<AtomicMemSetInst>(I); | ||||||||||||
5984 | SDValue Dst = getValue(MI.getRawDest()); | ||||||||||||
5985 | SDValue Val = getValue(MI.getValue()); | ||||||||||||
5986 | SDValue Length = getValue(MI.getLength()); | ||||||||||||
5987 | |||||||||||||
5988 | unsigned DstAlign = MI.getDestAlignment(); | ||||||||||||
5989 | Type *LengthTy = MI.getLength()->getType(); | ||||||||||||
5990 | unsigned ElemSz = MI.getElementSizeInBytes(); | ||||||||||||
5991 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||||||
5992 | SDValue MC = DAG.getAtomicMemset(getRoot(), sdl, Dst, DstAlign, Val, Length, | ||||||||||||
5993 | LengthTy, ElemSz, isTC, | ||||||||||||
5994 | MachinePointerInfo(MI.getRawDest())); | ||||||||||||
5995 | updateDAGForMaybeTailCall(MC); | ||||||||||||
5996 | return; | ||||||||||||
5997 | } | ||||||||||||
5998 | case Intrinsic::call_preallocated_setup: { | ||||||||||||
5999 | const CallBase *PreallocatedCall = FindPreallocatedCall(&I); | ||||||||||||
6000 | SDValue SrcValue = DAG.getSrcValue(PreallocatedCall); | ||||||||||||
6001 | SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other, | ||||||||||||
6002 | getRoot(), SrcValue); | ||||||||||||
6003 | setValue(&I, Res); | ||||||||||||
6004 | DAG.setRoot(Res); | ||||||||||||
6005 | return; | ||||||||||||
6006 | } | ||||||||||||
6007 | case Intrinsic::call_preallocated_arg: { | ||||||||||||
6008 | const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0)); | ||||||||||||
6009 | SDValue SrcValue = DAG.getSrcValue(PreallocatedCall); | ||||||||||||
6010 | SDValue Ops[3]; | ||||||||||||
6011 | Ops[0] = getRoot(); | ||||||||||||
6012 | Ops[1] = SrcValue; | ||||||||||||
6013 | Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl, | ||||||||||||
6014 | MVT::i32); // arg index | ||||||||||||
6015 | SDValue Res = DAG.getNode( | ||||||||||||
6016 | ISD::PREALLOCATED_ARG, sdl, | ||||||||||||
6017 | DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops); | ||||||||||||
6018 | setValue(&I, Res); | ||||||||||||
6019 | DAG.setRoot(Res.getValue(1)); | ||||||||||||
6020 | return; | ||||||||||||
6021 | } | ||||||||||||
6022 | case Intrinsic::dbg_addr: | ||||||||||||
6023 | case Intrinsic::dbg_declare: { | ||||||||||||
6024 | // Assume dbg.addr and dbg.declare can not currently use DIArgList, i.e. | ||||||||||||
6025 | // they are non-variadic. | ||||||||||||
6026 | const auto &DI = cast<DbgVariableIntrinsic>(I); | ||||||||||||
6027 | assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList")(static_cast <bool> (!DI.hasArgList() && "Only dbg.value should currently use DIArgList" ) ? void (0) : __assert_fail ("!DI.hasArgList() && \"Only dbg.value should currently use DIArgList\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6027, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
6028 | DILocalVariable *Variable = DI.getVariable(); | ||||||||||||
6029 | DIExpression *Expression = DI.getExpression(); | ||||||||||||
6030 | dropDanglingDebugInfo(Variable, Expression); | ||||||||||||
6031 | assert(Variable && "Missing variable")(static_cast <bool> (Variable && "Missing variable" ) ? void (0) : __assert_fail ("Variable && \"Missing variable\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6031, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
6032 | LLVM_DEBUG(dbgs() << "SelectionDAG visiting debug intrinsic: " << DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "SelectionDAG visiting debug intrinsic: " << DI << "\n"; } } while (false) | ||||||||||||
6033 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "SelectionDAG visiting debug intrinsic: " << DI << "\n"; } } while (false); | ||||||||||||
6034 | // Check if address has undef value. | ||||||||||||
6035 | const Value *Address = DI.getVariableLocationOp(0); | ||||||||||||
6036 | if (!Address || isa<UndefValue>(Address) || | ||||||||||||
6037 | (Address->use_empty() && !isa<Argument>(Address))) { | ||||||||||||
6038 | LLVM_DEBUG(dbgs() << "Dropping debug info for " << DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug info for " << DI << " (bad/undef/unused-arg address)\n"; } } while ( false) | ||||||||||||
6039 | << " (bad/undef/unused-arg address)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug info for " << DI << " (bad/undef/unused-arg address)\n"; } } while ( false); | ||||||||||||
6040 | return; | ||||||||||||
6041 | } | ||||||||||||
6042 | |||||||||||||
6043 | bool isParameter = Variable->isParameter() || isa<Argument>(Address); | ||||||||||||
6044 | |||||||||||||
6045 | // Check if this variable can be described by a frame index, typically | ||||||||||||
6046 | // either as a static alloca or a byval parameter. | ||||||||||||
6047 | int FI = std::numeric_limits<int>::max(); | ||||||||||||
6048 | if (const auto *AI = | ||||||||||||
6049 | dyn_cast<AllocaInst>(Address->stripInBoundsConstantOffsets())) { | ||||||||||||
6050 | if (AI->isStaticAlloca()) { | ||||||||||||
6051 | auto I = FuncInfo.StaticAllocaMap.find(AI); | ||||||||||||
6052 | if (I != FuncInfo.StaticAllocaMap.end()) | ||||||||||||
6053 | FI = I->second; | ||||||||||||
6054 | } | ||||||||||||
6055 | } else if (const auto *Arg = dyn_cast<Argument>( | ||||||||||||
6056 | Address->stripInBoundsConstantOffsets())) { | ||||||||||||
6057 | FI = FuncInfo.getArgumentFrameIndex(Arg); | ||||||||||||
6058 | } | ||||||||||||
6059 | |||||||||||||
6060 | // llvm.dbg.addr is control dependent and always generates indirect | ||||||||||||
6061 | // DBG_VALUE instructions. llvm.dbg.declare is handled as a frame index in | ||||||||||||
6062 | // the MachineFunction variable table. | ||||||||||||
6063 | if (FI != std::numeric_limits<int>::max()) { | ||||||||||||
6064 | if (Intrinsic == Intrinsic::dbg_addr) { | ||||||||||||
6065 | SDDbgValue *SDV = DAG.getFrameIndexDbgValue( | ||||||||||||
6066 | Variable, Expression, FI, getRoot().getNode(), /*IsIndirect*/ true, | ||||||||||||
6067 | dl, SDNodeOrder); | ||||||||||||
6068 | DAG.AddDbgValue(SDV, isParameter); | ||||||||||||
6069 | } else { | ||||||||||||
6070 | LLVM_DEBUG(dbgs() << "Skipping " << DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Skipping " << DI << " (variable info stashed in MF side table)\n" ; } } while (false) | ||||||||||||
6071 | << " (variable info stashed in MF side table)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Skipping " << DI << " (variable info stashed in MF side table)\n" ; } } while (false); | ||||||||||||
6072 | } | ||||||||||||
6073 | return; | ||||||||||||
6074 | } | ||||||||||||
6075 | |||||||||||||
6076 | SDValue &N = NodeMap[Address]; | ||||||||||||
6077 | if (!N.getNode() && isa<Argument>(Address)) | ||||||||||||
6078 | // Check unused arguments map. | ||||||||||||
6079 | N = UnusedArgNodeMap[Address]; | ||||||||||||
6080 | SDDbgValue *SDV; | ||||||||||||
6081 | if (N.getNode()) { | ||||||||||||
6082 | if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address)) | ||||||||||||
6083 | Address = BCI->getOperand(0); | ||||||||||||
6084 | // Parameters are handled specially. | ||||||||||||
6085 | auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode()); | ||||||||||||
6086 | if (isParameter && FINode) { | ||||||||||||
6087 | // Byval parameter. We have a frame index at this point. | ||||||||||||
6088 | SDV = | ||||||||||||
6089 | DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(), | ||||||||||||
6090 | /*IsIndirect*/ true, dl, SDNodeOrder); | ||||||||||||
6091 | } else if (isa<Argument>(Address)) { | ||||||||||||
6092 | // Address is an argument, so try to emit its dbg value using | ||||||||||||
6093 | // virtual register info from the FuncInfo.ValueMap. | ||||||||||||
6094 | EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, true, N); | ||||||||||||
6095 | return; | ||||||||||||
6096 | } else { | ||||||||||||
6097 | SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(), | ||||||||||||
6098 | true, dl, SDNodeOrder); | ||||||||||||
6099 | } | ||||||||||||
6100 | DAG.AddDbgValue(SDV, isParameter); | ||||||||||||
6101 | } else { | ||||||||||||
6102 | // If Address is an argument then try to emit its dbg value using | ||||||||||||
6103 | // virtual register info from the FuncInfo.ValueMap. | ||||||||||||
6104 | if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, true, | ||||||||||||
6105 | N)) { | ||||||||||||
6106 | LLVM_DEBUG(dbgs() << "Dropping debug info for " << DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug info for " << DI << " (could not emit func-arg dbg_value)\n"; } } while (false) | ||||||||||||
6107 | << " (could not emit func-arg dbg_value)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug info for " << DI << " (could not emit func-arg dbg_value)\n"; } } while (false); | ||||||||||||
6108 | } | ||||||||||||
6109 | } | ||||||||||||
6110 | return; | ||||||||||||
6111 | } | ||||||||||||
6112 | case Intrinsic::dbg_label: { | ||||||||||||
6113 | const DbgLabelInst &DI = cast<DbgLabelInst>(I); | ||||||||||||
6114 | DILabel *Label = DI.getLabel(); | ||||||||||||
6115 | assert(Label && "Missing label")(static_cast <bool> (Label && "Missing label") ? void (0) : __assert_fail ("Label && \"Missing label\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6115, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
6116 | |||||||||||||
6117 | SDDbgLabel *SDV; | ||||||||||||
6118 | SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder); | ||||||||||||
6119 | DAG.AddDbgLabel(SDV); | ||||||||||||
6120 | return; | ||||||||||||
6121 | } | ||||||||||||
6122 | case Intrinsic::dbg_value: { | ||||||||||||
6123 | const DbgValueInst &DI = cast<DbgValueInst>(I); | ||||||||||||
6124 | assert(DI.getVariable() && "Missing variable")(static_cast <bool> (DI.getVariable() && "Missing variable" ) ? void (0) : __assert_fail ("DI.getVariable() && \"Missing variable\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6124, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
6125 | |||||||||||||
6126 | DILocalVariable *Variable = DI.getVariable(); | ||||||||||||
6127 | DIExpression *Expression = DI.getExpression(); | ||||||||||||
6128 | dropDanglingDebugInfo(Variable, Expression); | ||||||||||||
6129 | SmallVector<Value *, 4> Values(DI.getValues()); | ||||||||||||
6130 | if (Values.empty()) | ||||||||||||
6131 | return; | ||||||||||||
6132 | |||||||||||||
6133 | if (std::count(Values.begin(), Values.end(), nullptr)) | ||||||||||||
6134 | return; | ||||||||||||
6135 | |||||||||||||
6136 | bool IsVariadic = DI.hasArgList(); | ||||||||||||
6137 | if (!handleDebugValue(Values, Variable, Expression, dl, DI.getDebugLoc(), | ||||||||||||
6138 | SDNodeOrder, IsVariadic)) | ||||||||||||
6139 | addDanglingDebugInfo(&DI, dl, SDNodeOrder); | ||||||||||||
6140 | return; | ||||||||||||
6141 | } | ||||||||||||
6142 | |||||||||||||
6143 | case Intrinsic::eh_typeid_for: { | ||||||||||||
6144 | // Find the type id for the given typeinfo. | ||||||||||||
6145 | GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0)); | ||||||||||||
6146 | unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV); | ||||||||||||
6147 | Res = DAG.getConstant(TypeID, sdl, MVT::i32); | ||||||||||||
6148 | setValue(&I, Res); | ||||||||||||
6149 | return; | ||||||||||||
6150 | } | ||||||||||||
6151 | |||||||||||||
6152 | case Intrinsic::eh_return_i32: | ||||||||||||
6153 | case Intrinsic::eh_return_i64: | ||||||||||||
6154 | DAG.getMachineFunction().setCallsEHReturn(true); | ||||||||||||
6155 | DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl, | ||||||||||||
6156 | MVT::Other, | ||||||||||||
6157 | getControlRoot(), | ||||||||||||
6158 | getValue(I.getArgOperand(0)), | ||||||||||||
6159 | getValue(I.getArgOperand(1)))); | ||||||||||||
6160 | return; | ||||||||||||
6161 | case Intrinsic::eh_unwind_init: | ||||||||||||
6162 | DAG.getMachineFunction().setCallsUnwindInit(true); | ||||||||||||
6163 | return; | ||||||||||||
6164 | case Intrinsic::eh_dwarf_cfa: | ||||||||||||
6165 | setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl, | ||||||||||||
6166 | TLI.getPointerTy(DAG.getDataLayout()), | ||||||||||||
6167 | getValue(I.getArgOperand(0)))); | ||||||||||||
6168 | return; | ||||||||||||
6169 | case Intrinsic::eh_sjlj_callsite: { | ||||||||||||
6170 | MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); | ||||||||||||
6171 | ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0)); | ||||||||||||
6172 | assert(CI && "Non-constant call site value in eh.sjlj.callsite!")(static_cast <bool> (CI && "Non-constant call site value in eh.sjlj.callsite!" ) ? void (0) : __assert_fail ("CI && \"Non-constant call site value in eh.sjlj.callsite!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6172, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
6173 | assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!")(static_cast <bool> (MMI.getCurrentCallSite() == 0 && "Overlapping call sites!") ? void (0) : __assert_fail ("MMI.getCurrentCallSite() == 0 && \"Overlapping call sites!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6173, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
6174 | |||||||||||||
6175 | MMI.setCurrentCallSite(CI->getZExtValue()); | ||||||||||||
6176 | return; | ||||||||||||
6177 | } | ||||||||||||
6178 | case Intrinsic::eh_sjlj_functioncontext: { | ||||||||||||
6179 | // Get and store the index of the function context. | ||||||||||||
6180 | MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); | ||||||||||||
6181 | AllocaInst *FnCtx = | ||||||||||||
6182 | cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts()); | ||||||||||||
6183 | int FI = FuncInfo.StaticAllocaMap[FnCtx]; | ||||||||||||
6184 | MFI.setFunctionContextIndex(FI); | ||||||||||||
6185 | return; | ||||||||||||
6186 | } | ||||||||||||
6187 | case Intrinsic::eh_sjlj_setjmp: { | ||||||||||||
6188 | SDValue Ops[2]; | ||||||||||||
6189 | Ops[0] = getRoot(); | ||||||||||||
6190 | Ops[1] = getValue(I.getArgOperand(0)); | ||||||||||||
6191 | SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl, | ||||||||||||
6192 | DAG.getVTList(MVT::i32, MVT::Other), Ops); | ||||||||||||
6193 | setValue(&I, Op.getValue(0)); | ||||||||||||
6194 | DAG.setRoot(Op.getValue(1)); | ||||||||||||
6195 | return; | ||||||||||||
6196 | } | ||||||||||||
6197 | case Intrinsic::eh_sjlj_longjmp: | ||||||||||||
6198 | DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other, | ||||||||||||
6199 | getRoot(), getValue(I.getArgOperand(0)))); | ||||||||||||
6200 | return; | ||||||||||||
6201 | case Intrinsic::eh_sjlj_setup_dispatch: | ||||||||||||
6202 | DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other, | ||||||||||||
6203 | getRoot())); | ||||||||||||
6204 | return; | ||||||||||||
6205 | case Intrinsic::masked_gather: | ||||||||||||
6206 | visitMaskedGather(I); | ||||||||||||
6207 | return; | ||||||||||||
6208 | case Intrinsic::masked_load: | ||||||||||||
6209 | visitMaskedLoad(I); | ||||||||||||
6210 | return; | ||||||||||||
6211 | case Intrinsic::masked_scatter: | ||||||||||||
6212 | visitMaskedScatter(I); | ||||||||||||
6213 | return; | ||||||||||||
6214 | case Intrinsic::masked_store: | ||||||||||||
6215 | visitMaskedStore(I); | ||||||||||||
6216 | return; | ||||||||||||
6217 | case Intrinsic::masked_expandload: | ||||||||||||
6218 | visitMaskedLoad(I, true /* IsExpanding */); | ||||||||||||
6219 | return; | ||||||||||||
6220 | case Intrinsic::masked_compressstore: | ||||||||||||
6221 | visitMaskedStore(I, true /* IsCompressing */); | ||||||||||||
6222 | return; | ||||||||||||
6223 | case Intrinsic::powi: | ||||||||||||
6224 | setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)), | ||||||||||||
6225 | getValue(I.getArgOperand(1)), DAG)); | ||||||||||||
6226 | return; | ||||||||||||
6227 | case Intrinsic::log: | ||||||||||||
6228 | setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); | ||||||||||||
6229 | return; | ||||||||||||
6230 | case Intrinsic::log2: | ||||||||||||
6231 | setValue(&I, | ||||||||||||
6232 | expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); | ||||||||||||
6233 | return; | ||||||||||||
6234 | case Intrinsic::log10: | ||||||||||||
6235 | setValue(&I, | ||||||||||||
6236 | expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); | ||||||||||||
6237 | return; | ||||||||||||
6238 | case Intrinsic::exp: | ||||||||||||
6239 | setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); | ||||||||||||
6240 | return; | ||||||||||||
6241 | case Intrinsic::exp2: | ||||||||||||
6242 | setValue(&I, | ||||||||||||
6243 | expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); | ||||||||||||
6244 | return; | ||||||||||||
6245 | case Intrinsic::pow: | ||||||||||||
6246 | setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)), | ||||||||||||
6247 | getValue(I.getArgOperand(1)), DAG, TLI, Flags)); | ||||||||||||
6248 | return; | ||||||||||||
6249 | case Intrinsic::sqrt: | ||||||||||||
6250 | case Intrinsic::fabs: | ||||||||||||
6251 | case Intrinsic::sin: | ||||||||||||
6252 | case Intrinsic::cos: | ||||||||||||
6253 | case Intrinsic::floor: | ||||||||||||
6254 | case Intrinsic::ceil: | ||||||||||||
6255 | case Intrinsic::trunc: | ||||||||||||
6256 | case Intrinsic::rint: | ||||||||||||
6257 | case Intrinsic::nearbyint: | ||||||||||||
6258 | case Intrinsic::round: | ||||||||||||
6259 | case Intrinsic::roundeven: | ||||||||||||
6260 | case Intrinsic::canonicalize: { | ||||||||||||
6261 | unsigned Opcode; | ||||||||||||
6262 | switch (Intrinsic) { | ||||||||||||
6263 | default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6263); // Can't reach here. | ||||||||||||
6264 | case Intrinsic::sqrt: Opcode = ISD::FSQRT; break; | ||||||||||||
6265 | case Intrinsic::fabs: Opcode = ISD::FABS; break; | ||||||||||||
6266 | case Intrinsic::sin: Opcode = ISD::FSIN; break; | ||||||||||||
6267 | case Intrinsic::cos: Opcode = ISD::FCOS; break; | ||||||||||||
6268 | case Intrinsic::floor: Opcode = ISD::FFLOOR; break; | ||||||||||||
6269 | case Intrinsic::ceil: Opcode = ISD::FCEIL; break; | ||||||||||||
6270 | case Intrinsic::trunc: Opcode = ISD::FTRUNC; break; | ||||||||||||
6271 | case Intrinsic::rint: Opcode = ISD::FRINT; break; | ||||||||||||
6272 | case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break; | ||||||||||||
6273 | case Intrinsic::round: Opcode = ISD::FROUND; break; | ||||||||||||
6274 | case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break; | ||||||||||||
6275 | case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break; | ||||||||||||
6276 | } | ||||||||||||
6277 | |||||||||||||
6278 | setValue(&I, DAG.getNode(Opcode, sdl, | ||||||||||||
6279 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||||||
6280 | getValue(I.getArgOperand(0)), Flags)); | ||||||||||||
6281 | return; | ||||||||||||
6282 | } | ||||||||||||
6283 | case Intrinsic::lround: | ||||||||||||
6284 | case Intrinsic::llround: | ||||||||||||
6285 | case Intrinsic::lrint: | ||||||||||||
6286 | case Intrinsic::llrint: { | ||||||||||||
6287 | unsigned Opcode; | ||||||||||||
6288 | switch (Intrinsic) { | ||||||||||||
6289 | default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6289); // Can't reach here. | ||||||||||||
6290 | case Intrinsic::lround: Opcode = ISD::LROUND; break; | ||||||||||||
6291 | case Intrinsic::llround: Opcode = ISD::LLROUND; break; | ||||||||||||
6292 | case Intrinsic::lrint: Opcode = ISD::LRINT; break; | ||||||||||||
6293 | case Intrinsic::llrint: Opcode = ISD::LLRINT; break; | ||||||||||||
6294 | } | ||||||||||||
6295 | |||||||||||||
6296 | EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
6297 | setValue(&I, DAG.getNode(Opcode, sdl, RetVT, | ||||||||||||
6298 | getValue(I.getArgOperand(0)))); | ||||||||||||
6299 | return; | ||||||||||||
6300 | } | ||||||||||||
6301 | case Intrinsic::minnum: | ||||||||||||
6302 | setValue(&I, DAG.getNode(ISD::FMINNUM, sdl, | ||||||||||||
6303 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||||||
6304 | getValue(I.getArgOperand(0)), | ||||||||||||
6305 | getValue(I.getArgOperand(1)), Flags)); | ||||||||||||
6306 | return; | ||||||||||||
6307 | case Intrinsic::maxnum: | ||||||||||||
6308 | setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl, | ||||||||||||
6309 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||||||
6310 | getValue(I.getArgOperand(0)), | ||||||||||||
6311 | getValue(I.getArgOperand(1)), Flags)); | ||||||||||||
6312 | return; | ||||||||||||
6313 | case Intrinsic::minimum: | ||||||||||||
6314 | setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl, | ||||||||||||
6315 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||||||
6316 | getValue(I.getArgOperand(0)), | ||||||||||||
6317 | getValue(I.getArgOperand(1)), Flags)); | ||||||||||||
6318 | return; | ||||||||||||
6319 | case Intrinsic::maximum: | ||||||||||||
6320 | setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl, | ||||||||||||
6321 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||||||
6322 | getValue(I.getArgOperand(0)), | ||||||||||||
6323 | getValue(I.getArgOperand(1)), Flags)); | ||||||||||||
6324 | return; | ||||||||||||
6325 | case Intrinsic::copysign: | ||||||||||||
6326 | setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl, | ||||||||||||
6327 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||||||
6328 | getValue(I.getArgOperand(0)), | ||||||||||||
6329 | getValue(I.getArgOperand(1)), Flags)); | ||||||||||||
6330 | return; | ||||||||||||
6331 | case Intrinsic::arithmetic_fence: { | ||||||||||||
6332 | setValue(&I, DAG.getNode(ISD::ARITH_FENCE, sdl, | ||||||||||||
6333 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||||||
6334 | getValue(I.getArgOperand(0)), Flags)); | ||||||||||||
6335 | return; | ||||||||||||
6336 | } | ||||||||||||
6337 | case Intrinsic::fma: | ||||||||||||
6338 | setValue(&I, DAG.getNode( | ||||||||||||
6339 | ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(), | ||||||||||||
6340 | getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), | ||||||||||||
6341 | getValue(I.getArgOperand(2)), Flags)); | ||||||||||||
6342 | return; | ||||||||||||
6343 | #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ | ||||||||||||
6344 | case Intrinsic::INTRINSIC: | ||||||||||||
6345 | #include "llvm/IR/ConstrainedOps.def" | ||||||||||||
6346 | visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I)); | ||||||||||||
6347 | return; | ||||||||||||
6348 | #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID: | ||||||||||||
6349 | #include "llvm/IR/VPIntrinsics.def" | ||||||||||||
6350 | visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I)); | ||||||||||||
6351 | return; | ||||||||||||
6352 | case Intrinsic::fmuladd: { | ||||||||||||
6353 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
6354 | if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict && | ||||||||||||
6355 | TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) { | ||||||||||||
6356 | setValue(&I, DAG.getNode(ISD::FMA, sdl, | ||||||||||||
6357 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||||||
6358 | getValue(I.getArgOperand(0)), | ||||||||||||
6359 | getValue(I.getArgOperand(1)), | ||||||||||||
6360 | getValue(I.getArgOperand(2)), Flags)); | ||||||||||||
6361 | } else { | ||||||||||||
6362 | // TODO: Intrinsic calls should have fast-math-flags. | ||||||||||||
6363 | SDValue Mul = DAG.getNode( | ||||||||||||
6364 | ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(), | ||||||||||||
6365 | getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags); | ||||||||||||
6366 | SDValue Add = DAG.getNode(ISD::FADD, sdl, | ||||||||||||
6367 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||||||
6368 | Mul, getValue(I.getArgOperand(2)), Flags); | ||||||||||||
6369 | setValue(&I, Add); | ||||||||||||
6370 | } | ||||||||||||
6371 | return; | ||||||||||||
6372 | } | ||||||||||||
6373 | case Intrinsic::convert_to_fp16: | ||||||||||||
6374 | setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16, | ||||||||||||
6375 | DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16, | ||||||||||||
6376 | getValue(I.getArgOperand(0)), | ||||||||||||
6377 | DAG.getTargetConstant(0, sdl, | ||||||||||||
6378 | MVT::i32)))); | ||||||||||||
6379 | return; | ||||||||||||
6380 | case Intrinsic::convert_from_fp16: | ||||||||||||
6381 | setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl, | ||||||||||||
6382 | TLI.getValueType(DAG.getDataLayout(), I.getType()), | ||||||||||||
6383 | DAG.getNode(ISD::BITCAST, sdl, MVT::f16, | ||||||||||||
6384 | getValue(I.getArgOperand(0))))); | ||||||||||||
6385 | return; | ||||||||||||
6386 | case Intrinsic::fptosi_sat: { | ||||||||||||
6387 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
6388 | setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT, | ||||||||||||
6389 | getValue(I.getArgOperand(0)), | ||||||||||||
6390 | DAG.getValueType(VT.getScalarType()))); | ||||||||||||
6391 | return; | ||||||||||||
6392 | } | ||||||||||||
6393 | case Intrinsic::fptoui_sat: { | ||||||||||||
6394 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
6395 | setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT, | ||||||||||||
6396 | getValue(I.getArgOperand(0)), | ||||||||||||
6397 | DAG.getValueType(VT.getScalarType()))); | ||||||||||||
6398 | return; | ||||||||||||
6399 | } | ||||||||||||
6400 | case Intrinsic::set_rounding: | ||||||||||||
6401 | Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other, | ||||||||||||
6402 | {getRoot(), getValue(I.getArgOperand(0))}); | ||||||||||||
6403 | setValue(&I, Res); | ||||||||||||
6404 | DAG.setRoot(Res.getValue(0)); | ||||||||||||
6405 | return; | ||||||||||||
6406 | case Intrinsic::pcmarker: { | ||||||||||||
6407 | SDValue Tmp = getValue(I.getArgOperand(0)); | ||||||||||||
6408 | DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp)); | ||||||||||||
6409 | return; | ||||||||||||
6410 | } | ||||||||||||
6411 | case Intrinsic::isnan: { | ||||||||||||
6412 | const DataLayout DLayout = DAG.getDataLayout(); | ||||||||||||
6413 | EVT DestVT = TLI.getValueType(DLayout, I.getType()); | ||||||||||||
6414 | EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType()); | ||||||||||||
6415 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
6416 | const Function &F = MF.getFunction(); | ||||||||||||
6417 | SDValue Op = getValue(I.getArgOperand(0)); | ||||||||||||
6418 | SDNodeFlags Flags; | ||||||||||||
6419 | Flags.setNoFPExcept( | ||||||||||||
6420 | !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP)); | ||||||||||||
6421 | |||||||||||||
6422 | // If ISD::ISNAN should be expanded, do it right now, because the expansion | ||||||||||||
6423 | // can use illegal types. Making expansion early allows to legalize these | ||||||||||||
6424 | // types prior to selection. | ||||||||||||
6425 | if (!TLI.isOperationLegalOrCustom(ISD::ISNAN, ArgVT)) { | ||||||||||||
6426 | SDValue Result = TLI.expandISNAN(DestVT, Op, Flags, sdl, DAG); | ||||||||||||
6427 | setValue(&I, Result); | ||||||||||||
6428 | return; | ||||||||||||
6429 | } | ||||||||||||
6430 | |||||||||||||
6431 | SDValue V = DAG.getNode(ISD::ISNAN, sdl, DestVT, Op, Flags); | ||||||||||||
6432 | setValue(&I, V); | ||||||||||||
6433 | return; | ||||||||||||
6434 | } | ||||||||||||
6435 | case Intrinsic::readcyclecounter: { | ||||||||||||
6436 | SDValue Op = getRoot(); | ||||||||||||
6437 | Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl, | ||||||||||||
6438 | DAG.getVTList(MVT::i64, MVT::Other), Op); | ||||||||||||
6439 | setValue(&I, Res); | ||||||||||||
6440 | DAG.setRoot(Res.getValue(1)); | ||||||||||||
6441 | return; | ||||||||||||
6442 | } | ||||||||||||
6443 | case Intrinsic::bitreverse: | ||||||||||||
6444 | setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl, | ||||||||||||
6445 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||||||
6446 | getValue(I.getArgOperand(0)))); | ||||||||||||
6447 | return; | ||||||||||||
6448 | case Intrinsic::bswap: | ||||||||||||
6449 | setValue(&I, DAG.getNode(ISD::BSWAP, sdl, | ||||||||||||
6450 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||||||
6451 | getValue(I.getArgOperand(0)))); | ||||||||||||
6452 | return; | ||||||||||||
6453 | case Intrinsic::cttz: { | ||||||||||||
6454 | SDValue Arg = getValue(I.getArgOperand(0)); | ||||||||||||
6455 | ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1)); | ||||||||||||
6456 | EVT Ty = Arg.getValueType(); | ||||||||||||
6457 | setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF, | ||||||||||||
6458 | sdl, Ty, Arg)); | ||||||||||||
6459 | return; | ||||||||||||
6460 | } | ||||||||||||
6461 | case Intrinsic::ctlz: { | ||||||||||||
6462 | SDValue Arg = getValue(I.getArgOperand(0)); | ||||||||||||
6463 | ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1)); | ||||||||||||
6464 | EVT Ty = Arg.getValueType(); | ||||||||||||
6465 | setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF, | ||||||||||||
6466 | sdl, Ty, Arg)); | ||||||||||||
6467 | return; | ||||||||||||
6468 | } | ||||||||||||
6469 | case Intrinsic::ctpop: { | ||||||||||||
6470 | SDValue Arg = getValue(I.getArgOperand(0)); | ||||||||||||
6471 | EVT Ty = Arg.getValueType(); | ||||||||||||
6472 | setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg)); | ||||||||||||
6473 | return; | ||||||||||||
6474 | } | ||||||||||||
6475 | case Intrinsic::fshl: | ||||||||||||
6476 | case Intrinsic::fshr: { | ||||||||||||
6477 | bool IsFSHL = Intrinsic == Intrinsic::fshl; | ||||||||||||
6478 | SDValue X = getValue(I.getArgOperand(0)); | ||||||||||||
6479 | SDValue Y = getValue(I.getArgOperand(1)); | ||||||||||||
6480 | SDValue Z = getValue(I.getArgOperand(2)); | ||||||||||||
6481 | EVT VT = X.getValueType(); | ||||||||||||
6482 | |||||||||||||
6483 | if (X == Y) { | ||||||||||||
6484 | auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR; | ||||||||||||
6485 | setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z)); | ||||||||||||
6486 | } else { | ||||||||||||
6487 | auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR; | ||||||||||||
6488 | setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z)); | ||||||||||||
6489 | } | ||||||||||||
6490 | return; | ||||||||||||
6491 | } | ||||||||||||
6492 | case Intrinsic::sadd_sat: { | ||||||||||||
6493 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
6494 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
6495 | setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||||||
6496 | return; | ||||||||||||
6497 | } | ||||||||||||
6498 | case Intrinsic::uadd_sat: { | ||||||||||||
6499 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
6500 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
6501 | setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||||||
6502 | return; | ||||||||||||
6503 | } | ||||||||||||
6504 | case Intrinsic::ssub_sat: { | ||||||||||||
6505 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
6506 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
6507 | setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||||||
6508 | return; | ||||||||||||
6509 | } | ||||||||||||
6510 | case Intrinsic::usub_sat: { | ||||||||||||
6511 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
6512 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
6513 | setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||||||
6514 | return; | ||||||||||||
6515 | } | ||||||||||||
6516 | case Intrinsic::sshl_sat: { | ||||||||||||
6517 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
6518 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
6519 | setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||||||
6520 | return; | ||||||||||||
6521 | } | ||||||||||||
6522 | case Intrinsic::ushl_sat: { | ||||||||||||
6523 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
6524 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
6525 | setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||||||
6526 | return; | ||||||||||||
6527 | } | ||||||||||||
6528 | case Intrinsic::smul_fix: | ||||||||||||
6529 | case Intrinsic::umul_fix: | ||||||||||||
6530 | case Intrinsic::smul_fix_sat: | ||||||||||||
6531 | case Intrinsic::umul_fix_sat: { | ||||||||||||
6532 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
6533 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
6534 | SDValue Op3 = getValue(I.getArgOperand(2)); | ||||||||||||
6535 | setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl, | ||||||||||||
6536 | Op1.getValueType(), Op1, Op2, Op3)); | ||||||||||||
6537 | return; | ||||||||||||
6538 | } | ||||||||||||
6539 | case Intrinsic::sdiv_fix: | ||||||||||||
6540 | case Intrinsic::udiv_fix: | ||||||||||||
6541 | case Intrinsic::sdiv_fix_sat: | ||||||||||||
6542 | case Intrinsic::udiv_fix_sat: { | ||||||||||||
6543 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
6544 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
6545 | SDValue Op3 = getValue(I.getArgOperand(2)); | ||||||||||||
6546 | setValue(&I, expandDivFix(FixedPointIntrinsicToOpcode(Intrinsic), sdl, | ||||||||||||
6547 | Op1, Op2, Op3, DAG, TLI)); | ||||||||||||
6548 | return; | ||||||||||||
6549 | } | ||||||||||||
6550 | case Intrinsic::smax: { | ||||||||||||
6551 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
6552 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
6553 | setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||||||
6554 | return; | ||||||||||||
6555 | } | ||||||||||||
6556 | case Intrinsic::smin: { | ||||||||||||
6557 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
6558 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
6559 | setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||||||
6560 | return; | ||||||||||||
6561 | } | ||||||||||||
6562 | case Intrinsic::umax: { | ||||||||||||
6563 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
6564 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
6565 | setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||||||
6566 | return; | ||||||||||||
6567 | } | ||||||||||||
6568 | case Intrinsic::umin: { | ||||||||||||
6569 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
6570 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
6571 | setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||||||
6572 | return; | ||||||||||||
6573 | } | ||||||||||||
6574 | case Intrinsic::abs: { | ||||||||||||
6575 | // TODO: Preserve "int min is poison" arg in SDAG? | ||||||||||||
6576 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
6577 | setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1)); | ||||||||||||
6578 | return; | ||||||||||||
6579 | } | ||||||||||||
6580 | case Intrinsic::stacksave: { | ||||||||||||
6581 | SDValue Op = getRoot(); | ||||||||||||
6582 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
6583 | Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op); | ||||||||||||
6584 | setValue(&I, Res); | ||||||||||||
6585 | DAG.setRoot(Res.getValue(1)); | ||||||||||||
6586 | return; | ||||||||||||
6587 | } | ||||||||||||
6588 | case Intrinsic::stackrestore: | ||||||||||||
6589 | Res = getValue(I.getArgOperand(0)); | ||||||||||||
6590 | DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res)); | ||||||||||||
6591 | return; | ||||||||||||
6592 | case Intrinsic::get_dynamic_area_offset: { | ||||||||||||
6593 | SDValue Op = getRoot(); | ||||||||||||
6594 | EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout()); | ||||||||||||
6595 | EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
6596 | // Result type for @llvm.get.dynamic.area.offset should match PtrTy for | ||||||||||||
6597 | // target. | ||||||||||||
6598 | if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits()) | ||||||||||||
6599 | report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset" | ||||||||||||
6600 | " intrinsic!"); | ||||||||||||
6601 | Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy), | ||||||||||||
6602 | Op); | ||||||||||||
6603 | DAG.setRoot(Op); | ||||||||||||
6604 | setValue(&I, Res); | ||||||||||||
6605 | return; | ||||||||||||
6606 | } | ||||||||||||
6607 | case Intrinsic::stackguard: { | ||||||||||||
6608 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
6609 | const Module &M = *MF.getFunction().getParent(); | ||||||||||||
6610 | SDValue Chain = getRoot(); | ||||||||||||
6611 | if (TLI.useLoadStackGuardNode()) { | ||||||||||||
6612 | Res = getLoadStackGuard(DAG, sdl, Chain); | ||||||||||||
6613 | } else { | ||||||||||||
6614 | EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
6615 | const Value *Global = TLI.getSDagStackGuard(M); | ||||||||||||
6616 | Align Align = DL->getPrefTypeAlign(Global->getType()); | ||||||||||||
6617 | Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global), | ||||||||||||
6618 | MachinePointerInfo(Global, 0), Align, | ||||||||||||
6619 | MachineMemOperand::MOVolatile); | ||||||||||||
6620 | } | ||||||||||||
6621 | if (TLI.useStackGuardXorFP()) | ||||||||||||
6622 | Res = TLI.emitStackGuardXorFP(DAG, Res, sdl); | ||||||||||||
6623 | DAG.setRoot(Chain); | ||||||||||||
6624 | setValue(&I, Res); | ||||||||||||
6625 | return; | ||||||||||||
6626 | } | ||||||||||||
6627 | case Intrinsic::stackprotector: { | ||||||||||||
6628 | // Emit code into the DAG to store the stack guard onto the stack. | ||||||||||||
6629 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
6630 | MachineFrameInfo &MFI = MF.getFrameInfo(); | ||||||||||||
6631 | SDValue Src, Chain = getRoot(); | ||||||||||||
6632 | |||||||||||||
6633 | if (TLI.useLoadStackGuardNode()) | ||||||||||||
6634 | Src = getLoadStackGuard(DAG, sdl, Chain); | ||||||||||||
6635 | else | ||||||||||||
6636 | Src = getValue(I.getArgOperand(0)); // The guard's value. | ||||||||||||
6637 | |||||||||||||
6638 | AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1)); | ||||||||||||
6639 | |||||||||||||
6640 | int FI = FuncInfo.StaticAllocaMap[Slot]; | ||||||||||||
6641 | MFI.setStackProtectorIndex(FI); | ||||||||||||
6642 | EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout()); | ||||||||||||
6643 | |||||||||||||
6644 | SDValue FIN = DAG.getFrameIndex(FI, PtrTy); | ||||||||||||
6645 | |||||||||||||
6646 | // Store the stack protector onto the stack. | ||||||||||||
6647 | Res = DAG.getStore( | ||||||||||||
6648 | Chain, sdl, Src, FIN, | ||||||||||||
6649 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), | ||||||||||||
6650 | MaybeAlign(), MachineMemOperand::MOVolatile); | ||||||||||||
6651 | setValue(&I, Res); | ||||||||||||
6652 | DAG.setRoot(Res); | ||||||||||||
6653 | return; | ||||||||||||
6654 | } | ||||||||||||
6655 | case Intrinsic::objectsize: | ||||||||||||
6656 | llvm_unreachable("llvm.objectsize.* should have been lowered already")::llvm::llvm_unreachable_internal("llvm.objectsize.* should have been lowered already" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6656); | ||||||||||||
6657 | |||||||||||||
6658 | case Intrinsic::is_constant: | ||||||||||||
6659 | llvm_unreachable("llvm.is.constant.* should have been lowered already")::llvm::llvm_unreachable_internal("llvm.is.constant.* should have been lowered already" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6659); | ||||||||||||
6660 | |||||||||||||
6661 | case Intrinsic::annotation: | ||||||||||||
6662 | case Intrinsic::ptr_annotation: | ||||||||||||
6663 | case Intrinsic::launder_invariant_group: | ||||||||||||
6664 | case Intrinsic::strip_invariant_group: | ||||||||||||
6665 | // Drop the intrinsic, but forward the value | ||||||||||||
6666 | setValue(&I, getValue(I.getOperand(0))); | ||||||||||||
6667 | return; | ||||||||||||
6668 | |||||||||||||
6669 | case Intrinsic::assume: | ||||||||||||
6670 | case Intrinsic::experimental_noalias_scope_decl: | ||||||||||||
6671 | case Intrinsic::var_annotation: | ||||||||||||
6672 | case Intrinsic::sideeffect: | ||||||||||||
6673 | // Discard annotate attributes, noalias scope declarations, assumptions, and | ||||||||||||
6674 | // artificial side-effects. | ||||||||||||
6675 | return; | ||||||||||||
6676 | |||||||||||||
6677 | case Intrinsic::codeview_annotation: { | ||||||||||||
6678 | // Emit a label associated with this metadata. | ||||||||||||
6679 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
6680 | MCSymbol *Label = | ||||||||||||
6681 | MF.getMMI().getContext().createTempSymbol("annotation", true); | ||||||||||||
6682 | Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata(); | ||||||||||||
6683 | MF.addCodeViewAnnotation(Label, cast<MDNode>(MD)); | ||||||||||||
6684 | Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label); | ||||||||||||
6685 | DAG.setRoot(Res); | ||||||||||||
6686 | return; | ||||||||||||
6687 | } | ||||||||||||
6688 | |||||||||||||
6689 | case Intrinsic::init_trampoline: { | ||||||||||||
6690 | const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts()); | ||||||||||||
6691 | |||||||||||||
6692 | SDValue Ops[6]; | ||||||||||||
6693 | Ops[0] = getRoot(); | ||||||||||||
6694 | Ops[1] = getValue(I.getArgOperand(0)); | ||||||||||||
6695 | Ops[2] = getValue(I.getArgOperand(1)); | ||||||||||||
6696 | Ops[3] = getValue(I.getArgOperand(2)); | ||||||||||||
6697 | Ops[4] = DAG.getSrcValue(I.getArgOperand(0)); | ||||||||||||
6698 | Ops[5] = DAG.getSrcValue(F); | ||||||||||||
6699 | |||||||||||||
6700 | Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops); | ||||||||||||
6701 | |||||||||||||
6702 | DAG.setRoot(Res); | ||||||||||||
6703 | return; | ||||||||||||
6704 | } | ||||||||||||
6705 | case Intrinsic::adjust_trampoline: | ||||||||||||
6706 | setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl, | ||||||||||||
6707 | TLI.getPointerTy(DAG.getDataLayout()), | ||||||||||||
6708 | getValue(I.getArgOperand(0)))); | ||||||||||||
6709 | return; | ||||||||||||
6710 | case Intrinsic::gcroot: { | ||||||||||||
6711 | assert(DAG.getMachineFunction().getFunction().hasGC() &&(static_cast <bool> (DAG.getMachineFunction().getFunction ().hasGC() && "only valid in functions with gc specified, enforced by Verifier" ) ? void (0) : __assert_fail ("DAG.getMachineFunction().getFunction().hasGC() && \"only valid in functions with gc specified, enforced by Verifier\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6712, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
6712 | "only valid in functions with gc specified, enforced by Verifier")(static_cast <bool> (DAG.getMachineFunction().getFunction ().hasGC() && "only valid in functions with gc specified, enforced by Verifier" ) ? void (0) : __assert_fail ("DAG.getMachineFunction().getFunction().hasGC() && \"only valid in functions with gc specified, enforced by Verifier\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6712, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
6713 | assert(GFI && "implied by previous")(static_cast <bool> (GFI && "implied by previous" ) ? void (0) : __assert_fail ("GFI && \"implied by previous\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6713, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
6714 | const Value *Alloca = I.getArgOperand(0)->stripPointerCasts(); | ||||||||||||
6715 | const Constant *TypeMap = cast<Constant>(I.getArgOperand(1)); | ||||||||||||
6716 | |||||||||||||
6717 | FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode()); | ||||||||||||
6718 | GFI->addStackRoot(FI->getIndex(), TypeMap); | ||||||||||||
6719 | return; | ||||||||||||
6720 | } | ||||||||||||
6721 | case Intrinsic::gcread: | ||||||||||||
6722 | case Intrinsic::gcwrite: | ||||||||||||
6723 | llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!")::llvm::llvm_unreachable_internal("GC failed to lower gcread/gcwrite intrinsics!" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6723); | ||||||||||||
6724 | case Intrinsic::flt_rounds: | ||||||||||||
6725 | Res = DAG.getNode(ISD::FLT_ROUNDS_, sdl, {MVT::i32, MVT::Other}, getRoot()); | ||||||||||||
6726 | setValue(&I, Res); | ||||||||||||
6727 | DAG.setRoot(Res.getValue(1)); | ||||||||||||
6728 | return; | ||||||||||||
6729 | |||||||||||||
6730 | case Intrinsic::expect: | ||||||||||||
6731 | // Just replace __builtin_expect(exp, c) with EXP. | ||||||||||||
6732 | setValue(&I, getValue(I.getArgOperand(0))); | ||||||||||||
6733 | return; | ||||||||||||
6734 | |||||||||||||
6735 | case Intrinsic::ubsantrap: | ||||||||||||
6736 | case Intrinsic::debugtrap: | ||||||||||||
6737 | case Intrinsic::trap: { | ||||||||||||
6738 | StringRef TrapFuncName = | ||||||||||||
6739 | I.getAttributes().getFnAttr("trap-func-name").getValueAsString(); | ||||||||||||
6740 | if (TrapFuncName.empty()) { | ||||||||||||
6741 | switch (Intrinsic) { | ||||||||||||
6742 | case Intrinsic::trap: | ||||||||||||
6743 | DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot())); | ||||||||||||
6744 | break; | ||||||||||||
6745 | case Intrinsic::debugtrap: | ||||||||||||
6746 | DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot())); | ||||||||||||
6747 | break; | ||||||||||||
6748 | case Intrinsic::ubsantrap: | ||||||||||||
6749 | DAG.setRoot(DAG.getNode( | ||||||||||||
6750 | ISD::UBSANTRAP, sdl, MVT::Other, getRoot(), | ||||||||||||
6751 | DAG.getTargetConstant( | ||||||||||||
6752 | cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl, | ||||||||||||
6753 | MVT::i32))); | ||||||||||||
6754 | break; | ||||||||||||
6755 | default: llvm_unreachable("unknown trap intrinsic")::llvm::llvm_unreachable_internal("unknown trap intrinsic", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6755); | ||||||||||||
6756 | } | ||||||||||||
6757 | return; | ||||||||||||
6758 | } | ||||||||||||
6759 | TargetLowering::ArgListTy Args; | ||||||||||||
6760 | if (Intrinsic == Intrinsic::ubsantrap) { | ||||||||||||
6761 | Args.push_back(TargetLoweringBase::ArgListEntry()); | ||||||||||||
6762 | Args[0].Val = I.getArgOperand(0); | ||||||||||||
6763 | Args[0].Node = getValue(Args[0].Val); | ||||||||||||
6764 | Args[0].Ty = Args[0].Val->getType(); | ||||||||||||
6765 | } | ||||||||||||
6766 | |||||||||||||
6767 | TargetLowering::CallLoweringInfo CLI(DAG); | ||||||||||||
6768 | CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee( | ||||||||||||
6769 | CallingConv::C, I.getType(), | ||||||||||||
6770 | DAG.getExternalSymbol(TrapFuncName.data(), | ||||||||||||
6771 | TLI.getPointerTy(DAG.getDataLayout())), | ||||||||||||
6772 | std::move(Args)); | ||||||||||||
6773 | |||||||||||||
6774 | std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); | ||||||||||||
6775 | DAG.setRoot(Result.second); | ||||||||||||
6776 | return; | ||||||||||||
6777 | } | ||||||||||||
6778 | |||||||||||||
6779 | case Intrinsic::uadd_with_overflow: | ||||||||||||
6780 | case Intrinsic::sadd_with_overflow: | ||||||||||||
6781 | case Intrinsic::usub_with_overflow: | ||||||||||||
6782 | case Intrinsic::ssub_with_overflow: | ||||||||||||
6783 | case Intrinsic::umul_with_overflow: | ||||||||||||
6784 | case Intrinsic::smul_with_overflow: { | ||||||||||||
6785 | ISD::NodeType Op; | ||||||||||||
6786 | switch (Intrinsic) { | ||||||||||||
6787 | default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6787); // Can't reach here. | ||||||||||||
6788 | case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break; | ||||||||||||
6789 | case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break; | ||||||||||||
6790 | case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break; | ||||||||||||
6791 | case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break; | ||||||||||||
6792 | case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break; | ||||||||||||
6793 | case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break; | ||||||||||||
6794 | } | ||||||||||||
6795 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
6796 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
6797 | |||||||||||||
6798 | EVT ResultVT = Op1.getValueType(); | ||||||||||||
6799 | EVT OverflowVT = MVT::i1; | ||||||||||||
6800 | if (ResultVT.isVector()) | ||||||||||||
6801 | OverflowVT = EVT::getVectorVT( | ||||||||||||
6802 | *Context, OverflowVT, ResultVT.getVectorElementCount()); | ||||||||||||
6803 | |||||||||||||
6804 | SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT); | ||||||||||||
6805 | setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2)); | ||||||||||||
6806 | return; | ||||||||||||
6807 | } | ||||||||||||
6808 | case Intrinsic::prefetch: { | ||||||||||||
6809 | SDValue Ops[5]; | ||||||||||||
6810 | unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue(); | ||||||||||||
6811 | auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore; | ||||||||||||
6812 | Ops[0] = DAG.getRoot(); | ||||||||||||
6813 | Ops[1] = getValue(I.getArgOperand(0)); | ||||||||||||
6814 | Ops[2] = getValue(I.getArgOperand(1)); | ||||||||||||
6815 | Ops[3] = getValue(I.getArgOperand(2)); | ||||||||||||
6816 | Ops[4] = getValue(I.getArgOperand(3)); | ||||||||||||
6817 | SDValue Result = DAG.getMemIntrinsicNode( | ||||||||||||
6818 | ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops, | ||||||||||||
6819 | EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)), | ||||||||||||
6820 | /* align */ None, Flags); | ||||||||||||
6821 | |||||||||||||
6822 | // Chain the prefetch in parallell with any pending loads, to stay out of | ||||||||||||
6823 | // the way of later optimizations. | ||||||||||||
6824 | PendingLoads.push_back(Result); | ||||||||||||
6825 | Result = getRoot(); | ||||||||||||
6826 | DAG.setRoot(Result); | ||||||||||||
6827 | return; | ||||||||||||
6828 | } | ||||||||||||
6829 | case Intrinsic::lifetime_start: | ||||||||||||
6830 | case Intrinsic::lifetime_end: { | ||||||||||||
6831 | bool IsStart = (Intrinsic == Intrinsic::lifetime_start); | ||||||||||||
6832 | // Stack coloring is not enabled in O0, discard region information. | ||||||||||||
6833 | if (TM.getOptLevel() == CodeGenOpt::None) | ||||||||||||
6834 | return; | ||||||||||||
6835 | |||||||||||||
6836 | const int64_t ObjectSize = | ||||||||||||
6837 | cast<ConstantInt>(I.getArgOperand(0))->getSExtValue(); | ||||||||||||
6838 | Value *const ObjectPtr = I.getArgOperand(1); | ||||||||||||
6839 | SmallVector<const Value *, 4> Allocas; | ||||||||||||
6840 | getUnderlyingObjects(ObjectPtr, Allocas); | ||||||||||||
6841 | |||||||||||||
6842 | for (const Value *Alloca : Allocas) { | ||||||||||||
6843 | const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca); | ||||||||||||
6844 | |||||||||||||
6845 | // Could not find an Alloca. | ||||||||||||
6846 | if (!LifetimeObject) | ||||||||||||
6847 | continue; | ||||||||||||
6848 | |||||||||||||
6849 | // First check that the Alloca is static, otherwise it won't have a | ||||||||||||
6850 | // valid frame index. | ||||||||||||
6851 | auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject); | ||||||||||||
6852 | if (SI == FuncInfo.StaticAllocaMap.end()) | ||||||||||||
6853 | return; | ||||||||||||
6854 | |||||||||||||
6855 | const int FrameIndex = SI->second; | ||||||||||||
6856 | int64_t Offset; | ||||||||||||
6857 | if (GetPointerBaseWithConstantOffset( | ||||||||||||
6858 | ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject) | ||||||||||||
6859 | Offset = -1; // Cannot determine offset from alloca to lifetime object. | ||||||||||||
6860 | Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize, | ||||||||||||
6861 | Offset); | ||||||||||||
6862 | DAG.setRoot(Res); | ||||||||||||
6863 | } | ||||||||||||
6864 | return; | ||||||||||||
6865 | } | ||||||||||||
6866 | case Intrinsic::pseudoprobe: { | ||||||||||||
6867 | auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(); | ||||||||||||
6868 | auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue(); | ||||||||||||
6869 | auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue(); | ||||||||||||
6870 | Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr); | ||||||||||||
6871 | DAG.setRoot(Res); | ||||||||||||
6872 | return; | ||||||||||||
6873 | } | ||||||||||||
6874 | case Intrinsic::invariant_start: | ||||||||||||
6875 | // Discard region information. | ||||||||||||
6876 | setValue(&I, DAG.getUNDEF(TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||||||
6877 | return; | ||||||||||||
6878 | case Intrinsic::invariant_end: | ||||||||||||
6879 | // Discard region information. | ||||||||||||
6880 | return; | ||||||||||||
6881 | case Intrinsic::clear_cache: | ||||||||||||
6882 | /// FunctionName may be null. | ||||||||||||
6883 | if (const char *FunctionName = TLI.getClearCacheBuiltinName()) | ||||||||||||
6884 | lowerCallToExternalSymbol(I, FunctionName); | ||||||||||||
6885 | return; | ||||||||||||
6886 | case Intrinsic::donothing: | ||||||||||||
6887 | case Intrinsic::seh_try_begin: | ||||||||||||
6888 | case Intrinsic::seh_scope_begin: | ||||||||||||
6889 | case Intrinsic::seh_try_end: | ||||||||||||
6890 | case Intrinsic::seh_scope_end: | ||||||||||||
6891 | // ignore | ||||||||||||
6892 | return; | ||||||||||||
6893 | case Intrinsic::experimental_stackmap: | ||||||||||||
6894 | visitStackmap(I); | ||||||||||||
6895 | return; | ||||||||||||
6896 | case Intrinsic::experimental_patchpoint_void: | ||||||||||||
6897 | case Intrinsic::experimental_patchpoint_i64: | ||||||||||||
6898 | visitPatchpoint(I); | ||||||||||||
6899 | return; | ||||||||||||
6900 | case Intrinsic::experimental_gc_statepoint: | ||||||||||||
6901 | LowerStatepoint(cast<GCStatepointInst>(I)); | ||||||||||||
6902 | return; | ||||||||||||
6903 | case Intrinsic::experimental_gc_result: | ||||||||||||
6904 | visitGCResult(cast<GCResultInst>(I)); | ||||||||||||
6905 | return; | ||||||||||||
6906 | case Intrinsic::experimental_gc_relocate: | ||||||||||||
6907 | visitGCRelocate(cast<GCRelocateInst>(I)); | ||||||||||||
6908 | return; | ||||||||||||
6909 | case Intrinsic::instrprof_increment: | ||||||||||||
6910 | llvm_unreachable("instrprof failed to lower an increment")::llvm::llvm_unreachable_internal("instrprof failed to lower an increment" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6910); | ||||||||||||
6911 | case Intrinsic::instrprof_value_profile: | ||||||||||||
6912 | llvm_unreachable("instrprof failed to lower a value profiling call")::llvm::llvm_unreachable_internal("instrprof failed to lower a value profiling call" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6912); | ||||||||||||
6913 | case Intrinsic::localescape: { | ||||||||||||
6914 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
6915 | const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); | ||||||||||||
6916 | |||||||||||||
6917 | // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission | ||||||||||||
6918 | // is the same on all targets. | ||||||||||||
6919 | for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) { | ||||||||||||
6920 | Value *Arg = I.getArgOperand(Idx)->stripPointerCasts(); | ||||||||||||
6921 | if (isa<ConstantPointerNull>(Arg)) | ||||||||||||
6922 | continue; // Skip null pointers. They represent a hole in index space. | ||||||||||||
6923 | AllocaInst *Slot = cast<AllocaInst>(Arg); | ||||||||||||
6924 | assert(FuncInfo.StaticAllocaMap.count(Slot) &&(static_cast <bool> (FuncInfo.StaticAllocaMap.count(Slot ) && "can only escape static allocas") ? void (0) : __assert_fail ("FuncInfo.StaticAllocaMap.count(Slot) && \"can only escape static allocas\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6925, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
6925 | "can only escape static allocas")(static_cast <bool> (FuncInfo.StaticAllocaMap.count(Slot ) && "can only escape static allocas") ? void (0) : __assert_fail ("FuncInfo.StaticAllocaMap.count(Slot) && \"can only escape static allocas\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6925, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
6926 | int FI = FuncInfo.StaticAllocaMap[Slot]; | ||||||||||||
6927 | MCSymbol *FrameAllocSym = | ||||||||||||
6928 | MF.getMMI().getContext().getOrCreateFrameAllocSymbol( | ||||||||||||
6929 | GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx); | ||||||||||||
6930 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl, | ||||||||||||
6931 | TII->get(TargetOpcode::LOCAL_ESCAPE)) | ||||||||||||
6932 | .addSym(FrameAllocSym) | ||||||||||||
6933 | .addFrameIndex(FI); | ||||||||||||
6934 | } | ||||||||||||
6935 | |||||||||||||
6936 | return; | ||||||||||||
6937 | } | ||||||||||||
6938 | |||||||||||||
6939 | case Intrinsic::localrecover: { | ||||||||||||
6940 | // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx) | ||||||||||||
6941 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
6942 | |||||||||||||
6943 | // Get the symbol that defines the frame offset. | ||||||||||||
6944 | auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts()); | ||||||||||||
6945 | auto *Idx = cast<ConstantInt>(I.getArgOperand(2)); | ||||||||||||
6946 | unsigned IdxVal = | ||||||||||||
6947 | unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max())); | ||||||||||||
6948 | MCSymbol *FrameAllocSym = | ||||||||||||
6949 | MF.getMMI().getContext().getOrCreateFrameAllocSymbol( | ||||||||||||
6950 | GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal); | ||||||||||||
6951 | |||||||||||||
6952 | Value *FP = I.getArgOperand(1); | ||||||||||||
6953 | SDValue FPVal = getValue(FP); | ||||||||||||
6954 | EVT PtrVT = FPVal.getValueType(); | ||||||||||||
6955 | |||||||||||||
6956 | // Create a MCSymbol for the label to avoid any target lowering | ||||||||||||
6957 | // that would make this PC relative. | ||||||||||||
6958 | SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT); | ||||||||||||
6959 | SDValue OffsetVal = | ||||||||||||
6960 | DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym); | ||||||||||||
6961 | |||||||||||||
6962 | // Add the offset to the FP. | ||||||||||||
6963 | SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl); | ||||||||||||
6964 | setValue(&I, Add); | ||||||||||||
6965 | |||||||||||||
6966 | return; | ||||||||||||
6967 | } | ||||||||||||
6968 | |||||||||||||
6969 | case Intrinsic::eh_exceptionpointer: | ||||||||||||
6970 | case Intrinsic::eh_exceptioncode: { | ||||||||||||
6971 | // Get the exception pointer vreg, copy from it, and resize it to fit. | ||||||||||||
6972 | const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0)); | ||||||||||||
6973 | MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); | ||||||||||||
6974 | const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT); | ||||||||||||
6975 | unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC); | ||||||||||||
6976 | SDValue N = | ||||||||||||
6977 | DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), VReg, PtrVT); | ||||||||||||
6978 | if (Intrinsic == Intrinsic::eh_exceptioncode) | ||||||||||||
6979 | N = DAG.getZExtOrTrunc(N, getCurSDLoc(), MVT::i32); | ||||||||||||
6980 | setValue(&I, N); | ||||||||||||
6981 | return; | ||||||||||||
6982 | } | ||||||||||||
6983 | case Intrinsic::xray_customevent: { | ||||||||||||
6984 | // Here we want to make sure that the intrinsic behaves as if it has a | ||||||||||||
6985 | // specific calling convention, and only for x86_64. | ||||||||||||
6986 | // FIXME: Support other platforms later. | ||||||||||||
6987 | const auto &Triple = DAG.getTarget().getTargetTriple(); | ||||||||||||
6988 | if (Triple.getArch() != Triple::x86_64) | ||||||||||||
6989 | return; | ||||||||||||
6990 | |||||||||||||
6991 | SDLoc DL = getCurSDLoc(); | ||||||||||||
6992 | SmallVector<SDValue, 8> Ops; | ||||||||||||
6993 | |||||||||||||
6994 | // We want to say that we always want the arguments in registers. | ||||||||||||
6995 | SDValue LogEntryVal = getValue(I.getArgOperand(0)); | ||||||||||||
6996 | SDValue StrSizeVal = getValue(I.getArgOperand(1)); | ||||||||||||
6997 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | ||||||||||||
6998 | SDValue Chain = getRoot(); | ||||||||||||
6999 | Ops.push_back(LogEntryVal); | ||||||||||||
7000 | Ops.push_back(StrSizeVal); | ||||||||||||
7001 | Ops.push_back(Chain); | ||||||||||||
7002 | |||||||||||||
7003 | // We need to enforce the calling convention for the callsite, so that | ||||||||||||
7004 | // argument ordering is enforced correctly, and that register allocation can | ||||||||||||
7005 | // see that some registers may be assumed clobbered and have to preserve | ||||||||||||
7006 | // them across calls to the intrinsic. | ||||||||||||
7007 | MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL, | ||||||||||||
7008 | DL, NodeTys, Ops); | ||||||||||||
7009 | SDValue patchableNode = SDValue(MN, 0); | ||||||||||||
7010 | DAG.setRoot(patchableNode); | ||||||||||||
7011 | setValue(&I, patchableNode); | ||||||||||||
7012 | return; | ||||||||||||
7013 | } | ||||||||||||
7014 | case Intrinsic::xray_typedevent: { | ||||||||||||
7015 | // Here we want to make sure that the intrinsic behaves as if it has a | ||||||||||||
7016 | // specific calling convention, and only for x86_64. | ||||||||||||
7017 | // FIXME: Support other platforms later. | ||||||||||||
7018 | const auto &Triple = DAG.getTarget().getTargetTriple(); | ||||||||||||
7019 | if (Triple.getArch() != Triple::x86_64) | ||||||||||||
7020 | return; | ||||||||||||
7021 | |||||||||||||
7022 | SDLoc DL = getCurSDLoc(); | ||||||||||||
7023 | SmallVector<SDValue, 8> Ops; | ||||||||||||
7024 | |||||||||||||
7025 | // We want to say that we always want the arguments in registers. | ||||||||||||
7026 | // It's unclear to me how manipulating the selection DAG here forces callers | ||||||||||||
7027 | // to provide arguments in registers instead of on the stack. | ||||||||||||
7028 | SDValue LogTypeId = getValue(I.getArgOperand(0)); | ||||||||||||
7029 | SDValue LogEntryVal = getValue(I.getArgOperand(1)); | ||||||||||||
7030 | SDValue StrSizeVal = getValue(I.getArgOperand(2)); | ||||||||||||
7031 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | ||||||||||||
7032 | SDValue Chain = getRoot(); | ||||||||||||
7033 | Ops.push_back(LogTypeId); | ||||||||||||
7034 | Ops.push_back(LogEntryVal); | ||||||||||||
7035 | Ops.push_back(StrSizeVal); | ||||||||||||
7036 | Ops.push_back(Chain); | ||||||||||||
7037 | |||||||||||||
7038 | // We need to enforce the calling convention for the callsite, so that | ||||||||||||
7039 | // argument ordering is enforced correctly, and that register allocation can | ||||||||||||
7040 | // see that some registers may be assumed clobbered and have to preserve | ||||||||||||
7041 | // them across calls to the intrinsic. | ||||||||||||
7042 | MachineSDNode *MN = DAG.getMachineNode( | ||||||||||||
7043 | TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, DL, NodeTys, Ops); | ||||||||||||
7044 | SDValue patchableNode = SDValue(MN, 0); | ||||||||||||
7045 | DAG.setRoot(patchableNode); | ||||||||||||
7046 | setValue(&I, patchableNode); | ||||||||||||
7047 | return; | ||||||||||||
7048 | } | ||||||||||||
7049 | case Intrinsic::experimental_deoptimize: | ||||||||||||
7050 | LowerDeoptimizeCall(&I); | ||||||||||||
7051 | return; | ||||||||||||
7052 | case Intrinsic::experimental_stepvector: | ||||||||||||
7053 | visitStepVector(I); | ||||||||||||
7054 | return; | ||||||||||||
7055 | case Intrinsic::vector_reduce_fadd: | ||||||||||||
7056 | case Intrinsic::vector_reduce_fmul: | ||||||||||||
7057 | case Intrinsic::vector_reduce_add: | ||||||||||||
7058 | case Intrinsic::vector_reduce_mul: | ||||||||||||
7059 | case Intrinsic::vector_reduce_and: | ||||||||||||
7060 | case Intrinsic::vector_reduce_or: | ||||||||||||
7061 | case Intrinsic::vector_reduce_xor: | ||||||||||||
7062 | case Intrinsic::vector_reduce_smax: | ||||||||||||
7063 | case Intrinsic::vector_reduce_smin: | ||||||||||||
7064 | case Intrinsic::vector_reduce_umax: | ||||||||||||
7065 | case Intrinsic::vector_reduce_umin: | ||||||||||||
7066 | case Intrinsic::vector_reduce_fmax: | ||||||||||||
7067 | case Intrinsic::vector_reduce_fmin: | ||||||||||||
7068 | visitVectorReduce(I, Intrinsic); | ||||||||||||
7069 | return; | ||||||||||||
7070 | |||||||||||||
7071 | case Intrinsic::icall_branch_funnel: { | ||||||||||||
7072 | SmallVector<SDValue, 16> Ops; | ||||||||||||
7073 | Ops.push_back(getValue(I.getArgOperand(0))); | ||||||||||||
7074 | |||||||||||||
7075 | int64_t Offset; | ||||||||||||
7076 | auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset( | ||||||||||||
7077 | I.getArgOperand(1), Offset, DAG.getDataLayout())); | ||||||||||||
7078 | if (!Base) | ||||||||||||
7079 | report_fatal_error( | ||||||||||||
7080 | "llvm.icall.branch.funnel operand must be a GlobalValue"); | ||||||||||||
7081 | Ops.push_back(DAG.getTargetGlobalAddress(Base, getCurSDLoc(), MVT::i64, 0)); | ||||||||||||
7082 | |||||||||||||
7083 | struct BranchFunnelTarget { | ||||||||||||
7084 | int64_t Offset; | ||||||||||||
7085 | SDValue Target; | ||||||||||||
7086 | }; | ||||||||||||
7087 | SmallVector<BranchFunnelTarget, 8> Targets; | ||||||||||||
7088 | |||||||||||||
7089 | for (unsigned Op = 1, N = I.getNumArgOperands(); Op != N; Op += 2) { | ||||||||||||
7090 | auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset( | ||||||||||||
7091 | I.getArgOperand(Op), Offset, DAG.getDataLayout())); | ||||||||||||
7092 | if (ElemBase != Base) | ||||||||||||
7093 | report_fatal_error("all llvm.icall.branch.funnel operands must refer " | ||||||||||||
7094 | "to the same GlobalValue"); | ||||||||||||
7095 | |||||||||||||
7096 | SDValue Val = getValue(I.getArgOperand(Op + 1)); | ||||||||||||
7097 | auto *GA = dyn_cast<GlobalAddressSDNode>(Val); | ||||||||||||
7098 | if (!GA) | ||||||||||||
7099 | report_fatal_error( | ||||||||||||
7100 | "llvm.icall.branch.funnel operand must be a GlobalValue"); | ||||||||||||
7101 | Targets.push_back({Offset, DAG.getTargetGlobalAddress( | ||||||||||||
7102 | GA->getGlobal(), getCurSDLoc(), | ||||||||||||
7103 | Val.getValueType(), GA->getOffset())}); | ||||||||||||
7104 | } | ||||||||||||
7105 | llvm::sort(Targets, | ||||||||||||
7106 | [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) { | ||||||||||||
7107 | return T1.Offset < T2.Offset; | ||||||||||||
7108 | }); | ||||||||||||
7109 | |||||||||||||
7110 | for (auto &T : Targets) { | ||||||||||||
7111 | Ops.push_back(DAG.getTargetConstant(T.Offset, getCurSDLoc(), MVT::i32)); | ||||||||||||
7112 | Ops.push_back(T.Target); | ||||||||||||
7113 | } | ||||||||||||
7114 | |||||||||||||
7115 | Ops.push_back(DAG.getRoot()); // Chain | ||||||||||||
7116 | SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, | ||||||||||||
7117 | getCurSDLoc(), MVT::Other, Ops), | ||||||||||||
7118 | 0); | ||||||||||||
7119 | DAG.setRoot(N); | ||||||||||||
7120 | setValue(&I, N); | ||||||||||||
7121 | HasTailCall = true; | ||||||||||||
7122 | return; | ||||||||||||
7123 | } | ||||||||||||
7124 | |||||||||||||
7125 | case Intrinsic::wasm_landingpad_index: | ||||||||||||
7126 | // Information this intrinsic contained has been transferred to | ||||||||||||
7127 | // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely | ||||||||||||
7128 | // delete it now. | ||||||||||||
7129 | return; | ||||||||||||
7130 | |||||||||||||
7131 | case Intrinsic::aarch64_settag: | ||||||||||||
7132 | case Intrinsic::aarch64_settag_zero: { | ||||||||||||
7133 | const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); | ||||||||||||
7134 | bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero; | ||||||||||||
7135 | SDValue Val = TSI.EmitTargetCodeForSetTag( | ||||||||||||
7136 | DAG, getCurSDLoc(), getRoot(), getValue(I.getArgOperand(0)), | ||||||||||||
7137 | getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)), | ||||||||||||
7138 | ZeroMemory); | ||||||||||||
7139 | DAG.setRoot(Val); | ||||||||||||
7140 | setValue(&I, Val); | ||||||||||||
7141 | return; | ||||||||||||
7142 | } | ||||||||||||
7143 | case Intrinsic::ptrmask: { | ||||||||||||
7144 | SDValue Ptr = getValue(I.getOperand(0)); | ||||||||||||
7145 | SDValue Const = getValue(I.getOperand(1)); | ||||||||||||
7146 | |||||||||||||
7147 | EVT PtrVT = Ptr.getValueType(); | ||||||||||||
7148 | setValue(&I, DAG.getNode(ISD::AND, getCurSDLoc(), PtrVT, Ptr, | ||||||||||||
7149 | DAG.getZExtOrTrunc(Const, getCurSDLoc(), PtrVT))); | ||||||||||||
7150 | return; | ||||||||||||
7151 | } | ||||||||||||
7152 | case Intrinsic::get_active_lane_mask: { | ||||||||||||
7153 | auto DL = getCurSDLoc(); | ||||||||||||
7154 | SDValue Index = getValue(I.getOperand(0)); | ||||||||||||
7155 | SDValue TripCount = getValue(I.getOperand(1)); | ||||||||||||
7156 | Type *ElementTy = I.getOperand(0)->getType(); | ||||||||||||
7157 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
7158 | unsigned VecWidth = VT.getVectorNumElements(); | ||||||||||||
7159 | |||||||||||||
7160 | SmallVector<SDValue, 16> OpsTripCount; | ||||||||||||
7161 | SmallVector<SDValue, 16> OpsIndex; | ||||||||||||
7162 | SmallVector<SDValue, 16> OpsStepConstants; | ||||||||||||
7163 | for (unsigned i = 0; i < VecWidth; i++) { | ||||||||||||
7164 | OpsTripCount.push_back(TripCount); | ||||||||||||
7165 | OpsIndex.push_back(Index); | ||||||||||||
7166 | OpsStepConstants.push_back( | ||||||||||||
7167 | DAG.getConstant(i, DL, EVT::getEVT(ElementTy))); | ||||||||||||
7168 | } | ||||||||||||
7169 | |||||||||||||
7170 | EVT CCVT = EVT::getVectorVT(I.getContext(), MVT::i1, VecWidth); | ||||||||||||
7171 | |||||||||||||
7172 | auto VecTy = EVT::getEVT(FixedVectorType::get(ElementTy, VecWidth)); | ||||||||||||
7173 | SDValue VectorIndex = DAG.getBuildVector(VecTy, DL, OpsIndex); | ||||||||||||
7174 | SDValue VectorStep = DAG.getBuildVector(VecTy, DL, OpsStepConstants); | ||||||||||||
7175 | SDValue VectorInduction = DAG.getNode( | ||||||||||||
7176 | ISD::UADDO, DL, DAG.getVTList(VecTy, CCVT), VectorIndex, VectorStep); | ||||||||||||
7177 | SDValue VectorTripCount = DAG.getBuildVector(VecTy, DL, OpsTripCount); | ||||||||||||
7178 | SDValue SetCC = DAG.getSetCC(DL, CCVT, VectorInduction.getValue(0), | ||||||||||||
7179 | VectorTripCount, ISD::CondCode::SETULT); | ||||||||||||
7180 | setValue(&I, DAG.getNode(ISD::AND, DL, CCVT, | ||||||||||||
7181 | DAG.getNOT(DL, VectorInduction.getValue(1), CCVT), | ||||||||||||
7182 | SetCC)); | ||||||||||||
7183 | return; | ||||||||||||
7184 | } | ||||||||||||
7185 | case Intrinsic::experimental_vector_insert: { | ||||||||||||
7186 | auto DL = getCurSDLoc(); | ||||||||||||
7187 | |||||||||||||
7188 | SDValue Vec = getValue(I.getOperand(0)); | ||||||||||||
7189 | SDValue SubVec = getValue(I.getOperand(1)); | ||||||||||||
7190 | SDValue Index = getValue(I.getOperand(2)); | ||||||||||||
7191 | |||||||||||||
7192 | // The intrinsic's index type is i64, but the SDNode requires an index type | ||||||||||||
7193 | // suitable for the target. Convert the index as required. | ||||||||||||
7194 | MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); | ||||||||||||
7195 | if (Index.getValueType() != VectorIdxTy) | ||||||||||||
7196 | Index = DAG.getVectorIdxConstant( | ||||||||||||
7197 | cast<ConstantSDNode>(Index)->getZExtValue(), DL); | ||||||||||||
7198 | |||||||||||||
7199 | EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
7200 | setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ResultVT, Vec, SubVec, | ||||||||||||
7201 | Index)); | ||||||||||||
7202 | return; | ||||||||||||
7203 | } | ||||||||||||
7204 | case Intrinsic::experimental_vector_extract: { | ||||||||||||
7205 | auto DL = getCurSDLoc(); | ||||||||||||
7206 | |||||||||||||
7207 | SDValue Vec = getValue(I.getOperand(0)); | ||||||||||||
7208 | SDValue Index = getValue(I.getOperand(1)); | ||||||||||||
7209 | EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
7210 | |||||||||||||
7211 | // The intrinsic's index type is i64, but the SDNode requires an index type | ||||||||||||
7212 | // suitable for the target. Convert the index as required. | ||||||||||||
7213 | MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); | ||||||||||||
7214 | if (Index.getValueType() != VectorIdxTy) | ||||||||||||
7215 | Index = DAG.getVectorIdxConstant( | ||||||||||||
7216 | cast<ConstantSDNode>(Index)->getZExtValue(), DL); | ||||||||||||
7217 | |||||||||||||
7218 | setValue(&I, DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResultVT, Vec, Index)); | ||||||||||||
7219 | return; | ||||||||||||
7220 | } | ||||||||||||
7221 | case Intrinsic::experimental_vector_reverse: | ||||||||||||
7222 | visitVectorReverse(I); | ||||||||||||
7223 | return; | ||||||||||||
7224 | case Intrinsic::experimental_vector_splice: | ||||||||||||
7225 | visitVectorSplice(I); | ||||||||||||
7226 | return; | ||||||||||||
7227 | } | ||||||||||||
7228 | } | ||||||||||||
7229 | |||||||||||||
7230 | void SelectionDAGBuilder::visitConstrainedFPIntrinsic( | ||||||||||||
7231 | const ConstrainedFPIntrinsic &FPI) { | ||||||||||||
7232 | SDLoc sdl = getCurSDLoc(); | ||||||||||||
7233 | |||||||||||||
7234 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
7235 | SmallVector<EVT, 4> ValueVTs; | ||||||||||||
7236 | ComputeValueVTs(TLI, DAG.getDataLayout(), FPI.getType(), ValueVTs); | ||||||||||||
7237 | ValueVTs.push_back(MVT::Other); // Out chain | ||||||||||||
7238 | |||||||||||||
7239 | // We do not need to serialize constrained FP intrinsics against | ||||||||||||
7240 | // each other or against (nonvolatile) loads, so they can be | ||||||||||||
7241 | // chained like loads. | ||||||||||||
7242 | SDValue Chain = DAG.getRoot(); | ||||||||||||
7243 | SmallVector<SDValue, 4> Opers; | ||||||||||||
7244 | Opers.push_back(Chain); | ||||||||||||
7245 | if (FPI.isUnaryOp()) { | ||||||||||||
7246 | Opers.push_back(getValue(FPI.getArgOperand(0))); | ||||||||||||
7247 | } else if (FPI.isTernaryOp()) { | ||||||||||||
7248 | Opers.push_back(getValue(FPI.getArgOperand(0))); | ||||||||||||
7249 | Opers.push_back(getValue(FPI.getArgOperand(1))); | ||||||||||||
7250 | Opers.push_back(getValue(FPI.getArgOperand(2))); | ||||||||||||
7251 | } else { | ||||||||||||
7252 | Opers.push_back(getValue(FPI.getArgOperand(0))); | ||||||||||||
7253 | Opers.push_back(getValue(FPI.getArgOperand(1))); | ||||||||||||
7254 | } | ||||||||||||
7255 | |||||||||||||
7256 | auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) { | ||||||||||||
7257 | assert(Result.getNode()->getNumValues() == 2)(static_cast <bool> (Result.getNode()->getNumValues( ) == 2) ? void (0) : __assert_fail ("Result.getNode()->getNumValues() == 2" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7257, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
7258 | |||||||||||||
7259 | // Push node to the appropriate list so that future instructions can be | ||||||||||||
7260 | // chained up correctly. | ||||||||||||
7261 | SDValue OutChain = Result.getValue(1); | ||||||||||||
7262 | switch (EB) { | ||||||||||||
7263 | case fp::ExceptionBehavior::ebIgnore: | ||||||||||||
7264 | // The only reason why ebIgnore nodes still need to be chained is that | ||||||||||||
7265 | // they might depend on the current rounding mode, and therefore must | ||||||||||||
7266 | // not be moved across instruction that may change that mode. | ||||||||||||
7267 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||||||||
7268 | case fp::ExceptionBehavior::ebMayTrap: | ||||||||||||
7269 | // These must not be moved across calls or instructions that may change | ||||||||||||
7270 | // floating-point exception masks. | ||||||||||||
7271 | PendingConstrainedFP.push_back(OutChain); | ||||||||||||
7272 | break; | ||||||||||||
7273 | case fp::ExceptionBehavior::ebStrict: | ||||||||||||
7274 | // These must not be moved across calls or instructions that may change | ||||||||||||
7275 | // floating-point exception masks or read floating-point exception flags. | ||||||||||||
7276 | // In addition, they cannot be optimized out even if unused. | ||||||||||||
7277 | PendingConstrainedFPStrict.push_back(OutChain); | ||||||||||||
7278 | break; | ||||||||||||
7279 | } | ||||||||||||
7280 | }; | ||||||||||||
7281 | |||||||||||||
7282 | SDVTList VTs = DAG.getVTList(ValueVTs); | ||||||||||||
7283 | fp::ExceptionBehavior EB = FPI.getExceptionBehavior().getValue(); | ||||||||||||
7284 | |||||||||||||
7285 | SDNodeFlags Flags; | ||||||||||||
7286 | if (EB == fp::ExceptionBehavior::ebIgnore) | ||||||||||||
7287 | Flags.setNoFPExcept(true); | ||||||||||||
7288 | |||||||||||||
7289 | if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI)) | ||||||||||||
7290 | Flags.copyFMF(*FPOp); | ||||||||||||
7291 | |||||||||||||
7292 | unsigned Opcode; | ||||||||||||
7293 | switch (FPI.getIntrinsicID()) { | ||||||||||||
7294 | default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7294); // Can't reach here. | ||||||||||||
7295 | #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ | ||||||||||||
7296 | case Intrinsic::INTRINSIC: \ | ||||||||||||
7297 | Opcode = ISD::STRICT_##DAGN; \ | ||||||||||||
7298 | break; | ||||||||||||
7299 | #include "llvm/IR/ConstrainedOps.def" | ||||||||||||
7300 | case Intrinsic::experimental_constrained_fmuladd: { | ||||||||||||
7301 | Opcode = ISD::STRICT_FMA; | ||||||||||||
7302 | // Break fmuladd into fmul and fadd. | ||||||||||||
7303 | if (TM.Options.AllowFPOpFusion == FPOpFusion::Strict || | ||||||||||||
7304 | !TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), | ||||||||||||
7305 | ValueVTs[0])) { | ||||||||||||
7306 | Opers.pop_back(); | ||||||||||||
7307 | SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags); | ||||||||||||
7308 | pushOutChain(Mul, EB); | ||||||||||||
7309 | Opcode = ISD::STRICT_FADD; | ||||||||||||
7310 | Opers.clear(); | ||||||||||||
7311 | Opers.push_back(Mul.getValue(1)); | ||||||||||||
7312 | Opers.push_back(Mul.getValue(0)); | ||||||||||||
7313 | Opers.push_back(getValue(FPI.getArgOperand(2))); | ||||||||||||
7314 | } | ||||||||||||
7315 | break; | ||||||||||||
7316 | } | ||||||||||||
7317 | } | ||||||||||||
7318 | |||||||||||||
7319 | // A few strict DAG nodes carry additional operands that are not | ||||||||||||
7320 | // set up by the default code above. | ||||||||||||
7321 | switch (Opcode) { | ||||||||||||
7322 | default: break; | ||||||||||||
7323 | case ISD::STRICT_FP_ROUND: | ||||||||||||
7324 | Opers.push_back( | ||||||||||||
7325 | DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||||||
7326 | break; | ||||||||||||
7327 | case ISD::STRICT_FSETCC: | ||||||||||||
7328 | case ISD::STRICT_FSETCCS: { | ||||||||||||
7329 | auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI); | ||||||||||||
7330 | ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate()); | ||||||||||||
7331 | if (TM.Options.NoNaNsFPMath) | ||||||||||||
7332 | Condition = getFCmpCodeWithoutNaN(Condition); | ||||||||||||
7333 | Opers.push_back(DAG.getCondCode(Condition)); | ||||||||||||
7334 | break; | ||||||||||||
7335 | } | ||||||||||||
7336 | } | ||||||||||||
7337 | |||||||||||||
7338 | SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags); | ||||||||||||
7339 | pushOutChain(Result, EB); | ||||||||||||
7340 | |||||||||||||
7341 | SDValue FPResult = Result.getValue(0); | ||||||||||||
7342 | setValue(&FPI, FPResult); | ||||||||||||
7343 | } | ||||||||||||
7344 | |||||||||||||
7345 | static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) { | ||||||||||||
7346 | Optional<unsigned> ResOPC; | ||||||||||||
7347 | switch (VPIntrin.getIntrinsicID()) { | ||||||||||||
7348 | #define BEGIN_REGISTER_VP_INTRINSIC(INTRIN, ...) case Intrinsic::INTRIN: | ||||||||||||
7349 | #define BEGIN_REGISTER_VP_SDNODE(VPSDID, ...) ResOPC = ISD::VPSDID; | ||||||||||||
7350 | #define END_REGISTER_VP_INTRINSIC(...) break; | ||||||||||||
7351 | #include "llvm/IR/VPIntrinsics.def" | ||||||||||||
7352 | } | ||||||||||||
7353 | |||||||||||||
7354 | if (!ResOPC.hasValue()) | ||||||||||||
7355 | llvm_unreachable(::llvm::llvm_unreachable_internal("Inconsistency: no SDNode available for this VPIntrinsic!" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7356) | ||||||||||||
7356 | "Inconsistency: no SDNode available for this VPIntrinsic!")::llvm::llvm_unreachable_internal("Inconsistency: no SDNode available for this VPIntrinsic!" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7356); | ||||||||||||
7357 | |||||||||||||
7358 | if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD || | ||||||||||||
7359 | *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) { | ||||||||||||
7360 | if (VPIntrin.getFastMathFlags().allowReassoc()) | ||||||||||||
7361 | return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD | ||||||||||||
7362 | : ISD::VP_REDUCE_FMUL; | ||||||||||||
7363 | } | ||||||||||||
7364 | |||||||||||||
7365 | return ResOPC.getValue(); | ||||||||||||
7366 | } | ||||||||||||
7367 | |||||||||||||
7368 | void SelectionDAGBuilder::visitVectorPredicationIntrinsic( | ||||||||||||
7369 | const VPIntrinsic &VPIntrin) { | ||||||||||||
7370 | SDLoc DL = getCurSDLoc(); | ||||||||||||
7371 | unsigned Opcode = getISDForVPIntrinsic(VPIntrin); | ||||||||||||
7372 | |||||||||||||
7373 | SmallVector<EVT, 4> ValueVTs; | ||||||||||||
7374 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
7375 | ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs); | ||||||||||||
7376 | SDVTList VTs = DAG.getVTList(ValueVTs); | ||||||||||||
7377 | |||||||||||||
7378 | auto EVLParamPos = | ||||||||||||
7379 | VPIntrinsic::getVectorLengthParamPos(VPIntrin.getIntrinsicID()); | ||||||||||||
7380 | |||||||||||||
7381 | MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy(); | ||||||||||||
7382 | assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&(static_cast <bool> (EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && "Unexpected target EVL type" ) ? void (0) : __assert_fail ("EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && \"Unexpected target EVL type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7383, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
7383 | "Unexpected target EVL type")(static_cast <bool> (EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && "Unexpected target EVL type" ) ? void (0) : __assert_fail ("EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && \"Unexpected target EVL type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7383, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
7384 | |||||||||||||
7385 | // Request operands. | ||||||||||||
7386 | SmallVector<SDValue, 7> OpValues; | ||||||||||||
7387 | for (unsigned I = 0; I < VPIntrin.getNumArgOperands(); ++I) { | ||||||||||||
7388 | auto Op = getValue(VPIntrin.getArgOperand(I)); | ||||||||||||
7389 | if (I == EVLParamPos) | ||||||||||||
7390 | Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op); | ||||||||||||
7391 | OpValues.push_back(Op); | ||||||||||||
7392 | } | ||||||||||||
7393 | |||||||||||||
7394 | SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues); | ||||||||||||
7395 | setValue(&VPIntrin, Result); | ||||||||||||
7396 | } | ||||||||||||
7397 | |||||||||||||
7398 | SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain, | ||||||||||||
7399 | const BasicBlock *EHPadBB, | ||||||||||||
7400 | MCSymbol *&BeginLabel) { | ||||||||||||
7401 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
7402 | MachineModuleInfo &MMI = MF.getMMI(); | ||||||||||||
7403 | |||||||||||||
7404 | // Insert a label before the invoke call to mark the try range. This can be | ||||||||||||
7405 | // used to detect deletion of the invoke via the MachineModuleInfo. | ||||||||||||
7406 | BeginLabel = MMI.getContext().createTempSymbol(); | ||||||||||||
7407 | |||||||||||||
7408 | // For SjLj, keep track of which landing pads go with which invokes | ||||||||||||
7409 | // so as to maintain the ordering of pads in the LSDA. | ||||||||||||
7410 | unsigned CallSiteIndex = MMI.getCurrentCallSite(); | ||||||||||||
7411 | if (CallSiteIndex) { | ||||||||||||
7412 | MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex); | ||||||||||||
7413 | LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex); | ||||||||||||
7414 | |||||||||||||
7415 | // Now that the call site is handled, stop tracking it. | ||||||||||||
7416 | MMI.setCurrentCallSite(0); | ||||||||||||
7417 | } | ||||||||||||
7418 | |||||||||||||
7419 | return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel); | ||||||||||||
7420 | } | ||||||||||||
7421 | |||||||||||||
7422 | SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II, | ||||||||||||
7423 | const BasicBlock *EHPadBB, | ||||||||||||
7424 | MCSymbol *BeginLabel) { | ||||||||||||
7425 | assert(BeginLabel && "BeginLabel should've been set")(static_cast <bool> (BeginLabel && "BeginLabel should've been set" ) ? void (0) : __assert_fail ("BeginLabel && \"BeginLabel should've been set\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7425, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
7426 | |||||||||||||
7427 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
7428 | MachineModuleInfo &MMI = MF.getMMI(); | ||||||||||||
7429 | |||||||||||||
7430 | // Insert a label at the end of the invoke call to mark the try range. This | ||||||||||||
7431 | // can be used to detect deletion of the invoke via the MachineModuleInfo. | ||||||||||||
7432 | MCSymbol *EndLabel = MMI.getContext().createTempSymbol(); | ||||||||||||
7433 | Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel); | ||||||||||||
7434 | |||||||||||||
7435 | // Inform MachineModuleInfo of range. | ||||||||||||
7436 | auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); | ||||||||||||
7437 | // There is a platform (e.g. wasm) that uses funclet style IR but does not | ||||||||||||
7438 | // actually use outlined funclets and their LSDA info style. | ||||||||||||
7439 | if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) { | ||||||||||||
7440 | assert(II && "II should've been set")(static_cast <bool> (II && "II should've been set" ) ? void (0) : __assert_fail ("II && \"II should've been set\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7440, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
7441 | WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo(); | ||||||||||||
7442 | EHInfo->addIPToStateRange(II, BeginLabel, EndLabel); | ||||||||||||
7443 | } else if (!isScopedEHPersonality(Pers)) { | ||||||||||||
7444 | assert(EHPadBB)(static_cast <bool> (EHPadBB) ? void (0) : __assert_fail ("EHPadBB", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7444, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
7445 | MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel); | ||||||||||||
7446 | } | ||||||||||||
7447 | |||||||||||||
7448 | return Chain; | ||||||||||||
7449 | } | ||||||||||||
7450 | |||||||||||||
7451 | std::pair<SDValue, SDValue> | ||||||||||||
7452 | SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI, | ||||||||||||
7453 | const BasicBlock *EHPadBB) { | ||||||||||||
7454 | MCSymbol *BeginLabel = nullptr; | ||||||||||||
7455 | |||||||||||||
7456 | if (EHPadBB) { | ||||||||||||
7457 | // Both PendingLoads and PendingExports must be flushed here; | ||||||||||||
7458 | // this call might not return. | ||||||||||||
7459 | (void)getRoot(); | ||||||||||||
7460 | DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel)); | ||||||||||||
7461 | CLI.setChain(getRoot()); | ||||||||||||
7462 | } | ||||||||||||
7463 | |||||||||||||
7464 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
7465 | std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); | ||||||||||||
7466 | |||||||||||||
7467 | assert((CLI.IsTailCall || Result.second.getNode()) &&(static_cast <bool> ((CLI.IsTailCall || Result.second.getNode ()) && "Non-null chain expected with non-tail call!") ? void (0) : __assert_fail ("(CLI.IsTailCall || Result.second.getNode()) && \"Non-null chain expected with non-tail call!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7468, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
7468 | "Non-null chain expected with non-tail call!")(static_cast <bool> ((CLI.IsTailCall || Result.second.getNode ()) && "Non-null chain expected with non-tail call!") ? void (0) : __assert_fail ("(CLI.IsTailCall || Result.second.getNode()) && \"Non-null chain expected with non-tail call!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7468, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
7469 | assert((Result.second.getNode() || !Result.first.getNode()) &&(static_cast <bool> ((Result.second.getNode() || !Result .first.getNode()) && "Null value expected with tail call!" ) ? void (0) : __assert_fail ("(Result.second.getNode() || !Result.first.getNode()) && \"Null value expected with tail call!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7470, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
7470 | "Null value expected with tail call!")(static_cast <bool> ((Result.second.getNode() || !Result .first.getNode()) && "Null value expected with tail call!" ) ? void (0) : __assert_fail ("(Result.second.getNode() || !Result.first.getNode()) && \"Null value expected with tail call!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7470, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
7471 | |||||||||||||
7472 | if (!Result.second.getNode()) { | ||||||||||||
7473 | // As a special case, a null chain means that a tail call has been emitted | ||||||||||||
7474 | // and the DAG root is already updated. | ||||||||||||
7475 | HasTailCall = true; | ||||||||||||
7476 | |||||||||||||
7477 | // Since there's no actual continuation from this block, nothing can be | ||||||||||||
7478 | // relying on us setting vregs for them. | ||||||||||||
7479 | PendingExports.clear(); | ||||||||||||
7480 | } else { | ||||||||||||
7481 | DAG.setRoot(Result.second); | ||||||||||||
7482 | } | ||||||||||||
7483 | |||||||||||||
7484 | if (EHPadBB) { | ||||||||||||
7485 | DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB, | ||||||||||||
7486 | BeginLabel)); | ||||||||||||
7487 | } | ||||||||||||
7488 | |||||||||||||
7489 | return Result; | ||||||||||||
7490 | } | ||||||||||||
7491 | |||||||||||||
7492 | void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, | ||||||||||||
7493 | bool isTailCall, | ||||||||||||
7494 | bool isMustTailCall, | ||||||||||||
7495 | const BasicBlock *EHPadBB) { | ||||||||||||
7496 | auto &DL = DAG.getDataLayout(); | ||||||||||||
7497 | FunctionType *FTy = CB.getFunctionType(); | ||||||||||||
7498 | Type *RetTy = CB.getType(); | ||||||||||||
7499 | |||||||||||||
7500 | TargetLowering::ArgListTy Args; | ||||||||||||
7501 | Args.reserve(CB.arg_size()); | ||||||||||||
7502 | |||||||||||||
7503 | const Value *SwiftErrorVal = nullptr; | ||||||||||||
7504 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
7505 | |||||||||||||
7506 | if (isTailCall) { | ||||||||||||
7507 | // Avoid emitting tail calls in functions with the disable-tail-calls | ||||||||||||
7508 | // attribute. | ||||||||||||
7509 | auto *Caller = CB.getParent()->getParent(); | ||||||||||||
7510 | if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() == | ||||||||||||
7511 | "true" && !isMustTailCall) | ||||||||||||
7512 | isTailCall = false; | ||||||||||||
7513 | |||||||||||||
7514 | // We can't tail call inside a function with a swifterror argument. Lowering | ||||||||||||
7515 | // does not support this yet. It would have to move into the swifterror | ||||||||||||
7516 | // register before the call. | ||||||||||||
7517 | if (TLI.supportSwiftError() && | ||||||||||||
7518 | Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) | ||||||||||||
7519 | isTailCall = false; | ||||||||||||
7520 | } | ||||||||||||
7521 | |||||||||||||
7522 | for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) { | ||||||||||||
7523 | TargetLowering::ArgListEntry Entry; | ||||||||||||
7524 | const Value *V = *I; | ||||||||||||
7525 | |||||||||||||
7526 | // Skip empty types | ||||||||||||
7527 | if (V->getType()->isEmptyTy()) | ||||||||||||
7528 | continue; | ||||||||||||
7529 | |||||||||||||
7530 | SDValue ArgNode = getValue(V); | ||||||||||||
7531 | Entry.Node = ArgNode; Entry.Ty = V->getType(); | ||||||||||||
7532 | |||||||||||||
7533 | Entry.setAttributes(&CB, I - CB.arg_begin()); | ||||||||||||
7534 | |||||||||||||
7535 | // Use swifterror virtual register as input to the call. | ||||||||||||
7536 | if (Entry.IsSwiftError && TLI.supportSwiftError()) { | ||||||||||||
7537 | SwiftErrorVal = V; | ||||||||||||
7538 | // We find the virtual register for the actual swifterror argument. | ||||||||||||
7539 | // Instead of using the Value, we use the virtual register instead. | ||||||||||||
7540 | Entry.Node = | ||||||||||||
7541 | DAG.getRegister(SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V), | ||||||||||||
7542 | EVT(TLI.getPointerTy(DL))); | ||||||||||||
7543 | } | ||||||||||||
7544 | |||||||||||||
7545 | Args.push_back(Entry); | ||||||||||||
7546 | |||||||||||||
7547 | // If we have an explicit sret argument that is an Instruction, (i.e., it | ||||||||||||
7548 | // might point to function-local memory), we can't meaningfully tail-call. | ||||||||||||
7549 | if (Entry.IsSRet && isa<Instruction>(V)) | ||||||||||||
7550 | isTailCall = false; | ||||||||||||
7551 | } | ||||||||||||
7552 | |||||||||||||
7553 | // If call site has a cfguardtarget operand bundle, create and add an | ||||||||||||
7554 | // additional ArgListEntry. | ||||||||||||
7555 | if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) { | ||||||||||||
7556 | TargetLowering::ArgListEntry Entry; | ||||||||||||
7557 | Value *V = Bundle->Inputs[0]; | ||||||||||||
7558 | SDValue ArgNode = getValue(V); | ||||||||||||
7559 | Entry.Node = ArgNode; | ||||||||||||
7560 | Entry.Ty = V->getType(); | ||||||||||||
7561 | Entry.IsCFGuardTarget = true; | ||||||||||||
7562 | Args.push_back(Entry); | ||||||||||||
7563 | } | ||||||||||||
7564 | |||||||||||||
7565 | // Check if target-independent constraints permit a tail call here. | ||||||||||||
7566 | // Target-dependent constraints are checked within TLI->LowerCallTo. | ||||||||||||
7567 | if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget())) | ||||||||||||
7568 | isTailCall = false; | ||||||||||||
7569 | |||||||||||||
7570 | // Disable tail calls if there is an swifterror argument. Targets have not | ||||||||||||
7571 | // been updated to support tail calls. | ||||||||||||
7572 | if (TLI.supportSwiftError() && SwiftErrorVal) | ||||||||||||
7573 | isTailCall = false; | ||||||||||||
7574 | |||||||||||||
7575 | TargetLowering::CallLoweringInfo CLI(DAG); | ||||||||||||
7576 | CLI.setDebugLoc(getCurSDLoc()) | ||||||||||||
7577 | .setChain(getRoot()) | ||||||||||||
7578 | .setCallee(RetTy, FTy, Callee, std::move(Args), CB) | ||||||||||||
7579 | .setTailCall(isTailCall) | ||||||||||||
7580 | .setConvergent(CB.isConvergent()) | ||||||||||||
7581 | .setIsPreallocated( | ||||||||||||
7582 | CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0); | ||||||||||||
7583 | std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB); | ||||||||||||
7584 | |||||||||||||
7585 | if (Result.first.getNode()) { | ||||||||||||
7586 | Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first); | ||||||||||||
7587 | setValue(&CB, Result.first); | ||||||||||||
7588 | } | ||||||||||||
7589 | |||||||||||||
7590 | // The last element of CLI.InVals has the SDValue for swifterror return. | ||||||||||||
7591 | // Here we copy it to a virtual register and update SwiftErrorMap for | ||||||||||||
7592 | // book-keeping. | ||||||||||||
7593 | if (SwiftErrorVal && TLI.supportSwiftError()) { | ||||||||||||
7594 | // Get the last element of InVals. | ||||||||||||
7595 | SDValue Src = CLI.InVals.back(); | ||||||||||||
7596 | Register VReg = | ||||||||||||
7597 | SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal); | ||||||||||||
7598 | SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src); | ||||||||||||
7599 | DAG.setRoot(CopyNode); | ||||||||||||
7600 | } | ||||||||||||
7601 | } | ||||||||||||
7602 | |||||||||||||
7603 | static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, | ||||||||||||
7604 | SelectionDAGBuilder &Builder) { | ||||||||||||
7605 | // Check to see if this load can be trivially constant folded, e.g. if the | ||||||||||||
7606 | // input is from a string literal. | ||||||||||||
7607 | if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) { | ||||||||||||
7608 | // Cast pointer to the type we really want to load. | ||||||||||||
7609 | Type *LoadTy = | ||||||||||||
7610 | Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits()); | ||||||||||||
7611 | if (LoadVT.isVector()) | ||||||||||||
7612 | LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements()); | ||||||||||||
7613 | |||||||||||||
7614 | LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput), | ||||||||||||
7615 | PointerType::getUnqual(LoadTy)); | ||||||||||||
7616 | |||||||||||||
7617 | if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr( | ||||||||||||
7618 | const_cast<Constant *>(LoadInput), LoadTy, *Builder.DL)) | ||||||||||||
7619 | return Builder.getValue(LoadCst); | ||||||||||||
7620 | } | ||||||||||||
7621 | |||||||||||||
7622 | // Otherwise, we have to emit the load. If the pointer is to unfoldable but | ||||||||||||
7623 | // still constant memory, the input chain can be the entry node. | ||||||||||||
7624 | SDValue Root; | ||||||||||||
7625 | bool ConstantMemory = false; | ||||||||||||
7626 | |||||||||||||
7627 | // Do not serialize (non-volatile) loads of constant memory with anything. | ||||||||||||
7628 | if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) { | ||||||||||||
7629 | Root = Builder.DAG.getEntryNode(); | ||||||||||||
7630 | ConstantMemory = true; | ||||||||||||
7631 | } else { | ||||||||||||
7632 | // Do not serialize non-volatile loads against each other. | ||||||||||||
7633 | Root = Builder.DAG.getRoot(); | ||||||||||||
7634 | } | ||||||||||||
7635 | |||||||||||||
7636 | SDValue Ptr = Builder.getValue(PtrVal); | ||||||||||||
7637 | SDValue LoadVal = | ||||||||||||
7638 | Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr, | ||||||||||||
7639 | MachinePointerInfo(PtrVal), Align(1)); | ||||||||||||
7640 | |||||||||||||
7641 | if (!ConstantMemory) | ||||||||||||
7642 | Builder.PendingLoads.push_back(LoadVal.getValue(1)); | ||||||||||||
7643 | return LoadVal; | ||||||||||||
7644 | } | ||||||||||||
7645 | |||||||||||||
7646 | /// Record the value for an instruction that produces an integer result, | ||||||||||||
7647 | /// converting the type where necessary. | ||||||||||||
7648 | void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I, | ||||||||||||
7649 | SDValue Value, | ||||||||||||
7650 | bool IsSigned) { | ||||||||||||
7651 | EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||||||
7652 | I.getType(), true); | ||||||||||||
7653 | if (IsSigned) | ||||||||||||
7654 | Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT); | ||||||||||||
7655 | else | ||||||||||||
7656 | Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT); | ||||||||||||
7657 | setValue(&I, Value); | ||||||||||||
7658 | } | ||||||||||||
7659 | |||||||||||||
7660 | /// See if we can lower a memcmp/bcmp call into an optimized form. If so, return | ||||||||||||
7661 | /// true and lower it. Otherwise return false, and it will be lowered like a | ||||||||||||
7662 | /// normal call. | ||||||||||||
7663 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||||||
7664 | /// correct prototype. | ||||||||||||
7665 | bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) { | ||||||||||||
7666 | const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1); | ||||||||||||
7667 | const Value *Size = I.getArgOperand(2); | ||||||||||||
7668 | const ConstantInt *CSize = dyn_cast<ConstantInt>(Size); | ||||||||||||
7669 | if (CSize && CSize->getZExtValue() == 0) { | ||||||||||||
7670 | EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||||||
7671 | I.getType(), true); | ||||||||||||
7672 | setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT)); | ||||||||||||
7673 | return true; | ||||||||||||
7674 | } | ||||||||||||
7675 | |||||||||||||
7676 | const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); | ||||||||||||
7677 | std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp( | ||||||||||||
7678 | DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS), | ||||||||||||
7679 | getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS)); | ||||||||||||
7680 | if (Res.first.getNode()) { | ||||||||||||
7681 | processIntegerCallValue(I, Res.first, true); | ||||||||||||
7682 | PendingLoads.push_back(Res.second); | ||||||||||||
7683 | return true; | ||||||||||||
7684 | } | ||||||||||||
7685 | |||||||||||||
7686 | // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0 | ||||||||||||
7687 | // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0 | ||||||||||||
7688 | if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I)) | ||||||||||||
7689 | return false; | ||||||||||||
7690 | |||||||||||||
7691 | // If the target has a fast compare for the given size, it will return a | ||||||||||||
7692 | // preferred load type for that size. Require that the load VT is legal and | ||||||||||||
7693 | // that the target supports unaligned loads of that type. Otherwise, return | ||||||||||||
7694 | // INVALID. | ||||||||||||
7695 | auto hasFastLoadsAndCompare = [&](unsigned NumBits) { | ||||||||||||
7696 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
7697 | MVT LVT = TLI.hasFastEqualityCompare(NumBits); | ||||||||||||
7698 | if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) { | ||||||||||||
7699 | // TODO: Handle 5 byte compare as 4-byte + 1 byte. | ||||||||||||
7700 | // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads. | ||||||||||||
7701 | // TODO: Check alignment of src and dest ptrs. | ||||||||||||
7702 | unsigned DstAS = LHS->getType()->getPointerAddressSpace(); | ||||||||||||
7703 | unsigned SrcAS = RHS->getType()->getPointerAddressSpace(); | ||||||||||||
7704 | if (!TLI.isTypeLegal(LVT) || | ||||||||||||
7705 | !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) || | ||||||||||||
7706 | !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS)) | ||||||||||||
7707 | LVT = MVT::INVALID_SIMPLE_VALUE_TYPE; | ||||||||||||
7708 | } | ||||||||||||
7709 | |||||||||||||
7710 | return LVT; | ||||||||||||
7711 | }; | ||||||||||||
7712 | |||||||||||||
7713 | // This turns into unaligned loads. We only do this if the target natively | ||||||||||||
7714 | // supports the MVT we'll be loading or if it is small enough (<= 4) that | ||||||||||||
7715 | // we'll only produce a small number of byte loads. | ||||||||||||
7716 | MVT LoadVT; | ||||||||||||
7717 | unsigned NumBitsToCompare = CSize->getZExtValue() * 8; | ||||||||||||
7718 | switch (NumBitsToCompare) { | ||||||||||||
7719 | default: | ||||||||||||
7720 | return false; | ||||||||||||
7721 | case 16: | ||||||||||||
7722 | LoadVT = MVT::i16; | ||||||||||||
7723 | break; | ||||||||||||
7724 | case 32: | ||||||||||||
7725 | LoadVT = MVT::i32; | ||||||||||||
7726 | break; | ||||||||||||
7727 | case 64: | ||||||||||||
7728 | case 128: | ||||||||||||
7729 | case 256: | ||||||||||||
7730 | LoadVT = hasFastLoadsAndCompare(NumBitsToCompare); | ||||||||||||
7731 | break; | ||||||||||||
7732 | } | ||||||||||||
7733 | |||||||||||||
7734 | if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE) | ||||||||||||
7735 | return false; | ||||||||||||
7736 | |||||||||||||
7737 | SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this); | ||||||||||||
7738 | SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this); | ||||||||||||
7739 | |||||||||||||
7740 | // Bitcast to a wide integer type if the loads are vectors. | ||||||||||||
7741 | if (LoadVT.isVector()) { | ||||||||||||
7742 | EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits()); | ||||||||||||
7743 | LoadL = DAG.getBitcast(CmpVT, LoadL); | ||||||||||||
7744 | LoadR = DAG.getBitcast(CmpVT, LoadR); | ||||||||||||
7745 | } | ||||||||||||
7746 | |||||||||||||
7747 | SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE); | ||||||||||||
7748 | processIntegerCallValue(I, Cmp, false); | ||||||||||||
7749 | return true; | ||||||||||||
7750 | } | ||||||||||||
7751 | |||||||||||||
7752 | /// See if we can lower a memchr call into an optimized form. If so, return | ||||||||||||
7753 | /// true and lower it. Otherwise return false, and it will be lowered like a | ||||||||||||
7754 | /// normal call. | ||||||||||||
7755 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||||||
7756 | /// correct prototype. | ||||||||||||
7757 | bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) { | ||||||||||||
7758 | const Value *Src = I.getArgOperand(0); | ||||||||||||
7759 | const Value *Char = I.getArgOperand(1); | ||||||||||||
7760 | const Value *Length = I.getArgOperand(2); | ||||||||||||
7761 | |||||||||||||
7762 | const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); | ||||||||||||
7763 | std::pair<SDValue, SDValue> Res = | ||||||||||||
7764 | TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(), | ||||||||||||
7765 | getValue(Src), getValue(Char), getValue(Length), | ||||||||||||
7766 | MachinePointerInfo(Src)); | ||||||||||||
7767 | if (Res.first.getNode()) { | ||||||||||||
7768 | setValue(&I, Res.first); | ||||||||||||
7769 | PendingLoads.push_back(Res.second); | ||||||||||||
7770 | return true; | ||||||||||||
7771 | } | ||||||||||||
7772 | |||||||||||||
7773 | return false; | ||||||||||||
7774 | } | ||||||||||||
7775 | |||||||||||||
7776 | /// See if we can lower a mempcpy call into an optimized form. If so, return | ||||||||||||
7777 | /// true and lower it. Otherwise return false, and it will be lowered like a | ||||||||||||
7778 | /// normal call. | ||||||||||||
7779 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||||||
7780 | /// correct prototype. | ||||||||||||
7781 | bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) { | ||||||||||||
7782 | SDValue Dst = getValue(I.getArgOperand(0)); | ||||||||||||
7783 | SDValue Src = getValue(I.getArgOperand(1)); | ||||||||||||
7784 | SDValue Size = getValue(I.getArgOperand(2)); | ||||||||||||
7785 | |||||||||||||
7786 | Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne(); | ||||||||||||
7787 | Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne(); | ||||||||||||
7788 | // DAG::getMemcpy needs Alignment to be defined. | ||||||||||||
7789 | Align Alignment = std::min(DstAlign, SrcAlign); | ||||||||||||
7790 | |||||||||||||
7791 | bool isVol = false; | ||||||||||||
7792 | SDLoc sdl = getCurSDLoc(); | ||||||||||||
7793 | |||||||||||||
7794 | // In the mempcpy context we need to pass in a false value for isTailCall | ||||||||||||
7795 | // because the return pointer needs to be adjusted by the size of | ||||||||||||
7796 | // the copied memory. | ||||||||||||
7797 | SDValue Root = isVol ? getRoot() : getMemoryRoot(); | ||||||||||||
7798 | AAMDNodes AAInfo; | ||||||||||||
7799 | I.getAAMetadata(AAInfo); | ||||||||||||
7800 | SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, isVol, false, | ||||||||||||
7801 | /*isTailCall=*/false, | ||||||||||||
7802 | MachinePointerInfo(I.getArgOperand(0)), | ||||||||||||
7803 | MachinePointerInfo(I.getArgOperand(1)), AAInfo); | ||||||||||||
7804 | assert(MC.getNode() != nullptr &&(static_cast <bool> (MC.getNode() != nullptr && "** memcpy should not be lowered as TailCall in mempcpy context **" ) ? void (0) : __assert_fail ("MC.getNode() != nullptr && \"** memcpy should not be lowered as TailCall in mempcpy context **\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7805, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
7805 | "** memcpy should not be lowered as TailCall in mempcpy context **")(static_cast <bool> (MC.getNode() != nullptr && "** memcpy should not be lowered as TailCall in mempcpy context **" ) ? void (0) : __assert_fail ("MC.getNode() != nullptr && \"** memcpy should not be lowered as TailCall in mempcpy context **\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7805, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
7806 | DAG.setRoot(MC); | ||||||||||||
7807 | |||||||||||||
7808 | // Check if Size needs to be truncated or extended. | ||||||||||||
7809 | Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType()); | ||||||||||||
7810 | |||||||||||||
7811 | // Adjust return pointer to point just past the last dst byte. | ||||||||||||
7812 | SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(), | ||||||||||||
7813 | Dst, Size); | ||||||||||||
7814 | setValue(&I, DstPlusSize); | ||||||||||||
7815 | return true; | ||||||||||||
7816 | } | ||||||||||||
7817 | |||||||||||||
7818 | /// See if we can lower a strcpy call into an optimized form. If so, return | ||||||||||||
7819 | /// true and lower it, otherwise return false and it will be lowered like a | ||||||||||||
7820 | /// normal call. | ||||||||||||
7821 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||||||
7822 | /// correct prototype. | ||||||||||||
7823 | bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) { | ||||||||||||
7824 | const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); | ||||||||||||
7825 | |||||||||||||
7826 | const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); | ||||||||||||
7827 | std::pair<SDValue, SDValue> Res = | ||||||||||||
7828 | TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(), | ||||||||||||
7829 | getValue(Arg0), getValue(Arg1), | ||||||||||||
7830 | MachinePointerInfo(Arg0), | ||||||||||||
7831 | MachinePointerInfo(Arg1), isStpcpy); | ||||||||||||
7832 | if (Res.first.getNode()) { | ||||||||||||
7833 | setValue(&I, Res.first); | ||||||||||||
7834 | DAG.setRoot(Res.second); | ||||||||||||
7835 | return true; | ||||||||||||
7836 | } | ||||||||||||
7837 | |||||||||||||
7838 | return false; | ||||||||||||
7839 | } | ||||||||||||
7840 | |||||||||||||
7841 | /// See if we can lower a strcmp call into an optimized form. If so, return | ||||||||||||
7842 | /// true and lower it, otherwise return false and it will be lowered like a | ||||||||||||
7843 | /// normal call. | ||||||||||||
7844 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||||||
7845 | /// correct prototype. | ||||||||||||
7846 | bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) { | ||||||||||||
7847 | const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); | ||||||||||||
7848 | |||||||||||||
7849 | const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); | ||||||||||||
7850 | std::pair<SDValue, SDValue> Res = | ||||||||||||
7851 | TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(), | ||||||||||||
7852 | getValue(Arg0), getValue(Arg1), | ||||||||||||
7853 | MachinePointerInfo(Arg0), | ||||||||||||
7854 | MachinePointerInfo(Arg1)); | ||||||||||||
7855 | if (Res.first.getNode()) { | ||||||||||||
7856 | processIntegerCallValue(I, Res.first, true); | ||||||||||||
7857 | PendingLoads.push_back(Res.second); | ||||||||||||
7858 | return true; | ||||||||||||
7859 | } | ||||||||||||
7860 | |||||||||||||
7861 | return false; | ||||||||||||
7862 | } | ||||||||||||
7863 | |||||||||||||
7864 | /// See if we can lower a strlen call into an optimized form. If so, return | ||||||||||||
7865 | /// true and lower it, otherwise return false and it will be lowered like a | ||||||||||||
7866 | /// normal call. | ||||||||||||
7867 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||||||
7868 | /// correct prototype. | ||||||||||||
7869 | bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) { | ||||||||||||
7870 | const Value *Arg0 = I.getArgOperand(0); | ||||||||||||
7871 | |||||||||||||
7872 | const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); | ||||||||||||
7873 | std::pair<SDValue, SDValue> Res = | ||||||||||||
7874 | TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(), | ||||||||||||
7875 | getValue(Arg0), MachinePointerInfo(Arg0)); | ||||||||||||
7876 | if (Res.first.getNode()) { | ||||||||||||
7877 | processIntegerCallValue(I, Res.first, false); | ||||||||||||
7878 | PendingLoads.push_back(Res.second); | ||||||||||||
7879 | return true; | ||||||||||||
7880 | } | ||||||||||||
7881 | |||||||||||||
7882 | return false; | ||||||||||||
7883 | } | ||||||||||||
7884 | |||||||||||||
7885 | /// See if we can lower a strnlen call into an optimized form. If so, return | ||||||||||||
7886 | /// true and lower it, otherwise return false and it will be lowered like a | ||||||||||||
7887 | /// normal call. | ||||||||||||
7888 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||||||
7889 | /// correct prototype. | ||||||||||||
7890 | bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) { | ||||||||||||
7891 | const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); | ||||||||||||
7892 | |||||||||||||
7893 | const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); | ||||||||||||
7894 | std::pair<SDValue, SDValue> Res = | ||||||||||||
7895 | TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(), | ||||||||||||
7896 | getValue(Arg0), getValue(Arg1), | ||||||||||||
7897 | MachinePointerInfo(Arg0)); | ||||||||||||
7898 | if (Res.first.getNode()) { | ||||||||||||
7899 | processIntegerCallValue(I, Res.first, false); | ||||||||||||
7900 | PendingLoads.push_back(Res.second); | ||||||||||||
7901 | return true; | ||||||||||||
7902 | } | ||||||||||||
7903 | |||||||||||||
7904 | return false; | ||||||||||||
7905 | } | ||||||||||||
7906 | |||||||||||||
7907 | /// See if we can lower a unary floating-point operation into an SDNode with | ||||||||||||
7908 | /// the specified Opcode. If so, return true and lower it, otherwise return | ||||||||||||
7909 | /// false and it will be lowered like a normal call. | ||||||||||||
7910 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||||||
7911 | /// correct prototype. | ||||||||||||
7912 | bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I, | ||||||||||||
7913 | unsigned Opcode) { | ||||||||||||
7914 | // We already checked this call's prototype; verify it doesn't modify errno. | ||||||||||||
7915 | if (!I.onlyReadsMemory()) | ||||||||||||
7916 | return false; | ||||||||||||
7917 | |||||||||||||
7918 | SDNodeFlags Flags; | ||||||||||||
7919 | Flags.copyFMF(cast<FPMathOperator>(I)); | ||||||||||||
7920 | |||||||||||||
7921 | SDValue Tmp = getValue(I.getArgOperand(0)); | ||||||||||||
7922 | setValue(&I, | ||||||||||||
7923 | DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags)); | ||||||||||||
7924 | return true; | ||||||||||||
7925 | } | ||||||||||||
7926 | |||||||||||||
7927 | /// See if we can lower a binary floating-point operation into an SDNode with | ||||||||||||
7928 | /// the specified Opcode. If so, return true and lower it. Otherwise return | ||||||||||||
7929 | /// false, and it will be lowered like a normal call. | ||||||||||||
7930 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||||||
7931 | /// correct prototype. | ||||||||||||
7932 | bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I, | ||||||||||||
7933 | unsigned Opcode) { | ||||||||||||
7934 | // We already checked this call's prototype; verify it doesn't modify errno. | ||||||||||||
7935 | if (!I.onlyReadsMemory()) | ||||||||||||
7936 | return false; | ||||||||||||
7937 | |||||||||||||
7938 | SDNodeFlags Flags; | ||||||||||||
7939 | Flags.copyFMF(cast<FPMathOperator>(I)); | ||||||||||||
7940 | |||||||||||||
7941 | SDValue Tmp0 = getValue(I.getArgOperand(0)); | ||||||||||||
7942 | SDValue Tmp1 = getValue(I.getArgOperand(1)); | ||||||||||||
7943 | EVT VT = Tmp0.getValueType(); | ||||||||||||
7944 | setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags)); | ||||||||||||
7945 | return true; | ||||||||||||
7946 | } | ||||||||||||
7947 | |||||||||||||
7948 | void SelectionDAGBuilder::visitCall(const CallInst &I) { | ||||||||||||
7949 | // Handle inline assembly differently. | ||||||||||||
7950 | if (I.isInlineAsm()) { | ||||||||||||
7951 | visitInlineAsm(I); | ||||||||||||
7952 | return; | ||||||||||||
7953 | } | ||||||||||||
7954 | |||||||||||||
7955 | if (Function *F = I.getCalledFunction()) { | ||||||||||||
7956 | if (F->hasFnAttribute("dontcall")) { | ||||||||||||
7957 | unsigned LocCookie = 0; | ||||||||||||
7958 | if (MDNode *MD = I.getMetadata("srcloc")) | ||||||||||||
7959 | LocCookie = | ||||||||||||
7960 | mdconst::extract<ConstantInt>(MD->getOperand(0))->getZExtValue(); | ||||||||||||
7961 | DiagnosticInfoDontCall D(F->getName(), LocCookie); | ||||||||||||
7962 | DAG.getContext()->diagnose(D); | ||||||||||||
7963 | } | ||||||||||||
7964 | |||||||||||||
7965 | if (F->isDeclaration()) { | ||||||||||||
7966 | // Is this an LLVM intrinsic or a target-specific intrinsic? | ||||||||||||
7967 | unsigned IID = F->getIntrinsicID(); | ||||||||||||
7968 | if (!IID) | ||||||||||||
7969 | if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) | ||||||||||||
7970 | IID = II->getIntrinsicID(F); | ||||||||||||
7971 | |||||||||||||
7972 | if (IID) { | ||||||||||||
7973 | visitIntrinsicCall(I, IID); | ||||||||||||
7974 | return; | ||||||||||||
7975 | } | ||||||||||||
7976 | } | ||||||||||||
7977 | |||||||||||||
7978 | // Check for well-known libc/libm calls. If the function is internal, it | ||||||||||||
7979 | // can't be a library call. Don't do the check if marked as nobuiltin for | ||||||||||||
7980 | // some reason or the call site requires strict floating point semantics. | ||||||||||||
7981 | LibFunc Func; | ||||||||||||
7982 | if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() && | ||||||||||||
7983 | F->hasName() && LibInfo->getLibFunc(*F, Func) && | ||||||||||||
7984 | LibInfo->hasOptimizedCodeGen(Func)) { | ||||||||||||
7985 | switch (Func) { | ||||||||||||
7986 | default: break; | ||||||||||||
7987 | case LibFunc_bcmp: | ||||||||||||
7988 | if (visitMemCmpBCmpCall(I)) | ||||||||||||
7989 | return; | ||||||||||||
7990 | break; | ||||||||||||
7991 | case LibFunc_copysign: | ||||||||||||
7992 | case LibFunc_copysignf: | ||||||||||||
7993 | case LibFunc_copysignl: | ||||||||||||
7994 | // We already checked this call's prototype; verify it doesn't modify | ||||||||||||
7995 | // errno. | ||||||||||||
7996 | if (I.onlyReadsMemory()) { | ||||||||||||
7997 | SDValue LHS = getValue(I.getArgOperand(0)); | ||||||||||||
7998 | SDValue RHS = getValue(I.getArgOperand(1)); | ||||||||||||
7999 | setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(), | ||||||||||||
8000 | LHS.getValueType(), LHS, RHS)); | ||||||||||||
8001 | return; | ||||||||||||
8002 | } | ||||||||||||
8003 | break; | ||||||||||||
8004 | case LibFunc_fabs: | ||||||||||||
8005 | case LibFunc_fabsf: | ||||||||||||
8006 | case LibFunc_fabsl: | ||||||||||||
8007 | if (visitUnaryFloatCall(I, ISD::FABS)) | ||||||||||||
8008 | return; | ||||||||||||
8009 | break; | ||||||||||||
8010 | case LibFunc_fmin: | ||||||||||||
8011 | case LibFunc_fminf: | ||||||||||||
8012 | case LibFunc_fminl: | ||||||||||||
8013 | if (visitBinaryFloatCall(I, ISD::FMINNUM)) | ||||||||||||
8014 | return; | ||||||||||||
8015 | break; | ||||||||||||
8016 | case LibFunc_fmax: | ||||||||||||
8017 | case LibFunc_fmaxf: | ||||||||||||
8018 | case LibFunc_fmaxl: | ||||||||||||
8019 | if (visitBinaryFloatCall(I, ISD::FMAXNUM)) | ||||||||||||
8020 | return; | ||||||||||||
8021 | break; | ||||||||||||
8022 | case LibFunc_sin: | ||||||||||||
8023 | case LibFunc_sinf: | ||||||||||||
8024 | case LibFunc_sinl: | ||||||||||||
8025 | if (visitUnaryFloatCall(I, ISD::FSIN)) | ||||||||||||
8026 | return; | ||||||||||||
8027 | break; | ||||||||||||
8028 | case LibFunc_cos: | ||||||||||||
8029 | case LibFunc_cosf: | ||||||||||||
8030 | case LibFunc_cosl: | ||||||||||||
8031 | if (visitUnaryFloatCall(I, ISD::FCOS)) | ||||||||||||
8032 | return; | ||||||||||||
8033 | break; | ||||||||||||
8034 | case LibFunc_sqrt: | ||||||||||||
8035 | case LibFunc_sqrtf: | ||||||||||||
8036 | case LibFunc_sqrtl: | ||||||||||||
8037 | case LibFunc_sqrt_finite: | ||||||||||||
8038 | case LibFunc_sqrtf_finite: | ||||||||||||
8039 | case LibFunc_sqrtl_finite: | ||||||||||||
8040 | if (visitUnaryFloatCall(I, ISD::FSQRT)) | ||||||||||||
8041 | return; | ||||||||||||
8042 | break; | ||||||||||||
8043 | case LibFunc_floor: | ||||||||||||
8044 | case LibFunc_floorf: | ||||||||||||
8045 | case LibFunc_floorl: | ||||||||||||
8046 | if (visitUnaryFloatCall(I, ISD::FFLOOR)) | ||||||||||||
8047 | return; | ||||||||||||
8048 | break; | ||||||||||||
8049 | case LibFunc_nearbyint: | ||||||||||||
8050 | case LibFunc_nearbyintf: | ||||||||||||
8051 | case LibFunc_nearbyintl: | ||||||||||||
8052 | if (visitUnaryFloatCall(I, ISD::FNEARBYINT)) | ||||||||||||
8053 | return; | ||||||||||||
8054 | break; | ||||||||||||
8055 | case LibFunc_ceil: | ||||||||||||
8056 | case LibFunc_ceilf: | ||||||||||||
8057 | case LibFunc_ceill: | ||||||||||||
8058 | if (visitUnaryFloatCall(I, ISD::FCEIL)) | ||||||||||||
8059 | return; | ||||||||||||
8060 | break; | ||||||||||||
8061 | case LibFunc_rint: | ||||||||||||
8062 | case LibFunc_rintf: | ||||||||||||
8063 | case LibFunc_rintl: | ||||||||||||
8064 | if (visitUnaryFloatCall(I, ISD::FRINT)) | ||||||||||||
8065 | return; | ||||||||||||
8066 | break; | ||||||||||||
8067 | case LibFunc_round: | ||||||||||||
8068 | case LibFunc_roundf: | ||||||||||||
8069 | case LibFunc_roundl: | ||||||||||||
8070 | if (visitUnaryFloatCall(I, ISD::FROUND)) | ||||||||||||
8071 | return; | ||||||||||||
8072 | break; | ||||||||||||
8073 | case LibFunc_trunc: | ||||||||||||
8074 | case LibFunc_truncf: | ||||||||||||
8075 | case LibFunc_truncl: | ||||||||||||
8076 | if (visitUnaryFloatCall(I, ISD::FTRUNC)) | ||||||||||||
8077 | return; | ||||||||||||
8078 | break; | ||||||||||||
8079 | case LibFunc_log2: | ||||||||||||
8080 | case LibFunc_log2f: | ||||||||||||
8081 | case LibFunc_log2l: | ||||||||||||
8082 | if (visitUnaryFloatCall(I, ISD::FLOG2)) | ||||||||||||
8083 | return; | ||||||||||||
8084 | break; | ||||||||||||
8085 | case LibFunc_exp2: | ||||||||||||
8086 | case LibFunc_exp2f: | ||||||||||||
8087 | case LibFunc_exp2l: | ||||||||||||
8088 | if (visitUnaryFloatCall(I, ISD::FEXP2)) | ||||||||||||
8089 | return; | ||||||||||||
8090 | break; | ||||||||||||
8091 | case LibFunc_memcmp: | ||||||||||||
8092 | if (visitMemCmpBCmpCall(I)) | ||||||||||||
8093 | return; | ||||||||||||
8094 | break; | ||||||||||||
8095 | case LibFunc_mempcpy: | ||||||||||||
8096 | if (visitMemPCpyCall(I)) | ||||||||||||
8097 | return; | ||||||||||||
8098 | break; | ||||||||||||
8099 | case LibFunc_memchr: | ||||||||||||
8100 | if (visitMemChrCall(I)) | ||||||||||||
8101 | return; | ||||||||||||
8102 | break; | ||||||||||||
8103 | case LibFunc_strcpy: | ||||||||||||
8104 | if (visitStrCpyCall(I, false)) | ||||||||||||
8105 | return; | ||||||||||||
8106 | break; | ||||||||||||
8107 | case LibFunc_stpcpy: | ||||||||||||
8108 | if (visitStrCpyCall(I, true)) | ||||||||||||
8109 | return; | ||||||||||||
8110 | break; | ||||||||||||
8111 | case LibFunc_strcmp: | ||||||||||||
8112 | if (visitStrCmpCall(I)) | ||||||||||||
8113 | return; | ||||||||||||
8114 | break; | ||||||||||||
8115 | case LibFunc_strlen: | ||||||||||||
8116 | if (visitStrLenCall(I)) | ||||||||||||
8117 | return; | ||||||||||||
8118 | break; | ||||||||||||
8119 | case LibFunc_strnlen: | ||||||||||||
8120 | if (visitStrNLenCall(I)) | ||||||||||||
8121 | return; | ||||||||||||
8122 | break; | ||||||||||||
8123 | } | ||||||||||||
8124 | } | ||||||||||||
8125 | } | ||||||||||||
8126 | |||||||||||||
8127 | // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't | ||||||||||||
8128 | // have to do anything here to lower funclet bundles. | ||||||||||||
8129 | // CFGuardTarget bundles are lowered in LowerCallTo. | ||||||||||||
8130 | assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget , LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall }) && "Cannot lower calls with arbitrary operand bundles!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower calls with arbitrary operand bundles!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8134, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8131 | {LLVMContext::OB_deopt, LLVMContext::OB_funclet,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget , LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall }) && "Cannot lower calls with arbitrary operand bundles!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower calls with arbitrary operand bundles!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8134, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8132 | LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget , LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall }) && "Cannot lower calls with arbitrary operand bundles!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower calls with arbitrary operand bundles!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8134, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8133 | LLVMContext::OB_clang_arc_attachedcall}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget , LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall }) && "Cannot lower calls with arbitrary operand bundles!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower calls with arbitrary operand bundles!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8134, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8134 | "Cannot lower calls with arbitrary operand bundles!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget , LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall }) && "Cannot lower calls with arbitrary operand bundles!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower calls with arbitrary operand bundles!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8134, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8135 | |||||||||||||
8136 | SDValue Callee = getValue(I.getCalledOperand()); | ||||||||||||
8137 | |||||||||||||
8138 | if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) | ||||||||||||
8139 | LowerCallSiteWithDeoptBundle(&I, Callee, nullptr); | ||||||||||||
8140 | else | ||||||||||||
8141 | // Check if we can potentially perform a tail call. More detailed checking | ||||||||||||
8142 | // is be done within LowerCallTo, after more information about the call is | ||||||||||||
8143 | // known. | ||||||||||||
8144 | LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall()); | ||||||||||||
8145 | } | ||||||||||||
8146 | |||||||||||||
8147 | namespace { | ||||||||||||
8148 | |||||||||||||
8149 | /// AsmOperandInfo - This contains information for each constraint that we are | ||||||||||||
8150 | /// lowering. | ||||||||||||
8151 | class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo { | ||||||||||||
8152 | public: | ||||||||||||
8153 | /// CallOperand - If this is the result output operand or a clobber | ||||||||||||
8154 | /// this is null, otherwise it is the incoming operand to the CallInst. | ||||||||||||
8155 | /// This gets modified as the asm is processed. | ||||||||||||
8156 | SDValue CallOperand; | ||||||||||||
8157 | |||||||||||||
8158 | /// AssignedRegs - If this is a register or register class operand, this | ||||||||||||
8159 | /// contains the set of register corresponding to the operand. | ||||||||||||
8160 | RegsForValue AssignedRegs; | ||||||||||||
8161 | |||||||||||||
8162 | explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info) | ||||||||||||
8163 | : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) { | ||||||||||||
8164 | } | ||||||||||||
8165 | |||||||||||||
8166 | /// Whether or not this operand accesses memory | ||||||||||||
8167 | bool hasMemory(const TargetLowering &TLI) const { | ||||||||||||
8168 | // Indirect operand accesses access memory. | ||||||||||||
8169 | if (isIndirect) | ||||||||||||
8170 | return true; | ||||||||||||
8171 | |||||||||||||
8172 | for (const auto &Code : Codes) | ||||||||||||
8173 | if (TLI.getConstraintType(Code) == TargetLowering::C_Memory) | ||||||||||||
8174 | return true; | ||||||||||||
8175 | |||||||||||||
8176 | return false; | ||||||||||||
8177 | } | ||||||||||||
8178 | |||||||||||||
8179 | /// getCallOperandValEVT - Return the EVT of the Value* that this operand | ||||||||||||
8180 | /// corresponds to. If there is no Value* for this operand, it returns | ||||||||||||
8181 | /// MVT::Other. | ||||||||||||
8182 | EVT getCallOperandValEVT(LLVMContext &Context, const TargetLowering &TLI, | ||||||||||||
8183 | const DataLayout &DL) const { | ||||||||||||
8184 | if (!CallOperandVal) return MVT::Other; | ||||||||||||
8185 | |||||||||||||
8186 | if (isa<BasicBlock>(CallOperandVal)) | ||||||||||||
8187 | return TLI.getProgramPointerTy(DL); | ||||||||||||
8188 | |||||||||||||
8189 | llvm::Type *OpTy = CallOperandVal->getType(); | ||||||||||||
8190 | |||||||||||||
8191 | // FIXME: code duplicated from TargetLowering::ParseConstraints(). | ||||||||||||
8192 | // If this is an indirect operand, the operand is a pointer to the | ||||||||||||
8193 | // accessed type. | ||||||||||||
8194 | if (isIndirect) { | ||||||||||||
8195 | PointerType *PtrTy = dyn_cast<PointerType>(OpTy); | ||||||||||||
8196 | if (!PtrTy) | ||||||||||||
8197 | report_fatal_error("Indirect operand for inline asm not a pointer!"); | ||||||||||||
8198 | OpTy = PtrTy->getElementType(); | ||||||||||||
8199 | } | ||||||||||||
8200 | |||||||||||||
8201 | // Look for vector wrapped in a struct. e.g. { <16 x i8> }. | ||||||||||||
8202 | if (StructType *STy = dyn_cast<StructType>(OpTy)) | ||||||||||||
8203 | if (STy->getNumElements() == 1) | ||||||||||||
8204 | OpTy = STy->getElementType(0); | ||||||||||||
8205 | |||||||||||||
8206 | // If OpTy is not a single value, it may be a struct/union that we | ||||||||||||
8207 | // can tile with integers. | ||||||||||||
8208 | if (!OpTy->isSingleValueType() && OpTy->isSized()) { | ||||||||||||
8209 | unsigned BitSize = DL.getTypeSizeInBits(OpTy); | ||||||||||||
8210 | switch (BitSize) { | ||||||||||||
8211 | default: break; | ||||||||||||
8212 | case 1: | ||||||||||||
8213 | case 8: | ||||||||||||
8214 | case 16: | ||||||||||||
8215 | case 32: | ||||||||||||
8216 | case 64: | ||||||||||||
8217 | case 128: | ||||||||||||
8218 | OpTy = IntegerType::get(Context, BitSize); | ||||||||||||
8219 | break; | ||||||||||||
8220 | } | ||||||||||||
8221 | } | ||||||||||||
8222 | |||||||||||||
8223 | return TLI.getAsmOperandValueType(DL, OpTy, true); | ||||||||||||
8224 | } | ||||||||||||
8225 | }; | ||||||||||||
8226 | |||||||||||||
8227 | |||||||||||||
8228 | } // end anonymous namespace | ||||||||||||
8229 | |||||||||||||
8230 | /// Make sure that the output operand \p OpInfo and its corresponding input | ||||||||||||
8231 | /// operand \p MatchingOpInfo have compatible constraint types (otherwise error | ||||||||||||
8232 | /// out). | ||||||||||||
8233 | static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, | ||||||||||||
8234 | SDISelAsmOperandInfo &MatchingOpInfo, | ||||||||||||
8235 | SelectionDAG &DAG) { | ||||||||||||
8236 | if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT) | ||||||||||||
8237 | return; | ||||||||||||
8238 | |||||||||||||
8239 | const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo(); | ||||||||||||
8240 | const auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
8241 | |||||||||||||
8242 | std::pair<unsigned, const TargetRegisterClass *> MatchRC = | ||||||||||||
8243 | TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode, | ||||||||||||
8244 | OpInfo.ConstraintVT); | ||||||||||||
8245 | std::pair<unsigned, const TargetRegisterClass *> InputRC = | ||||||||||||
8246 | TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode, | ||||||||||||
8247 | MatchingOpInfo.ConstraintVT); | ||||||||||||
8248 | if ((OpInfo.ConstraintVT.isInteger() != | ||||||||||||
8249 | MatchingOpInfo.ConstraintVT.isInteger()) || | ||||||||||||
8250 | (MatchRC.second != InputRC.second)) { | ||||||||||||
8251 | // FIXME: error out in a more elegant fashion | ||||||||||||
8252 | report_fatal_error("Unsupported asm: input constraint" | ||||||||||||
8253 | " with a matching output constraint of" | ||||||||||||
8254 | " incompatible type!"); | ||||||||||||
8255 | } | ||||||||||||
8256 | MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT; | ||||||||||||
8257 | } | ||||||||||||
8258 | |||||||||||||
8259 | /// Get a direct memory input to behave well as an indirect operand. | ||||||||||||
8260 | /// This may introduce stores, hence the need for a \p Chain. | ||||||||||||
8261 | /// \return The (possibly updated) chain. | ||||||||||||
8262 | static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, | ||||||||||||
8263 | SDISelAsmOperandInfo &OpInfo, | ||||||||||||
8264 | SelectionDAG &DAG) { | ||||||||||||
8265 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
8266 | |||||||||||||
8267 | // If we don't have an indirect input, put it in the constpool if we can, | ||||||||||||
8268 | // otherwise spill it to a stack slot. | ||||||||||||
8269 | // TODO: This isn't quite right. We need to handle these according to | ||||||||||||
8270 | // the addressing mode that the constraint wants. Also, this may take | ||||||||||||
8271 | // an additional register for the computation and we don't want that | ||||||||||||
8272 | // either. | ||||||||||||
8273 | |||||||||||||
8274 | // If the operand is a float, integer, or vector constant, spill to a | ||||||||||||
8275 | // constant pool entry to get its address. | ||||||||||||
8276 | const Value *OpVal = OpInfo.CallOperandVal; | ||||||||||||
8277 | if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) || | ||||||||||||
8278 | isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) { | ||||||||||||
8279 | OpInfo.CallOperand = DAG.getConstantPool( | ||||||||||||
8280 | cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout())); | ||||||||||||
8281 | return Chain; | ||||||||||||
8282 | } | ||||||||||||
8283 | |||||||||||||
8284 | // Otherwise, create a stack slot and emit a store to it before the asm. | ||||||||||||
8285 | Type *Ty = OpVal->getType(); | ||||||||||||
8286 | auto &DL = DAG.getDataLayout(); | ||||||||||||
8287 | uint64_t TySize = DL.getTypeAllocSize(Ty); | ||||||||||||
8288 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
8289 | int SSFI = MF.getFrameInfo().CreateStackObject( | ||||||||||||
8290 | TySize, DL.getPrefTypeAlign(Ty), false); | ||||||||||||
8291 | SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL)); | ||||||||||||
8292 | Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot, | ||||||||||||
8293 | MachinePointerInfo::getFixedStack(MF, SSFI), | ||||||||||||
8294 | TLI.getMemValueType(DL, Ty)); | ||||||||||||
8295 | OpInfo.CallOperand = StackSlot; | ||||||||||||
8296 | |||||||||||||
8297 | return Chain; | ||||||||||||
8298 | } | ||||||||||||
8299 | |||||||||||||
8300 | /// GetRegistersForValue - Assign registers (virtual or physical) for the | ||||||||||||
8301 | /// specified operand. We prefer to assign virtual registers, to allow the | ||||||||||||
8302 | /// register allocator to handle the assignment process. However, if the asm | ||||||||||||
8303 | /// uses features that we can't model on machineinstrs, we have SDISel do the | ||||||||||||
8304 | /// allocation. This produces generally horrible, but correct, code. | ||||||||||||
8305 | /// | ||||||||||||
8306 | /// OpInfo describes the operand | ||||||||||||
8307 | /// RefOpInfo describes the matching operand if any, the operand otherwise | ||||||||||||
8308 | static void GetRegistersForValue(SelectionDAG &DAG, const SDLoc &DL, | ||||||||||||
8309 | SDISelAsmOperandInfo &OpInfo, | ||||||||||||
8310 | SDISelAsmOperandInfo &RefOpInfo) { | ||||||||||||
8311 | LLVMContext &Context = *DAG.getContext(); | ||||||||||||
8312 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
8313 | |||||||||||||
8314 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
8315 | SmallVector<unsigned, 4> Regs; | ||||||||||||
8316 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | ||||||||||||
8317 | |||||||||||||
8318 | // No work to do for memory operations. | ||||||||||||
8319 | if (OpInfo.ConstraintType == TargetLowering::C_Memory) | ||||||||||||
8320 | return; | ||||||||||||
8321 | |||||||||||||
8322 | // If this is a constraint for a single physreg, or a constraint for a | ||||||||||||
8323 | // register class, find it. | ||||||||||||
8324 | unsigned AssignedReg; | ||||||||||||
8325 | const TargetRegisterClass *RC; | ||||||||||||
8326 | std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint( | ||||||||||||
8327 | &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT); | ||||||||||||
8328 | // RC is unset only on failure. Return immediately. | ||||||||||||
8329 | if (!RC) | ||||||||||||
8330 | return; | ||||||||||||
8331 | |||||||||||||
8332 | // Get the actual register value type. This is important, because the user | ||||||||||||
8333 | // may have asked for (e.g.) the AX register in i32 type. We need to | ||||||||||||
8334 | // remember that AX is actually i16 to get the right extension. | ||||||||||||
8335 | const MVT RegVT = *TRI.legalclasstypes_begin(*RC); | ||||||||||||
8336 | |||||||||||||
8337 | if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) { | ||||||||||||
8338 | // If this is an FP operand in an integer register (or visa versa), or more | ||||||||||||
8339 | // generally if the operand value disagrees with the register class we plan | ||||||||||||
8340 | // to stick it in, fix the operand type. | ||||||||||||
8341 | // | ||||||||||||
8342 | // If this is an input value, the bitcast to the new type is done now. | ||||||||||||
8343 | // Bitcast for output value is done at the end of visitInlineAsm(). | ||||||||||||
8344 | if ((OpInfo.Type == InlineAsm::isOutput || | ||||||||||||
8345 | OpInfo.Type == InlineAsm::isInput) && | ||||||||||||
8346 | !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) { | ||||||||||||
8347 | // Try to convert to the first EVT that the reg class contains. If the | ||||||||||||
8348 | // types are identical size, use a bitcast to convert (e.g. two differing | ||||||||||||
8349 | // vector types). Note: output bitcast is done at the end of | ||||||||||||
8350 | // visitInlineAsm(). | ||||||||||||
8351 | if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) { | ||||||||||||
8352 | // Exclude indirect inputs while they are unsupported because the code | ||||||||||||
8353 | // to perform the load is missing and thus OpInfo.CallOperand still | ||||||||||||
8354 | // refers to the input address rather than the pointed-to value. | ||||||||||||
8355 | if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect) | ||||||||||||
8356 | OpInfo.CallOperand = | ||||||||||||
8357 | DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand); | ||||||||||||
8358 | OpInfo.ConstraintVT = RegVT; | ||||||||||||
8359 | // If the operand is an FP value and we want it in integer registers, | ||||||||||||
8360 | // use the corresponding integer type. This turns an f64 value into | ||||||||||||
8361 | // i64, which can be passed with two i32 values on a 32-bit machine. | ||||||||||||
8362 | } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) { | ||||||||||||
8363 | MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits()); | ||||||||||||
8364 | if (OpInfo.Type == InlineAsm::isInput) | ||||||||||||
8365 | OpInfo.CallOperand = | ||||||||||||
8366 | DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand); | ||||||||||||
8367 | OpInfo.ConstraintVT = VT; | ||||||||||||
8368 | } | ||||||||||||
8369 | } | ||||||||||||
8370 | } | ||||||||||||
8371 | |||||||||||||
8372 | // No need to allocate a matching input constraint since the constraint it's | ||||||||||||
8373 | // matching to has already been allocated. | ||||||||||||
8374 | if (OpInfo.isMatchingInputConstraint()) | ||||||||||||
8375 | return; | ||||||||||||
8376 | |||||||||||||
8377 | EVT ValueVT = OpInfo.ConstraintVT; | ||||||||||||
8378 | if (OpInfo.ConstraintVT == MVT::Other) | ||||||||||||
8379 | ValueVT = RegVT; | ||||||||||||
8380 | |||||||||||||
8381 | // Initialize NumRegs. | ||||||||||||
8382 | unsigned NumRegs = 1; | ||||||||||||
8383 | if (OpInfo.ConstraintVT != MVT::Other) | ||||||||||||
8384 | NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT); | ||||||||||||
8385 | |||||||||||||
8386 | // If this is a constraint for a specific physical register, like {r17}, | ||||||||||||
8387 | // assign it now. | ||||||||||||
8388 | |||||||||||||
8389 | // If this associated to a specific register, initialize iterator to correct | ||||||||||||
8390 | // place. If virtual, make sure we have enough registers | ||||||||||||
8391 | |||||||||||||
8392 | // Initialize iterator if necessary | ||||||||||||
8393 | TargetRegisterClass::iterator I = RC->begin(); | ||||||||||||
8394 | MachineRegisterInfo &RegInfo = MF.getRegInfo(); | ||||||||||||
8395 | |||||||||||||
8396 | // Do not check for single registers. | ||||||||||||
8397 | if (AssignedReg) { | ||||||||||||
8398 | for (; *I != AssignedReg; ++I) | ||||||||||||
8399 | assert(I != RC->end() && "AssignedReg should be member of RC")(static_cast <bool> (I != RC->end() && "AssignedReg should be member of RC" ) ? void (0) : __assert_fail ("I != RC->end() && \"AssignedReg should be member of RC\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8399, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8400 | } | ||||||||||||
8401 | |||||||||||||
8402 | for (; NumRegs; --NumRegs, ++I) { | ||||||||||||
8403 | assert(I != RC->end() && "Ran out of registers to allocate!")(static_cast <bool> (I != RC->end() && "Ran out of registers to allocate!" ) ? void (0) : __assert_fail ("I != RC->end() && \"Ran out of registers to allocate!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8403, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8404 | Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC); | ||||||||||||
8405 | Regs.push_back(R); | ||||||||||||
8406 | } | ||||||||||||
8407 | |||||||||||||
8408 | OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT); | ||||||||||||
8409 | } | ||||||||||||
8410 | |||||||||||||
8411 | static unsigned | ||||||||||||
8412 | findMatchingInlineAsmOperand(unsigned OperandNo, | ||||||||||||
8413 | const std::vector<SDValue> &AsmNodeOperands) { | ||||||||||||
8414 | // Scan until we find the definition we already emitted of this operand. | ||||||||||||
8415 | unsigned CurOp = InlineAsm::Op_FirstOperand; | ||||||||||||
8416 | for (; OperandNo; --OperandNo) { | ||||||||||||
8417 | // Advance to the next operand. | ||||||||||||
8418 | unsigned OpFlag = | ||||||||||||
8419 | cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); | ||||||||||||
8420 | assert((InlineAsm::isRegDefKind(OpFlag) ||(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind (OpFlag)) && "Skipped past definitions?") ? void (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8423, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8421 | InlineAsm::isRegDefEarlyClobberKind(OpFlag) ||(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind (OpFlag)) && "Skipped past definitions?") ? void (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8423, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8422 | InlineAsm::isMemKind(OpFlag)) &&(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind (OpFlag)) && "Skipped past definitions?") ? void (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8423, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8423 | "Skipped past definitions?")(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind (OpFlag)) && "Skipped past definitions?") ? void (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8423, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8424 | CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1; | ||||||||||||
8425 | } | ||||||||||||
8426 | return CurOp; | ||||||||||||
8427 | } | ||||||||||||
8428 | |||||||||||||
8429 | namespace { | ||||||||||||
8430 | |||||||||||||
8431 | class ExtraFlags { | ||||||||||||
8432 | unsigned Flags = 0; | ||||||||||||
8433 | |||||||||||||
8434 | public: | ||||||||||||
8435 | explicit ExtraFlags(const CallBase &Call) { | ||||||||||||
8436 | const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand()); | ||||||||||||
8437 | if (IA->hasSideEffects()) | ||||||||||||
8438 | Flags |= InlineAsm::Extra_HasSideEffects; | ||||||||||||
8439 | if (IA->isAlignStack()) | ||||||||||||
8440 | Flags |= InlineAsm::Extra_IsAlignStack; | ||||||||||||
8441 | if (Call.isConvergent()) | ||||||||||||
8442 | Flags |= InlineAsm::Extra_IsConvergent; | ||||||||||||
8443 | Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect; | ||||||||||||
8444 | } | ||||||||||||
8445 | |||||||||||||
8446 | void update(const TargetLowering::AsmOperandInfo &OpInfo) { | ||||||||||||
8447 | // Ideally, we would only check against memory constraints. However, the | ||||||||||||
8448 | // meaning of an Other constraint can be target-specific and we can't easily | ||||||||||||
8449 | // reason about it. Therefore, be conservative and set MayLoad/MayStore | ||||||||||||
8450 | // for Other constraints as well. | ||||||||||||
8451 | if (OpInfo.ConstraintType == TargetLowering::C_Memory || | ||||||||||||
8452 | OpInfo.ConstraintType == TargetLowering::C_Other) { | ||||||||||||
8453 | if (OpInfo.Type == InlineAsm::isInput) | ||||||||||||
8454 | Flags |= InlineAsm::Extra_MayLoad; | ||||||||||||
8455 | else if (OpInfo.Type == InlineAsm::isOutput) | ||||||||||||
8456 | Flags |= InlineAsm::Extra_MayStore; | ||||||||||||
8457 | else if (OpInfo.Type == InlineAsm::isClobber) | ||||||||||||
8458 | Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore); | ||||||||||||
8459 | } | ||||||||||||
8460 | } | ||||||||||||
8461 | |||||||||||||
8462 | unsigned get() const { return Flags; } | ||||||||||||
8463 | }; | ||||||||||||
8464 | |||||||||||||
8465 | } // end anonymous namespace | ||||||||||||
8466 | |||||||||||||
8467 | /// visitInlineAsm - Handle a call to an InlineAsm object. | ||||||||||||
8468 | void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call, | ||||||||||||
8469 | const BasicBlock *EHPadBB) { | ||||||||||||
8470 | const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand()); | ||||||||||||
| |||||||||||||
8471 | |||||||||||||
8472 | /// ConstraintOperands - Information about all of the constraints. | ||||||||||||
8473 | SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands; | ||||||||||||
8474 | |||||||||||||
8475 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
8476 | TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints( | ||||||||||||
8477 | DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), Call); | ||||||||||||
8478 | |||||||||||||
8479 | // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack, | ||||||||||||
8480 | // AsmDialect, MayLoad, MayStore). | ||||||||||||
8481 | bool HasSideEffect = IA->hasSideEffects(); | ||||||||||||
8482 | ExtraFlags ExtraInfo(Call); | ||||||||||||
8483 | |||||||||||||
8484 | unsigned ArgNo = 0; // ArgNo - The argument of the CallInst. | ||||||||||||
8485 | unsigned ResNo = 0; // ResNo - The result number of the next output. | ||||||||||||
8486 | unsigned NumMatchingOps = 0; | ||||||||||||
8487 | for (auto &T : TargetConstraints) { | ||||||||||||
8488 | ConstraintOperands.push_back(SDISelAsmOperandInfo(T)); | ||||||||||||
8489 | SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back(); | ||||||||||||
8490 | |||||||||||||
8491 | // Compute the value type for each operand. | ||||||||||||
8492 | if (OpInfo.Type == InlineAsm::isInput || | ||||||||||||
8493 | (OpInfo.Type == InlineAsm::isOutput && OpInfo.isIndirect)) { | ||||||||||||
8494 | OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++); | ||||||||||||
8495 | |||||||||||||
8496 | // Process the call argument. BasicBlocks are labels, currently appearing | ||||||||||||
8497 | // only in asm's. | ||||||||||||
8498 | if (isa<CallBrInst>(Call) && | ||||||||||||
8499 | ArgNo - 1 >= (cast<CallBrInst>(&Call)->getNumArgOperands() - | ||||||||||||
8500 | cast<CallBrInst>(&Call)->getNumIndirectDests() - | ||||||||||||
8501 | NumMatchingOps) && | ||||||||||||
8502 | (NumMatchingOps == 0 || | ||||||||||||
8503 | ArgNo - 1 < (cast<CallBrInst>(&Call)->getNumArgOperands() - | ||||||||||||
8504 | NumMatchingOps))) { | ||||||||||||
8505 | const auto *BA = cast<BlockAddress>(OpInfo.CallOperandVal); | ||||||||||||
8506 | EVT VT = TLI.getValueType(DAG.getDataLayout(), BA->getType(), true); | ||||||||||||
8507 | OpInfo.CallOperand = DAG.getTargetBlockAddress(BA, VT); | ||||||||||||
8508 | } else if (const auto *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) { | ||||||||||||
8509 | OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]); | ||||||||||||
8510 | } else { | ||||||||||||
8511 | OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); | ||||||||||||
8512 | } | ||||||||||||
8513 | |||||||||||||
8514 | EVT VT = OpInfo.getCallOperandValEVT(*DAG.getContext(), TLI, | ||||||||||||
8515 | DAG.getDataLayout()); | ||||||||||||
8516 | OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other; | ||||||||||||
8517 | } else if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) { | ||||||||||||
8518 | // The return value of the call is this value. As such, there is no | ||||||||||||
8519 | // corresponding argument. | ||||||||||||
8520 | assert(!Call.getType()->isVoidTy() && "Bad inline asm!")(static_cast <bool> (!Call.getType()->isVoidTy() && "Bad inline asm!") ? void (0) : __assert_fail ("!Call.getType()->isVoidTy() && \"Bad inline asm!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8520, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8521 | if (StructType *STy = dyn_cast<StructType>(Call.getType())) { | ||||||||||||
8522 | OpInfo.ConstraintVT = TLI.getSimpleValueType( | ||||||||||||
8523 | DAG.getDataLayout(), STy->getElementType(ResNo)); | ||||||||||||
8524 | } else { | ||||||||||||
8525 | assert(ResNo == 0 && "Asm only has one result!")(static_cast <bool> (ResNo == 0 && "Asm only has one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8525, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8526 | OpInfo.ConstraintVT = TLI.getAsmOperandValueType( | ||||||||||||
8527 | DAG.getDataLayout(), Call.getType()).getSimpleVT(); | ||||||||||||
8528 | } | ||||||||||||
8529 | ++ResNo; | ||||||||||||
8530 | } else { | ||||||||||||
8531 | OpInfo.ConstraintVT = MVT::Other; | ||||||||||||
8532 | } | ||||||||||||
8533 | |||||||||||||
8534 | if (OpInfo.hasMatchingInput()) | ||||||||||||
8535 | ++NumMatchingOps; | ||||||||||||
8536 | |||||||||||||
8537 | if (!HasSideEffect) | ||||||||||||
8538 | HasSideEffect = OpInfo.hasMemory(TLI); | ||||||||||||
8539 | |||||||||||||
8540 | // Determine if this InlineAsm MayLoad or MayStore based on the constraints. | ||||||||||||
8541 | // FIXME: Could we compute this on OpInfo rather than T? | ||||||||||||
8542 | |||||||||||||
8543 | // Compute the constraint code and ConstraintType to use. | ||||||||||||
8544 | TLI.ComputeConstraintToUse(T, SDValue()); | ||||||||||||
8545 | |||||||||||||
8546 | if (T.ConstraintType == TargetLowering::C_Immediate && | ||||||||||||
8547 | OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand)) | ||||||||||||
8548 | // We've delayed emitting a diagnostic like the "n" constraint because | ||||||||||||
8549 | // inlining could cause an integer showing up. | ||||||||||||
8550 | return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) + | ||||||||||||
8551 | "' expects an integer constant " | ||||||||||||
8552 | "expression"); | ||||||||||||
8553 | |||||||||||||
8554 | ExtraInfo.update(T); | ||||||||||||
8555 | } | ||||||||||||
8556 | |||||||||||||
8557 | // We won't need to flush pending loads if this asm doesn't touch | ||||||||||||
8558 | // memory and is nonvolatile. | ||||||||||||
8559 | SDValue Flag, Chain = (HasSideEffect
| ||||||||||||
8560 | |||||||||||||
8561 | bool EmitEHLabels = isa<InvokeInst>(Call) && IA->canThrow(); | ||||||||||||
8562 | if (EmitEHLabels
| ||||||||||||
8563 | assert(EHPadBB && "InvokeInst must have an EHPadBB")(static_cast <bool> (EHPadBB && "InvokeInst must have an EHPadBB" ) ? void (0) : __assert_fail ("EHPadBB && \"InvokeInst must have an EHPadBB\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8563, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8564 | } | ||||||||||||
8565 | bool IsCallBr = isa<CallBrInst>(Call); | ||||||||||||
8566 | |||||||||||||
8567 | if (IsCallBr
| ||||||||||||
8568 | // If this is a callbr or invoke we need to flush pending exports since | ||||||||||||
8569 | // inlineasm_br and invoke are terminators. | ||||||||||||
8570 | // We need to do this before nodes are glued to the inlineasm_br node. | ||||||||||||
8571 | Chain = getControlRoot(); | ||||||||||||
8572 | } | ||||||||||||
8573 | |||||||||||||
8574 | MCSymbol *BeginLabel = nullptr; | ||||||||||||
8575 | if (EmitEHLabels
| ||||||||||||
8576 | Chain = lowerStartEH(Chain, EHPadBB, BeginLabel); | ||||||||||||
8577 | } | ||||||||||||
8578 | |||||||||||||
8579 | // Second pass over the constraints: compute which constraint option to use. | ||||||||||||
8580 | for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) { | ||||||||||||
8581 | // If this is an output operand with a matching input operand, look up the | ||||||||||||
8582 | // matching input. If their types mismatch, e.g. one is an integer, the | ||||||||||||
8583 | // other is floating point, or their sizes are different, flag it as an | ||||||||||||
8584 | // error. | ||||||||||||
8585 | if (OpInfo.hasMatchingInput()) { | ||||||||||||
8586 | SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput]; | ||||||||||||
8587 | patchMatchingInput(OpInfo, Input, DAG); | ||||||||||||
8588 | } | ||||||||||||
8589 | |||||||||||||
8590 | // Compute the constraint code and ConstraintType to use. | ||||||||||||
8591 | TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG); | ||||||||||||
8592 | |||||||||||||
8593 | if (OpInfo.ConstraintType == TargetLowering::C_Memory && | ||||||||||||
8594 | OpInfo.Type == InlineAsm::isClobber) | ||||||||||||
8595 | continue; | ||||||||||||
8596 | |||||||||||||
8597 | // If this is a memory input, and if the operand is not indirect, do what we | ||||||||||||
8598 | // need to provide an address for the memory input. | ||||||||||||
8599 | if (OpInfo.ConstraintType == TargetLowering::C_Memory && | ||||||||||||
8600 | !OpInfo.isIndirect) { | ||||||||||||
8601 | assert((OpInfo.isMultipleAlternative ||(static_cast <bool> ((OpInfo.isMultipleAlternative || ( OpInfo.Type == InlineAsm::isInput)) && "Can only indirectify direct input operands!" ) ? void (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8603, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8602 | (OpInfo.Type == InlineAsm::isInput)) &&(static_cast <bool> ((OpInfo.isMultipleAlternative || ( OpInfo.Type == InlineAsm::isInput)) && "Can only indirectify direct input operands!" ) ? void (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8603, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8603 | "Can only indirectify direct input operands!")(static_cast <bool> ((OpInfo.isMultipleAlternative || ( OpInfo.Type == InlineAsm::isInput)) && "Can only indirectify direct input operands!" ) ? void (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8603, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8604 | |||||||||||||
8605 | // Memory operands really want the address of the value. | ||||||||||||
8606 | Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG); | ||||||||||||
8607 | |||||||||||||
8608 | // There is no longer a Value* corresponding to this operand. | ||||||||||||
8609 | OpInfo.CallOperandVal = nullptr; | ||||||||||||
8610 | |||||||||||||
8611 | // It is now an indirect operand. | ||||||||||||
8612 | OpInfo.isIndirect = true; | ||||||||||||
8613 | } | ||||||||||||
8614 | |||||||||||||
8615 | } | ||||||||||||
8616 | |||||||||||||
8617 | // AsmNodeOperands - The operands for the ISD::INLINEASM node. | ||||||||||||
8618 | std::vector<SDValue> AsmNodeOperands; | ||||||||||||
8619 | AsmNodeOperands.push_back(SDValue()); // reserve space for input chain | ||||||||||||
8620 | AsmNodeOperands.push_back(DAG.getTargetExternalSymbol( | ||||||||||||
8621 | IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout()))); | ||||||||||||
8622 | |||||||||||||
8623 | // If we have a !srcloc metadata node associated with it, we want to attach | ||||||||||||
8624 | // this to the ultimately generated inline asm machineinstr. To do this, we | ||||||||||||
8625 | // pass in the third operand as this (potentially null) inline asm MDNode. | ||||||||||||
8626 | const MDNode *SrcLoc = Call.getMetadata("srcloc"); | ||||||||||||
8627 | AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc)); | ||||||||||||
8628 | |||||||||||||
8629 | // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore | ||||||||||||
8630 | // bits as operand 3. | ||||||||||||
8631 | AsmNodeOperands.push_back(DAG.getTargetConstant( | ||||||||||||
8632 | ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||||||
8633 | |||||||||||||
8634 | // Third pass: Loop over operands to prepare DAG-level operands.. As part of | ||||||||||||
8635 | // this, assign virtual and physical registers for inputs and otput. | ||||||||||||
8636 | for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) { | ||||||||||||
8637 | // Assign Registers. | ||||||||||||
8638 | SDISelAsmOperandInfo &RefOpInfo = | ||||||||||||
8639 | OpInfo.isMatchingInputConstraint() | ||||||||||||
8640 | ? ConstraintOperands[OpInfo.getMatchedOperand()] | ||||||||||||
8641 | : OpInfo; | ||||||||||||
8642 | GetRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo); | ||||||||||||
8643 | |||||||||||||
8644 | auto DetectWriteToReservedRegister = [&]() { | ||||||||||||
8645 | const MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
8646 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | ||||||||||||
8647 | for (unsigned Reg : OpInfo.AssignedRegs.Regs) { | ||||||||||||
8648 | if (Register::isPhysicalRegister(Reg) && | ||||||||||||
8649 | TRI.isInlineAsmReadOnlyReg(MF, Reg)) { | ||||||||||||
8650 | const char *RegName = TRI.getName(Reg); | ||||||||||||
8651 | emitInlineAsmError(Call, "write to reserved register '" + | ||||||||||||
8652 | Twine(RegName) + "'"); | ||||||||||||
8653 | return true; | ||||||||||||
8654 | } | ||||||||||||
8655 | } | ||||||||||||
8656 | return false; | ||||||||||||
8657 | }; | ||||||||||||
8658 | |||||||||||||
8659 | switch (OpInfo.Type) { | ||||||||||||
8660 | case InlineAsm::isOutput: | ||||||||||||
8661 | if (OpInfo.ConstraintType == TargetLowering::C_Memory) { | ||||||||||||
8662 | unsigned ConstraintID = | ||||||||||||
8663 | TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode); | ||||||||||||
8664 | assert(ConstraintID != InlineAsm::Constraint_Unknown &&(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id." ) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8665, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8665 | "Failed to convert memory constraint code to constraint id.")(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id." ) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8665, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8666 | |||||||||||||
8667 | // Add information to the INLINEASM node to know about this output. | ||||||||||||
8668 | unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); | ||||||||||||
8669 | OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID); | ||||||||||||
8670 | AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(), | ||||||||||||
8671 | MVT::i32)); | ||||||||||||
8672 | AsmNodeOperands.push_back(OpInfo.CallOperand); | ||||||||||||
8673 | } else { | ||||||||||||
8674 | // Otherwise, this outputs to a register (directly for C_Register / | ||||||||||||
8675 | // C_RegisterClass, and a target-defined fashion for | ||||||||||||
8676 | // C_Immediate/C_Other). Find a register that we can use. | ||||||||||||
8677 | if (OpInfo.AssignedRegs.Regs.empty()) { | ||||||||||||
8678 | emitInlineAsmError( | ||||||||||||
8679 | Call, "couldn't allocate output register for constraint '" + | ||||||||||||
8680 | Twine(OpInfo.ConstraintCode) + "'"); | ||||||||||||
8681 | return; | ||||||||||||
8682 | } | ||||||||||||
8683 | |||||||||||||
8684 | if (DetectWriteToReservedRegister()) | ||||||||||||
8685 | return; | ||||||||||||
8686 | |||||||||||||
8687 | // Add information to the INLINEASM node to know that this register is | ||||||||||||
8688 | // set. | ||||||||||||
8689 | OpInfo.AssignedRegs.AddInlineAsmOperands( | ||||||||||||
8690 | OpInfo.isEarlyClobber ? InlineAsm::Kind_RegDefEarlyClobber | ||||||||||||
8691 | : InlineAsm::Kind_RegDef, | ||||||||||||
8692 | false, 0, getCurSDLoc(), DAG, AsmNodeOperands); | ||||||||||||
8693 | } | ||||||||||||
8694 | break; | ||||||||||||
8695 | |||||||||||||
8696 | case InlineAsm::isInput: { | ||||||||||||
8697 | SDValue InOperandVal = OpInfo.CallOperand; | ||||||||||||
8698 | |||||||||||||
8699 | if (OpInfo.isMatchingInputConstraint()) { | ||||||||||||
8700 | // If this is required to match an output register we have already set, | ||||||||||||
8701 | // just use its register. | ||||||||||||
8702 | auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(), | ||||||||||||
8703 | AsmNodeOperands); | ||||||||||||
8704 | unsigned OpFlag = | ||||||||||||
8705 | cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); | ||||||||||||
8706 | if (InlineAsm::isRegDefKind(OpFlag) || | ||||||||||||
8707 | InlineAsm::isRegDefEarlyClobberKind(OpFlag)) { | ||||||||||||
8708 | // Add (OpFlag&0xffff)>>3 registers to MatchedRegs. | ||||||||||||
8709 | if (OpInfo.isIndirect) { | ||||||||||||
8710 | // This happens on gcc/testsuite/gcc.dg/pr8788-1.c | ||||||||||||
8711 | emitInlineAsmError(Call, "inline asm not supported yet: " | ||||||||||||
8712 | "don't know how to handle tied " | ||||||||||||
8713 | "indirect register inputs"); | ||||||||||||
8714 | return; | ||||||||||||
8715 | } | ||||||||||||
8716 | |||||||||||||
8717 | SmallVector<unsigned, 4> Regs; | ||||||||||||
8718 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||||||
8719 | MachineRegisterInfo &MRI = MF.getRegInfo(); | ||||||||||||
8720 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | ||||||||||||
8721 | auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]); | ||||||||||||
8722 | Register TiedReg = R->getReg(); | ||||||||||||
8723 | MVT RegVT = R->getSimpleValueType(0); | ||||||||||||
8724 | const TargetRegisterClass *RC = TiedReg.isVirtual() ? | ||||||||||||
8725 | MRI.getRegClass(TiedReg) : TRI.getMinimalPhysRegClass(TiedReg); | ||||||||||||
8726 | unsigned NumRegs = InlineAsm::getNumOperandRegisters(OpFlag); | ||||||||||||
8727 | for (unsigned i = 0; i != NumRegs; ++i) | ||||||||||||
8728 | Regs.push_back(MRI.createVirtualRegister(RC)); | ||||||||||||
8729 | |||||||||||||
8730 | RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType()); | ||||||||||||
8731 | |||||||||||||
8732 | SDLoc dl = getCurSDLoc(); | ||||||||||||
8733 | // Use the produced MatchedRegs object to | ||||||||||||
8734 | MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag, &Call); | ||||||||||||
8735 | MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, | ||||||||||||
8736 | true, OpInfo.getMatchedOperand(), dl, | ||||||||||||
8737 | DAG, AsmNodeOperands); | ||||||||||||
8738 | break; | ||||||||||||
8739 | } | ||||||||||||
8740 | |||||||||||||
8741 | assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!")(static_cast <bool> (InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!") ? void (0) : __assert_fail ( "InlineAsm::isMemKind(OpFlag) && \"Unknown matching constraint!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8741, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8742 | assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&(static_cast <bool> (InlineAsm::getNumOperandRegisters( OpFlag) == 1 && "Unexpected number of operands") ? void (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(OpFlag) == 1 && \"Unexpected number of operands\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8743, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8743 | "Unexpected number of operands")(static_cast <bool> (InlineAsm::getNumOperandRegisters( OpFlag) == 1 && "Unexpected number of operands") ? void (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(OpFlag) == 1 && \"Unexpected number of operands\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8743, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8744 | // Add information to the INLINEASM node to know about this input. | ||||||||||||
8745 | // See InlineAsm.h isUseOperandTiedToDef. | ||||||||||||
8746 | OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag); | ||||||||||||
8747 | OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag, | ||||||||||||
8748 | OpInfo.getMatchedOperand()); | ||||||||||||
8749 | AsmNodeOperands.push_back(DAG.getTargetConstant( | ||||||||||||
8750 | OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||||||
8751 | AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]); | ||||||||||||
8752 | break; | ||||||||||||
8753 | } | ||||||||||||
8754 | |||||||||||||
8755 | // Treat indirect 'X' constraint as memory. | ||||||||||||
8756 | if (OpInfo.ConstraintType == TargetLowering::C_Other && | ||||||||||||
8757 | OpInfo.isIndirect) | ||||||||||||
8758 | OpInfo.ConstraintType = TargetLowering::C_Memory; | ||||||||||||
8759 | |||||||||||||
8760 | if (OpInfo.ConstraintType == TargetLowering::C_Immediate || | ||||||||||||
8761 | OpInfo.ConstraintType == TargetLowering::C_Other) { | ||||||||||||
8762 | std::vector<SDValue> Ops; | ||||||||||||
8763 | TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode, | ||||||||||||
8764 | Ops, DAG); | ||||||||||||
8765 | if (Ops.empty()) { | ||||||||||||
8766 | if (OpInfo.ConstraintType == TargetLowering::C_Immediate) | ||||||||||||
8767 | if (isa<ConstantSDNode>(InOperandVal)) { | ||||||||||||
8768 | emitInlineAsmError(Call, "value out of range for constraint '" + | ||||||||||||
8769 | Twine(OpInfo.ConstraintCode) + "'"); | ||||||||||||
8770 | return; | ||||||||||||
8771 | } | ||||||||||||
8772 | |||||||||||||
8773 | emitInlineAsmError(Call, | ||||||||||||
8774 | "invalid operand for inline asm constraint '" + | ||||||||||||
8775 | Twine(OpInfo.ConstraintCode) + "'"); | ||||||||||||
8776 | return; | ||||||||||||
8777 | } | ||||||||||||
8778 | |||||||||||||
8779 | // Add information to the INLINEASM node to know about this input. | ||||||||||||
8780 | unsigned ResOpType = | ||||||||||||
8781 | InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size()); | ||||||||||||
8782 | AsmNodeOperands.push_back(DAG.getTargetConstant( | ||||||||||||
8783 | ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||||||
8784 | llvm::append_range(AsmNodeOperands, Ops); | ||||||||||||
8785 | break; | ||||||||||||
8786 | } | ||||||||||||
8787 | |||||||||||||
8788 | if (OpInfo.ConstraintType == TargetLowering::C_Memory) { | ||||||||||||
8789 | assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!")(static_cast <bool> (OpInfo.isIndirect && "Operand must be indirect to be a mem!" ) ? void (0) : __assert_fail ("OpInfo.isIndirect && \"Operand must be indirect to be a mem!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8789, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8790 | assert(InOperandVal.getValueType() ==(static_cast <bool> (InOperandVal.getValueType() == TLI .getPointerTy(DAG.getDataLayout()) && "Memory operands expect pointer values" ) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8792, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8791 | TLI.getPointerTy(DAG.getDataLayout()) &&(static_cast <bool> (InOperandVal.getValueType() == TLI .getPointerTy(DAG.getDataLayout()) && "Memory operands expect pointer values" ) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8792, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8792 | "Memory operands expect pointer values")(static_cast <bool> (InOperandVal.getValueType() == TLI .getPointerTy(DAG.getDataLayout()) && "Memory operands expect pointer values" ) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8792, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8793 | |||||||||||||
8794 | unsigned ConstraintID = | ||||||||||||
8795 | TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode); | ||||||||||||
8796 | assert(ConstraintID != InlineAsm::Constraint_Unknown &&(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id." ) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8797, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8797 | "Failed to convert memory constraint code to constraint id.")(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id." ) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8797, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8798 | |||||||||||||
8799 | // Add information to the INLINEASM node to know about this input. | ||||||||||||
8800 | unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); | ||||||||||||
8801 | ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID); | ||||||||||||
8802 | AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, | ||||||||||||
8803 | getCurSDLoc(), | ||||||||||||
8804 | MVT::i32)); | ||||||||||||
8805 | AsmNodeOperands.push_back(InOperandVal); | ||||||||||||
8806 | break; | ||||||||||||
8807 | } | ||||||||||||
8808 | |||||||||||||
8809 | assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||(static_cast <bool> ((OpInfo.ConstraintType == TargetLowering ::C_RegisterClass || OpInfo.ConstraintType == TargetLowering:: C_Register) && "Unknown constraint type!") ? void (0) : __assert_fail ("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8811, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8810 | OpInfo.ConstraintType == TargetLowering::C_Register) &&(static_cast <bool> ((OpInfo.ConstraintType == TargetLowering ::C_RegisterClass || OpInfo.ConstraintType == TargetLowering:: C_Register) && "Unknown constraint type!") ? void (0) : __assert_fail ("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8811, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8811 | "Unknown constraint type!")(static_cast <bool> ((OpInfo.ConstraintType == TargetLowering ::C_RegisterClass || OpInfo.ConstraintType == TargetLowering:: C_Register) && "Unknown constraint type!") ? void (0) : __assert_fail ("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8811, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8812 | |||||||||||||
8813 | // TODO: Support this. | ||||||||||||
8814 | if (OpInfo.isIndirect) { | ||||||||||||
8815 | emitInlineAsmError( | ||||||||||||
8816 | Call, "Don't know how to handle indirect register inputs yet " | ||||||||||||
8817 | "for constraint '" + | ||||||||||||
8818 | Twine(OpInfo.ConstraintCode) + "'"); | ||||||||||||
8819 | return; | ||||||||||||
8820 | } | ||||||||||||
8821 | |||||||||||||
8822 | // Copy the input into the appropriate registers. | ||||||||||||
8823 | if (OpInfo.AssignedRegs.Regs.empty()) { | ||||||||||||
8824 | emitInlineAsmError(Call, | ||||||||||||
8825 | "couldn't allocate input reg for constraint '" + | ||||||||||||
8826 | Twine(OpInfo.ConstraintCode) + "'"); | ||||||||||||
8827 | return; | ||||||||||||
8828 | } | ||||||||||||
8829 | |||||||||||||
8830 | if (DetectWriteToReservedRegister()) | ||||||||||||
8831 | return; | ||||||||||||
8832 | |||||||||||||
8833 | SDLoc dl = getCurSDLoc(); | ||||||||||||
8834 | |||||||||||||
8835 | OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag, | ||||||||||||
8836 | &Call); | ||||||||||||
8837 | |||||||||||||
8838 | OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0, | ||||||||||||
8839 | dl, DAG, AsmNodeOperands); | ||||||||||||
8840 | break; | ||||||||||||
8841 | } | ||||||||||||
8842 | case InlineAsm::isClobber: | ||||||||||||
8843 | // Add the clobbered value to the operand list, so that the register | ||||||||||||
8844 | // allocator is aware that the physreg got clobbered. | ||||||||||||
8845 | if (!OpInfo.AssignedRegs.Regs.empty()) | ||||||||||||
8846 | OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber, | ||||||||||||
8847 | false, 0, getCurSDLoc(), DAG, | ||||||||||||
8848 | AsmNodeOperands); | ||||||||||||
8849 | break; | ||||||||||||
8850 | } | ||||||||||||
8851 | } | ||||||||||||
8852 | |||||||||||||
8853 | // Finish up input operands. Set the input chain and add the flag last. | ||||||||||||
8854 | AsmNodeOperands[InlineAsm::Op_InputChain] = Chain; | ||||||||||||
8855 | if (Flag.getNode()) AsmNodeOperands.push_back(Flag); | ||||||||||||
8856 | |||||||||||||
8857 | unsigned ISDOpc = IsCallBr
| ||||||||||||
8858 | Chain = DAG.getNode(ISDOpc, getCurSDLoc(), | ||||||||||||
8859 | DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands); | ||||||||||||
8860 | Flag = Chain.getValue(1); | ||||||||||||
8861 | |||||||||||||
8862 | // Do additional work to generate outputs. | ||||||||||||
8863 | |||||||||||||
8864 | SmallVector<EVT, 1> ResultVTs; | ||||||||||||
8865 | SmallVector<SDValue, 1> ResultValues; | ||||||||||||
8866 | SmallVector<SDValue, 8> OutChains; | ||||||||||||
8867 | |||||||||||||
8868 | llvm::Type *CallResultType = Call.getType(); | ||||||||||||
8869 | ArrayRef<Type *> ResultTypes; | ||||||||||||
8870 | if (StructType *StructResult
| ||||||||||||
8871 | ResultTypes = StructResult->elements(); | ||||||||||||
8872 | else if (!CallResultType->isVoidTy()) | ||||||||||||
8873 | ResultTypes = makeArrayRef(CallResultType); | ||||||||||||
8874 | |||||||||||||
8875 | auto CurResultType = ResultTypes.begin(); | ||||||||||||
8876 | auto handleRegAssign = [&](SDValue V) { | ||||||||||||
8877 | assert(CurResultType != ResultTypes.end() && "Unexpected value")(static_cast <bool> (CurResultType != ResultTypes.end() && "Unexpected value") ? void (0) : __assert_fail ("CurResultType != ResultTypes.end() && \"Unexpected value\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8877, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8878 | assert((*CurResultType)->isSized() && "Unexpected unsized type")(static_cast <bool> ((*CurResultType)->isSized() && "Unexpected unsized type") ? void (0) : __assert_fail ("(*CurResultType)->isSized() && \"Unexpected unsized type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8878, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8879 | EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType); | ||||||||||||
8880 | ++CurResultType; | ||||||||||||
8881 | // If the type of the inline asm call site return value is different but has | ||||||||||||
8882 | // same size as the type of the asm output bitcast it. One example of this | ||||||||||||
8883 | // is for vectors with different width / number of elements. This can | ||||||||||||
8884 | // happen for register classes that can contain multiple different value | ||||||||||||
8885 | // types. The preg or vreg allocated may not have the same VT as was | ||||||||||||
8886 | // expected. | ||||||||||||
8887 | // | ||||||||||||
8888 | // This can also happen for a return value that disagrees with the register | ||||||||||||
8889 | // class it is put in, eg. a double in a general-purpose register on a | ||||||||||||
8890 | // 32-bit machine. | ||||||||||||
8891 | if (ResultVT != V.getValueType() && | ||||||||||||
8892 | ResultVT.getSizeInBits() == V.getValueSizeInBits()) | ||||||||||||
8893 | V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V); | ||||||||||||
8894 | else if (ResultVT != V.getValueType() && ResultVT.isInteger() && | ||||||||||||
8895 | V.getValueType().isInteger()) { | ||||||||||||
8896 | // If a result value was tied to an input value, the computed result | ||||||||||||
8897 | // may have a wider width than the expected result. Extract the | ||||||||||||
8898 | // relevant portion. | ||||||||||||
8899 | V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V); | ||||||||||||
8900 | } | ||||||||||||
8901 | assert(ResultVT == V.getValueType() && "Asm result value mismatch!")(static_cast <bool> (ResultVT == V.getValueType() && "Asm result value mismatch!") ? void (0) : __assert_fail ("ResultVT == V.getValueType() && \"Asm result value mismatch!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8901, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8902 | ResultVTs.push_back(ResultVT); | ||||||||||||
8903 | ResultValues.push_back(V); | ||||||||||||
8904 | }; | ||||||||||||
8905 | |||||||||||||
8906 | // Deal with output operands. | ||||||||||||
8907 | for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) { | ||||||||||||
8908 | if (OpInfo.Type == InlineAsm::isOutput) { | ||||||||||||
8909 | SDValue Val; | ||||||||||||
8910 | // Skip trivial output operands. | ||||||||||||
8911 | if (OpInfo.AssignedRegs.Regs.empty()) | ||||||||||||
8912 | continue; | ||||||||||||
8913 | |||||||||||||
8914 | switch (OpInfo.ConstraintType) { | ||||||||||||
8915 | case TargetLowering::C_Register: | ||||||||||||
8916 | case TargetLowering::C_RegisterClass: | ||||||||||||
8917 | Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), | ||||||||||||
8918 | Chain, &Flag, &Call); | ||||||||||||
8919 | break; | ||||||||||||
8920 | case TargetLowering::C_Immediate: | ||||||||||||
8921 | case TargetLowering::C_Other: | ||||||||||||
8922 | Val = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(), | ||||||||||||
8923 | OpInfo, DAG); | ||||||||||||
8924 | break; | ||||||||||||
8925 | case TargetLowering::C_Memory: | ||||||||||||
8926 | break; // Already handled. | ||||||||||||
8927 | case TargetLowering::C_Unknown: | ||||||||||||
8928 | assert(false && "Unexpected unknown constraint")(static_cast <bool> (false && "Unexpected unknown constraint" ) ? void (0) : __assert_fail ("false && \"Unexpected unknown constraint\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8928, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8929 | } | ||||||||||||
8930 | |||||||||||||
8931 | // Indirect output manifest as stores. Record output chains. | ||||||||||||
8932 | if (OpInfo.isIndirect) { | ||||||||||||
8933 | const Value *Ptr = OpInfo.CallOperandVal; | ||||||||||||
8934 | assert(Ptr && "Expected value CallOperandVal for indirect asm operand")(static_cast <bool> (Ptr && "Expected value CallOperandVal for indirect asm operand" ) ? void (0) : __assert_fail ("Ptr && \"Expected value CallOperandVal for indirect asm operand\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8934, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8935 | SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr), | ||||||||||||
8936 | MachinePointerInfo(Ptr)); | ||||||||||||
8937 | OutChains.push_back(Store); | ||||||||||||
8938 | } else { | ||||||||||||
8939 | // generate CopyFromRegs to associated registers. | ||||||||||||
8940 | assert(!Call.getType()->isVoidTy() && "Bad inline asm!")(static_cast <bool> (!Call.getType()->isVoidTy() && "Bad inline asm!") ? void (0) : __assert_fail ("!Call.getType()->isVoidTy() && \"Bad inline asm!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8940, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8941 | if (Val.getOpcode() == ISD::MERGE_VALUES) { | ||||||||||||
8942 | for (const SDValue &V : Val->op_values()) | ||||||||||||
8943 | handleRegAssign(V); | ||||||||||||
8944 | } else | ||||||||||||
8945 | handleRegAssign(Val); | ||||||||||||
8946 | } | ||||||||||||
8947 | } | ||||||||||||
8948 | } | ||||||||||||
8949 | |||||||||||||
8950 | // Set results. | ||||||||||||
8951 | if (!ResultValues.empty()) { | ||||||||||||
8952 | assert(CurResultType == ResultTypes.end() &&(static_cast <bool> (CurResultType == ResultTypes.end() && "Mismatch in number of ResultTypes") ? void (0) : __assert_fail ("CurResultType == ResultTypes.end() && \"Mismatch in number of ResultTypes\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8953, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8953 | "Mismatch in number of ResultTypes")(static_cast <bool> (CurResultType == ResultTypes.end() && "Mismatch in number of ResultTypes") ? void (0) : __assert_fail ("CurResultType == ResultTypes.end() && \"Mismatch in number of ResultTypes\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8953, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8954 | assert(ResultValues.size() == ResultTypes.size() &&(static_cast <bool> (ResultValues.size() == ResultTypes .size() && "Mismatch in number of output operands in asm result" ) ? void (0) : __assert_fail ("ResultValues.size() == ResultTypes.size() && \"Mismatch in number of output operands in asm result\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8955, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
8955 | "Mismatch in number of output operands in asm result")(static_cast <bool> (ResultValues.size() == ResultTypes .size() && "Mismatch in number of output operands in asm result" ) ? void (0) : __assert_fail ("ResultValues.size() == ResultTypes.size() && \"Mismatch in number of output operands in asm result\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 8955, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
8956 | |||||||||||||
8957 | SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), | ||||||||||||
8958 | DAG.getVTList(ResultVTs), ResultValues); | ||||||||||||
8959 | setValue(&Call, V); | ||||||||||||
8960 | } | ||||||||||||
8961 | |||||||||||||
8962 | // Collect store chains. | ||||||||||||
8963 | if (!OutChains.empty()) | ||||||||||||
8964 | Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains); | ||||||||||||
8965 | |||||||||||||
8966 | if (EmitEHLabels) { | ||||||||||||
8967 | Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel); | ||||||||||||
8968 | } | ||||||||||||
8969 | |||||||||||||
8970 | // Only Update Root if inline assembly has a memory effect. | ||||||||||||
8971 | if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr || | ||||||||||||
8972 | EmitEHLabels) | ||||||||||||
8973 | DAG.setRoot(Chain); | ||||||||||||
8974 | } | ||||||||||||
8975 | |||||||||||||
8976 | void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call, | ||||||||||||
8977 | const Twine &Message) { | ||||||||||||
8978 | LLVMContext &Ctx = *DAG.getContext(); | ||||||||||||
8979 | Ctx.emitError(&Call, Message); | ||||||||||||
8980 | |||||||||||||
8981 | // Make sure we leave the DAG in a valid state | ||||||||||||
8982 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
8983 | SmallVector<EVT, 1> ValueVTs; | ||||||||||||
8984 | ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs); | ||||||||||||
8985 | |||||||||||||
8986 | if (ValueVTs.empty()) | ||||||||||||
8987 | return; | ||||||||||||
8988 | |||||||||||||
8989 | SmallVector<SDValue, 1> Ops; | ||||||||||||
8990 | for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i) | ||||||||||||
8991 | Ops.push_back(DAG.getUNDEF(ValueVTs[i])); | ||||||||||||
8992 | |||||||||||||
8993 | setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc())); | ||||||||||||
8994 | } | ||||||||||||
8995 | |||||||||||||
8996 | void SelectionDAGBuilder::visitVAStart(const CallInst &I) { | ||||||||||||
8997 | DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(), | ||||||||||||
8998 | MVT::Other, getRoot(), | ||||||||||||
8999 | getValue(I.getArgOperand(0)), | ||||||||||||
9000 | DAG.getSrcValue(I.getArgOperand(0)))); | ||||||||||||
9001 | } | ||||||||||||
9002 | |||||||||||||
9003 | void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) { | ||||||||||||
9004 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
9005 | const DataLayout &DL = DAG.getDataLayout(); | ||||||||||||
9006 | SDValue V = DAG.getVAArg( | ||||||||||||
9007 | TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(), | ||||||||||||
9008 | getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)), | ||||||||||||
9009 | DL.getABITypeAlign(I.getType()).value()); | ||||||||||||
9010 | DAG.setRoot(V.getValue(1)); | ||||||||||||
9011 | |||||||||||||
9012 | if (I.getType()->isPointerTy()) | ||||||||||||
9013 | V = DAG.getPtrExtOrTrunc( | ||||||||||||
9014 | V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType())); | ||||||||||||
9015 | setValue(&I, V); | ||||||||||||
9016 | } | ||||||||||||
9017 | |||||||||||||
9018 | void SelectionDAGBuilder::visitVAEnd(const CallInst &I) { | ||||||||||||
9019 | DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(), | ||||||||||||
9020 | MVT::Other, getRoot(), | ||||||||||||
9021 | getValue(I.getArgOperand(0)), | ||||||||||||
9022 | DAG.getSrcValue(I.getArgOperand(0)))); | ||||||||||||
9023 | } | ||||||||||||
9024 | |||||||||||||
9025 | void SelectionDAGBuilder::visitVACopy(const CallInst &I) { | ||||||||||||
9026 | DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(), | ||||||||||||
9027 | MVT::Other, getRoot(), | ||||||||||||
9028 | getValue(I.getArgOperand(0)), | ||||||||||||
9029 | getValue(I.getArgOperand(1)), | ||||||||||||
9030 | DAG.getSrcValue(I.getArgOperand(0)), | ||||||||||||
9031 | DAG.getSrcValue(I.getArgOperand(1)))); | ||||||||||||
9032 | } | ||||||||||||
9033 | |||||||||||||
9034 | SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG, | ||||||||||||
9035 | const Instruction &I, | ||||||||||||
9036 | SDValue Op) { | ||||||||||||
9037 | const MDNode *Range = I.getMetadata(LLVMContext::MD_range); | ||||||||||||
9038 | if (!Range) | ||||||||||||
9039 | return Op; | ||||||||||||
9040 | |||||||||||||
9041 | ConstantRange CR = getConstantRangeFromMetadata(*Range); | ||||||||||||
9042 | if (CR.isFullSet() || CR.isEmptySet() || CR.isUpperWrapped()) | ||||||||||||
9043 | return Op; | ||||||||||||
9044 | |||||||||||||
9045 | APInt Lo = CR.getUnsignedMin(); | ||||||||||||
9046 | if (!Lo.isMinValue()) | ||||||||||||
9047 | return Op; | ||||||||||||
9048 | |||||||||||||
9049 | APInt Hi = CR.getUnsignedMax(); | ||||||||||||
9050 | unsigned Bits = std::max(Hi.getActiveBits(), | ||||||||||||
9051 | static_cast<unsigned>(IntegerType::MIN_INT_BITS)); | ||||||||||||
9052 | |||||||||||||
9053 | EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits); | ||||||||||||
9054 | |||||||||||||
9055 | SDLoc SL = getCurSDLoc(); | ||||||||||||
9056 | |||||||||||||
9057 | SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op, | ||||||||||||
9058 | DAG.getValueType(SmallVT)); | ||||||||||||
9059 | unsigned NumVals = Op.getNode()->getNumValues(); | ||||||||||||
9060 | if (NumVals == 1) | ||||||||||||
9061 | return ZExt; | ||||||||||||
9062 | |||||||||||||
9063 | SmallVector<SDValue, 4> Ops; | ||||||||||||
9064 | |||||||||||||
9065 | Ops.push_back(ZExt); | ||||||||||||
9066 | for (unsigned I = 1; I != NumVals; ++I) | ||||||||||||
9067 | Ops.push_back(Op.getValue(I)); | ||||||||||||
9068 | |||||||||||||
9069 | return DAG.getMergeValues(Ops, SL); | ||||||||||||
9070 | } | ||||||||||||
9071 | |||||||||||||
9072 | /// Populate a CallLowerinInfo (into \p CLI) based on the properties of | ||||||||||||
9073 | /// the call being lowered. | ||||||||||||
9074 | /// | ||||||||||||
9075 | /// This is a helper for lowering intrinsics that follow a target calling | ||||||||||||
9076 | /// convention or require stack pointer adjustment. Only a subset of the | ||||||||||||
9077 | /// intrinsic's operands need to participate in the calling convention. | ||||||||||||
9078 | void SelectionDAGBuilder::populateCallLoweringInfo( | ||||||||||||
9079 | TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, | ||||||||||||
9080 | unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, | ||||||||||||
9081 | bool IsPatchPoint) { | ||||||||||||
9082 | TargetLowering::ArgListTy Args; | ||||||||||||
9083 | Args.reserve(NumArgs); | ||||||||||||
9084 | |||||||||||||
9085 | // Populate the argument list. | ||||||||||||
9086 | // Attributes for args start at offset 1, after the return attribute. | ||||||||||||
9087 | for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; | ||||||||||||
9088 | ArgI != ArgE; ++ArgI) { | ||||||||||||
9089 | const Value *V = Call->getOperand(ArgI); | ||||||||||||
9090 | |||||||||||||
9091 | assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.")(static_cast <bool> (!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.") ? void (0) : __assert_fail ("!V->getType()->isEmptyTy() && \"Empty type passed to intrinsic.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9091, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9092 | |||||||||||||
9093 | TargetLowering::ArgListEntry Entry; | ||||||||||||
9094 | Entry.Node = getValue(V); | ||||||||||||
9095 | Entry.Ty = V->getType(); | ||||||||||||
9096 | Entry.setAttributes(Call, ArgI); | ||||||||||||
9097 | Args.push_back(Entry); | ||||||||||||
9098 | } | ||||||||||||
9099 | |||||||||||||
9100 | CLI.setDebugLoc(getCurSDLoc()) | ||||||||||||
9101 | .setChain(getRoot()) | ||||||||||||
9102 | .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args)) | ||||||||||||
9103 | .setDiscardResult(Call->use_empty()) | ||||||||||||
9104 | .setIsPatchPoint(IsPatchPoint) | ||||||||||||
9105 | .setIsPreallocated( | ||||||||||||
9106 | Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0); | ||||||||||||
9107 | } | ||||||||||||
9108 | |||||||||||||
9109 | /// Add a stack map intrinsic call's live variable operands to a stackmap | ||||||||||||
9110 | /// or patchpoint target node's operand list. | ||||||||||||
9111 | /// | ||||||||||||
9112 | /// Constants are converted to TargetConstants purely as an optimization to | ||||||||||||
9113 | /// avoid constant materialization and register allocation. | ||||||||||||
9114 | /// | ||||||||||||
9115 | /// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not | ||||||||||||
9116 | /// generate addess computation nodes, and so FinalizeISel can convert the | ||||||||||||
9117 | /// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids | ||||||||||||
9118 | /// address materialization and register allocation, but may also be required | ||||||||||||
9119 | /// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an | ||||||||||||
9120 | /// alloca in the entry block, then the runtime may assume that the alloca's | ||||||||||||
9121 | /// StackMap location can be read immediately after compilation and that the | ||||||||||||
9122 | /// location is valid at any point during execution (this is similar to the | ||||||||||||
9123 | /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were | ||||||||||||
9124 | /// only available in a register, then the runtime would need to trap when | ||||||||||||
9125 | /// execution reaches the StackMap in order to read the alloca's location. | ||||||||||||
9126 | static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, | ||||||||||||
9127 | const SDLoc &DL, SmallVectorImpl<SDValue> &Ops, | ||||||||||||
9128 | SelectionDAGBuilder &Builder) { | ||||||||||||
9129 | for (unsigned i = StartIdx, e = Call.arg_size(); i != e; ++i) { | ||||||||||||
9130 | SDValue OpVal = Builder.getValue(Call.getArgOperand(i)); | ||||||||||||
9131 | if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) { | ||||||||||||
9132 | Ops.push_back( | ||||||||||||
9133 | Builder.DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); | ||||||||||||
9134 | Ops.push_back( | ||||||||||||
9135 | Builder.DAG.getTargetConstant(C->getSExtValue(), DL, MVT::i64)); | ||||||||||||
9136 | } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) { | ||||||||||||
9137 | const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo(); | ||||||||||||
9138 | Ops.push_back(Builder.DAG.getTargetFrameIndex( | ||||||||||||
9139 | FI->getIndex(), TLI.getFrameIndexTy(Builder.DAG.getDataLayout()))); | ||||||||||||
9140 | } else | ||||||||||||
9141 | Ops.push_back(OpVal); | ||||||||||||
9142 | } | ||||||||||||
9143 | } | ||||||||||||
9144 | |||||||||||||
9145 | /// Lower llvm.experimental.stackmap directly to its target opcode. | ||||||||||||
9146 | void SelectionDAGBuilder::visitStackmap(const CallInst &CI) { | ||||||||||||
9147 | // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>, | ||||||||||||
9148 | // [live variables...]) | ||||||||||||
9149 | |||||||||||||
9150 | assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.")(static_cast <bool> (CI.getType()->isVoidTy() && "Stackmap cannot return a value.") ? void (0) : __assert_fail ("CI.getType()->isVoidTy() && \"Stackmap cannot return a value.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9150, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9151 | |||||||||||||
9152 | SDValue Chain, InFlag, Callee, NullPtr; | ||||||||||||
9153 | SmallVector<SDValue, 32> Ops; | ||||||||||||
9154 | |||||||||||||
9155 | SDLoc DL = getCurSDLoc(); | ||||||||||||
9156 | Callee = getValue(CI.getCalledOperand()); | ||||||||||||
9157 | NullPtr = DAG.getIntPtrConstant(0, DL, true); | ||||||||||||
9158 | |||||||||||||
9159 | // The stackmap intrinsic only records the live variables (the arguments | ||||||||||||
9160 | // passed to it) and emits NOPS (if requested). Unlike the patchpoint | ||||||||||||
9161 | // intrinsic, this won't be lowered to a function call. This means we don't | ||||||||||||
9162 | // have to worry about calling conventions and target specific lowering code. | ||||||||||||
9163 | // Instead we perform the call lowering right here. | ||||||||||||
9164 | // | ||||||||||||
9165 | // chain, flag = CALLSEQ_START(chain, 0, 0) | ||||||||||||
9166 | // chain, flag = STACKMAP(id, nbytes, ..., chain, flag) | ||||||||||||
9167 | // chain, flag = CALLSEQ_END(chain, 0, 0, flag) | ||||||||||||
9168 | // | ||||||||||||
9169 | Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL); | ||||||||||||
9170 | InFlag = Chain.getValue(1); | ||||||||||||
9171 | |||||||||||||
9172 | // Add the <id> and <numBytes> constants. | ||||||||||||
9173 | SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos)); | ||||||||||||
9174 | Ops.push_back(DAG.getTargetConstant( | ||||||||||||
9175 | cast<ConstantSDNode>(IDVal)->getZExtValue(), DL, MVT::i64)); | ||||||||||||
9176 | SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos)); | ||||||||||||
9177 | Ops.push_back(DAG.getTargetConstant( | ||||||||||||
9178 | cast<ConstantSDNode>(NBytesVal)->getZExtValue(), DL, | ||||||||||||
9179 | MVT::i32)); | ||||||||||||
9180 | |||||||||||||
9181 | // Push live variables for the stack map. | ||||||||||||
9182 | addStackMapLiveVars(CI, 2, DL, Ops, *this); | ||||||||||||
9183 | |||||||||||||
9184 | // We are not pushing any register mask info here on the operands list, | ||||||||||||
9185 | // because the stackmap doesn't clobber anything. | ||||||||||||
9186 | |||||||||||||
9187 | // Push the chain and the glue flag. | ||||||||||||
9188 | Ops.push_back(Chain); | ||||||||||||
9189 | Ops.push_back(InFlag); | ||||||||||||
9190 | |||||||||||||
9191 | // Create the STACKMAP node. | ||||||||||||
9192 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | ||||||||||||
9193 | SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops); | ||||||||||||
9194 | Chain = SDValue(SM, 0); | ||||||||||||
9195 | InFlag = Chain.getValue(1); | ||||||||||||
9196 | |||||||||||||
9197 | Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL); | ||||||||||||
9198 | |||||||||||||
9199 | // Stackmaps don't generate values, so nothing goes into the NodeMap. | ||||||||||||
9200 | |||||||||||||
9201 | // Set the root to the target-lowered call chain. | ||||||||||||
9202 | DAG.setRoot(Chain); | ||||||||||||
9203 | |||||||||||||
9204 | // Inform the Frame Information that we have a stackmap in this function. | ||||||||||||
9205 | FuncInfo.MF->getFrameInfo().setHasStackMap(); | ||||||||||||
9206 | } | ||||||||||||
9207 | |||||||||||||
9208 | /// Lower llvm.experimental.patchpoint directly to its target opcode. | ||||||||||||
9209 | void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB, | ||||||||||||
9210 | const BasicBlock *EHPadBB) { | ||||||||||||
9211 | // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>, | ||||||||||||
9212 | // i32 <numBytes>, | ||||||||||||
9213 | // i8* <target>, | ||||||||||||
9214 | // i32 <numArgs>, | ||||||||||||
9215 | // [Args...], | ||||||||||||
9216 | // [live variables...]) | ||||||||||||
9217 | |||||||||||||
9218 | CallingConv::ID CC = CB.getCallingConv(); | ||||||||||||
9219 | bool IsAnyRegCC = CC == CallingConv::AnyReg; | ||||||||||||
9220 | bool HasDef = !CB.getType()->isVoidTy(); | ||||||||||||
9221 | SDLoc dl = getCurSDLoc(); | ||||||||||||
9222 | SDValue Callee = getValue(CB.getArgOperand(PatchPointOpers::TargetPos)); | ||||||||||||
9223 | |||||||||||||
9224 | // Handle immediate and symbolic callees. | ||||||||||||
9225 | if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee)) | ||||||||||||
9226 | Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl, | ||||||||||||
9227 | /*isTarget=*/true); | ||||||||||||
9228 | else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee)) | ||||||||||||
9229 | Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(), | ||||||||||||
9230 | SDLoc(SymbolicCallee), | ||||||||||||
9231 | SymbolicCallee->getValueType(0)); | ||||||||||||
9232 | |||||||||||||
9233 | // Get the real number of arguments participating in the call <numArgs> | ||||||||||||
9234 | SDValue NArgVal = getValue(CB.getArgOperand(PatchPointOpers::NArgPos)); | ||||||||||||
9235 | unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue(); | ||||||||||||
9236 | |||||||||||||
9237 | // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs> | ||||||||||||
9238 | // Intrinsics include all meta-operands up to but not including CC. | ||||||||||||
9239 | unsigned NumMetaOpers = PatchPointOpers::CCPos; | ||||||||||||
9240 | assert(CB.arg_size() >= NumMetaOpers + NumArgs &&(static_cast <bool> (CB.arg_size() >= NumMetaOpers + NumArgs && "Not enough arguments provided to the patchpoint intrinsic" ) ? void (0) : __assert_fail ("CB.arg_size() >= NumMetaOpers + NumArgs && \"Not enough arguments provided to the patchpoint intrinsic\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9241, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
9241 | "Not enough arguments provided to the patchpoint intrinsic")(static_cast <bool> (CB.arg_size() >= NumMetaOpers + NumArgs && "Not enough arguments provided to the patchpoint intrinsic" ) ? void (0) : __assert_fail ("CB.arg_size() >= NumMetaOpers + NumArgs && \"Not enough arguments provided to the patchpoint intrinsic\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9241, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9242 | |||||||||||||
9243 | // For AnyRegCC the arguments are lowered later on manually. | ||||||||||||
9244 | unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs; | ||||||||||||
9245 | Type *ReturnTy = | ||||||||||||
9246 | IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType(); | ||||||||||||
9247 | |||||||||||||
9248 | TargetLowering::CallLoweringInfo CLI(DAG); | ||||||||||||
9249 | populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee, | ||||||||||||
9250 | ReturnTy, true); | ||||||||||||
9251 | std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB); | ||||||||||||
9252 | |||||||||||||
9253 | SDNode *CallEnd = Result.second.getNode(); | ||||||||||||
9254 | if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg)) | ||||||||||||
9255 | CallEnd = CallEnd->getOperand(0).getNode(); | ||||||||||||
9256 | |||||||||||||
9257 | /// Get a call instruction from the call sequence chain. | ||||||||||||
9258 | /// Tail calls are not allowed. | ||||||||||||
9259 | assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&(static_cast <bool> (CallEnd->getOpcode() == ISD::CALLSEQ_END && "Expected a callseq node.") ? void (0) : __assert_fail ("CallEnd->getOpcode() == ISD::CALLSEQ_END && \"Expected a callseq node.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9260, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
9260 | "Expected a callseq node.")(static_cast <bool> (CallEnd->getOpcode() == ISD::CALLSEQ_END && "Expected a callseq node.") ? void (0) : __assert_fail ("CallEnd->getOpcode() == ISD::CALLSEQ_END && \"Expected a callseq node.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9260, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9261 | SDNode *Call = CallEnd->getOperand(0).getNode(); | ||||||||||||
9262 | bool HasGlue = Call->getGluedNode(); | ||||||||||||
9263 | |||||||||||||
9264 | // Replace the target specific call node with the patchable intrinsic. | ||||||||||||
9265 | SmallVector<SDValue, 8> Ops; | ||||||||||||
9266 | |||||||||||||
9267 | // Add the <id> and <numBytes> constants. | ||||||||||||
9268 | SDValue IDVal = getValue(CB.getArgOperand(PatchPointOpers::IDPos)); | ||||||||||||
9269 | Ops.push_back(DAG.getTargetConstant( | ||||||||||||
9270 | cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64)); | ||||||||||||
9271 | SDValue NBytesVal = getValue(CB.getArgOperand(PatchPointOpers::NBytesPos)); | ||||||||||||
9272 | Ops.push_back(DAG.getTargetConstant( | ||||||||||||
9273 | cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl, | ||||||||||||
9274 | MVT::i32)); | ||||||||||||
9275 | |||||||||||||
9276 | // Add the callee. | ||||||||||||
9277 | Ops.push_back(Callee); | ||||||||||||
9278 | |||||||||||||
9279 | // Adjust <numArgs> to account for any arguments that have been passed on the | ||||||||||||
9280 | // stack instead. | ||||||||||||
9281 | // Call Node: Chain, Target, {Args}, RegMask, [Glue] | ||||||||||||
9282 | unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3); | ||||||||||||
9283 | NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs; | ||||||||||||
9284 | Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32)); | ||||||||||||
9285 | |||||||||||||
9286 | // Add the calling convention | ||||||||||||
9287 | Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32)); | ||||||||||||
9288 | |||||||||||||
9289 | // Add the arguments we omitted previously. The register allocator should | ||||||||||||
9290 | // place these in any free register. | ||||||||||||
9291 | if (IsAnyRegCC) | ||||||||||||
9292 | for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) | ||||||||||||
9293 | Ops.push_back(getValue(CB.getArgOperand(i))); | ||||||||||||
9294 | |||||||||||||
9295 | // Push the arguments from the call instruction up to the register mask. | ||||||||||||
9296 | SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1; | ||||||||||||
9297 | Ops.append(Call->op_begin() + 2, e); | ||||||||||||
9298 | |||||||||||||
9299 | // Push live variables for the stack map. | ||||||||||||
9300 | addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this); | ||||||||||||
9301 | |||||||||||||
9302 | // Push the register mask info. | ||||||||||||
9303 | if (HasGlue) | ||||||||||||
9304 | Ops.push_back(*(Call->op_end()-2)); | ||||||||||||
9305 | else | ||||||||||||
9306 | Ops.push_back(*(Call->op_end()-1)); | ||||||||||||
9307 | |||||||||||||
9308 | // Push the chain (this is originally the first operand of the call, but | ||||||||||||
9309 | // becomes now the last or second to last operand). | ||||||||||||
9310 | Ops.push_back(*(Call->op_begin())); | ||||||||||||
9311 | |||||||||||||
9312 | // Push the glue flag (last operand). | ||||||||||||
9313 | if (HasGlue) | ||||||||||||
9314 | Ops.push_back(*(Call->op_end()-1)); | ||||||||||||
9315 | |||||||||||||
9316 | SDVTList NodeTys; | ||||||||||||
9317 | if (IsAnyRegCC && HasDef) { | ||||||||||||
9318 | // Create the return types based on the intrinsic definition | ||||||||||||
9319 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
9320 | SmallVector<EVT, 3> ValueVTs; | ||||||||||||
9321 | ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs); | ||||||||||||
9322 | assert(ValueVTs.size() == 1 && "Expected only one return value type.")(static_cast <bool> (ValueVTs.size() == 1 && "Expected only one return value type." ) ? void (0) : __assert_fail ("ValueVTs.size() == 1 && \"Expected only one return value type.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9322, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9323 | |||||||||||||
9324 | // There is always a chain and a glue type at the end | ||||||||||||
9325 | ValueVTs.push_back(MVT::Other); | ||||||||||||
9326 | ValueVTs.push_back(MVT::Glue); | ||||||||||||
9327 | NodeTys = DAG.getVTList(ValueVTs); | ||||||||||||
9328 | } else | ||||||||||||
9329 | NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | ||||||||||||
9330 | |||||||||||||
9331 | // Replace the target specific call node with a PATCHPOINT node. | ||||||||||||
9332 | MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT, | ||||||||||||
9333 | dl, NodeTys, Ops); | ||||||||||||
9334 | |||||||||||||
9335 | // Update the NodeMap. | ||||||||||||
9336 | if (HasDef) { | ||||||||||||
9337 | if (IsAnyRegCC) | ||||||||||||
9338 | setValue(&CB, SDValue(MN, 0)); | ||||||||||||
9339 | else | ||||||||||||
9340 | setValue(&CB, Result.first); | ||||||||||||
9341 | } | ||||||||||||
9342 | |||||||||||||
9343 | // Fixup the consumers of the intrinsic. The chain and glue may be used in the | ||||||||||||
9344 | // call sequence. Furthermore the location of the chain and glue can change | ||||||||||||
9345 | // when the AnyReg calling convention is used and the intrinsic returns a | ||||||||||||
9346 | // value. | ||||||||||||
9347 | if (IsAnyRegCC && HasDef) { | ||||||||||||
9348 | SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)}; | ||||||||||||
9349 | SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)}; | ||||||||||||
9350 | DAG.ReplaceAllUsesOfValuesWith(From, To, 2); | ||||||||||||
9351 | } else | ||||||||||||
9352 | DAG.ReplaceAllUsesWith(Call, MN); | ||||||||||||
9353 | DAG.DeleteNode(Call); | ||||||||||||
9354 | |||||||||||||
9355 | // Inform the Frame Information that we have a patchpoint in this function. | ||||||||||||
9356 | FuncInfo.MF->getFrameInfo().setHasPatchPoint(); | ||||||||||||
9357 | } | ||||||||||||
9358 | |||||||||||||
9359 | void SelectionDAGBuilder::visitVectorReduce(const CallInst &I, | ||||||||||||
9360 | unsigned Intrinsic) { | ||||||||||||
9361 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
9362 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||||||
9363 | SDValue Op2; | ||||||||||||
9364 | if (I.getNumArgOperands() > 1) | ||||||||||||
9365 | Op2 = getValue(I.getArgOperand(1)); | ||||||||||||
9366 | SDLoc dl = getCurSDLoc(); | ||||||||||||
9367 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
9368 | SDValue Res; | ||||||||||||
9369 | SDNodeFlags SDFlags; | ||||||||||||
9370 | if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) | ||||||||||||
9371 | SDFlags.copyFMF(*FPMO); | ||||||||||||
9372 | |||||||||||||
9373 | switch (Intrinsic) { | ||||||||||||
9374 | case Intrinsic::vector_reduce_fadd: | ||||||||||||
9375 | if (SDFlags.hasAllowReassociation()) | ||||||||||||
9376 | Res = DAG.getNode(ISD::FADD, dl, VT, Op1, | ||||||||||||
9377 | DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags), | ||||||||||||
9378 | SDFlags); | ||||||||||||
9379 | else | ||||||||||||
9380 | Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags); | ||||||||||||
9381 | break; | ||||||||||||
9382 | case Intrinsic::vector_reduce_fmul: | ||||||||||||
9383 | if (SDFlags.hasAllowReassociation()) | ||||||||||||
9384 | Res = DAG.getNode(ISD::FMUL, dl, VT, Op1, | ||||||||||||
9385 | DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags), | ||||||||||||
9386 | SDFlags); | ||||||||||||
9387 | else | ||||||||||||
9388 | Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags); | ||||||||||||
9389 | break; | ||||||||||||
9390 | case Intrinsic::vector_reduce_add: | ||||||||||||
9391 | Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1); | ||||||||||||
9392 | break; | ||||||||||||
9393 | case Intrinsic::vector_reduce_mul: | ||||||||||||
9394 | Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1); | ||||||||||||
9395 | break; | ||||||||||||
9396 | case Intrinsic::vector_reduce_and: | ||||||||||||
9397 | Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1); | ||||||||||||
9398 | break; | ||||||||||||
9399 | case Intrinsic::vector_reduce_or: | ||||||||||||
9400 | Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1); | ||||||||||||
9401 | break; | ||||||||||||
9402 | case Intrinsic::vector_reduce_xor: | ||||||||||||
9403 | Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1); | ||||||||||||
9404 | break; | ||||||||||||
9405 | case Intrinsic::vector_reduce_smax: | ||||||||||||
9406 | Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1); | ||||||||||||
9407 | break; | ||||||||||||
9408 | case Intrinsic::vector_reduce_smin: | ||||||||||||
9409 | Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1); | ||||||||||||
9410 | break; | ||||||||||||
9411 | case Intrinsic::vector_reduce_umax: | ||||||||||||
9412 | Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1); | ||||||||||||
9413 | break; | ||||||||||||
9414 | case Intrinsic::vector_reduce_umin: | ||||||||||||
9415 | Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1); | ||||||||||||
9416 | break; | ||||||||||||
9417 | case Intrinsic::vector_reduce_fmax: | ||||||||||||
9418 | Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags); | ||||||||||||
9419 | break; | ||||||||||||
9420 | case Intrinsic::vector_reduce_fmin: | ||||||||||||
9421 | Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags); | ||||||||||||
9422 | break; | ||||||||||||
9423 | default: | ||||||||||||
9424 | llvm_unreachable("Unhandled vector reduce intrinsic")::llvm::llvm_unreachable_internal("Unhandled vector reduce intrinsic" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9424); | ||||||||||||
9425 | } | ||||||||||||
9426 | setValue(&I, Res); | ||||||||||||
9427 | } | ||||||||||||
9428 | |||||||||||||
9429 | /// Returns an AttributeList representing the attributes applied to the return | ||||||||||||
9430 | /// value of the given call. | ||||||||||||
9431 | static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) { | ||||||||||||
9432 | SmallVector<Attribute::AttrKind, 2> Attrs; | ||||||||||||
9433 | if (CLI.RetSExt) | ||||||||||||
9434 | Attrs.push_back(Attribute::SExt); | ||||||||||||
9435 | if (CLI.RetZExt) | ||||||||||||
9436 | Attrs.push_back(Attribute::ZExt); | ||||||||||||
9437 | if (CLI.IsInReg) | ||||||||||||
9438 | Attrs.push_back(Attribute::InReg); | ||||||||||||
9439 | |||||||||||||
9440 | return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex, | ||||||||||||
9441 | Attrs); | ||||||||||||
9442 | } | ||||||||||||
9443 | |||||||||||||
9444 | /// TargetLowering::LowerCallTo - This is the default LowerCallTo | ||||||||||||
9445 | /// implementation, which just calls LowerCall. | ||||||||||||
9446 | /// FIXME: When all targets are | ||||||||||||
9447 | /// migrated to using LowerCall, this hook should be integrated into SDISel. | ||||||||||||
9448 | std::pair<SDValue, SDValue> | ||||||||||||
9449 | TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { | ||||||||||||
9450 | // Handle the incoming return values from the call. | ||||||||||||
9451 | CLI.Ins.clear(); | ||||||||||||
9452 | Type *OrigRetTy = CLI.RetTy; | ||||||||||||
9453 | SmallVector<EVT, 4> RetTys; | ||||||||||||
9454 | SmallVector<uint64_t, 4> Offsets; | ||||||||||||
9455 | auto &DL = CLI.DAG.getDataLayout(); | ||||||||||||
9456 | ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets); | ||||||||||||
9457 | |||||||||||||
9458 | if (CLI.IsPostTypeLegalization) { | ||||||||||||
9459 | // If we are lowering a libcall after legalization, split the return type. | ||||||||||||
9460 | SmallVector<EVT, 4> OldRetTys; | ||||||||||||
9461 | SmallVector<uint64_t, 4> OldOffsets; | ||||||||||||
9462 | RetTys.swap(OldRetTys); | ||||||||||||
9463 | Offsets.swap(OldOffsets); | ||||||||||||
9464 | |||||||||||||
9465 | for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) { | ||||||||||||
9466 | EVT RetVT = OldRetTys[i]; | ||||||||||||
9467 | uint64_t Offset = OldOffsets[i]; | ||||||||||||
9468 | MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT); | ||||||||||||
9469 | unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT); | ||||||||||||
9470 | unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8; | ||||||||||||
9471 | RetTys.append(NumRegs, RegisterVT); | ||||||||||||
9472 | for (unsigned j = 0; j != NumRegs; ++j) | ||||||||||||
9473 | Offsets.push_back(Offset + j * RegisterVTByteSZ); | ||||||||||||
9474 | } | ||||||||||||
9475 | } | ||||||||||||
9476 | |||||||||||||
9477 | SmallVector<ISD::OutputArg, 4> Outs; | ||||||||||||
9478 | GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL); | ||||||||||||
9479 | |||||||||||||
9480 | bool CanLowerReturn = | ||||||||||||
9481 | this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(), | ||||||||||||
9482 | CLI.IsVarArg, Outs, CLI.RetTy->getContext()); | ||||||||||||
9483 | |||||||||||||
9484 | SDValue DemoteStackSlot; | ||||||||||||
9485 | int DemoteStackIdx = -100; | ||||||||||||
9486 | if (!CanLowerReturn) { | ||||||||||||
9487 | // FIXME: equivalent assert? | ||||||||||||
9488 | // assert(!CS.hasInAllocaArgument() && | ||||||||||||
9489 | // "sret demotion is incompatible with inalloca"); | ||||||||||||
9490 | uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy); | ||||||||||||
9491 | Align Alignment = DL.getPrefTypeAlign(CLI.RetTy); | ||||||||||||
9492 | MachineFunction &MF = CLI.DAG.getMachineFunction(); | ||||||||||||
9493 | DemoteStackIdx = | ||||||||||||
9494 | MF.getFrameInfo().CreateStackObject(TySize, Alignment, false); | ||||||||||||
9495 | Type *StackSlotPtrType = PointerType::get(CLI.RetTy, | ||||||||||||
9496 | DL.getAllocaAddrSpace()); | ||||||||||||
9497 | |||||||||||||
9498 | DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL)); | ||||||||||||
9499 | ArgListEntry Entry; | ||||||||||||
9500 | Entry.Node = DemoteStackSlot; | ||||||||||||
9501 | Entry.Ty = StackSlotPtrType; | ||||||||||||
9502 | Entry.IsSExt = false; | ||||||||||||
9503 | Entry.IsZExt = false; | ||||||||||||
9504 | Entry.IsInReg = false; | ||||||||||||
9505 | Entry.IsSRet = true; | ||||||||||||
9506 | Entry.IsNest = false; | ||||||||||||
9507 | Entry.IsByVal = false; | ||||||||||||
9508 | Entry.IsByRef = false; | ||||||||||||
9509 | Entry.IsReturned = false; | ||||||||||||
9510 | Entry.IsSwiftSelf = false; | ||||||||||||
9511 | Entry.IsSwiftAsync = false; | ||||||||||||
9512 | Entry.IsSwiftError = false; | ||||||||||||
9513 | Entry.IsCFGuardTarget = false; | ||||||||||||
9514 | Entry.Alignment = Alignment; | ||||||||||||
9515 | CLI.getArgs().insert(CLI.getArgs().begin(), Entry); | ||||||||||||
9516 | CLI.NumFixedArgs += 1; | ||||||||||||
9517 | CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext()); | ||||||||||||
9518 | |||||||||||||
9519 | // sret demotion isn't compatible with tail-calls, since the sret argument | ||||||||||||
9520 | // points into the callers stack frame. | ||||||||||||
9521 | CLI.IsTailCall = false; | ||||||||||||
9522 | } else { | ||||||||||||
9523 | bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters( | ||||||||||||
9524 | CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL); | ||||||||||||
9525 | for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { | ||||||||||||
9526 | ISD::ArgFlagsTy Flags; | ||||||||||||
9527 | if (NeedsRegBlock) { | ||||||||||||
9528 | Flags.setInConsecutiveRegs(); | ||||||||||||
9529 | if (I == RetTys.size() - 1) | ||||||||||||
9530 | Flags.setInConsecutiveRegsLast(); | ||||||||||||
9531 | } | ||||||||||||
9532 | EVT VT = RetTys[I]; | ||||||||||||
9533 | MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(), | ||||||||||||
9534 | CLI.CallConv, VT); | ||||||||||||
9535 | unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(), | ||||||||||||
9536 | CLI.CallConv, VT); | ||||||||||||
9537 | for (unsigned i = 0; i != NumRegs; ++i) { | ||||||||||||
9538 | ISD::InputArg MyFlags; | ||||||||||||
9539 | MyFlags.Flags = Flags; | ||||||||||||
9540 | MyFlags.VT = RegisterVT; | ||||||||||||
9541 | MyFlags.ArgVT = VT; | ||||||||||||
9542 | MyFlags.Used = CLI.IsReturnValueUsed; | ||||||||||||
9543 | if (CLI.RetTy->isPointerTy()) { | ||||||||||||
9544 | MyFlags.Flags.setPointer(); | ||||||||||||
9545 | MyFlags.Flags.setPointerAddrSpace( | ||||||||||||
9546 | cast<PointerType>(CLI.RetTy)->getAddressSpace()); | ||||||||||||
9547 | } | ||||||||||||
9548 | if (CLI.RetSExt) | ||||||||||||
9549 | MyFlags.Flags.setSExt(); | ||||||||||||
9550 | if (CLI.RetZExt) | ||||||||||||
9551 | MyFlags.Flags.setZExt(); | ||||||||||||
9552 | if (CLI.IsInReg) | ||||||||||||
9553 | MyFlags.Flags.setInReg(); | ||||||||||||
9554 | CLI.Ins.push_back(MyFlags); | ||||||||||||
9555 | } | ||||||||||||
9556 | } | ||||||||||||
9557 | } | ||||||||||||
9558 | |||||||||||||
9559 | // We push in swifterror return as the last element of CLI.Ins. | ||||||||||||
9560 | ArgListTy &Args = CLI.getArgs(); | ||||||||||||
9561 | if (supportSwiftError()) { | ||||||||||||
9562 | for (unsigned i = 0, e = Args.size(); i != e; ++i) { | ||||||||||||
9563 | if (Args[i].IsSwiftError) { | ||||||||||||
9564 | ISD::InputArg MyFlags; | ||||||||||||
9565 | MyFlags.VT = getPointerTy(DL); | ||||||||||||
9566 | MyFlags.ArgVT = EVT(getPointerTy(DL)); | ||||||||||||
9567 | MyFlags.Flags.setSwiftError(); | ||||||||||||
9568 | CLI.Ins.push_back(MyFlags); | ||||||||||||
9569 | } | ||||||||||||
9570 | } | ||||||||||||
9571 | } | ||||||||||||
9572 | |||||||||||||
9573 | // Handle all of the outgoing arguments. | ||||||||||||
9574 | CLI.Outs.clear(); | ||||||||||||
9575 | CLI.OutVals.clear(); | ||||||||||||
9576 | for (unsigned i = 0, e = Args.size(); i != e; ++i) { | ||||||||||||
9577 | SmallVector<EVT, 4> ValueVTs; | ||||||||||||
9578 | ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs); | ||||||||||||
9579 | // FIXME: Split arguments if CLI.IsPostTypeLegalization | ||||||||||||
9580 | Type *FinalType = Args[i].Ty; | ||||||||||||
9581 | if (Args[i].IsByVal) | ||||||||||||
9582 | FinalType = Args[i].IndirectType; | ||||||||||||
9583 | bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters( | ||||||||||||
9584 | FinalType, CLI.CallConv, CLI.IsVarArg, DL); | ||||||||||||
9585 | for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues; | ||||||||||||
9586 | ++Value) { | ||||||||||||
9587 | EVT VT = ValueVTs[Value]; | ||||||||||||
9588 | Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext()); | ||||||||||||
9589 | SDValue Op = SDValue(Args[i].Node.getNode(), | ||||||||||||
9590 | Args[i].Node.getResNo() + Value); | ||||||||||||
9591 | ISD::ArgFlagsTy Flags; | ||||||||||||
9592 | |||||||||||||
9593 | // Certain targets (such as MIPS), may have a different ABI alignment | ||||||||||||
9594 | // for a type depending on the context. Give the target a chance to | ||||||||||||
9595 | // specify the alignment it wants. | ||||||||||||
9596 | const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL)); | ||||||||||||
9597 | Flags.setOrigAlign(OriginalAlignment); | ||||||||||||
9598 | |||||||||||||
9599 | if (Args[i].Ty->isPointerTy()) { | ||||||||||||
9600 | Flags.setPointer(); | ||||||||||||
9601 | Flags.setPointerAddrSpace( | ||||||||||||
9602 | cast<PointerType>(Args[i].Ty)->getAddressSpace()); | ||||||||||||
9603 | } | ||||||||||||
9604 | if (Args[i].IsZExt) | ||||||||||||
9605 | Flags.setZExt(); | ||||||||||||
9606 | if (Args[i].IsSExt) | ||||||||||||
9607 | Flags.setSExt(); | ||||||||||||
9608 | if (Args[i].IsInReg) { | ||||||||||||
9609 | // If we are using vectorcall calling convention, a structure that is | ||||||||||||
9610 | // passed InReg - is surely an HVA | ||||||||||||
9611 | if (CLI.CallConv == CallingConv::X86_VectorCall && | ||||||||||||
9612 | isa<StructType>(FinalType)) { | ||||||||||||
9613 | // The first value of a structure is marked | ||||||||||||
9614 | if (0 == Value) | ||||||||||||
9615 | Flags.setHvaStart(); | ||||||||||||
9616 | Flags.setHva(); | ||||||||||||
9617 | } | ||||||||||||
9618 | // Set InReg Flag | ||||||||||||
9619 | Flags.setInReg(); | ||||||||||||
9620 | } | ||||||||||||
9621 | if (Args[i].IsSRet) | ||||||||||||
9622 | Flags.setSRet(); | ||||||||||||
9623 | if (Args[i].IsSwiftSelf) | ||||||||||||
9624 | Flags.setSwiftSelf(); | ||||||||||||
9625 | if (Args[i].IsSwiftAsync) | ||||||||||||
9626 | Flags.setSwiftAsync(); | ||||||||||||
9627 | if (Args[i].IsSwiftError) | ||||||||||||
9628 | Flags.setSwiftError(); | ||||||||||||
9629 | if (Args[i].IsCFGuardTarget) | ||||||||||||
9630 | Flags.setCFGuardTarget(); | ||||||||||||
9631 | if (Args[i].IsByVal) | ||||||||||||
9632 | Flags.setByVal(); | ||||||||||||
9633 | if (Args[i].IsByRef) | ||||||||||||
9634 | Flags.setByRef(); | ||||||||||||
9635 | if (Args[i].IsPreallocated) { | ||||||||||||
9636 | Flags.setPreallocated(); | ||||||||||||
9637 | // Set the byval flag for CCAssignFn callbacks that don't know about | ||||||||||||
9638 | // preallocated. This way we can know how many bytes we should've | ||||||||||||
9639 | // allocated and how many bytes a callee cleanup function will pop. If | ||||||||||||
9640 | // we port preallocated to more targets, we'll have to add custom | ||||||||||||
9641 | // preallocated handling in the various CC lowering callbacks. | ||||||||||||
9642 | Flags.setByVal(); | ||||||||||||
9643 | } | ||||||||||||
9644 | if (Args[i].IsInAlloca) { | ||||||||||||
9645 | Flags.setInAlloca(); | ||||||||||||
9646 | // Set the byval flag for CCAssignFn callbacks that don't know about | ||||||||||||
9647 | // inalloca. This way we can know how many bytes we should've allocated | ||||||||||||
9648 | // and how many bytes a callee cleanup function will pop. If we port | ||||||||||||
9649 | // inalloca to more targets, we'll have to add custom inalloca handling | ||||||||||||
9650 | // in the various CC lowering callbacks. | ||||||||||||
9651 | Flags.setByVal(); | ||||||||||||
9652 | } | ||||||||||||
9653 | Align MemAlign; | ||||||||||||
9654 | if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) { | ||||||||||||
9655 | unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType); | ||||||||||||
9656 | Flags.setByValSize(FrameSize); | ||||||||||||
9657 | |||||||||||||
9658 | // info is not there but there are cases it cannot get right. | ||||||||||||
9659 | if (auto MA = Args[i].Alignment) | ||||||||||||
9660 | MemAlign = *MA; | ||||||||||||
9661 | else | ||||||||||||
9662 | MemAlign = Align(getByValTypeAlignment(Args[i].IndirectType, DL)); | ||||||||||||
9663 | } else if (auto MA = Args[i].Alignment) { | ||||||||||||
9664 | MemAlign = *MA; | ||||||||||||
9665 | } else { | ||||||||||||
9666 | MemAlign = OriginalAlignment; | ||||||||||||
9667 | } | ||||||||||||
9668 | Flags.setMemAlign(MemAlign); | ||||||||||||
9669 | if (Args[i].IsNest) | ||||||||||||
9670 | Flags.setNest(); | ||||||||||||
9671 | if (NeedsRegBlock) | ||||||||||||
9672 | Flags.setInConsecutiveRegs(); | ||||||||||||
9673 | |||||||||||||
9674 | MVT PartVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(), | ||||||||||||
9675 | CLI.CallConv, VT); | ||||||||||||
9676 | unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(), | ||||||||||||
9677 | CLI.CallConv, VT); | ||||||||||||
9678 | SmallVector<SDValue, 4> Parts(NumParts); | ||||||||||||
9679 | ISD::NodeType ExtendKind = ISD::ANY_EXTEND; | ||||||||||||
9680 | |||||||||||||
9681 | if (Args[i].IsSExt) | ||||||||||||
9682 | ExtendKind = ISD::SIGN_EXTEND; | ||||||||||||
9683 | else if (Args[i].IsZExt) | ||||||||||||
9684 | ExtendKind = ISD::ZERO_EXTEND; | ||||||||||||
9685 | |||||||||||||
9686 | // Conservatively only handle 'returned' on non-vectors that can be lowered, | ||||||||||||
9687 | // for now. | ||||||||||||
9688 | if (Args[i].IsReturned && !Op.getValueType().isVector() && | ||||||||||||
9689 | CanLowerReturn) { | ||||||||||||
9690 | assert((CLI.RetTy == Args[i].Ty ||(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy ->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace ())) && RetTys.size() == NumValues && "unexpected use of 'returned'" ) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9694, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
9691 | (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy ->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace ())) && RetTys.size() == NumValues && "unexpected use of 'returned'" ) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9694, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
9692 | CLI.RetTy->getPointerAddressSpace() ==(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy ->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace ())) && RetTys.size() == NumValues && "unexpected use of 'returned'" ) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9694, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
9693 | Args[i].Ty->getPointerAddressSpace())) &&(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy ->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace ())) && RetTys.size() == NumValues && "unexpected use of 'returned'" ) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9694, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
9694 | RetTys.size() == NumValues && "unexpected use of 'returned'")(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy ->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace ())) && RetTys.size() == NumValues && "unexpected use of 'returned'" ) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9694, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9695 | // Before passing 'returned' to the target lowering code, ensure that | ||||||||||||
9696 | // either the register MVT and the actual EVT are the same size or that | ||||||||||||
9697 | // the return value and argument are extended in the same way; in these | ||||||||||||
9698 | // cases it's safe to pass the argument register value unchanged as the | ||||||||||||
9699 | // return register value (although it's at the target's option whether | ||||||||||||
9700 | // to do so) | ||||||||||||
9701 | // TODO: allow code generation to take advantage of partially preserved | ||||||||||||
9702 | // registers rather than clobbering the entire register when the | ||||||||||||
9703 | // parameter extension method is not compatible with the return | ||||||||||||
9704 | // extension method | ||||||||||||
9705 | if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) || | ||||||||||||
9706 | (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt && | ||||||||||||
9707 | CLI.RetZExt == Args[i].IsZExt)) | ||||||||||||
9708 | Flags.setReturned(); | ||||||||||||
9709 | } | ||||||||||||
9710 | |||||||||||||
9711 | getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB, | ||||||||||||
9712 | CLI.CallConv, ExtendKind); | ||||||||||||
9713 | |||||||||||||
9714 | for (unsigned j = 0; j != NumParts; ++j) { | ||||||||||||
9715 | // if it isn't first piece, alignment must be 1 | ||||||||||||
9716 | // For scalable vectors the scalable part is currently handled | ||||||||||||
9717 | // by individual targets, so we just use the known minimum size here. | ||||||||||||
9718 | ISD::OutputArg MyFlags( | ||||||||||||
9719 | Flags, Parts[j].getValueType().getSimpleVT(), VT, | ||||||||||||
9720 | i < CLI.NumFixedArgs, i, | ||||||||||||
9721 | j * Parts[j].getValueType().getStoreSize().getKnownMinSize()); | ||||||||||||
9722 | if (NumParts > 1 && j == 0) | ||||||||||||
9723 | MyFlags.Flags.setSplit(); | ||||||||||||
9724 | else if (j != 0) { | ||||||||||||
9725 | MyFlags.Flags.setOrigAlign(Align(1)); | ||||||||||||
9726 | if (j == NumParts - 1) | ||||||||||||
9727 | MyFlags.Flags.setSplitEnd(); | ||||||||||||
9728 | } | ||||||||||||
9729 | |||||||||||||
9730 | CLI.Outs.push_back(MyFlags); | ||||||||||||
9731 | CLI.OutVals.push_back(Parts[j]); | ||||||||||||
9732 | } | ||||||||||||
9733 | |||||||||||||
9734 | if (NeedsRegBlock && Value == NumValues - 1) | ||||||||||||
9735 | CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast(); | ||||||||||||
9736 | } | ||||||||||||
9737 | } | ||||||||||||
9738 | |||||||||||||
9739 | SmallVector<SDValue, 4> InVals; | ||||||||||||
9740 | CLI.Chain = LowerCall(CLI, InVals); | ||||||||||||
9741 | |||||||||||||
9742 | // Update CLI.InVals to use outside of this function. | ||||||||||||
9743 | CLI.InVals = InVals; | ||||||||||||
9744 | |||||||||||||
9745 | // Verify that the target's LowerCall behaved as expected. | ||||||||||||
9746 | assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&(static_cast <bool> (CLI.Chain.getNode() && CLI .Chain.getValueType() == MVT::Other && "LowerCall didn't return a valid chain!" ) ? void (0) : __assert_fail ("CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other && \"LowerCall didn't return a valid chain!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9747, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
9747 | "LowerCall didn't return a valid chain!")(static_cast <bool> (CLI.Chain.getNode() && CLI .Chain.getValueType() == MVT::Other && "LowerCall didn't return a valid chain!" ) ? void (0) : __assert_fail ("CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other && \"LowerCall didn't return a valid chain!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9747, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9748 | assert((!CLI.IsTailCall || InVals.empty()) &&(static_cast <bool> ((!CLI.IsTailCall || InVals.empty() ) && "LowerCall emitted a return value for a tail call!" ) ? void (0) : __assert_fail ("(!CLI.IsTailCall || InVals.empty()) && \"LowerCall emitted a return value for a tail call!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9749, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
9749 | "LowerCall emitted a return value for a tail call!")(static_cast <bool> ((!CLI.IsTailCall || InVals.empty() ) && "LowerCall emitted a return value for a tail call!" ) ? void (0) : __assert_fail ("(!CLI.IsTailCall || InVals.empty()) && \"LowerCall emitted a return value for a tail call!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9749, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9750 | assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&(static_cast <bool> ((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && "LowerCall didn't emit the correct number of values!" ) ? void (0) : __assert_fail ("(CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && \"LowerCall didn't emit the correct number of values!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9751, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
9751 | "LowerCall didn't emit the correct number of values!")(static_cast <bool> ((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && "LowerCall didn't emit the correct number of values!" ) ? void (0) : __assert_fail ("(CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && \"LowerCall didn't emit the correct number of values!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9751, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9752 | |||||||||||||
9753 | // For a tail call, the return value is merely live-out and there aren't | ||||||||||||
9754 | // any nodes in the DAG representing it. Return a special value to | ||||||||||||
9755 | // indicate that a tail call has been emitted and no more Instructions | ||||||||||||
9756 | // should be processed in the current block. | ||||||||||||
9757 | if (CLI.IsTailCall) { | ||||||||||||
9758 | CLI.DAG.setRoot(CLI.Chain); | ||||||||||||
9759 | return std::make_pair(SDValue(), SDValue()); | ||||||||||||
9760 | } | ||||||||||||
9761 | |||||||||||||
9762 | #ifndef NDEBUG | ||||||||||||
9763 | for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) { | ||||||||||||
9764 | assert(InVals[i].getNode() && "LowerCall emitted a null value!")(static_cast <bool> (InVals[i].getNode() && "LowerCall emitted a null value!" ) ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerCall emitted a null value!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9764, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9765 | assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&(static_cast <bool> (EVT(CLI.Ins[i].VT) == InVals[i].getValueType () && "LowerCall emitted a value with the wrong type!" ) ? void (0) : __assert_fail ("EVT(CLI.Ins[i].VT) == InVals[i].getValueType() && \"LowerCall emitted a value with the wrong type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9766, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
9766 | "LowerCall emitted a value with the wrong type!")(static_cast <bool> (EVT(CLI.Ins[i].VT) == InVals[i].getValueType () && "LowerCall emitted a value with the wrong type!" ) ? void (0) : __assert_fail ("EVT(CLI.Ins[i].VT) == InVals[i].getValueType() && \"LowerCall emitted a value with the wrong type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9766, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9767 | } | ||||||||||||
9768 | #endif | ||||||||||||
9769 | |||||||||||||
9770 | SmallVector<SDValue, 4> ReturnValues; | ||||||||||||
9771 | if (!CanLowerReturn) { | ||||||||||||
9772 | // The instruction result is the result of loading from the | ||||||||||||
9773 | // hidden sret parameter. | ||||||||||||
9774 | SmallVector<EVT, 1> PVTs; | ||||||||||||
9775 | Type *PtrRetTy = OrigRetTy->getPointerTo(DL.getAllocaAddrSpace()); | ||||||||||||
9776 | |||||||||||||
9777 | ComputeValueVTs(*this, DL, PtrRetTy, PVTs); | ||||||||||||
9778 | assert(PVTs.size() == 1 && "Pointers should fit in one register")(static_cast <bool> (PVTs.size() == 1 && "Pointers should fit in one register" ) ? void (0) : __assert_fail ("PVTs.size() == 1 && \"Pointers should fit in one register\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9778, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9779 | EVT PtrVT = PVTs[0]; | ||||||||||||
9780 | |||||||||||||
9781 | unsigned NumValues = RetTys.size(); | ||||||||||||
9782 | ReturnValues.resize(NumValues); | ||||||||||||
9783 | SmallVector<SDValue, 4> Chains(NumValues); | ||||||||||||
9784 | |||||||||||||
9785 | // An aggregate return value cannot wrap around the address space, so | ||||||||||||
9786 | // offsets to its parts don't wrap either. | ||||||||||||
9787 | SDNodeFlags Flags; | ||||||||||||
9788 | Flags.setNoUnsignedWrap(true); | ||||||||||||
9789 | |||||||||||||
9790 | MachineFunction &MF = CLI.DAG.getMachineFunction(); | ||||||||||||
9791 | Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx); | ||||||||||||
9792 | for (unsigned i = 0; i < NumValues; ++i) { | ||||||||||||
9793 | SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot, | ||||||||||||
9794 | CLI.DAG.getConstant(Offsets[i], CLI.DL, | ||||||||||||
9795 | PtrVT), Flags); | ||||||||||||
9796 | SDValue L = CLI.DAG.getLoad( | ||||||||||||
9797 | RetTys[i], CLI.DL, CLI.Chain, Add, | ||||||||||||
9798 | MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(), | ||||||||||||
9799 | DemoteStackIdx, Offsets[i]), | ||||||||||||
9800 | HiddenSRetAlign); | ||||||||||||
9801 | ReturnValues[i] = L; | ||||||||||||
9802 | Chains[i] = L.getValue(1); | ||||||||||||
9803 | } | ||||||||||||
9804 | |||||||||||||
9805 | CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains); | ||||||||||||
9806 | } else { | ||||||||||||
9807 | // Collect the legal value parts into potentially illegal values | ||||||||||||
9808 | // that correspond to the original function's return values. | ||||||||||||
9809 | Optional<ISD::NodeType> AssertOp; | ||||||||||||
9810 | if (CLI.RetSExt) | ||||||||||||
9811 | AssertOp = ISD::AssertSext; | ||||||||||||
9812 | else if (CLI.RetZExt) | ||||||||||||
9813 | AssertOp = ISD::AssertZext; | ||||||||||||
9814 | unsigned CurReg = 0; | ||||||||||||
9815 | for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { | ||||||||||||
9816 | EVT VT = RetTys[I]; | ||||||||||||
9817 | MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(), | ||||||||||||
9818 | CLI.CallConv, VT); | ||||||||||||
9819 | unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(), | ||||||||||||
9820 | CLI.CallConv, VT); | ||||||||||||
9821 | |||||||||||||
9822 | ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg], | ||||||||||||
9823 | NumRegs, RegisterVT, VT, nullptr, | ||||||||||||
9824 | CLI.CallConv, AssertOp)); | ||||||||||||
9825 | CurReg += NumRegs; | ||||||||||||
9826 | } | ||||||||||||
9827 | |||||||||||||
9828 | // For a function returning void, there is no return value. We can't create | ||||||||||||
9829 | // such a node, so we just return a null return value in that case. In | ||||||||||||
9830 | // that case, nothing will actually look at the value. | ||||||||||||
9831 | if (ReturnValues.empty()) | ||||||||||||
9832 | return std::make_pair(SDValue(), CLI.Chain); | ||||||||||||
9833 | } | ||||||||||||
9834 | |||||||||||||
9835 | SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL, | ||||||||||||
9836 | CLI.DAG.getVTList(RetTys), ReturnValues); | ||||||||||||
9837 | return std::make_pair(Res, CLI.Chain); | ||||||||||||
9838 | } | ||||||||||||
9839 | |||||||||||||
9840 | /// Places new result values for the node in Results (their number | ||||||||||||
9841 | /// and types must exactly match those of the original return values of | ||||||||||||
9842 | /// the node), or leaves Results empty, which indicates that the node is not | ||||||||||||
9843 | /// to be custom lowered after all. | ||||||||||||
9844 | void TargetLowering::LowerOperationWrapper(SDNode *N, | ||||||||||||
9845 | SmallVectorImpl<SDValue> &Results, | ||||||||||||
9846 | SelectionDAG &DAG) const { | ||||||||||||
9847 | SDValue Res = LowerOperation(SDValue(N, 0), DAG); | ||||||||||||
9848 | |||||||||||||
9849 | if (!Res.getNode()) | ||||||||||||
9850 | return; | ||||||||||||
9851 | |||||||||||||
9852 | // If the original node has one result, take the return value from | ||||||||||||
9853 | // LowerOperation as is. It might not be result number 0. | ||||||||||||
9854 | if (N->getNumValues() == 1) { | ||||||||||||
9855 | Results.push_back(Res); | ||||||||||||
9856 | return; | ||||||||||||
9857 | } | ||||||||||||
9858 | |||||||||||||
9859 | // If the original node has multiple results, then the return node should | ||||||||||||
9860 | // have the same number of results. | ||||||||||||
9861 | assert((N->getNumValues() == Res->getNumValues()) &&(static_cast <bool> ((N->getNumValues() == Res->getNumValues ()) && "Lowering returned the wrong number of results!" ) ? void (0) : __assert_fail ("(N->getNumValues() == Res->getNumValues()) && \"Lowering returned the wrong number of results!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9862, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
9862 | "Lowering returned the wrong number of results!")(static_cast <bool> ((N->getNumValues() == Res->getNumValues ()) && "Lowering returned the wrong number of results!" ) ? void (0) : __assert_fail ("(N->getNumValues() == Res->getNumValues()) && \"Lowering returned the wrong number of results!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9862, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9863 | |||||||||||||
9864 | // Places new result values base on N result number. | ||||||||||||
9865 | for (unsigned I = 0, E = N->getNumValues(); I != E; ++I) | ||||||||||||
9866 | Results.push_back(Res.getValue(I)); | ||||||||||||
9867 | } | ||||||||||||
9868 | |||||||||||||
9869 | SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { | ||||||||||||
9870 | llvm_unreachable("LowerOperation not implemented for this target!")::llvm::llvm_unreachable_internal("LowerOperation not implemented for this target!" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9870); | ||||||||||||
9871 | } | ||||||||||||
9872 | |||||||||||||
9873 | void | ||||||||||||
9874 | SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, unsigned Reg) { | ||||||||||||
9875 | SDValue Op = getNonRegisterValue(V); | ||||||||||||
9876 | assert((Op.getOpcode() != ISD::CopyFromReg ||(static_cast <bool> ((Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && "Copy from a reg to the same reg!") ? void (0) : __assert_fail ("(Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && \"Copy from a reg to the same reg!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9878, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
9877 | cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&(static_cast <bool> ((Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && "Copy from a reg to the same reg!") ? void (0) : __assert_fail ("(Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && \"Copy from a reg to the same reg!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9878, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
9878 | "Copy from a reg to the same reg!")(static_cast <bool> ((Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && "Copy from a reg to the same reg!") ? void (0) : __assert_fail ("(Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && \"Copy from a reg to the same reg!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9878, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9879 | assert(!Register::isPhysicalRegister(Reg) && "Is a physreg")(static_cast <bool> (!Register::isPhysicalRegister(Reg) && "Is a physreg") ? void (0) : __assert_fail ("!Register::isPhysicalRegister(Reg) && \"Is a physreg\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9879, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
9880 | |||||||||||||
9881 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
9882 | // If this is an InlineAsm we have to match the registers required, not the | ||||||||||||
9883 | // notional registers required by the type. | ||||||||||||
9884 | |||||||||||||
9885 | RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(), | ||||||||||||
9886 | None); // This is not an ABI copy. | ||||||||||||
9887 | SDValue Chain = DAG.getEntryNode(); | ||||||||||||
9888 | |||||||||||||
9889 | ISD::NodeType ExtendType = ISD::ANY_EXTEND; | ||||||||||||
9890 | auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V); | ||||||||||||
9891 | if (PreferredExtendIt != FuncInfo.PreferredExtendType.end()) | ||||||||||||
9892 | ExtendType = PreferredExtendIt->second; | ||||||||||||
9893 | RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType); | ||||||||||||
9894 | PendingExports.push_back(Chain); | ||||||||||||
9895 | } | ||||||||||||
9896 | |||||||||||||
9897 | #include "llvm/CodeGen/SelectionDAGISel.h" | ||||||||||||
9898 | |||||||||||||
9899 | /// isOnlyUsedInEntryBlock - If the specified argument is only used in the | ||||||||||||
9900 | /// entry block, return true. This includes arguments used by switches, since | ||||||||||||
9901 | /// the switch may expand into multiple basic blocks. | ||||||||||||
9902 | static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) { | ||||||||||||
9903 | // With FastISel active, we may be splitting blocks, so force creation | ||||||||||||
9904 | // of virtual registers for all non-dead arguments. | ||||||||||||
9905 | if (FastISel) | ||||||||||||
9906 | return A->use_empty(); | ||||||||||||
9907 | |||||||||||||
9908 | const BasicBlock &Entry = A->getParent()->front(); | ||||||||||||
9909 | for (const User *U : A->users()) | ||||||||||||
9910 | if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U)) | ||||||||||||
9911 | return false; // Use not in entry block. | ||||||||||||
9912 | |||||||||||||
9913 | return true; | ||||||||||||
9914 | } | ||||||||||||
9915 | |||||||||||||
9916 | using ArgCopyElisionMapTy = | ||||||||||||
9917 | DenseMap<const Argument *, | ||||||||||||
9918 | std::pair<const AllocaInst *, const StoreInst *>>; | ||||||||||||
9919 | |||||||||||||
9920 | /// Scan the entry block of the function in FuncInfo for arguments that look | ||||||||||||
9921 | /// like copies into a local alloca. Record any copied arguments in | ||||||||||||
9922 | /// ArgCopyElisionCandidates. | ||||||||||||
9923 | static void | ||||||||||||
9924 | findArgumentCopyElisionCandidates(const DataLayout &DL, | ||||||||||||
9925 | FunctionLoweringInfo *FuncInfo, | ||||||||||||
9926 | ArgCopyElisionMapTy &ArgCopyElisionCandidates) { | ||||||||||||
9927 | // Record the state of every static alloca used in the entry block. Argument | ||||||||||||
9928 | // allocas are all used in the entry block, so we need approximately as many | ||||||||||||
9929 | // entries as we have arguments. | ||||||||||||
9930 | enum StaticAllocaInfo { Unknown, Clobbered, Elidable }; | ||||||||||||
9931 | SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas; | ||||||||||||
9932 | unsigned NumArgs = FuncInfo->Fn->arg_size(); | ||||||||||||
9933 | StaticAllocas.reserve(NumArgs * 2); | ||||||||||||
9934 | |||||||||||||
9935 | auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * { | ||||||||||||
9936 | if (!V) | ||||||||||||
9937 | return nullptr; | ||||||||||||
9938 | V = V->stripPointerCasts(); | ||||||||||||
9939 | const auto *AI = dyn_cast<AllocaInst>(V); | ||||||||||||
9940 | if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI)) | ||||||||||||
9941 | return nullptr; | ||||||||||||
9942 | auto Iter = StaticAllocas.insert({AI, Unknown}); | ||||||||||||
9943 | return &Iter.first->second; | ||||||||||||
9944 | }; | ||||||||||||
9945 | |||||||||||||
9946 | // Look for stores of arguments to static allocas. Look through bitcasts and | ||||||||||||
9947 | // GEPs to handle type coercions, as long as the alloca is fully initialized | ||||||||||||
9948 | // by the store. Any non-store use of an alloca escapes it and any subsequent | ||||||||||||
9949 | // unanalyzed store might write it. | ||||||||||||
9950 | // FIXME: Handle structs initialized with multiple stores. | ||||||||||||
9951 | for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) { | ||||||||||||
9952 | // Look for stores, and handle non-store uses conservatively. | ||||||||||||
9953 | const auto *SI = dyn_cast<StoreInst>(&I); | ||||||||||||
9954 | if (!SI) { | ||||||||||||
9955 | // We will look through cast uses, so ignore them completely. | ||||||||||||
9956 | if (I.isCast()) | ||||||||||||
9957 | continue; | ||||||||||||
9958 | // Ignore debug info and pseudo op intrinsics, they don't escape or store | ||||||||||||
9959 | // to allocas. | ||||||||||||
9960 | if (I.isDebugOrPseudoInst()) | ||||||||||||
9961 | continue; | ||||||||||||
9962 | // This is an unknown instruction. Assume it escapes or writes to all | ||||||||||||
9963 | // static alloca operands. | ||||||||||||
9964 | for (const Use &U : I.operands()) { | ||||||||||||
9965 | if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U)) | ||||||||||||
9966 | *Info = StaticAllocaInfo::Clobbered; | ||||||||||||
9967 | } | ||||||||||||
9968 | continue; | ||||||||||||
9969 | } | ||||||||||||
9970 | |||||||||||||
9971 | // If the stored value is a static alloca, mark it as escaped. | ||||||||||||
9972 | if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand())) | ||||||||||||
9973 | *Info = StaticAllocaInfo::Clobbered; | ||||||||||||
9974 | |||||||||||||
9975 | // Check if the destination is a static alloca. | ||||||||||||
9976 | const Value *Dst = SI->getPointerOperand()->stripPointerCasts(); | ||||||||||||
9977 | StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst); | ||||||||||||
9978 | if (!Info) | ||||||||||||
9979 | continue; | ||||||||||||
9980 | const AllocaInst *AI = cast<AllocaInst>(Dst); | ||||||||||||
9981 | |||||||||||||
9982 | // Skip allocas that have been initialized or clobbered. | ||||||||||||
9983 | if (*Info != StaticAllocaInfo::Unknown) | ||||||||||||
9984 | continue; | ||||||||||||
9985 | |||||||||||||
9986 | // Check if the stored value is an argument, and that this store fully | ||||||||||||
9987 | // initializes the alloca. | ||||||||||||
9988 | // If the argument type has padding bits we can't directly forward a pointer | ||||||||||||
9989 | // as the upper bits may contain garbage. | ||||||||||||
9990 | // Don't elide copies from the same argument twice. | ||||||||||||
9991 | const Value *Val = SI->getValueOperand()->stripPointerCasts(); | ||||||||||||
9992 | const auto *Arg = dyn_cast<Argument>(Val); | ||||||||||||
9993 | if (!Arg || Arg->hasPassPointeeByValueCopyAttr() || | ||||||||||||
9994 | Arg->getType()->isEmptyTy() || | ||||||||||||
9995 | DL.getTypeStoreSize(Arg->getType()) != | ||||||||||||
9996 | DL.getTypeAllocSize(AI->getAllocatedType()) || | ||||||||||||
9997 | !DL.typeSizeEqualsStoreSize(Arg->getType()) || | ||||||||||||
9998 | ArgCopyElisionCandidates.count(Arg)) { | ||||||||||||
9999 | *Info = StaticAllocaInfo::Clobbered; | ||||||||||||
10000 | continue; | ||||||||||||
10001 | } | ||||||||||||
10002 | |||||||||||||
10003 | LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Found argument copy elision candidate: " << *AI << '\n'; } } while (false) | ||||||||||||
10004 | << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Found argument copy elision candidate: " << *AI << '\n'; } } while (false); | ||||||||||||
10005 | |||||||||||||
10006 | // Mark this alloca and store for argument copy elision. | ||||||||||||
10007 | *Info = StaticAllocaInfo::Elidable; | ||||||||||||
10008 | ArgCopyElisionCandidates.insert({Arg, {AI, SI}}); | ||||||||||||
10009 | |||||||||||||
10010 | // Stop scanning if we've seen all arguments. This will happen early in -O0 | ||||||||||||
10011 | // builds, which is useful, because -O0 builds have large entry blocks and | ||||||||||||
10012 | // many allocas. | ||||||||||||
10013 | if (ArgCopyElisionCandidates.size() == NumArgs) | ||||||||||||
10014 | break; | ||||||||||||
10015 | } | ||||||||||||
10016 | } | ||||||||||||
10017 | |||||||||||||
10018 | /// Try to elide argument copies from memory into a local alloca. Succeeds if | ||||||||||||
10019 | /// ArgVal is a load from a suitable fixed stack object. | ||||||||||||
10020 | static void tryToElideArgumentCopy( | ||||||||||||
10021 | FunctionLoweringInfo &FuncInfo, SmallVectorImpl<SDValue> &Chains, | ||||||||||||
10022 | DenseMap<int, int> &ArgCopyElisionFrameIndexMap, | ||||||||||||
10023 | SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs, | ||||||||||||
10024 | ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, | ||||||||||||
10025 | SDValue ArgVal, bool &ArgHasUses) { | ||||||||||||
10026 | // Check if this is a load from a fixed stack object. | ||||||||||||
10027 | auto *LNode = dyn_cast<LoadSDNode>(ArgVal); | ||||||||||||
10028 | if (!LNode) | ||||||||||||
10029 | return; | ||||||||||||
10030 | auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()); | ||||||||||||
10031 | if (!FINode) | ||||||||||||
10032 | return; | ||||||||||||
10033 | |||||||||||||
10034 | // Check that the fixed stack object is the right size and alignment. | ||||||||||||
10035 | // Look at the alignment that the user wrote on the alloca instead of looking | ||||||||||||
10036 | // at the stack object. | ||||||||||||
10037 | auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg); | ||||||||||||
10038 | assert(ArgCopyIter != ArgCopyElisionCandidates.end())(static_cast <bool> (ArgCopyIter != ArgCopyElisionCandidates .end()) ? void (0) : __assert_fail ("ArgCopyIter != ArgCopyElisionCandidates.end()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10038, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
10039 | const AllocaInst *AI = ArgCopyIter->second.first; | ||||||||||||
10040 | int FixedIndex = FINode->getIndex(); | ||||||||||||
10041 | int &AllocaIndex = FuncInfo.StaticAllocaMap[AI]; | ||||||||||||
10042 | int OldIndex = AllocaIndex; | ||||||||||||
10043 | MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo(); | ||||||||||||
10044 | if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) { | ||||||||||||
10045 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " argument copy elision failed due to bad fixed stack " "object size\n"; } } while (false) | ||||||||||||
10046 | dbgs() << " argument copy elision failed due to bad fixed stack "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " argument copy elision failed due to bad fixed stack " "object size\n"; } } while (false) | ||||||||||||
10047 | "object size\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " argument copy elision failed due to bad fixed stack " "object size\n"; } } while (false); | ||||||||||||
10048 | return; | ||||||||||||
10049 | } | ||||||||||||
10050 | Align RequiredAlignment = AI->getAlign(); | ||||||||||||
10051 | if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) { | ||||||||||||
10052 | LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " argument copy elision failed: alignment of alloca " "greater than stack argument alignment (" << DebugStr( RequiredAlignment) << " vs " << DebugStr(MFI.getObjectAlign (FixedIndex)) << ")\n"; } } while (false) | ||||||||||||
10053 | "greater than stack argument alignment ("do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " argument copy elision failed: alignment of alloca " "greater than stack argument alignment (" << DebugStr( RequiredAlignment) << " vs " << DebugStr(MFI.getObjectAlign (FixedIndex)) << ")\n"; } } while (false) | ||||||||||||
10054 | << DebugStr(RequiredAlignment) << " vs "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " argument copy elision failed: alignment of alloca " "greater than stack argument alignment (" << DebugStr( RequiredAlignment) << " vs " << DebugStr(MFI.getObjectAlign (FixedIndex)) << ")\n"; } } while (false) | ||||||||||||
10055 | << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " argument copy elision failed: alignment of alloca " "greater than stack argument alignment (" << DebugStr( RequiredAlignment) << " vs " << DebugStr(MFI.getObjectAlign (FixedIndex)) << ")\n"; } } while (false); | ||||||||||||
10056 | return; | ||||||||||||
10057 | } | ||||||||||||
10058 | |||||||||||||
10059 | // Perform the elision. Delete the old stack object and replace its only use | ||||||||||||
10060 | // in the variable info map. Mark the stack object as mutable. | ||||||||||||
10061 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n' << " Replacing frame index " << OldIndex << " with " << FixedIndex << '\n'; }; } } while (false) | ||||||||||||
10062 | dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n' << " Replacing frame index " << OldIndex << " with " << FixedIndex << '\n'; }; } } while (false) | ||||||||||||
10063 | << " Replacing frame index " << OldIndex << " with " << FixedIndexdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n' << " Replacing frame index " << OldIndex << " with " << FixedIndex << '\n'; }; } } while (false) | ||||||||||||
10064 | << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n' << " Replacing frame index " << OldIndex << " with " << FixedIndex << '\n'; }; } } while (false) | ||||||||||||
10065 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n' << " Replacing frame index " << OldIndex << " with " << FixedIndex << '\n'; }; } } while (false); | ||||||||||||
10066 | MFI.RemoveStackObject(OldIndex); | ||||||||||||
10067 | MFI.setIsImmutableObjectIndex(FixedIndex, false); | ||||||||||||
10068 | AllocaIndex = FixedIndex; | ||||||||||||
10069 | ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex}); | ||||||||||||
10070 | Chains.push_back(ArgVal.getValue(1)); | ||||||||||||
10071 | |||||||||||||
10072 | // Avoid emitting code for the store implementing the copy. | ||||||||||||
10073 | const StoreInst *SI = ArgCopyIter->second.second; | ||||||||||||
10074 | ElidedArgCopyInstrs.insert(SI); | ||||||||||||
10075 | |||||||||||||
10076 | // Check for uses of the argument again so that we can avoid exporting ArgVal | ||||||||||||
10077 | // if it is't used by anything other than the store. | ||||||||||||
10078 | for (const Value *U : Arg.users()) { | ||||||||||||
10079 | if (U != SI) { | ||||||||||||
10080 | ArgHasUses = true; | ||||||||||||
10081 | break; | ||||||||||||
10082 | } | ||||||||||||
10083 | } | ||||||||||||
10084 | } | ||||||||||||
10085 | |||||||||||||
10086 | void SelectionDAGISel::LowerArguments(const Function &F) { | ||||||||||||
10087 | SelectionDAG &DAG = SDB->DAG; | ||||||||||||
10088 | SDLoc dl = SDB->getCurSDLoc(); | ||||||||||||
10089 | const DataLayout &DL = DAG.getDataLayout(); | ||||||||||||
10090 | SmallVector<ISD::InputArg, 16> Ins; | ||||||||||||
10091 | |||||||||||||
10092 | // In Naked functions we aren't going to save any registers. | ||||||||||||
10093 | if (F.hasFnAttribute(Attribute::Naked)) | ||||||||||||
10094 | return; | ||||||||||||
10095 | |||||||||||||
10096 | if (!FuncInfo->CanLowerReturn) { | ||||||||||||
10097 | // Put in an sret pointer parameter before all the other parameters. | ||||||||||||
10098 | SmallVector<EVT, 1> ValueVTs; | ||||||||||||
10099 | ComputeValueVTs(*TLI, DAG.getDataLayout(), | ||||||||||||
10100 | F.getReturnType()->getPointerTo( | ||||||||||||
10101 | DAG.getDataLayout().getAllocaAddrSpace()), | ||||||||||||
10102 | ValueVTs); | ||||||||||||
10103 | |||||||||||||
10104 | // NOTE: Assuming that a pointer will never break down to more than one VT | ||||||||||||
10105 | // or one register. | ||||||||||||
10106 | ISD::ArgFlagsTy Flags; | ||||||||||||
10107 | Flags.setSRet(); | ||||||||||||
10108 | MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]); | ||||||||||||
10109 | ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true, | ||||||||||||
10110 | ISD::InputArg::NoArgIndex, 0); | ||||||||||||
10111 | Ins.push_back(RetArg); | ||||||||||||
10112 | } | ||||||||||||
10113 | |||||||||||||
10114 | // Look for stores of arguments to static allocas. Mark such arguments with a | ||||||||||||
10115 | // flag to ask the target to give us the memory location of that argument if | ||||||||||||
10116 | // available. | ||||||||||||
10117 | ArgCopyElisionMapTy ArgCopyElisionCandidates; | ||||||||||||
10118 | findArgumentCopyElisionCandidates(DL, FuncInfo.get(), | ||||||||||||
10119 | ArgCopyElisionCandidates); | ||||||||||||
10120 | |||||||||||||
10121 | // Set up the incoming argument description vector. | ||||||||||||
10122 | for (const Argument &Arg : F.args()) { | ||||||||||||
10123 | unsigned ArgNo = Arg.getArgNo(); | ||||||||||||
10124 | SmallVector<EVT, 4> ValueVTs; | ||||||||||||
10125 | ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs); | ||||||||||||
10126 | bool isArgValueUsed = !Arg.use_empty(); | ||||||||||||
10127 | unsigned PartBase = 0; | ||||||||||||
10128 | Type *FinalType = Arg.getType(); | ||||||||||||
10129 | if (Arg.hasAttribute(Attribute::ByVal)) | ||||||||||||
10130 | FinalType = Arg.getParamByValType(); | ||||||||||||
10131 | bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters( | ||||||||||||
10132 | FinalType, F.getCallingConv(), F.isVarArg(), DL); | ||||||||||||
10133 | for (unsigned Value = 0, NumValues = ValueVTs.size(); | ||||||||||||
10134 | Value != NumValues; ++Value) { | ||||||||||||
10135 | EVT VT = ValueVTs[Value]; | ||||||||||||
10136 | Type *ArgTy = VT.getTypeForEVT(*DAG.getContext()); | ||||||||||||
10137 | ISD::ArgFlagsTy Flags; | ||||||||||||
10138 | |||||||||||||
10139 | |||||||||||||
10140 | if (Arg.getType()->isPointerTy()) { | ||||||||||||
10141 | Flags.setPointer(); | ||||||||||||
10142 | Flags.setPointerAddrSpace( | ||||||||||||
10143 | cast<PointerType>(Arg.getType())->getAddressSpace()); | ||||||||||||
10144 | } | ||||||||||||
10145 | if (Arg.hasAttribute(Attribute::ZExt)) | ||||||||||||
10146 | Flags.setZExt(); | ||||||||||||
10147 | if (Arg.hasAttribute(Attribute::SExt)) | ||||||||||||
10148 | Flags.setSExt(); | ||||||||||||
10149 | if (Arg.hasAttribute(Attribute::InReg)) { | ||||||||||||
10150 | // If we are using vectorcall calling convention, a structure that is | ||||||||||||
10151 | // passed InReg - is surely an HVA | ||||||||||||
10152 | if (F.getCallingConv() == CallingConv::X86_VectorCall && | ||||||||||||
10153 | isa<StructType>(Arg.getType())) { | ||||||||||||
10154 | // The first value of a structure is marked | ||||||||||||
10155 | if (0 == Value) | ||||||||||||
10156 | Flags.setHvaStart(); | ||||||||||||
10157 | Flags.setHva(); | ||||||||||||
10158 | } | ||||||||||||
10159 | // Set InReg Flag | ||||||||||||
10160 | Flags.setInReg(); | ||||||||||||
10161 | } | ||||||||||||
10162 | if (Arg.hasAttribute(Attribute::StructRet)) | ||||||||||||
10163 | Flags.setSRet(); | ||||||||||||
10164 | if (Arg.hasAttribute(Attribute::SwiftSelf)) | ||||||||||||
10165 | Flags.setSwiftSelf(); | ||||||||||||
10166 | if (Arg.hasAttribute(Attribute::SwiftAsync)) | ||||||||||||
10167 | Flags.setSwiftAsync(); | ||||||||||||
10168 | if (Arg.hasAttribute(Attribute::SwiftError)) | ||||||||||||
10169 | Flags.setSwiftError(); | ||||||||||||
10170 | if (Arg.hasAttribute(Attribute::ByVal)) | ||||||||||||
10171 | Flags.setByVal(); | ||||||||||||
10172 | if (Arg.hasAttribute(Attribute::ByRef)) | ||||||||||||
10173 | Flags.setByRef(); | ||||||||||||
10174 | if (Arg.hasAttribute(Attribute::InAlloca)) { | ||||||||||||
10175 | Flags.setInAlloca(); | ||||||||||||
10176 | // Set the byval flag for CCAssignFn callbacks that don't know about | ||||||||||||
10177 | // inalloca. This way we can know how many bytes we should've allocated | ||||||||||||
10178 | // and how many bytes a callee cleanup function will pop. If we port | ||||||||||||
10179 | // inalloca to more targets, we'll have to add custom inalloca handling | ||||||||||||
10180 | // in the various CC lowering callbacks. | ||||||||||||
10181 | Flags.setByVal(); | ||||||||||||
10182 | } | ||||||||||||
10183 | if (Arg.hasAttribute(Attribute::Preallocated)) { | ||||||||||||
10184 | Flags.setPreallocated(); | ||||||||||||
10185 | // Set the byval flag for CCAssignFn callbacks that don't know about | ||||||||||||
10186 | // preallocated. This way we can know how many bytes we should've | ||||||||||||
10187 | // allocated and how many bytes a callee cleanup function will pop. If | ||||||||||||
10188 | // we port preallocated to more targets, we'll have to add custom | ||||||||||||
10189 | // preallocated handling in the various CC lowering callbacks. | ||||||||||||
10190 | Flags.setByVal(); | ||||||||||||
10191 | } | ||||||||||||
10192 | |||||||||||||
10193 | // Certain targets (such as MIPS), may have a different ABI alignment | ||||||||||||
10194 | // for a type depending on the context. Give the target a chance to | ||||||||||||
10195 | // specify the alignment it wants. | ||||||||||||
10196 | const Align OriginalAlignment( | ||||||||||||
10197 | TLI->getABIAlignmentForCallingConv(ArgTy, DL)); | ||||||||||||
10198 | Flags.setOrigAlign(OriginalAlignment); | ||||||||||||
10199 | |||||||||||||
10200 | Align MemAlign; | ||||||||||||
10201 | Type *ArgMemTy = nullptr; | ||||||||||||
10202 | if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() || | ||||||||||||
10203 | Flags.isByRef()) { | ||||||||||||
10204 | if (!ArgMemTy) | ||||||||||||
10205 | ArgMemTy = Arg.getPointeeInMemoryValueType(); | ||||||||||||
10206 | |||||||||||||
10207 | uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy); | ||||||||||||
10208 | |||||||||||||
10209 | // For in-memory arguments, size and alignment should be passed from FE. | ||||||||||||
10210 | // BE will guess if this info is not there but there are cases it cannot | ||||||||||||
10211 | // get right. | ||||||||||||
10212 | if (auto ParamAlign = Arg.getParamStackAlign()) | ||||||||||||
10213 | MemAlign = *ParamAlign; | ||||||||||||
10214 | else if ((ParamAlign = Arg.getParamAlign())) | ||||||||||||
10215 | MemAlign = *ParamAlign; | ||||||||||||
10216 | else | ||||||||||||
10217 | MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL)); | ||||||||||||
10218 | if (Flags.isByRef()) | ||||||||||||
10219 | Flags.setByRefSize(MemSize); | ||||||||||||
10220 | else | ||||||||||||
10221 | Flags.setByValSize(MemSize); | ||||||||||||
10222 | } else if (auto ParamAlign = Arg.getParamStackAlign()) { | ||||||||||||
10223 | MemAlign = *ParamAlign; | ||||||||||||
10224 | } else { | ||||||||||||
10225 | MemAlign = OriginalAlignment; | ||||||||||||
10226 | } | ||||||||||||
10227 | Flags.setMemAlign(MemAlign); | ||||||||||||
10228 | |||||||||||||
10229 | if (Arg.hasAttribute(Attribute::Nest)) | ||||||||||||
10230 | Flags.setNest(); | ||||||||||||
10231 | if (NeedsRegBlock) | ||||||||||||
10232 | Flags.setInConsecutiveRegs(); | ||||||||||||
10233 | if (ArgCopyElisionCandidates.count(&Arg)) | ||||||||||||
10234 | Flags.setCopyElisionCandidate(); | ||||||||||||
10235 | if (Arg.hasAttribute(Attribute::Returned)) | ||||||||||||
10236 | Flags.setReturned(); | ||||||||||||
10237 | |||||||||||||
10238 | MVT RegisterVT = TLI->getRegisterTypeForCallingConv( | ||||||||||||
10239 | *CurDAG->getContext(), F.getCallingConv(), VT); | ||||||||||||
10240 | unsigned NumRegs = TLI->getNumRegistersForCallingConv( | ||||||||||||
10241 | *CurDAG->getContext(), F.getCallingConv(), VT); | ||||||||||||
10242 | for (unsigned i = 0; i != NumRegs; ++i) { | ||||||||||||
10243 | // For scalable vectors, use the minimum size; individual targets | ||||||||||||
10244 | // are responsible for handling scalable vector arguments and | ||||||||||||
10245 | // return values. | ||||||||||||
10246 | ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed, | ||||||||||||
10247 | ArgNo, PartBase+i*RegisterVT.getStoreSize().getKnownMinSize()); | ||||||||||||
10248 | if (NumRegs > 1 && i == 0) | ||||||||||||
10249 | MyFlags.Flags.setSplit(); | ||||||||||||
10250 | // if it isn't first piece, alignment must be 1 | ||||||||||||
10251 | else if (i > 0) { | ||||||||||||
10252 | MyFlags.Flags.setOrigAlign(Align(1)); | ||||||||||||
10253 | if (i == NumRegs - 1) | ||||||||||||
10254 | MyFlags.Flags.setSplitEnd(); | ||||||||||||
10255 | } | ||||||||||||
10256 | Ins.push_back(MyFlags); | ||||||||||||
10257 | } | ||||||||||||
10258 | if (NeedsRegBlock && Value == NumValues - 1) | ||||||||||||
10259 | Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast(); | ||||||||||||
10260 | PartBase += VT.getStoreSize().getKnownMinSize(); | ||||||||||||
10261 | } | ||||||||||||
10262 | } | ||||||||||||
10263 | |||||||||||||
10264 | // Call the target to set up the argument values. | ||||||||||||
10265 | SmallVector<SDValue, 8> InVals; | ||||||||||||
10266 | SDValue NewRoot = TLI->LowerFormalArguments( | ||||||||||||
10267 | DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals); | ||||||||||||
10268 | |||||||||||||
10269 | // Verify that the target's LowerFormalArguments behaved as expected. | ||||||||||||
10270 | assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&(static_cast <bool> (NewRoot.getNode() && NewRoot .getValueType() == MVT::Other && "LowerFormalArguments didn't return a valid chain!" ) ? void (0) : __assert_fail ("NewRoot.getNode() && NewRoot.getValueType() == MVT::Other && \"LowerFormalArguments didn't return a valid chain!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10271, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
10271 | "LowerFormalArguments didn't return a valid chain!")(static_cast <bool> (NewRoot.getNode() && NewRoot .getValueType() == MVT::Other && "LowerFormalArguments didn't return a valid chain!" ) ? void (0) : __assert_fail ("NewRoot.getNode() && NewRoot.getValueType() == MVT::Other && \"LowerFormalArguments didn't return a valid chain!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10271, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
10272 | assert(InVals.size() == Ins.size() &&(static_cast <bool> (InVals.size() == Ins.size() && "LowerFormalArguments didn't emit the correct number of values!" ) ? void (0) : __assert_fail ("InVals.size() == Ins.size() && \"LowerFormalArguments didn't emit the correct number of values!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10273, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
10273 | "LowerFormalArguments didn't emit the correct number of values!")(static_cast <bool> (InVals.size() == Ins.size() && "LowerFormalArguments didn't emit the correct number of values!" ) ? void (0) : __assert_fail ("InVals.size() == Ins.size() && \"LowerFormalArguments didn't emit the correct number of values!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10273, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
10274 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10277, __extension__ __PRETTY_FUNCTION__)); (static_cast < bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!") ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10279, __extension__ __PRETTY_FUNCTION__)); } }; } } while ( false) | ||||||||||||
10275 | for (unsigned i = 0, e = Ins.size(); i != e; ++i) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10277, __extension__ __PRETTY_FUNCTION__)); (static_cast < bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!") ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10279, __extension__ __PRETTY_FUNCTION__)); } }; } } while ( false) | ||||||||||||
10276 | assert(InVals[i].getNode() &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10277, __extension__ __PRETTY_FUNCTION__)); (static_cast < bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!") ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10279, __extension__ __PRETTY_FUNCTION__)); } }; } } while ( false) | ||||||||||||
10277 | "LowerFormalArguments emitted a null value!");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10277, __extension__ __PRETTY_FUNCTION__)); (static_cast < bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!") ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10279, __extension__ __PRETTY_FUNCTION__)); } }; } } while ( false) | ||||||||||||
10278 | assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10277, __extension__ __PRETTY_FUNCTION__)); (static_cast < bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!") ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10279, __extension__ __PRETTY_FUNCTION__)); } }; } } while ( false) | ||||||||||||
10279 | "LowerFormalArguments emitted a value with the wrong type!");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10277, __extension__ __PRETTY_FUNCTION__)); (static_cast < bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!") ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10279, __extension__ __PRETTY_FUNCTION__)); } }; } } while ( false) | ||||||||||||
10280 | }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10277, __extension__ __PRETTY_FUNCTION__)); (static_cast < bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!") ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10279, __extension__ __PRETTY_FUNCTION__)); } }; } } while ( false) | ||||||||||||
10281 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10277, __extension__ __PRETTY_FUNCTION__)); (static_cast < bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!") ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10279, __extension__ __PRETTY_FUNCTION__)); } }; } } while ( false); | ||||||||||||
10282 | |||||||||||||
10283 | // Update the DAG with the new chain value resulting from argument lowering. | ||||||||||||
10284 | DAG.setRoot(NewRoot); | ||||||||||||
10285 | |||||||||||||
10286 | // Set up the argument values. | ||||||||||||
10287 | unsigned i = 0; | ||||||||||||
10288 | if (!FuncInfo->CanLowerReturn) { | ||||||||||||
10289 | // Create a virtual register for the sret pointer, and put in a copy | ||||||||||||
10290 | // from the sret argument into it. | ||||||||||||
10291 | SmallVector<EVT, 1> ValueVTs; | ||||||||||||
10292 | ComputeValueVTs(*TLI, DAG.getDataLayout(), | ||||||||||||
10293 | F.getReturnType()->getPointerTo( | ||||||||||||
10294 | DAG.getDataLayout().getAllocaAddrSpace()), | ||||||||||||
10295 | ValueVTs); | ||||||||||||
10296 | MVT VT = ValueVTs[0].getSimpleVT(); | ||||||||||||
10297 | MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT); | ||||||||||||
10298 | Optional<ISD::NodeType> AssertOp = None; | ||||||||||||
10299 | SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, | ||||||||||||
10300 | nullptr, F.getCallingConv(), AssertOp); | ||||||||||||
10301 | |||||||||||||
10302 | MachineFunction& MF = SDB->DAG.getMachineFunction(); | ||||||||||||
10303 | MachineRegisterInfo& RegInfo = MF.getRegInfo(); | ||||||||||||
10304 | Register SRetReg = | ||||||||||||
10305 | RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT)); | ||||||||||||
10306 | FuncInfo->DemoteRegister = SRetReg; | ||||||||||||
10307 | NewRoot = | ||||||||||||
10308 | SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue); | ||||||||||||
10309 | DAG.setRoot(NewRoot); | ||||||||||||
10310 | |||||||||||||
10311 | // i indexes lowered arguments. Bump it past the hidden sret argument. | ||||||||||||
10312 | ++i; | ||||||||||||
10313 | } | ||||||||||||
10314 | |||||||||||||
10315 | SmallVector<SDValue, 4> Chains; | ||||||||||||
10316 | DenseMap<int, int> ArgCopyElisionFrameIndexMap; | ||||||||||||
10317 | for (const Argument &Arg : F.args()) { | ||||||||||||
10318 | SmallVector<SDValue, 4> ArgValues; | ||||||||||||
10319 | SmallVector<EVT, 4> ValueVTs; | ||||||||||||
10320 | ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs); | ||||||||||||
10321 | unsigned NumValues = ValueVTs.size(); | ||||||||||||
10322 | if (NumValues == 0) | ||||||||||||
10323 | continue; | ||||||||||||
10324 | |||||||||||||
10325 | bool ArgHasUses = !Arg.use_empty(); | ||||||||||||
10326 | |||||||||||||
10327 | // Elide the copying store if the target loaded this argument from a | ||||||||||||
10328 | // suitable fixed stack object. | ||||||||||||
10329 | if (Ins[i].Flags.isCopyElisionCandidate()) { | ||||||||||||
10330 | tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap, | ||||||||||||
10331 | ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg, | ||||||||||||
10332 | InVals[i], ArgHasUses); | ||||||||||||
10333 | } | ||||||||||||
10334 | |||||||||||||
10335 | // If this argument is unused then remember its value. It is used to generate | ||||||||||||
10336 | // debugging information. | ||||||||||||
10337 | bool isSwiftErrorArg = | ||||||||||||
10338 | TLI->supportSwiftError() && | ||||||||||||
10339 | Arg.hasAttribute(Attribute::SwiftError); | ||||||||||||
10340 | if (!ArgHasUses && !isSwiftErrorArg) { | ||||||||||||
10341 | SDB->setUnusedArgValue(&Arg, InVals[i]); | ||||||||||||
10342 | |||||||||||||
10343 | // Also remember any frame index for use in FastISel. | ||||||||||||
10344 | if (FrameIndexSDNode *FI = | ||||||||||||
10345 | dyn_cast<FrameIndexSDNode>(InVals[i].getNode())) | ||||||||||||
10346 | FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex()); | ||||||||||||
10347 | } | ||||||||||||
10348 | |||||||||||||
10349 | for (unsigned Val = 0; Val != NumValues; ++Val) { | ||||||||||||
10350 | EVT VT = ValueVTs[Val]; | ||||||||||||
10351 | MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(), | ||||||||||||
10352 | F.getCallingConv(), VT); | ||||||||||||
10353 | unsigned NumParts = TLI->getNumRegistersForCallingConv( | ||||||||||||
10354 | *CurDAG->getContext(), F.getCallingConv(), VT); | ||||||||||||
10355 | |||||||||||||
10356 | // Even an apparent 'unused' swifterror argument needs to be returned. So | ||||||||||||
10357 | // we do generate a copy for it that can be used on return from the | ||||||||||||
10358 | // function. | ||||||||||||
10359 | if (ArgHasUses || isSwiftErrorArg) { | ||||||||||||
10360 | Optional<ISD::NodeType> AssertOp; | ||||||||||||
10361 | if (Arg.hasAttribute(Attribute::SExt)) | ||||||||||||
10362 | AssertOp = ISD::AssertSext; | ||||||||||||
10363 | else if (Arg.hasAttribute(Attribute::ZExt)) | ||||||||||||
10364 | AssertOp = ISD::AssertZext; | ||||||||||||
10365 | |||||||||||||
10366 | ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts, | ||||||||||||
10367 | PartVT, VT, nullptr, | ||||||||||||
10368 | F.getCallingConv(), AssertOp)); | ||||||||||||
10369 | } | ||||||||||||
10370 | |||||||||||||
10371 | i += NumParts; | ||||||||||||
10372 | } | ||||||||||||
10373 | |||||||||||||
10374 | // We don't need to do anything else for unused arguments. | ||||||||||||
10375 | if (ArgValues.empty()) | ||||||||||||
10376 | continue; | ||||||||||||
10377 | |||||||||||||
10378 | // Note down frame index. | ||||||||||||
10379 | if (FrameIndexSDNode *FI = | ||||||||||||
10380 | dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode())) | ||||||||||||
10381 | FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex()); | ||||||||||||
10382 | |||||||||||||
10383 | SDValue Res = DAG.getMergeValues(makeArrayRef(ArgValues.data(), NumValues), | ||||||||||||
10384 | SDB->getCurSDLoc()); | ||||||||||||
10385 | |||||||||||||
10386 | SDB->setValue(&Arg, Res); | ||||||||||||
10387 | if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) { | ||||||||||||
10388 | // We want to associate the argument with the frame index, among | ||||||||||||
10389 | // involved operands, that correspond to the lowest address. The | ||||||||||||
10390 | // getCopyFromParts function, called earlier, is swapping the order of | ||||||||||||
10391 | // the operands to BUILD_PAIR depending on endianness. The result of | ||||||||||||
10392 | // that swapping is that the least significant bits of the argument will | ||||||||||||
10393 | // be in the first operand of the BUILD_PAIR node, and the most | ||||||||||||
10394 | // significant bits will be in the second operand. | ||||||||||||
10395 | unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0; | ||||||||||||
10396 | if (LoadSDNode *LNode = | ||||||||||||
10397 | dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode())) | ||||||||||||
10398 | if (FrameIndexSDNode *FI = | ||||||||||||
10399 | dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) | ||||||||||||
10400 | FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex()); | ||||||||||||
10401 | } | ||||||||||||
10402 | |||||||||||||
10403 | // Analyses past this point are naive and don't expect an assertion. | ||||||||||||
10404 | if (Res.getOpcode() == ISD::AssertZext) | ||||||||||||
10405 | Res = Res.getOperand(0); | ||||||||||||
10406 | |||||||||||||
10407 | // Update the SwiftErrorVRegDefMap. | ||||||||||||
10408 | if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) { | ||||||||||||
10409 | unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg(); | ||||||||||||
10410 | if (Register::isVirtualRegister(Reg)) | ||||||||||||
10411 | SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(), | ||||||||||||
10412 | Reg); | ||||||||||||
10413 | } | ||||||||||||
10414 | |||||||||||||
10415 | // If this argument is live outside of the entry block, insert a copy from | ||||||||||||
10416 | // wherever we got it to the vreg that other BB's will reference it as. | ||||||||||||
10417 | if (Res.getOpcode() == ISD::CopyFromReg) { | ||||||||||||
10418 | // If we can, though, try to skip creating an unnecessary vreg. | ||||||||||||
10419 | // FIXME: This isn't very clean... it would be nice to make this more | ||||||||||||
10420 | // general. | ||||||||||||
10421 | unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg(); | ||||||||||||
10422 | if (Register::isVirtualRegister(Reg)) { | ||||||||||||
10423 | FuncInfo->ValueMap[&Arg] = Reg; | ||||||||||||
10424 | continue; | ||||||||||||
10425 | } | ||||||||||||
10426 | } | ||||||||||||
10427 | if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) { | ||||||||||||
10428 | FuncInfo->InitializeRegForValue(&Arg); | ||||||||||||
10429 | SDB->CopyToExportRegsIfNeeded(&Arg); | ||||||||||||
10430 | } | ||||||||||||
10431 | } | ||||||||||||
10432 | |||||||||||||
10433 | if (!Chains.empty()) { | ||||||||||||
10434 | Chains.push_back(NewRoot); | ||||||||||||
10435 | NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); | ||||||||||||
10436 | } | ||||||||||||
10437 | |||||||||||||
10438 | DAG.setRoot(NewRoot); | ||||||||||||
10439 | |||||||||||||
10440 | assert(i == InVals.size() && "Argument register count mismatch!")(static_cast <bool> (i == InVals.size() && "Argument register count mismatch!" ) ? void (0) : __assert_fail ("i == InVals.size() && \"Argument register count mismatch!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10440, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
10441 | |||||||||||||
10442 | // If any argument copy elisions occurred and we have debug info, update the | ||||||||||||
10443 | // stale frame indices used in the dbg.declare variable info table. | ||||||||||||
10444 | MachineFunction::VariableDbgInfoMapTy &DbgDeclareInfo = MF->getVariableDbgInfo(); | ||||||||||||
10445 | if (!DbgDeclareInfo.empty() && !ArgCopyElisionFrameIndexMap.empty()) { | ||||||||||||
10446 | for (MachineFunction::VariableDbgInfo &VI : DbgDeclareInfo) { | ||||||||||||
10447 | auto I = ArgCopyElisionFrameIndexMap.find(VI.Slot); | ||||||||||||
10448 | if (I != ArgCopyElisionFrameIndexMap.end()) | ||||||||||||
10449 | VI.Slot = I->second; | ||||||||||||
10450 | } | ||||||||||||
10451 | } | ||||||||||||
10452 | |||||||||||||
10453 | // Finally, if the target has anything special to do, allow it to do so. | ||||||||||||
10454 | emitFunctionEntryCode(); | ||||||||||||
10455 | } | ||||||||||||
10456 | |||||||||||||
10457 | /// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to | ||||||||||||
10458 | /// ensure constants are generated when needed. Remember the virtual registers | ||||||||||||
10459 | /// that need to be added to the Machine PHI nodes as input. We cannot just | ||||||||||||
10460 | /// directly add them, because expansion might result in multiple MBB's for one | ||||||||||||
10461 | /// BB. As such, the start of the BB might correspond to a different MBB than | ||||||||||||
10462 | /// the end. | ||||||||||||
10463 | void | ||||||||||||
10464 | SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { | ||||||||||||
10465 | const Instruction *TI = LLVMBB->getTerminator(); | ||||||||||||
10466 | |||||||||||||
10467 | SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled; | ||||||||||||
10468 | |||||||||||||
10469 | // Check PHI nodes in successors that expect a value to be available from this | ||||||||||||
10470 | // block. | ||||||||||||
10471 | for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) { | ||||||||||||
10472 | const BasicBlock *SuccBB = TI->getSuccessor(succ); | ||||||||||||
10473 | if (!isa<PHINode>(SuccBB->begin())) continue; | ||||||||||||
10474 | MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB]; | ||||||||||||
10475 | |||||||||||||
10476 | // If this terminator has multiple identical successors (common for | ||||||||||||
10477 | // switches), only handle each succ once. | ||||||||||||
10478 | if (!SuccsHandled.insert(SuccMBB).second) | ||||||||||||
10479 | continue; | ||||||||||||
10480 | |||||||||||||
10481 | MachineBasicBlock::iterator MBBI = SuccMBB->begin(); | ||||||||||||
10482 | |||||||||||||
10483 | // At this point we know that there is a 1-1 correspondence between LLVM PHI | ||||||||||||
10484 | // nodes and Machine PHI nodes, but the incoming operands have not been | ||||||||||||
10485 | // emitted yet. | ||||||||||||
10486 | for (const PHINode &PN : SuccBB->phis()) { | ||||||||||||
10487 | // Ignore dead phi's. | ||||||||||||
10488 | if (PN.use_empty()) | ||||||||||||
10489 | continue; | ||||||||||||
10490 | |||||||||||||
10491 | // Skip empty types | ||||||||||||
10492 | if (PN.getType()->isEmptyTy()) | ||||||||||||
10493 | continue; | ||||||||||||
10494 | |||||||||||||
10495 | unsigned Reg; | ||||||||||||
10496 | const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB); | ||||||||||||
10497 | |||||||||||||
10498 | if (const Constant *C = dyn_cast<Constant>(PHIOp)) { | ||||||||||||
10499 | unsigned &RegOut = ConstantsOut[C]; | ||||||||||||
10500 | if (RegOut == 0) { | ||||||||||||
10501 | RegOut = FuncInfo.CreateRegs(C); | ||||||||||||
10502 | CopyValueToVirtualRegister(C, RegOut); | ||||||||||||
10503 | } | ||||||||||||
10504 | Reg = RegOut; | ||||||||||||
10505 | } else { | ||||||||||||
10506 | DenseMap<const Value *, Register>::iterator I = | ||||||||||||
10507 | FuncInfo.ValueMap.find(PHIOp); | ||||||||||||
10508 | if (I != FuncInfo.ValueMap.end()) | ||||||||||||
10509 | Reg = I->second; | ||||||||||||
10510 | else { | ||||||||||||
10511 | assert(isa<AllocaInst>(PHIOp) &&(static_cast <bool> (isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp) ) && "Didn't codegen value into a register!??") ? void (0) : __assert_fail ("isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && \"Didn't codegen value into a register!??\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10513, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
10512 | FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&(static_cast <bool> (isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp) ) && "Didn't codegen value into a register!??") ? void (0) : __assert_fail ("isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && \"Didn't codegen value into a register!??\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10513, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
10513 | "Didn't codegen value into a register!??")(static_cast <bool> (isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp) ) && "Didn't codegen value into a register!??") ? void (0) : __assert_fail ("isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && \"Didn't codegen value into a register!??\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10513, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
10514 | Reg = FuncInfo.CreateRegs(PHIOp); | ||||||||||||
10515 | CopyValueToVirtualRegister(PHIOp, Reg); | ||||||||||||
10516 | } | ||||||||||||
10517 | } | ||||||||||||
10518 | |||||||||||||
10519 | // Remember that this register needs to added to the machine PHI node as | ||||||||||||
10520 | // the input for this MBB. | ||||||||||||
10521 | SmallVector<EVT, 4> ValueVTs; | ||||||||||||
10522 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
10523 | ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs); | ||||||||||||
10524 | for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) { | ||||||||||||
10525 | EVT VT = ValueVTs[vti]; | ||||||||||||
10526 | unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT); | ||||||||||||
10527 | for (unsigned i = 0, e = NumRegisters; i != e; ++i) | ||||||||||||
10528 | FuncInfo.PHINodesToUpdate.push_back( | ||||||||||||
10529 | std::make_pair(&*MBBI++, Reg + i)); | ||||||||||||
10530 | Reg += NumRegisters; | ||||||||||||
10531 | } | ||||||||||||
10532 | } | ||||||||||||
10533 | } | ||||||||||||
10534 | |||||||||||||
10535 | ConstantsOut.clear(); | ||||||||||||
10536 | } | ||||||||||||
10537 | |||||||||||||
10538 | /// Add a successor MBB to ParentMBB< creating a new MachineBB for BB if SuccMBB | ||||||||||||
10539 | /// is 0. | ||||||||||||
10540 | MachineBasicBlock * | ||||||||||||
10541 | SelectionDAGBuilder::StackProtectorDescriptor:: | ||||||||||||
10542 | AddSuccessorMBB(const BasicBlock *BB, | ||||||||||||
10543 | MachineBasicBlock *ParentMBB, | ||||||||||||
10544 | bool IsLikely, | ||||||||||||
10545 | MachineBasicBlock *SuccMBB) { | ||||||||||||
10546 | // If SuccBB has not been created yet, create it. | ||||||||||||
10547 | if (!SuccMBB) { | ||||||||||||
10548 | MachineFunction *MF = ParentMBB->getParent(); | ||||||||||||
10549 | MachineFunction::iterator BBI(ParentMBB); | ||||||||||||
10550 | SuccMBB = MF->CreateMachineBasicBlock(BB); | ||||||||||||
10551 | MF->insert(++BBI, SuccMBB); | ||||||||||||
10552 | } | ||||||||||||
10553 | // Add it as a successor of ParentMBB. | ||||||||||||
10554 | ParentMBB->addSuccessor( | ||||||||||||
10555 | SuccMBB, BranchProbabilityInfo::getBranchProbStackProtector(IsLikely)); | ||||||||||||
10556 | return SuccMBB; | ||||||||||||
10557 | } | ||||||||||||
10558 | |||||||||||||
10559 | MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) { | ||||||||||||
10560 | MachineFunction::iterator I(MBB); | ||||||||||||
10561 | if (++I == FuncInfo.MF->end()) | ||||||||||||
10562 | return nullptr; | ||||||||||||
10563 | return &*I; | ||||||||||||
10564 | } | ||||||||||||
10565 | |||||||||||||
10566 | /// During lowering new call nodes can be created (such as memset, etc.). | ||||||||||||
10567 | /// Those will become new roots of the current DAG, but complications arise | ||||||||||||
10568 | /// when they are tail calls. In such cases, the call lowering will update | ||||||||||||
10569 | /// the root, but the builder still needs to know that a tail call has been | ||||||||||||
10570 | /// lowered in order to avoid generating an additional return. | ||||||||||||
10571 | void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) { | ||||||||||||
10572 | // If the node is null, we do have a tail call. | ||||||||||||
10573 | if (MaybeTC.getNode() != nullptr) | ||||||||||||
10574 | DAG.setRoot(MaybeTC); | ||||||||||||
10575 | else | ||||||||||||
10576 | HasTailCall = true; | ||||||||||||
10577 | } | ||||||||||||
10578 | |||||||||||||
10579 | void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, | ||||||||||||
10580 | MachineBasicBlock *SwitchMBB, | ||||||||||||
10581 | MachineBasicBlock *DefaultMBB) { | ||||||||||||
10582 | MachineFunction *CurMF = FuncInfo.MF; | ||||||||||||
10583 | MachineBasicBlock *NextMBB = nullptr; | ||||||||||||
10584 | MachineFunction::iterator BBI(W.MBB); | ||||||||||||
10585 | if (++BBI != FuncInfo.MF->end()) | ||||||||||||
10586 | NextMBB = &*BBI; | ||||||||||||
10587 | |||||||||||||
10588 | unsigned Size = W.LastCluster - W.FirstCluster + 1; | ||||||||||||
10589 | |||||||||||||
10590 | BranchProbabilityInfo *BPI = FuncInfo.BPI; | ||||||||||||
10591 | |||||||||||||
10592 | if (Size == 2 && W.MBB == SwitchMBB) { | ||||||||||||
10593 | // If any two of the cases has the same destination, and if one value | ||||||||||||
10594 | // is the same as the other, but has one bit unset that the other has set, | ||||||||||||
10595 | // use bit manipulation to do two compares at once. For example: | ||||||||||||
10596 | // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)" | ||||||||||||
10597 | // TODO: This could be extended to merge any 2 cases in switches with 3 | ||||||||||||
10598 | // cases. | ||||||||||||
10599 | // TODO: Handle cases where W.CaseBB != SwitchBB. | ||||||||||||
10600 | CaseCluster &Small = *W.FirstCluster; | ||||||||||||
10601 | CaseCluster &Big = *W.LastCluster; | ||||||||||||
10602 | |||||||||||||
10603 | if (Small.Low == Small.High && Big.Low == Big.High && | ||||||||||||
10604 | Small.MBB == Big.MBB) { | ||||||||||||
10605 | const APInt &SmallValue = Small.Low->getValue(); | ||||||||||||
10606 | const APInt &BigValue = Big.Low->getValue(); | ||||||||||||
10607 | |||||||||||||
10608 | // Check that there is only one bit different. | ||||||||||||
10609 | APInt CommonBit = BigValue ^ SmallValue; | ||||||||||||
10610 | if (CommonBit.isPowerOf2()) { | ||||||||||||
10611 | SDValue CondLHS = getValue(Cond); | ||||||||||||
10612 | EVT VT = CondLHS.getValueType(); | ||||||||||||
10613 | SDLoc DL = getCurSDLoc(); | ||||||||||||
10614 | |||||||||||||
10615 | SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS, | ||||||||||||
10616 | DAG.getConstant(CommonBit, DL, VT)); | ||||||||||||
10617 | SDValue Cond = DAG.getSetCC( | ||||||||||||
10618 | DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT), | ||||||||||||
10619 | ISD::SETEQ); | ||||||||||||
10620 | |||||||||||||
10621 | // Update successor info. | ||||||||||||
10622 | // Both Small and Big will jump to Small.BB, so we sum up the | ||||||||||||
10623 | // probabilities. | ||||||||||||
10624 | addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob); | ||||||||||||
10625 | if (BPI) | ||||||||||||
10626 | addSuccessorWithProb( | ||||||||||||
10627 | SwitchMBB, DefaultMBB, | ||||||||||||
10628 | // The default destination is the first successor in IR. | ||||||||||||
10629 | BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0)); | ||||||||||||
10630 | else | ||||||||||||
10631 | addSuccessorWithProb(SwitchMBB, DefaultMBB); | ||||||||||||
10632 | |||||||||||||
10633 | // Insert the true branch. | ||||||||||||
10634 | SDValue BrCond = | ||||||||||||
10635 | DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond, | ||||||||||||
10636 | DAG.getBasicBlock(Small.MBB)); | ||||||||||||
10637 | // Insert the false branch. | ||||||||||||
10638 | BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond, | ||||||||||||
10639 | DAG.getBasicBlock(DefaultMBB)); | ||||||||||||
10640 | |||||||||||||
10641 | DAG.setRoot(BrCond); | ||||||||||||
10642 | return; | ||||||||||||
10643 | } | ||||||||||||
10644 | } | ||||||||||||
10645 | } | ||||||||||||
10646 | |||||||||||||
10647 | if (TM.getOptLevel() != CodeGenOpt::None) { | ||||||||||||
10648 | // Here, we order cases by probability so the most likely case will be | ||||||||||||
10649 | // checked first. However, two clusters can have the same probability in | ||||||||||||
10650 | // which case their relative ordering is non-deterministic. So we use Low | ||||||||||||
10651 | // as a tie-breaker as clusters are guaranteed to never overlap. | ||||||||||||
10652 | llvm::sort(W.FirstCluster, W.LastCluster + 1, | ||||||||||||
10653 | [](const CaseCluster &a, const CaseCluster &b) { | ||||||||||||
10654 | return a.Prob != b.Prob ? | ||||||||||||
10655 | a.Prob > b.Prob : | ||||||||||||
10656 | a.Low->getValue().slt(b.Low->getValue()); | ||||||||||||
10657 | }); | ||||||||||||
10658 | |||||||||||||
10659 | // Rearrange the case blocks so that the last one falls through if possible | ||||||||||||
10660 | // without changing the order of probabilities. | ||||||||||||
10661 | for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) { | ||||||||||||
10662 | --I; | ||||||||||||
10663 | if (I->Prob > W.LastCluster->Prob) | ||||||||||||
10664 | break; | ||||||||||||
10665 | if (I->Kind == CC_Range && I->MBB == NextMBB) { | ||||||||||||
10666 | std::swap(*I, *W.LastCluster); | ||||||||||||
10667 | break; | ||||||||||||
10668 | } | ||||||||||||
10669 | } | ||||||||||||
10670 | } | ||||||||||||
10671 | |||||||||||||
10672 | // Compute total probability. | ||||||||||||
10673 | BranchProbability DefaultProb = W.DefaultProb; | ||||||||||||
10674 | BranchProbability UnhandledProbs = DefaultProb; | ||||||||||||
10675 | for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I) | ||||||||||||
10676 | UnhandledProbs += I->Prob; | ||||||||||||
10677 | |||||||||||||
10678 | MachineBasicBlock *CurMBB = W.MBB; | ||||||||||||
10679 | for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) { | ||||||||||||
10680 | bool FallthroughUnreachable = false; | ||||||||||||
10681 | MachineBasicBlock *Fallthrough; | ||||||||||||
10682 | if (I == W.LastCluster) { | ||||||||||||
10683 | // For the last cluster, fall through to the default destination. | ||||||||||||
10684 | Fallthrough = DefaultMBB; | ||||||||||||
10685 | FallthroughUnreachable = isa<UnreachableInst>( | ||||||||||||
10686 | DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg()); | ||||||||||||
10687 | } else { | ||||||||||||
10688 | Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock()); | ||||||||||||
10689 | CurMF->insert(BBI, Fallthrough); | ||||||||||||
10690 | // Put Cond in a virtual register to make it available from the new blocks. | ||||||||||||
10691 | ExportFromCurrentBlock(Cond); | ||||||||||||
10692 | } | ||||||||||||
10693 | UnhandledProbs -= I->Prob; | ||||||||||||
10694 | |||||||||||||
10695 | switch (I->Kind) { | ||||||||||||
10696 | case CC_JumpTable: { | ||||||||||||
10697 | // FIXME: Optimize away range check based on pivot comparisons. | ||||||||||||
10698 | JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first; | ||||||||||||
10699 | SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second; | ||||||||||||
10700 | |||||||||||||
10701 | // The jump block hasn't been inserted yet; insert it here. | ||||||||||||
10702 | MachineBasicBlock *JumpMBB = JT->MBB; | ||||||||||||
10703 | CurMF->insert(BBI, JumpMBB); | ||||||||||||
10704 | |||||||||||||
10705 | auto JumpProb = I->Prob; | ||||||||||||
10706 | auto FallthroughProb = UnhandledProbs; | ||||||||||||
10707 | |||||||||||||
10708 | // If the default statement is a target of the jump table, we evenly | ||||||||||||
10709 | // distribute the default probability to successors of CurMBB. Also | ||||||||||||
10710 | // update the probability on the edge from JumpMBB to Fallthrough. | ||||||||||||
10711 | for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(), | ||||||||||||
10712 | SE = JumpMBB->succ_end(); | ||||||||||||
10713 | SI != SE; ++SI) { | ||||||||||||
10714 | if (*SI == DefaultMBB) { | ||||||||||||
10715 | JumpProb += DefaultProb / 2; | ||||||||||||
10716 | FallthroughProb -= DefaultProb / 2; | ||||||||||||
10717 | JumpMBB->setSuccProbability(SI, DefaultProb / 2); | ||||||||||||
10718 | JumpMBB->normalizeSuccProbs(); | ||||||||||||
10719 | break; | ||||||||||||
10720 | } | ||||||||||||
10721 | } | ||||||||||||
10722 | |||||||||||||
10723 | if (FallthroughUnreachable) { | ||||||||||||
10724 | // Skip the range check if the fallthrough block is unreachable. | ||||||||||||
10725 | JTH->OmitRangeCheck = true; | ||||||||||||
10726 | } | ||||||||||||
10727 | |||||||||||||
10728 | if (!JTH->OmitRangeCheck) | ||||||||||||
10729 | addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb); | ||||||||||||
10730 | addSuccessorWithProb(CurMBB, JumpMBB, JumpProb); | ||||||||||||
10731 | CurMBB->normalizeSuccProbs(); | ||||||||||||
10732 | |||||||||||||
10733 | // The jump table header will be inserted in our current block, do the | ||||||||||||
10734 | // range check, and fall through to our fallthrough block. | ||||||||||||
10735 | JTH->HeaderBB = CurMBB; | ||||||||||||
10736 | JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader. | ||||||||||||
10737 | |||||||||||||
10738 | // If we're in the right place, emit the jump table header right now. | ||||||||||||
10739 | if (CurMBB == SwitchMBB) { | ||||||||||||
10740 | visitJumpTableHeader(*JT, *JTH, SwitchMBB); | ||||||||||||
10741 | JTH->Emitted = true; | ||||||||||||
10742 | } | ||||||||||||
10743 | break; | ||||||||||||
10744 | } | ||||||||||||
10745 | case CC_BitTests: { | ||||||||||||
10746 | // FIXME: Optimize away range check based on pivot comparisons. | ||||||||||||
10747 | BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex]; | ||||||||||||
10748 | |||||||||||||
10749 | // The bit test blocks haven't been inserted yet; insert them here. | ||||||||||||
10750 | for (BitTestCase &BTC : BTB->Cases) | ||||||||||||
10751 | CurMF->insert(BBI, BTC.ThisBB); | ||||||||||||
10752 | |||||||||||||
10753 | // Fill in fields of the BitTestBlock. | ||||||||||||
10754 | BTB->Parent = CurMBB; | ||||||||||||
10755 | BTB->Default = Fallthrough; | ||||||||||||
10756 | |||||||||||||
10757 | BTB->DefaultProb = UnhandledProbs; | ||||||||||||
10758 | // If the cases in bit test don't form a contiguous range, we evenly | ||||||||||||
10759 | // distribute the probability on the edge to Fallthrough to two | ||||||||||||
10760 | // successors of CurMBB. | ||||||||||||
10761 | if (!BTB->ContiguousRange) { | ||||||||||||
10762 | BTB->Prob += DefaultProb / 2; | ||||||||||||
10763 | BTB->DefaultProb -= DefaultProb / 2; | ||||||||||||
10764 | } | ||||||||||||
10765 | |||||||||||||
10766 | if (FallthroughUnreachable) { | ||||||||||||
10767 | // Skip the range check if the fallthrough block is unreachable. | ||||||||||||
10768 | BTB->OmitRangeCheck = true; | ||||||||||||
10769 | } | ||||||||||||
10770 | |||||||||||||
10771 | // If we're in the right place, emit the bit test header right now. | ||||||||||||
10772 | if (CurMBB == SwitchMBB) { | ||||||||||||
10773 | visitBitTestHeader(*BTB, SwitchMBB); | ||||||||||||
10774 | BTB->Emitted = true; | ||||||||||||
10775 | } | ||||||||||||
10776 | break; | ||||||||||||
10777 | } | ||||||||||||
10778 | case CC_Range: { | ||||||||||||
10779 | const Value *RHS, *LHS, *MHS; | ||||||||||||
10780 | ISD::CondCode CC; | ||||||||||||
10781 | if (I->Low == I->High) { | ||||||||||||
10782 | // Check Cond == I->Low. | ||||||||||||
10783 | CC = ISD::SETEQ; | ||||||||||||
10784 | LHS = Cond; | ||||||||||||
10785 | RHS=I->Low; | ||||||||||||
10786 | MHS = nullptr; | ||||||||||||
10787 | } else { | ||||||||||||
10788 | // Check I->Low <= Cond <= I->High. | ||||||||||||
10789 | CC = ISD::SETLE; | ||||||||||||
10790 | LHS = I->Low; | ||||||||||||
10791 | MHS = Cond; | ||||||||||||
10792 | RHS = I->High; | ||||||||||||
10793 | } | ||||||||||||
10794 | |||||||||||||
10795 | // If Fallthrough is unreachable, fold away the comparison. | ||||||||||||
10796 | if (FallthroughUnreachable) | ||||||||||||
10797 | CC = ISD::SETTRUE; | ||||||||||||
10798 | |||||||||||||
10799 | // The false probability is the sum of all unhandled cases. | ||||||||||||
10800 | CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB, | ||||||||||||
10801 | getCurSDLoc(), I->Prob, UnhandledProbs); | ||||||||||||
10802 | |||||||||||||
10803 | if (CurMBB == SwitchMBB) | ||||||||||||
10804 | visitSwitchCase(CB, SwitchMBB); | ||||||||||||
10805 | else | ||||||||||||
10806 | SL->SwitchCases.push_back(CB); | ||||||||||||
10807 | |||||||||||||
10808 | break; | ||||||||||||
10809 | } | ||||||||||||
10810 | } | ||||||||||||
10811 | CurMBB = Fallthrough; | ||||||||||||
10812 | } | ||||||||||||
10813 | } | ||||||||||||
10814 | |||||||||||||
10815 | unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC, | ||||||||||||
10816 | CaseClusterIt First, | ||||||||||||
10817 | CaseClusterIt Last) { | ||||||||||||
10818 | return std::count_if(First, Last + 1, [&](const CaseCluster &X) { | ||||||||||||
10819 | if (X.Prob != CC.Prob) | ||||||||||||
10820 | return X.Prob > CC.Prob; | ||||||||||||
10821 | |||||||||||||
10822 | // Ties are broken by comparing the case value. | ||||||||||||
10823 | return X.Low->getValue().slt(CC.Low->getValue()); | ||||||||||||
10824 | }); | ||||||||||||
10825 | } | ||||||||||||
10826 | |||||||||||||
10827 | void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList, | ||||||||||||
10828 | const SwitchWorkListItem &W, | ||||||||||||
10829 | Value *Cond, | ||||||||||||
10830 | MachineBasicBlock *SwitchMBB) { | ||||||||||||
10831 | assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&(static_cast <bool> (W.FirstCluster->Low->getValue ().slt(W.LastCluster->Low->getValue()) && "Clusters not sorted?" ) ? void (0) : __assert_fail ("W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) && \"Clusters not sorted?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10832, __extension__ __PRETTY_FUNCTION__)) | ||||||||||||
10832 | "Clusters not sorted?")(static_cast <bool> (W.FirstCluster->Low->getValue ().slt(W.LastCluster->Low->getValue()) && "Clusters not sorted?" ) ? void (0) : __assert_fail ("W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) && \"Clusters not sorted?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10832, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
10833 | |||||||||||||
10834 | assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!")(static_cast <bool> (W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!") ? void (0) : __assert_fail ("W.LastCluster - W.FirstCluster + 1 >= 2 && \"Too small to split!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10834, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
10835 | |||||||||||||
10836 | // Balance the tree based on branch probabilities to create a near-optimal (in | ||||||||||||
10837 | // terms of search time given key frequency) binary search tree. See e.g. Kurt | ||||||||||||
10838 | // Mehlhorn "Nearly Optimal Binary Search Trees" (1975). | ||||||||||||
10839 | CaseClusterIt LastLeft = W.FirstCluster; | ||||||||||||
10840 | CaseClusterIt FirstRight = W.LastCluster; | ||||||||||||
10841 | auto LeftProb = LastLeft->Prob + W.DefaultProb / 2; | ||||||||||||
10842 | auto RightProb = FirstRight->Prob + W.DefaultProb / 2; | ||||||||||||
10843 | |||||||||||||
10844 | // Move LastLeft and FirstRight towards each other from opposite directions to | ||||||||||||
10845 | // find a partitioning of the clusters which balances the probability on both | ||||||||||||
10846 | // sides. If LeftProb and RightProb are equal, alternate which side is | ||||||||||||
10847 | // taken to ensure 0-probability nodes are distributed evenly. | ||||||||||||
10848 | unsigned I = 0; | ||||||||||||
10849 | while (LastLeft + 1 < FirstRight) { | ||||||||||||
10850 | if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1))) | ||||||||||||
10851 | LeftProb += (++LastLeft)->Prob; | ||||||||||||
10852 | else | ||||||||||||
10853 | RightProb += (--FirstRight)->Prob; | ||||||||||||
10854 | I++; | ||||||||||||
10855 | } | ||||||||||||
10856 | |||||||||||||
10857 | while (true) { | ||||||||||||
10858 | // Our binary search tree differs from a typical BST in that ours can have up | ||||||||||||
10859 | // to three values in each leaf. The pivot selection above doesn't take that | ||||||||||||
10860 | // into account, which means the tree might require more nodes and be less | ||||||||||||
10861 | // efficient. We compensate for this here. | ||||||||||||
10862 | |||||||||||||
10863 | unsigned NumLeft = LastLeft - W.FirstCluster + 1; | ||||||||||||
10864 | unsigned NumRight = W.LastCluster - FirstRight + 1; | ||||||||||||
10865 | |||||||||||||
10866 | if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) { | ||||||||||||
10867 | // If one side has less than 3 clusters, and the other has more than 3, | ||||||||||||
10868 | // consider taking a cluster from the other side. | ||||||||||||
10869 | |||||||||||||
10870 | if (NumLeft < NumRight) { | ||||||||||||
10871 | // Consider moving the first cluster on the right to the left side. | ||||||||||||
10872 | CaseCluster &CC = *FirstRight; | ||||||||||||
10873 | unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster); | ||||||||||||
10874 | unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft); | ||||||||||||
10875 | if (LeftSideRank <= RightSideRank) { | ||||||||||||
10876 | // Moving the cluster to the left does not demote it. | ||||||||||||
10877 | ++LastLeft; | ||||||||||||
10878 | ++FirstRight; | ||||||||||||
10879 | continue; | ||||||||||||
10880 | } | ||||||||||||
10881 | } else { | ||||||||||||
10882 | assert(NumRight < NumLeft)(static_cast <bool> (NumRight < NumLeft) ? void (0) : __assert_fail ("NumRight < NumLeft", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10882, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
10883 | // Consider moving the last element on the left to the right side. | ||||||||||||
10884 | CaseCluster &CC = *LastLeft; | ||||||||||||
10885 | unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft); | ||||||||||||
10886 | unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster); | ||||||||||||
10887 | if (RightSideRank <= LeftSideRank) { | ||||||||||||
10888 | // Moving the cluster to the right does not demot it. | ||||||||||||
10889 | --LastLeft; | ||||||||||||
10890 | --FirstRight; | ||||||||||||
10891 | continue; | ||||||||||||
10892 | } | ||||||||||||
10893 | } | ||||||||||||
10894 | } | ||||||||||||
10895 | break; | ||||||||||||
10896 | } | ||||||||||||
10897 | |||||||||||||
10898 | assert(LastLeft + 1 == FirstRight)(static_cast <bool> (LastLeft + 1 == FirstRight) ? void (0) : __assert_fail ("LastLeft + 1 == FirstRight", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10898, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
10899 | assert(LastLeft >= W.FirstCluster)(static_cast <bool> (LastLeft >= W.FirstCluster) ? void (0) : __assert_fail ("LastLeft >= W.FirstCluster", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10899, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
10900 | assert(FirstRight <= W.LastCluster)(static_cast <bool> (FirstRight <= W.LastCluster) ? void (0) : __assert_fail ("FirstRight <= W.LastCluster", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10900, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
10901 | |||||||||||||
10902 | // Use the first element on the right as pivot since we will make less-than | ||||||||||||
10903 | // comparisons against it. | ||||||||||||
10904 | CaseClusterIt PivotCluster = FirstRight; | ||||||||||||
10905 | assert(PivotCluster > W.FirstCluster)(static_cast <bool> (PivotCluster > W.FirstCluster) ? void (0) : __assert_fail ("PivotCluster > W.FirstCluster" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10905, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
10906 | assert(PivotCluster <= W.LastCluster)(static_cast <bool> (PivotCluster <= W.LastCluster) ? void (0) : __assert_fail ("PivotCluster <= W.LastCluster" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 10906, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
10907 | |||||||||||||
10908 | CaseClusterIt FirstLeft = W.FirstCluster; | ||||||||||||
10909 | CaseClusterIt LastRight = W.LastCluster; | ||||||||||||
10910 | |||||||||||||
10911 | const ConstantInt *Pivot = PivotCluster->Low; | ||||||||||||
10912 | |||||||||||||
10913 | // New blocks will be inserted immediately after the current one. | ||||||||||||
10914 | MachineFunction::iterator BBI(W.MBB); | ||||||||||||
10915 | ++BBI; | ||||||||||||
10916 | |||||||||||||
10917 | // We will branch to the LHS if Value < Pivot. If LHS is a single cluster, | ||||||||||||
10918 | // we can branch to its destination directly if it's squeezed exactly in | ||||||||||||
10919 | // between the known lower bound and Pivot - 1. | ||||||||||||
10920 | MachineBasicBlock *LeftMBB; | ||||||||||||
10921 | if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range && | ||||||||||||
10922 | FirstLeft->Low == W.GE && | ||||||||||||
10923 | (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) { | ||||||||||||
10924 | LeftMBB = FirstLeft->MBB; | ||||||||||||
10925 | } else { | ||||||||||||
10926 | LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock()); | ||||||||||||
10927 | FuncInfo.MF->insert(BBI, LeftMBB); | ||||||||||||
10928 | WorkList.push_back( | ||||||||||||
10929 | {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2}); | ||||||||||||
10930 | // Put Cond in a virtual register to make it available from the new blocks. | ||||||||||||
10931 | ExportFromCurrentBlock(Cond); | ||||||||||||
10932 | } | ||||||||||||
10933 | |||||||||||||
10934 | // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a | ||||||||||||
10935 | // single cluster, RHS.Low == Pivot, and we can branch to its destination | ||||||||||||
10936 | // directly if RHS.High equals the current upper bound. | ||||||||||||
10937 | MachineBasicBlock *RightMBB; | ||||||||||||
10938 | if (FirstRight == LastRight && FirstRight->Kind == CC_Range && | ||||||||||||
10939 | W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) { | ||||||||||||
10940 | RightMBB = FirstRight->MBB; | ||||||||||||
10941 | } else { | ||||||||||||
10942 | RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock()); | ||||||||||||
10943 | FuncInfo.MF->insert(BBI, RightMBB); | ||||||||||||
10944 | WorkList.push_back( | ||||||||||||
10945 | {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2}); | ||||||||||||
10946 | // Put Cond in a virtual register to make it available from the new blocks. | ||||||||||||
10947 | ExportFromCurrentBlock(Cond); | ||||||||||||
10948 | } | ||||||||||||
10949 | |||||||||||||
10950 | // Create the CaseBlock record that will be used to lower the branch. | ||||||||||||
10951 | CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB, | ||||||||||||
10952 | getCurSDLoc(), LeftProb, RightProb); | ||||||||||||
10953 | |||||||||||||
10954 | if (W.MBB == SwitchMBB) | ||||||||||||
10955 | visitSwitchCase(CB, SwitchMBB); | ||||||||||||
10956 | else | ||||||||||||
10957 | SL->SwitchCases.push_back(CB); | ||||||||||||
10958 | } | ||||||||||||
10959 | |||||||||||||
10960 | // Scale CaseProb after peeling a case with the probablity of PeeledCaseProb | ||||||||||||
10961 | // from the swith statement. | ||||||||||||
10962 | static BranchProbability scaleCaseProbality(BranchProbability CaseProb, | ||||||||||||
10963 | BranchProbability PeeledCaseProb) { | ||||||||||||
10964 | if (PeeledCaseProb == BranchProbability::getOne()) | ||||||||||||
10965 | return BranchProbability::getZero(); | ||||||||||||
10966 | BranchProbability SwitchProb = PeeledCaseProb.getCompl(); | ||||||||||||
10967 | |||||||||||||
10968 | uint32_t Numerator = CaseProb.getNumerator(); | ||||||||||||
10969 | uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator()); | ||||||||||||
10970 | return BranchProbability(Numerator, std::max(Numerator, Denominator)); | ||||||||||||
10971 | } | ||||||||||||
10972 | |||||||||||||
10973 | // Try to peel the top probability case if it exceeds the threshold. | ||||||||||||
10974 | // Return current MachineBasicBlock for the switch statement if the peeling | ||||||||||||
10975 | // does not occur. | ||||||||||||
10976 | // If the peeling is performed, return the newly created MachineBasicBlock | ||||||||||||
10977 | // for the peeled switch statement. Also update Clusters to remove the peeled | ||||||||||||
10978 | // case. PeeledCaseProb is the BranchProbability for the peeled case. | ||||||||||||
10979 | MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster( | ||||||||||||
10980 | const SwitchInst &SI, CaseClusterVector &Clusters, | ||||||||||||
10981 | BranchProbability &PeeledCaseProb) { | ||||||||||||
10982 | MachineBasicBlock *SwitchMBB = FuncInfo.MBB; | ||||||||||||
10983 | // Don't perform if there is only one cluster or optimizing for size. | ||||||||||||
10984 | if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 || | ||||||||||||
10985 | TM.getOptLevel() == CodeGenOpt::None || | ||||||||||||
10986 | SwitchMBB->getParent()->getFunction().hasMinSize()) | ||||||||||||
10987 | return SwitchMBB; | ||||||||||||
10988 | |||||||||||||
10989 | BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100); | ||||||||||||
10990 | unsigned PeeledCaseIndex = 0; | ||||||||||||
10991 | bool SwitchPeeled = false; | ||||||||||||
10992 | for (unsigned Index = 0; Index < Clusters.size(); ++Index) { | ||||||||||||
10993 | CaseCluster &CC = Clusters[Index]; | ||||||||||||
10994 | if (CC.Prob < TopCaseProb) | ||||||||||||
10995 | continue; | ||||||||||||
10996 | TopCaseProb = CC.Prob; | ||||||||||||
10997 | PeeledCaseIndex = Index; | ||||||||||||
10998 | SwitchPeeled = true; | ||||||||||||
10999 | } | ||||||||||||
11000 | if (!SwitchPeeled) | ||||||||||||
11001 | return SwitchMBB; | ||||||||||||
11002 | |||||||||||||
11003 | LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Peeled one top case in switch stmt, prob: " << TopCaseProb << "\n"; } } while (false) | ||||||||||||
11004 | << TopCaseProb << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Peeled one top case in switch stmt, prob: " << TopCaseProb << "\n"; } } while (false); | ||||||||||||
11005 | |||||||||||||
11006 | // Record the MBB for the peeled switch statement. | ||||||||||||
11007 | MachineFunction::iterator BBI(SwitchMBB); | ||||||||||||
11008 | ++BBI; | ||||||||||||
11009 | MachineBasicBlock *PeeledSwitchMBB = | ||||||||||||
11010 | FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock()); | ||||||||||||
11011 | FuncInfo.MF->insert(BBI, PeeledSwitchMBB); | ||||||||||||
11012 | |||||||||||||
11013 | ExportFromCurrentBlock(SI.getCondition()); | ||||||||||||
11014 | auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex; | ||||||||||||
11015 | SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt, | ||||||||||||
11016 | nullptr, nullptr, TopCaseProb.getCompl()}; | ||||||||||||
11017 | lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB); | ||||||||||||
11018 | |||||||||||||
11019 | Clusters.erase(PeeledCaseIt); | ||||||||||||
11020 | for (CaseCluster &CC : Clusters) { | ||||||||||||
11021 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Scale the probablity for one cluster, before scaling: " << CC.Prob << "\n"; } } while (false) | ||||||||||||
11022 | dbgs() << "Scale the probablity for one cluster, before scaling: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Scale the probablity for one cluster, before scaling: " << CC.Prob << "\n"; } } while (false) | ||||||||||||
11023 | << CC.Prob << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Scale the probablity for one cluster, before scaling: " << CC.Prob << "\n"; } } while (false); | ||||||||||||
11024 | CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb); | ||||||||||||
11025 | LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "After scaling: " << CC.Prob << "\n"; } } while (false); | ||||||||||||
11026 | } | ||||||||||||
11027 | PeeledCaseProb = TopCaseProb; | ||||||||||||
11028 | return PeeledSwitchMBB; | ||||||||||||
11029 | } | ||||||||||||
11030 | |||||||||||||
11031 | void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { | ||||||||||||
11032 | // Extract cases from the switch. | ||||||||||||
11033 | BranchProbabilityInfo *BPI = FuncInfo.BPI; | ||||||||||||
11034 | CaseClusterVector Clusters; | ||||||||||||
11035 | Clusters.reserve(SI.getNumCases()); | ||||||||||||
11036 | for (auto I : SI.cases()) { | ||||||||||||
11037 | MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()]; | ||||||||||||
11038 | const ConstantInt *CaseVal = I.getCaseValue(); | ||||||||||||
11039 | BranchProbability Prob = | ||||||||||||
11040 | BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex()) | ||||||||||||
11041 | : BranchProbability(1, SI.getNumCases() + 1); | ||||||||||||
11042 | Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob)); | ||||||||||||
11043 | } | ||||||||||||
11044 | |||||||||||||
11045 | MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()]; | ||||||||||||
11046 | |||||||||||||
11047 | // Cluster adjacent cases with the same destination. We do this at all | ||||||||||||
11048 | // optimization levels because it's cheap to do and will make codegen faster | ||||||||||||
11049 | // if there are many clusters. | ||||||||||||
11050 | sortAndRangeify(Clusters); | ||||||||||||
11051 | |||||||||||||
11052 | // The branch probablity of the peeled case. | ||||||||||||
11053 | BranchProbability PeeledCaseProb = BranchProbability::getZero(); | ||||||||||||
11054 | MachineBasicBlock *PeeledSwitchMBB = | ||||||||||||
11055 | peelDominantCaseCluster(SI, Clusters, PeeledCaseProb); | ||||||||||||
11056 | |||||||||||||
11057 | // If there is only the default destination, jump there directly. | ||||||||||||
11058 | MachineBasicBlock *SwitchMBB = FuncInfo.MBB; | ||||||||||||
11059 | if (Clusters.empty()) { | ||||||||||||
11060 | assert(PeeledSwitchMBB == SwitchMBB)(static_cast <bool> (PeeledSwitchMBB == SwitchMBB) ? void (0) : __assert_fail ("PeeledSwitchMBB == SwitchMBB", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 11060, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
11061 | SwitchMBB->addSuccessor(DefaultMBB); | ||||||||||||
11062 | if (DefaultMBB != NextBlock(SwitchMBB)) { | ||||||||||||
11063 | DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, | ||||||||||||
11064 | getControlRoot(), DAG.getBasicBlock(DefaultMBB))); | ||||||||||||
11065 | } | ||||||||||||
11066 | return; | ||||||||||||
11067 | } | ||||||||||||
11068 | |||||||||||||
11069 | SL->findJumpTables(Clusters, &SI, DefaultMBB, DAG.getPSI(), DAG.getBFI()); | ||||||||||||
11070 | SL->findBitTestClusters(Clusters, &SI); | ||||||||||||
11071 | |||||||||||||
11072 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11073 | dbgs() << "Case clusters: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11074 | for (const CaseCluster &C : Clusters) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11075 | if (C.Kind == CC_JumpTable)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11076 | dbgs() << "JT:";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11077 | if (C.Kind == CC_BitTests)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11078 | dbgs() << "BT:";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11079 | |||||||||||||
11080 | C.Low->getValue().print(dbgs(), true);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11081 | if (C.Low != C.High) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11082 | dbgs() << '-';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11083 | C.High->getValue().print(dbgs(), true);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11084 | }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11085 | dbgs() << ' ';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11086 | }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11087 | dbgs() << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||||||
11088 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ); | ||||||||||||
11089 | |||||||||||||
11090 | assert(!Clusters.empty())(static_cast <bool> (!Clusters.empty()) ? void (0) : __assert_fail ("!Clusters.empty()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 11090, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
11091 | SwitchWorkList WorkList; | ||||||||||||
11092 | CaseClusterIt First = Clusters.begin(); | ||||||||||||
11093 | CaseClusterIt Last = Clusters.end() - 1; | ||||||||||||
11094 | auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB); | ||||||||||||
11095 | // Scale the branchprobability for DefaultMBB if the peel occurs and | ||||||||||||
11096 | // DefaultMBB is not replaced. | ||||||||||||
11097 | if (PeeledCaseProb != BranchProbability::getZero() && | ||||||||||||
11098 | DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()]) | ||||||||||||
11099 | DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb); | ||||||||||||
11100 | WorkList.push_back( | ||||||||||||
11101 | {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb}); | ||||||||||||
11102 | |||||||||||||
11103 | while (!WorkList.empty()) { | ||||||||||||
11104 | SwitchWorkListItem W = WorkList.pop_back_val(); | ||||||||||||
11105 | unsigned NumClusters = W.LastCluster - W.FirstCluster + 1; | ||||||||||||
11106 | |||||||||||||
11107 | if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None && | ||||||||||||
11108 | !DefaultMBB->getParent()->getFunction().hasMinSize()) { | ||||||||||||
11109 | // For optimized builds, lower large range as a balanced binary tree. | ||||||||||||
11110 | splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB); | ||||||||||||
11111 | continue; | ||||||||||||
11112 | } | ||||||||||||
11113 | |||||||||||||
11114 | lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB); | ||||||||||||
11115 | } | ||||||||||||
11116 | } | ||||||||||||
11117 | |||||||||||||
11118 | void SelectionDAGBuilder::visitStepVector(const CallInst &I) { | ||||||||||||
11119 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
11120 | auto DL = getCurSDLoc(); | ||||||||||||
11121 | EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
11122 | setValue(&I, DAG.getStepVector(DL, ResultVT)); | ||||||||||||
11123 | } | ||||||||||||
11124 | |||||||||||||
11125 | void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) { | ||||||||||||
11126 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
11127 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
11128 | |||||||||||||
11129 | SDLoc DL = getCurSDLoc(); | ||||||||||||
11130 | SDValue V = getValue(I.getOperand(0)); | ||||||||||||
11131 | assert(VT == V.getValueType() && "Malformed vector.reverse!")(static_cast <bool> (VT == V.getValueType() && "Malformed vector.reverse!" ) ? void (0) : __assert_fail ("VT == V.getValueType() && \"Malformed vector.reverse!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 11131, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||
11132 | |||||||||||||
11133 | if (VT.isScalableVector()) { | ||||||||||||
11134 | setValue(&I, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V)); | ||||||||||||
11135 | return; | ||||||||||||
11136 | } | ||||||||||||
11137 | |||||||||||||
11138 | // Use VECTOR_SHUFFLE for the fixed-length vector | ||||||||||||
11139 | // to maintain existing behavior. | ||||||||||||
11140 | SmallVector<int, 8> Mask; | ||||||||||||
11141 | unsigned NumElts = VT.getVectorMinNumElements(); | ||||||||||||
11142 | for (unsigned i = 0; i != NumElts; ++i) | ||||||||||||
11143 | Mask.push_back(NumElts - 1 - i); | ||||||||||||
11144 | |||||||||||||
11145 | setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask)); | ||||||||||||
11146 | } | ||||||||||||
11147 | |||||||||||||
11148 | void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) { | ||||||||||||
11149 | SmallVector<EVT, 4> ValueVTs; | ||||||||||||
11150 | ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(), | ||||||||||||
11151 | ValueVTs); | ||||||||||||
11152 | unsigned NumValues = ValueVTs.size(); | ||||||||||||
11153 | if (NumValues == 0) return; | ||||||||||||
11154 | |||||||||||||
11155 | SmallVector<SDValue, 4> Values(NumValues); | ||||||||||||
11156 | SDValue Op = getValue(I.getOperand(0)); | ||||||||||||
11157 | |||||||||||||
11158 | for (unsigned i = 0; i != NumValues; ++i) | ||||||||||||
11159 | Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i], | ||||||||||||
11160 | SDValue(Op.getNode(), Op.getResNo() + i)); | ||||||||||||
11161 | |||||||||||||
11162 | setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), | ||||||||||||
11163 | DAG.getVTList(ValueVTs), Values)); | ||||||||||||
11164 | } | ||||||||||||
11165 | |||||||||||||
11166 | void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) { | ||||||||||||
11167 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||||||
11168 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||||||
11169 | |||||||||||||
11170 | SDLoc DL = getCurSDLoc(); | ||||||||||||
11171 | SDValue V1 = getValue(I.getOperand(0)); | ||||||||||||
11172 | SDValue V2 = getValue(I.getOperand(1)); | ||||||||||||
11173 | int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue(); | ||||||||||||
11174 | |||||||||||||
11175 | // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node. | ||||||||||||
11176 | if (VT.isScalableVector()) { | ||||||||||||
11177 | MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout()); | ||||||||||||
11178 | setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2, | ||||||||||||
11179 | DAG.getConstant(Imm, DL, IdxVT))); | ||||||||||||
11180 | return; | ||||||||||||
11181 | } | ||||||||||||
11182 | |||||||||||||
11183 | unsigned NumElts = VT.getVectorNumElements(); | ||||||||||||
11184 | |||||||||||||
11185 | if ((-Imm > NumElts) || (Imm >= NumElts)) { | ||||||||||||
11186 | // Result is undefined if immediate is out-of-bounds. | ||||||||||||
11187 | setValue(&I, DAG.getUNDEF(VT)); | ||||||||||||
11188 | return; | ||||||||||||
11189 | } | ||||||||||||
11190 | |||||||||||||
11191 | uint64_t Idx = (NumElts + Imm) % NumElts; | ||||||||||||
11192 | |||||||||||||
11193 | // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors. | ||||||||||||
11194 | SmallVector<int, 8> Mask; | ||||||||||||
11195 | for (unsigned i = 0; i < NumElts; ++i) | ||||||||||||
11196 | Mask.push_back(Idx + i); | ||||||||||||
11197 | setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask)); | ||||||||||||
11198 | } |
1 | //===- llvm/CodeGen/SelectionDAG.h - InstSelection DAG ----------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file declares the SelectionDAG class, and transitively defines the |
10 | // SDNode class and subclasses. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #ifndef LLVM_CODEGEN_SELECTIONDAG_H |
15 | #define LLVM_CODEGEN_SELECTIONDAG_H |
16 | |
17 | #include "llvm/ADT/APFloat.h" |
18 | #include "llvm/ADT/APInt.h" |
19 | #include "llvm/ADT/ArrayRef.h" |
20 | #include "llvm/ADT/DenseMap.h" |
21 | #include "llvm/ADT/DenseSet.h" |
22 | #include "llvm/ADT/FoldingSet.h" |
23 | #include "llvm/ADT/SetVector.h" |
24 | #include "llvm/ADT/SmallVector.h" |
25 | #include "llvm/ADT/StringMap.h" |
26 | #include "llvm/ADT/ilist.h" |
27 | #include "llvm/ADT/iterator.h" |
28 | #include "llvm/ADT/iterator_range.h" |
29 | #include "llvm/CodeGen/DAGCombine.h" |
30 | #include "llvm/CodeGen/ISDOpcodes.h" |
31 | #include "llvm/CodeGen/MachineFunction.h" |
32 | #include "llvm/CodeGen/MachineMemOperand.h" |
33 | #include "llvm/CodeGen/SelectionDAGNodes.h" |
34 | #include "llvm/CodeGen/ValueTypes.h" |
35 | #include "llvm/IR/DebugLoc.h" |
36 | #include "llvm/IR/Instructions.h" |
37 | #include "llvm/IR/Metadata.h" |
38 | #include "llvm/Support/Allocator.h" |
39 | #include "llvm/Support/ArrayRecycler.h" |
40 | #include "llvm/Support/AtomicOrdering.h" |
41 | #include "llvm/Support/Casting.h" |
42 | #include "llvm/Support/CodeGen.h" |
43 | #include "llvm/Support/ErrorHandling.h" |
44 | #include "llvm/Support/MachineValueType.h" |
45 | #include "llvm/Support/RecyclingAllocator.h" |
46 | #include <algorithm> |
47 | #include <cassert> |
48 | #include <cstdint> |
49 | #include <functional> |
50 | #include <map> |
51 | #include <string> |
52 | #include <tuple> |
53 | #include <utility> |
54 | #include <vector> |
55 | |
56 | namespace llvm { |
57 | |
58 | class AAResults; |
59 | class BlockAddress; |
60 | class BlockFrequencyInfo; |
61 | class Constant; |
62 | class ConstantFP; |
63 | class ConstantInt; |
64 | class DataLayout; |
65 | struct fltSemantics; |
66 | class FunctionLoweringInfo; |
67 | class GlobalValue; |
68 | struct KnownBits; |
69 | class LegacyDivergenceAnalysis; |
70 | class LLVMContext; |
71 | class MachineBasicBlock; |
72 | class MachineConstantPoolValue; |
73 | class MCSymbol; |
74 | class OptimizationRemarkEmitter; |
75 | class ProfileSummaryInfo; |
76 | class SDDbgValue; |
77 | class SDDbgOperand; |
78 | class SDDbgLabel; |
79 | class SelectionDAG; |
80 | class SelectionDAGTargetInfo; |
81 | class TargetLibraryInfo; |
82 | class TargetLowering; |
83 | class TargetMachine; |
84 | class TargetSubtargetInfo; |
85 | class Value; |
86 | |
87 | class SDVTListNode : public FoldingSetNode { |
88 | friend struct FoldingSetTrait<SDVTListNode>; |
89 | |
90 | /// A reference to an Interned FoldingSetNodeID for this node. |
91 | /// The Allocator in SelectionDAG holds the data. |
92 | /// SDVTList contains all types which are frequently accessed in SelectionDAG. |
93 | /// The size of this list is not expected to be big so it won't introduce |
94 | /// a memory penalty. |
95 | FoldingSetNodeIDRef FastID; |
96 | const EVT *VTs; |
97 | unsigned int NumVTs; |
98 | /// The hash value for SDVTList is fixed, so cache it to avoid |
99 | /// hash calculation. |
100 | unsigned HashValue; |
101 | |
102 | public: |
103 | SDVTListNode(const FoldingSetNodeIDRef ID, const EVT *VT, unsigned int Num) : |
104 | FastID(ID), VTs(VT), NumVTs(Num) { |
105 | HashValue = ID.ComputeHash(); |
106 | } |
107 | |
108 | SDVTList getSDVTList() { |
109 | SDVTList result = {VTs, NumVTs}; |
110 | return result; |
111 | } |
112 | }; |
113 | |
114 | /// Specialize FoldingSetTrait for SDVTListNode |
115 | /// to avoid computing temp FoldingSetNodeID and hash value. |
116 | template<> struct FoldingSetTrait<SDVTListNode> : DefaultFoldingSetTrait<SDVTListNode> { |
117 | static void Profile(const SDVTListNode &X, FoldingSetNodeID& ID) { |
118 | ID = X.FastID; |
119 | } |
120 | |
121 | static bool Equals(const SDVTListNode &X, const FoldingSetNodeID &ID, |
122 | unsigned IDHash, FoldingSetNodeID &TempID) { |
123 | if (X.HashValue != IDHash) |
124 | return false; |
125 | return ID == X.FastID; |
126 | } |
127 | |
128 | static unsigned ComputeHash(const SDVTListNode &X, FoldingSetNodeID &TempID) { |
129 | return X.HashValue; |
130 | } |
131 | }; |
132 | |
133 | template <> struct ilist_alloc_traits<SDNode> { |
134 | static void deleteNode(SDNode *) { |
135 | llvm_unreachable("ilist_traits<SDNode> shouldn't see a deleteNode call!")::llvm::llvm_unreachable_internal("ilist_traits<SDNode> shouldn't see a deleteNode call!" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 135); |
136 | } |
137 | }; |
138 | |
139 | /// Keeps track of dbg_value information through SDISel. We do |
140 | /// not build SDNodes for these so as not to perturb the generated code; |
141 | /// instead the info is kept off to the side in this structure. Each SDNode may |
142 | /// have one or more associated dbg_value entries. This information is kept in |
143 | /// DbgValMap. |
144 | /// Byval parameters are handled separately because they don't use alloca's, |
145 | /// which busts the normal mechanism. There is good reason for handling all |
146 | /// parameters separately: they may not have code generated for them, they |
147 | /// should always go at the beginning of the function regardless of other code |
148 | /// motion, and debug info for them is potentially useful even if the parameter |
149 | /// is unused. Right now only byval parameters are handled separately. |
150 | class SDDbgInfo { |
151 | BumpPtrAllocator Alloc; |
152 | SmallVector<SDDbgValue*, 32> DbgValues; |
153 | SmallVector<SDDbgValue*, 32> ByvalParmDbgValues; |
154 | SmallVector<SDDbgLabel*, 4> DbgLabels; |
155 | using DbgValMapType = DenseMap<const SDNode *, SmallVector<SDDbgValue *, 2>>; |
156 | DbgValMapType DbgValMap; |
157 | |
158 | public: |
159 | SDDbgInfo() = default; |
160 | SDDbgInfo(const SDDbgInfo &) = delete; |
161 | SDDbgInfo &operator=(const SDDbgInfo &) = delete; |
162 | |
163 | void add(SDDbgValue *V, bool isParameter); |
164 | |
165 | void add(SDDbgLabel *L) { DbgLabels.push_back(L); } |
166 | |
167 | /// Invalidate all DbgValues attached to the node and remove |
168 | /// it from the Node-to-DbgValues map. |
169 | void erase(const SDNode *Node); |
170 | |
171 | void clear() { |
172 | DbgValMap.clear(); |
173 | DbgValues.clear(); |
174 | ByvalParmDbgValues.clear(); |
175 | DbgLabels.clear(); |
176 | Alloc.Reset(); |
177 | } |
178 | |
179 | BumpPtrAllocator &getAlloc() { return Alloc; } |
180 | |
181 | bool empty() const { |
182 | return DbgValues.empty() && ByvalParmDbgValues.empty() && DbgLabels.empty(); |
183 | } |
184 | |
185 | ArrayRef<SDDbgValue*> getSDDbgValues(const SDNode *Node) const { |
186 | auto I = DbgValMap.find(Node); |
187 | if (I != DbgValMap.end()) |
188 | return I->second; |
189 | return ArrayRef<SDDbgValue*>(); |
190 | } |
191 | |
192 | using DbgIterator = SmallVectorImpl<SDDbgValue*>::iterator; |
193 | using DbgLabelIterator = SmallVectorImpl<SDDbgLabel*>::iterator; |
194 | |
195 | DbgIterator DbgBegin() { return DbgValues.begin(); } |
196 | DbgIterator DbgEnd() { return DbgValues.end(); } |
197 | DbgIterator ByvalParmDbgBegin() { return ByvalParmDbgValues.begin(); } |
198 | DbgIterator ByvalParmDbgEnd() { return ByvalParmDbgValues.end(); } |
199 | DbgLabelIterator DbgLabelBegin() { return DbgLabels.begin(); } |
200 | DbgLabelIterator DbgLabelEnd() { return DbgLabels.end(); } |
201 | }; |
202 | |
203 | void checkForCycles(const SelectionDAG *DAG, bool force = false); |
204 | |
205 | /// This is used to represent a portion of an LLVM function in a low-level |
206 | /// Data Dependence DAG representation suitable for instruction selection. |
207 | /// This DAG is constructed as the first step of instruction selection in order |
208 | /// to allow implementation of machine specific optimizations |
209 | /// and code simplifications. |
210 | /// |
211 | /// The representation used by the SelectionDAG is a target-independent |
212 | /// representation, which has some similarities to the GCC RTL representation, |
213 | /// but is significantly more simple, powerful, and is a graph form instead of a |
214 | /// linear form. |
215 | /// |
216 | class SelectionDAG { |
217 | const TargetMachine &TM; |
218 | const SelectionDAGTargetInfo *TSI = nullptr; |
219 | const TargetLowering *TLI = nullptr; |
220 | const TargetLibraryInfo *LibInfo = nullptr; |
221 | MachineFunction *MF; |
222 | Pass *SDAGISelPass = nullptr; |
223 | LLVMContext *Context; |
224 | CodeGenOpt::Level OptLevel; |
225 | |
226 | LegacyDivergenceAnalysis * DA = nullptr; |
227 | FunctionLoweringInfo * FLI = nullptr; |
228 | |
229 | /// The function-level optimization remark emitter. Used to emit remarks |
230 | /// whenever manipulating the DAG. |
231 | OptimizationRemarkEmitter *ORE; |
232 | |
233 | ProfileSummaryInfo *PSI = nullptr; |
234 | BlockFrequencyInfo *BFI = nullptr; |
235 | |
236 | /// The starting token. |
237 | SDNode EntryNode; |
238 | |
239 | /// The root of the entire DAG. |
240 | SDValue Root; |
241 | |
242 | /// A linked list of nodes in the current DAG. |
243 | ilist<SDNode> AllNodes; |
244 | |
245 | /// The AllocatorType for allocating SDNodes. We use |
246 | /// pool allocation with recycling. |
247 | using NodeAllocatorType = RecyclingAllocator<BumpPtrAllocator, SDNode, |
248 | sizeof(LargestSDNode), |
249 | alignof(MostAlignedSDNode)>; |
250 | |
251 | /// Pool allocation for nodes. |
252 | NodeAllocatorType NodeAllocator; |
253 | |
254 | /// This structure is used to memoize nodes, automatically performing |
255 | /// CSE with existing nodes when a duplicate is requested. |
256 | FoldingSet<SDNode> CSEMap; |
257 | |
258 | /// Pool allocation for machine-opcode SDNode operands. |
259 | BumpPtrAllocator OperandAllocator; |
260 | ArrayRecycler<SDUse> OperandRecycler; |
261 | |
262 | /// Pool allocation for misc. objects that are created once per SelectionDAG. |
263 | BumpPtrAllocator Allocator; |
264 | |
265 | /// Tracks dbg_value and dbg_label information through SDISel. |
266 | SDDbgInfo *DbgInfo; |
267 | |
268 | using CallSiteInfo = MachineFunction::CallSiteInfo; |
269 | using CallSiteInfoImpl = MachineFunction::CallSiteInfoImpl; |
270 | |
271 | struct CallSiteDbgInfo { |
272 | CallSiteInfo CSInfo; |
273 | MDNode *HeapAllocSite = nullptr; |
274 | bool NoMerge = false; |
275 | }; |
276 | |
277 | DenseMap<const SDNode *, CallSiteDbgInfo> SDCallSiteDbgInfo; |
278 | |
279 | uint16_t NextPersistentId = 0; |
280 | |
281 | public: |
282 | /// Clients of various APIs that cause global effects on |
283 | /// the DAG can optionally implement this interface. This allows the clients |
284 | /// to handle the various sorts of updates that happen. |
285 | /// |
286 | /// A DAGUpdateListener automatically registers itself with DAG when it is |
287 | /// constructed, and removes itself when destroyed in RAII fashion. |
288 | struct DAGUpdateListener { |
289 | DAGUpdateListener *const Next; |
290 | SelectionDAG &DAG; |
291 | |
292 | explicit DAGUpdateListener(SelectionDAG &D) |
293 | : Next(D.UpdateListeners), DAG(D) { |
294 | DAG.UpdateListeners = this; |
295 | } |
296 | |
297 | virtual ~DAGUpdateListener() { |
298 | assert(DAG.UpdateListeners == this &&(static_cast <bool> (DAG.UpdateListeners == this && "DAGUpdateListeners must be destroyed in LIFO order") ? void (0) : __assert_fail ("DAG.UpdateListeners == this && \"DAGUpdateListeners must be destroyed in LIFO order\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 299, __extension__ __PRETTY_FUNCTION__)) |
299 | "DAGUpdateListeners must be destroyed in LIFO order")(static_cast <bool> (DAG.UpdateListeners == this && "DAGUpdateListeners must be destroyed in LIFO order") ? void (0) : __assert_fail ("DAG.UpdateListeners == this && \"DAGUpdateListeners must be destroyed in LIFO order\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 299, __extension__ __PRETTY_FUNCTION__)); |
300 | DAG.UpdateListeners = Next; |
301 | } |
302 | |
303 | /// The node N that was deleted and, if E is not null, an |
304 | /// equivalent node E that replaced it. |
305 | virtual void NodeDeleted(SDNode *N, SDNode *E); |
306 | |
307 | /// The node N that was updated. |
308 | virtual void NodeUpdated(SDNode *N); |
309 | |
310 | /// The node N that was inserted. |
311 | virtual void NodeInserted(SDNode *N); |
312 | }; |
313 | |
314 | struct DAGNodeDeletedListener : public DAGUpdateListener { |
315 | std::function<void(SDNode *, SDNode *)> Callback; |
316 | |
317 | DAGNodeDeletedListener(SelectionDAG &DAG, |
318 | std::function<void(SDNode *, SDNode *)> Callback) |
319 | : DAGUpdateListener(DAG), Callback(std::move(Callback)) {} |
320 | |
321 | void NodeDeleted(SDNode *N, SDNode *E) override { Callback(N, E); } |
322 | |
323 | private: |
324 | virtual void anchor(); |
325 | }; |
326 | |
327 | /// Help to insert SDNodeFlags automatically in transforming. Use |
328 | /// RAII to save and resume flags in current scope. |
329 | class FlagInserter { |
330 | SelectionDAG &DAG; |
331 | SDNodeFlags Flags; |
332 | FlagInserter *LastInserter; |
333 | |
334 | public: |
335 | FlagInserter(SelectionDAG &SDAG, SDNodeFlags Flags) |
336 | : DAG(SDAG), Flags(Flags), |
337 | LastInserter(SDAG.getFlagInserter()) { |
338 | SDAG.setFlagInserter(this); |
339 | } |
340 | FlagInserter(SelectionDAG &SDAG, SDNode *N) |
341 | : FlagInserter(SDAG, N->getFlags()) {} |
342 | |
343 | FlagInserter(const FlagInserter &) = delete; |
344 | FlagInserter &operator=(const FlagInserter &) = delete; |
345 | ~FlagInserter() { DAG.setFlagInserter(LastInserter); } |
346 | |
347 | SDNodeFlags getFlags() const { return Flags; } |
348 | }; |
349 | |
350 | /// When true, additional steps are taken to |
351 | /// ensure that getConstant() and similar functions return DAG nodes that |
352 | /// have legal types. This is important after type legalization since |
353 | /// any illegally typed nodes generated after this point will not experience |
354 | /// type legalization. |
355 | bool NewNodesMustHaveLegalTypes = false; |
356 | |
357 | private: |
358 | /// DAGUpdateListener is a friend so it can manipulate the listener stack. |
359 | friend struct DAGUpdateListener; |
360 | |
361 | /// Linked list of registered DAGUpdateListener instances. |
362 | /// This stack is maintained by DAGUpdateListener RAII. |
363 | DAGUpdateListener *UpdateListeners = nullptr; |
364 | |
365 | /// Implementation of setSubgraphColor. |
366 | /// Return whether we had to truncate the search. |
367 | bool setSubgraphColorHelper(SDNode *N, const char *Color, |
368 | DenseSet<SDNode *> &visited, |
369 | int level, bool &printed); |
370 | |
371 | template <typename SDNodeT, typename... ArgTypes> |
372 | SDNodeT *newSDNode(ArgTypes &&... Args) { |
373 | return new (NodeAllocator.template Allocate<SDNodeT>()) |
374 | SDNodeT(std::forward<ArgTypes>(Args)...); |
375 | } |
376 | |
377 | /// Build a synthetic SDNodeT with the given args and extract its subclass |
378 | /// data as an integer (e.g. for use in a folding set). |
379 | /// |
380 | /// The args to this function are the same as the args to SDNodeT's |
381 | /// constructor, except the second arg (assumed to be a const DebugLoc&) is |
382 | /// omitted. |
383 | template <typename SDNodeT, typename... ArgTypes> |
384 | static uint16_t getSyntheticNodeSubclassData(unsigned IROrder, |
385 | ArgTypes &&... Args) { |
386 | // The compiler can reduce this expression to a constant iff we pass an |
387 | // empty DebugLoc. Thankfully, the debug location doesn't have any bearing |
388 | // on the subclass data. |
389 | return SDNodeT(IROrder, DebugLoc(), std::forward<ArgTypes>(Args)...) |
390 | .getRawSubclassData(); |
391 | } |
392 | |
393 | template <typename SDNodeTy> |
394 | static uint16_t getSyntheticNodeSubclassData(unsigned Opc, unsigned Order, |
395 | SDVTList VTs, EVT MemoryVT, |
396 | MachineMemOperand *MMO) { |
397 | return SDNodeTy(Opc, Order, DebugLoc(), VTs, MemoryVT, MMO) |
398 | .getRawSubclassData(); |
399 | } |
400 | |
401 | void createOperands(SDNode *Node, ArrayRef<SDValue> Vals); |
402 | |
403 | void removeOperands(SDNode *Node) { |
404 | if (!Node->OperandList) |
405 | return; |
406 | OperandRecycler.deallocate( |
407 | ArrayRecycler<SDUse>::Capacity::get(Node->NumOperands), |
408 | Node->OperandList); |
409 | Node->NumOperands = 0; |
410 | Node->OperandList = nullptr; |
411 | } |
412 | void CreateTopologicalOrder(std::vector<SDNode*>& Order); |
413 | |
414 | public: |
415 | // Maximum depth for recursive analysis such as computeKnownBits, etc. |
416 | static constexpr unsigned MaxRecursionDepth = 6; |
417 | |
418 | explicit SelectionDAG(const TargetMachine &TM, CodeGenOpt::Level); |
419 | SelectionDAG(const SelectionDAG &) = delete; |
420 | SelectionDAG &operator=(const SelectionDAG &) = delete; |
421 | ~SelectionDAG(); |
422 | |
423 | /// Prepare this SelectionDAG to process code in the given MachineFunction. |
424 | void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, |
425 | Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, |
426 | LegacyDivergenceAnalysis * Divergence, |
427 | ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin); |
428 | |
429 | void setFunctionLoweringInfo(FunctionLoweringInfo * FuncInfo) { |
430 | FLI = FuncInfo; |
431 | } |
432 | |
433 | /// Clear state and free memory necessary to make this |
434 | /// SelectionDAG ready to process a new block. |
435 | void clear(); |
436 | |
437 | MachineFunction &getMachineFunction() const { return *MF; } |
438 | const Pass *getPass() const { return SDAGISelPass; } |
439 | |
440 | const DataLayout &getDataLayout() const { return MF->getDataLayout(); } |
441 | const TargetMachine &getTarget() const { return TM; } |
442 | const TargetSubtargetInfo &getSubtarget() const { return MF->getSubtarget(); } |
443 | const TargetLowering &getTargetLoweringInfo() const { return *TLI; } |
444 | const TargetLibraryInfo &getLibInfo() const { return *LibInfo; } |
445 | const SelectionDAGTargetInfo &getSelectionDAGInfo() const { return *TSI; } |
446 | const LegacyDivergenceAnalysis *getDivergenceAnalysis() const { return DA; } |
447 | LLVMContext *getContext() const { return Context; } |
448 | OptimizationRemarkEmitter &getORE() const { return *ORE; } |
449 | ProfileSummaryInfo *getPSI() const { return PSI; } |
450 | BlockFrequencyInfo *getBFI() const { return BFI; } |
451 | |
452 | FlagInserter *getFlagInserter() { return Inserter; } |
453 | void setFlagInserter(FlagInserter *FI) { Inserter = FI; } |
454 | |
455 | /// Just dump dot graph to a user-provided path and title. |
456 | /// This doesn't open the dot viewer program and |
457 | /// helps visualization when outside debugging session. |
458 | /// FileName expects absolute path. If provided |
459 | /// without any path separators then the file |
460 | /// will be created in the current directory. |
461 | /// Error will be emitted if the path is insane. |
462 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) |
463 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void dumpDotGraph(const Twine &FileName, const Twine &Title); |
464 | #endif |
465 | |
466 | /// Pop up a GraphViz/gv window with the DAG rendered using 'dot'. |
467 | void viewGraph(const std::string &Title); |
468 | void viewGraph(); |
469 | |
470 | #ifndef NDEBUG |
471 | std::map<const SDNode *, std::string> NodeGraphAttrs; |
472 | #endif |
473 | |
474 | /// Clear all previously defined node graph attributes. |
475 | /// Intended to be used from a debugging tool (eg. gdb). |
476 | void clearGraphAttrs(); |
477 | |
478 | /// Set graph attributes for a node. (eg. "color=red".) |
479 | void setGraphAttrs(const SDNode *N, const char *Attrs); |
480 | |
481 | /// Get graph attributes for a node. (eg. "color=red".) |
482 | /// Used from getNodeAttributes. |
483 | std::string getGraphAttrs(const SDNode *N) const; |
484 | |
485 | /// Convenience for setting node color attribute. |
486 | void setGraphColor(const SDNode *N, const char *Color); |
487 | |
488 | /// Convenience for setting subgraph color attribute. |
489 | void setSubgraphColor(SDNode *N, const char *Color); |
490 | |
491 | using allnodes_const_iterator = ilist<SDNode>::const_iterator; |
492 | |
493 | allnodes_const_iterator allnodes_begin() const { return AllNodes.begin(); } |
494 | allnodes_const_iterator allnodes_end() const { return AllNodes.end(); } |
495 | |
496 | using allnodes_iterator = ilist<SDNode>::iterator; |
497 | |
498 | allnodes_iterator allnodes_begin() { return AllNodes.begin(); } |
499 | allnodes_iterator allnodes_end() { return AllNodes.end(); } |
500 | |
501 | ilist<SDNode>::size_type allnodes_size() const { |
502 | return AllNodes.size(); |
503 | } |
504 | |
505 | iterator_range<allnodes_iterator> allnodes() { |
506 | return make_range(allnodes_begin(), allnodes_end()); |
507 | } |
508 | iterator_range<allnodes_const_iterator> allnodes() const { |
509 | return make_range(allnodes_begin(), allnodes_end()); |
510 | } |
511 | |
512 | /// Return the root tag of the SelectionDAG. |
513 | const SDValue &getRoot() const { return Root; } |
514 | |
515 | /// Return the token chain corresponding to the entry of the function. |
516 | SDValue getEntryNode() const { |
517 | return SDValue(const_cast<SDNode *>(&EntryNode), 0); |
518 | } |
519 | |
520 | /// Set the current root tag of the SelectionDAG. |
521 | /// |
522 | const SDValue &setRoot(SDValue N) { |
523 | assert((!N.getNode() || N.getValueType() == MVT::Other) &&(static_cast <bool> ((!N.getNode() || N.getValueType() == MVT::Other) && "DAG root value is not a chain!") ? void (0) : __assert_fail ("(!N.getNode() || N.getValueType() == MVT::Other) && \"DAG root value is not a chain!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 524, __extension__ __PRETTY_FUNCTION__)) |
524 | "DAG root value is not a chain!")(static_cast <bool> ((!N.getNode() || N.getValueType() == MVT::Other) && "DAG root value is not a chain!") ? void (0) : __assert_fail ("(!N.getNode() || N.getValueType() == MVT::Other) && \"DAG root value is not a chain!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 524, __extension__ __PRETTY_FUNCTION__)); |
525 | if (N.getNode()) |
526 | checkForCycles(N.getNode(), this); |
527 | Root = N; |
528 | if (N.getNode()) |
529 | checkForCycles(this); |
530 | return Root; |
531 | } |
532 | |
533 | #ifndef NDEBUG |
534 | void VerifyDAGDiverence(); |
535 | #endif |
536 | |
537 | /// This iterates over the nodes in the SelectionDAG, folding |
538 | /// certain types of nodes together, or eliminating superfluous nodes. The |
539 | /// Level argument controls whether Combine is allowed to produce nodes and |
540 | /// types that are illegal on the target. |
541 | void Combine(CombineLevel Level, AAResults *AA, |
542 | CodeGenOpt::Level OptLevel); |
543 | |
544 | /// This transforms the SelectionDAG into a SelectionDAG that |
545 | /// only uses types natively supported by the target. |
546 | /// Returns "true" if it made any changes. |
547 | /// |
548 | /// Note that this is an involved process that may invalidate pointers into |
549 | /// the graph. |
550 | bool LegalizeTypes(); |
551 | |
552 | /// This transforms the SelectionDAG into a SelectionDAG that is |
553 | /// compatible with the target instruction selector, as indicated by the |
554 | /// TargetLowering object. |
555 | /// |
556 | /// Note that this is an involved process that may invalidate pointers into |
557 | /// the graph. |
558 | void Legalize(); |
559 | |
560 | /// Transforms a SelectionDAG node and any operands to it into a node |
561 | /// that is compatible with the target instruction selector, as indicated by |
562 | /// the TargetLowering object. |
563 | /// |
564 | /// \returns true if \c N is a valid, legal node after calling this. |
565 | /// |
566 | /// This essentially runs a single recursive walk of the \c Legalize process |
567 | /// over the given node (and its operands). This can be used to incrementally |
568 | /// legalize the DAG. All of the nodes which are directly replaced, |
569 | /// potentially including N, are added to the output parameter \c |
570 | /// UpdatedNodes so that the delta to the DAG can be understood by the |
571 | /// caller. |
572 | /// |
573 | /// When this returns false, N has been legalized in a way that make the |
574 | /// pointer passed in no longer valid. It may have even been deleted from the |
575 | /// DAG, and so it shouldn't be used further. When this returns true, the |
576 | /// N passed in is a legal node, and can be immediately processed as such. |
577 | /// This may still have done some work on the DAG, and will still populate |
578 | /// UpdatedNodes with any new nodes replacing those originally in the DAG. |
579 | bool LegalizeOp(SDNode *N, SmallSetVector<SDNode *, 16> &UpdatedNodes); |
580 | |
581 | /// This transforms the SelectionDAG into a SelectionDAG |
582 | /// that only uses vector math operations supported by the target. This is |
583 | /// necessary as a separate step from Legalize because unrolling a vector |
584 | /// operation can introduce illegal types, which requires running |
585 | /// LegalizeTypes again. |
586 | /// |
587 | /// This returns true if it made any changes; in that case, LegalizeTypes |
588 | /// is called again before Legalize. |
589 | /// |
590 | /// Note that this is an involved process that may invalidate pointers into |
591 | /// the graph. |
592 | bool LegalizeVectors(); |
593 | |
594 | /// This method deletes all unreachable nodes in the SelectionDAG. |
595 | void RemoveDeadNodes(); |
596 | |
597 | /// Remove the specified node from the system. This node must |
598 | /// have no referrers. |
599 | void DeleteNode(SDNode *N); |
600 | |
601 | /// Return an SDVTList that represents the list of values specified. |
602 | SDVTList getVTList(EVT VT); |
603 | SDVTList getVTList(EVT VT1, EVT VT2); |
604 | SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3); |
605 | SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4); |
606 | SDVTList getVTList(ArrayRef<EVT> VTs); |
607 | |
608 | //===--------------------------------------------------------------------===// |
609 | // Node creation methods. |
610 | |
611 | /// Create a ConstantSDNode wrapping a constant value. |
612 | /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR. |
613 | /// |
614 | /// If only legal types can be produced, this does the necessary |
615 | /// transformations (e.g., if the vector element type is illegal). |
616 | /// @{ |
617 | SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, |
618 | bool isTarget = false, bool isOpaque = false); |
619 | SDValue getConstant(const APInt &Val, const SDLoc &DL, EVT VT, |
620 | bool isTarget = false, bool isOpaque = false); |
621 | |
622 | SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget = false, |
623 | bool IsOpaque = false) { |
624 | return getConstant(APInt::getAllOnesValue(VT.getScalarSizeInBits()), DL, |
625 | VT, IsTarget, IsOpaque); |
626 | } |
627 | |
628 | SDValue getConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT, |
629 | bool isTarget = false, bool isOpaque = false); |
630 | SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, |
631 | bool isTarget = false); |
632 | SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, |
633 | bool LegalTypes = true); |
634 | SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, |
635 | bool isTarget = false); |
636 | |
637 | SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, |
638 | bool isOpaque = false) { |
639 | return getConstant(Val, DL, VT, true, isOpaque); |
640 | } |
641 | SDValue getTargetConstant(const APInt &Val, const SDLoc &DL, EVT VT, |
642 | bool isOpaque = false) { |
643 | return getConstant(Val, DL, VT, true, isOpaque); |
644 | } |
645 | SDValue getTargetConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT, |
646 | bool isOpaque = false) { |
647 | return getConstant(Val, DL, VT, true, isOpaque); |
648 | } |
649 | |
650 | /// Create a true or false constant of type \p VT using the target's |
651 | /// BooleanContent for type \p OpVT. |
652 | SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT); |
653 | /// @} |
654 | |
655 | /// Create a ConstantFPSDNode wrapping a constant value. |
656 | /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR. |
657 | /// |
658 | /// If only legal types can be produced, this does the necessary |
659 | /// transformations (e.g., if the vector element type is illegal). |
660 | /// The forms that take a double should only be used for simple constants |
661 | /// that can be exactly represented in VT. No checks are made. |
662 | /// @{ |
663 | SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, |
664 | bool isTarget = false); |
665 | SDValue getConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT, |
666 | bool isTarget = false); |
667 | SDValue getConstantFP(const ConstantFP &V, const SDLoc &DL, EVT VT, |
668 | bool isTarget = false); |
669 | SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT) { |
670 | return getConstantFP(Val, DL, VT, true); |
671 | } |
672 | SDValue getTargetConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT) { |
673 | return getConstantFP(Val, DL, VT, true); |
674 | } |
675 | SDValue getTargetConstantFP(const ConstantFP &Val, const SDLoc &DL, EVT VT) { |
676 | return getConstantFP(Val, DL, VT, true); |
677 | } |
678 | /// @} |
679 | |
680 | SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, |
681 | int64_t offset = 0, bool isTargetGA = false, |
682 | unsigned TargetFlags = 0); |
683 | SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, |
684 | int64_t offset = 0, unsigned TargetFlags = 0) { |
685 | return getGlobalAddress(GV, DL, VT, offset, true, TargetFlags); |
686 | } |
687 | SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false); |
688 | SDValue getTargetFrameIndex(int FI, EVT VT) { |
689 | return getFrameIndex(FI, VT, true); |
690 | } |
691 | SDValue getJumpTable(int JTI, EVT VT, bool isTarget = false, |
692 | unsigned TargetFlags = 0); |
693 | SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags = 0) { |
694 | return getJumpTable(JTI, VT, true, TargetFlags); |
695 | } |
696 | SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align = None, |
697 | int Offs = 0, bool isT = false, |
698 | unsigned TargetFlags = 0); |
699 | SDValue getTargetConstantPool(const Constant *C, EVT VT, |
700 | MaybeAlign Align = None, int Offset = 0, |
701 | unsigned TargetFlags = 0) { |
702 | return getConstantPool(C, VT, Align, Offset, true, TargetFlags); |
703 | } |
704 | SDValue getConstantPool(MachineConstantPoolValue *C, EVT VT, |
705 | MaybeAlign Align = None, int Offs = 0, |
706 | bool isT = false, unsigned TargetFlags = 0); |
707 | SDValue getTargetConstantPool(MachineConstantPoolValue *C, EVT VT, |
708 | MaybeAlign Align = None, int Offset = 0, |
709 | unsigned TargetFlags = 0) { |
710 | return getConstantPool(C, VT, Align, Offset, true, TargetFlags); |
711 | } |
712 | SDValue getTargetIndex(int Index, EVT VT, int64_t Offset = 0, |
713 | unsigned TargetFlags = 0); |
714 | // When generating a branch to a BB, we don't in general know enough |
715 | // to provide debug info for the BB at that time, so keep this one around. |
716 | SDValue getBasicBlock(MachineBasicBlock *MBB); |
717 | SDValue getExternalSymbol(const char *Sym, EVT VT); |
718 | SDValue getTargetExternalSymbol(const char *Sym, EVT VT, |
719 | unsigned TargetFlags = 0); |
720 | SDValue getMCSymbol(MCSymbol *Sym, EVT VT); |
721 | |
722 | SDValue getValueType(EVT); |
723 | SDValue getRegister(unsigned Reg, EVT VT); |
724 | SDValue getRegisterMask(const uint32_t *RegMask); |
725 | SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label); |
726 | SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, |
727 | MCSymbol *Label); |
728 | SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset = 0, |
729 | bool isTarget = false, unsigned TargetFlags = 0); |
730 | SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, |
731 | int64_t Offset = 0, unsigned TargetFlags = 0) { |
732 | return getBlockAddress(BA, VT, Offset, true, TargetFlags); |
733 | } |
734 | |
735 | SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, |
736 | SDValue N) { |
737 | return getNode(ISD::CopyToReg, dl, MVT::Other, Chain, |
738 | getRegister(Reg, N.getValueType()), N); |
739 | } |
740 | |
741 | // This version of the getCopyToReg method takes an extra operand, which |
742 | // indicates that there is potentially an incoming glue value (if Glue is not |
743 | // null) and that there should be a glue result. |
744 | SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N, |
745 | SDValue Glue) { |
746 | SDVTList VTs = getVTList(MVT::Other, MVT::Glue); |
747 | SDValue Ops[] = { Chain, getRegister(Reg, N.getValueType()), N, Glue }; |
748 | return getNode(ISD::CopyToReg, dl, VTs, |
749 | makeArrayRef(Ops, Glue.getNode() ? 4 : 3)); |
750 | } |
751 | |
752 | // Similar to last getCopyToReg() except parameter Reg is a SDValue |
753 | SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, SDValue Reg, SDValue N, |
754 | SDValue Glue) { |
755 | SDVTList VTs = getVTList(MVT::Other, MVT::Glue); |
756 | SDValue Ops[] = { Chain, Reg, N, Glue }; |
757 | return getNode(ISD::CopyToReg, dl, VTs, |
758 | makeArrayRef(Ops, Glue.getNode() ? 4 : 3)); |
759 | } |
760 | |
761 | SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT) { |
762 | SDVTList VTs = getVTList(VT, MVT::Other); |
763 | SDValue Ops[] = { Chain, getRegister(Reg, VT) }; |
764 | return getNode(ISD::CopyFromReg, dl, VTs, Ops); |
765 | } |
766 | |
767 | // This version of the getCopyFromReg method takes an extra operand, which |
768 | // indicates that there is potentially an incoming glue value (if Glue is not |
769 | // null) and that there should be a glue result. |
770 | SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT, |
771 | SDValue Glue) { |
772 | SDVTList VTs = getVTList(VT, MVT::Other, MVT::Glue); |
773 | SDValue Ops[] = { Chain, getRegister(Reg, VT), Glue }; |
774 | return getNode(ISD::CopyFromReg, dl, VTs, |
775 | makeArrayRef(Ops, Glue.getNode() ? 3 : 2)); |
776 | } |
777 | |
778 | SDValue getCondCode(ISD::CondCode Cond); |
779 | |
780 | /// Return an ISD::VECTOR_SHUFFLE node. The number of elements in VT, |
781 | /// which must be a vector type, must match the number of mask elements |
782 | /// NumElts. An integer mask element equal to -1 is treated as undefined. |
783 | SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, |
784 | ArrayRef<int> Mask); |
785 | |
786 | /// Return an ISD::BUILD_VECTOR node. The number of elements in VT, |
787 | /// which must be a vector type, must match the number of operands in Ops. |
788 | /// The operands must have the same type as (or, for integers, a type wider |
789 | /// than) VT's element type. |
790 | SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDValue> Ops) { |
791 | // VerifySDNode (via InsertNode) checks BUILD_VECTOR later. |
792 | return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); |
793 | } |
794 | |
795 | /// Return an ISD::BUILD_VECTOR node. The number of elements in VT, |
796 | /// which must be a vector type, must match the number of operands in Ops. |
797 | /// The operands must have the same type as (or, for integers, a type wider |
798 | /// than) VT's element type. |
799 | SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDUse> Ops) { |
800 | // VerifySDNode (via InsertNode) checks BUILD_VECTOR later. |
801 | return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); |
802 | } |
803 | |
804 | /// Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all |
805 | /// elements. VT must be a vector type. Op's type must be the same as (or, |
806 | /// for integers, a type wider than) VT's element type. |
807 | SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op) { |
808 | // VerifySDNode (via InsertNode) checks BUILD_VECTOR later. |
809 | if (Op.getOpcode() == ISD::UNDEF) { |
810 | assert((VT.getVectorElementType() == Op.getValueType() ||(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType () || (VT.isInteger() && VT.getVectorElementType().bitsLE (Op.getValueType()))) && "A splatted value must have a width equal or (for integers) " "greater than the vector element type!") ? void (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 814, __extension__ __PRETTY_FUNCTION__)) |
811 | (VT.isInteger() &&(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType () || (VT.isInteger() && VT.getVectorElementType().bitsLE (Op.getValueType()))) && "A splatted value must have a width equal or (for integers) " "greater than the vector element type!") ? void (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 814, __extension__ __PRETTY_FUNCTION__)) |
812 | VT.getVectorElementType().bitsLE(Op.getValueType()))) &&(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType () || (VT.isInteger() && VT.getVectorElementType().bitsLE (Op.getValueType()))) && "A splatted value must have a width equal or (for integers) " "greater than the vector element type!") ? void (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 814, __extension__ __PRETTY_FUNCTION__)) |
813 | "A splatted value must have a width equal or (for integers) "(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType () || (VT.isInteger() && VT.getVectorElementType().bitsLE (Op.getValueType()))) && "A splatted value must have a width equal or (for integers) " "greater than the vector element type!") ? void (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 814, __extension__ __PRETTY_FUNCTION__)) |
814 | "greater than the vector element type!")(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType () || (VT.isInteger() && VT.getVectorElementType().bitsLE (Op.getValueType()))) && "A splatted value must have a width equal or (for integers) " "greater than the vector element type!") ? void (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 814, __extension__ __PRETTY_FUNCTION__)); |
815 | return getNode(ISD::UNDEF, SDLoc(), VT); |
816 | } |
817 | |
818 | SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Op); |
819 | return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); |
820 | } |
821 | |
822 | // Return a splat ISD::SPLAT_VECTOR node, consisting of Op splatted to all |
823 | // elements. |
824 | SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op) { |
825 | if (Op.getOpcode() == ISD::UNDEF) { |
826 | assert((VT.getVectorElementType() == Op.getValueType() ||(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType () || (VT.isInteger() && VT.getVectorElementType().bitsLE (Op.getValueType()))) && "A splatted value must have a width equal or (for integers) " "greater than the vector element type!") ? void (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 830, __extension__ __PRETTY_FUNCTION__)) |
827 | (VT.isInteger() &&(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType () || (VT.isInteger() && VT.getVectorElementType().bitsLE (Op.getValueType()))) && "A splatted value must have a width equal or (for integers) " "greater than the vector element type!") ? void (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 830, __extension__ __PRETTY_FUNCTION__)) |
828 | VT.getVectorElementType().bitsLE(Op.getValueType()))) &&(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType () || (VT.isInteger() && VT.getVectorElementType().bitsLE (Op.getValueType()))) && "A splatted value must have a width equal or (for integers) " "greater than the vector element type!") ? void (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 830, __extension__ __PRETTY_FUNCTION__)) |
829 | "A splatted value must have a width equal or (for integers) "(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType () || (VT.isInteger() && VT.getVectorElementType().bitsLE (Op.getValueType()))) && "A splatted value must have a width equal or (for integers) " "greater than the vector element type!") ? void (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 830, __extension__ __PRETTY_FUNCTION__)) |
830 | "greater than the vector element type!")(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType () || (VT.isInteger() && VT.getVectorElementType().bitsLE (Op.getValueType()))) && "A splatted value must have a width equal or (for integers) " "greater than the vector element type!") ? void (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 830, __extension__ __PRETTY_FUNCTION__)); |
831 | return getNode(ISD::UNDEF, SDLoc(), VT); |
832 | } |
833 | return getNode(ISD::SPLAT_VECTOR, DL, VT, Op); |
834 | } |
835 | |
836 | /// Returns a vector of type ResVT whose elements contain the linear sequence |
837 | /// <0, Step, Step * 2, Step * 3, ...> |
838 | SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal); |
839 | |
840 | /// Returns a vector of type ResVT whose elements contain the linear sequence |
841 | /// <0, 1, 2, 3, ...> |
842 | SDValue getStepVector(const SDLoc &DL, EVT ResVT); |
843 | |
844 | /// Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to |
845 | /// the shuffle node in input but with swapped operands. |
846 | /// |
847 | /// Example: shuffle A, B, <0,5,2,7> -> shuffle B, A, <4,1,6,3> |
848 | SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV); |
849 | |
850 | /// Convert Op, which must be of float type, to the |
851 | /// float type VT, by either extending or rounding (by truncation). |
852 | SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT); |
853 | |
854 | /// Convert Op, which must be a STRICT operation of float type, to the |
855 | /// float type VT, by either extending or rounding (by truncation). |
856 | std::pair<SDValue, SDValue> |
857 | getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT); |
858 | |
859 | /// Convert Op, which must be of integer type, to the |
860 | /// integer type VT, by either any-extending or truncating it. |
861 | SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); |
862 | |
863 | /// Convert Op, which must be of integer type, to the |
864 | /// integer type VT, by either sign-extending or truncating it. |
865 | SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); |
866 | |
867 | /// Convert Op, which must be of integer type, to the |
868 | /// integer type VT, by either zero-extending or truncating it. |
869 | SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); |
870 | |
871 | /// Return the expression required to zero extend the Op |
872 | /// value assuming it was the smaller SrcTy value. |
873 | SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT); |
874 | |
875 | /// Convert Op, which must be of integer type, to the integer type VT, by |
876 | /// either truncating it or performing either zero or sign extension as |
877 | /// appropriate extension for the pointer's semantics. |
878 | SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); |
879 | |
880 | /// Return the expression required to extend the Op as a pointer value |
881 | /// assuming it was the smaller SrcTy value. This may be either a zero extend |
882 | /// or a sign extend. |
883 | SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT); |
884 | |
885 | /// Convert Op, which must be of integer type, to the integer type VT, |
886 | /// by using an extension appropriate for the target's |
887 | /// BooleanContent for type OpVT or truncating it. |
888 | SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT); |
889 | |
890 | /// Create a bitwise NOT operation as (XOR Val, -1). |
891 | SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT); |
892 | |
893 | /// Create a logical NOT operation as (XOR Val, BooleanOne). |
894 | SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT); |
895 | |
896 | /// Returns sum of the base pointer and offset. |
897 | /// Unlike getObjectPtrOffset this does not set NoUnsignedWrap by default. |
898 | SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, |
899 | const SDNodeFlags Flags = SDNodeFlags()); |
900 | SDValue getMemBasePlusOffset(SDValue Base, SDValue Offset, const SDLoc &DL, |
901 | const SDNodeFlags Flags = SDNodeFlags()); |
902 | |
903 | /// Create an add instruction with appropriate flags when used for |
904 | /// addressing some offset of an object. i.e. if a load is split into multiple |
905 | /// components, create an add nuw from the base pointer to the offset. |
906 | SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset) { |
907 | SDNodeFlags Flags; |
908 | Flags.setNoUnsignedWrap(true); |
909 | return getMemBasePlusOffset(Ptr, Offset, SL, Flags); |
910 | } |
911 | |
912 | SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, SDValue Offset) { |
913 | // The object itself can't wrap around the address space, so it shouldn't be |
914 | // possible for the adds of the offsets to the split parts to overflow. |
915 | SDNodeFlags Flags; |
916 | Flags.setNoUnsignedWrap(true); |
917 | return getMemBasePlusOffset(Ptr, Offset, SL, Flags); |
918 | } |
919 | |
920 | /// Return a new CALLSEQ_START node, that starts new call frame, in which |
921 | /// InSize bytes are set up inside CALLSEQ_START..CALLSEQ_END sequence and |
922 | /// OutSize specifies part of the frame set up prior to the sequence. |
923 | SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, |
924 | const SDLoc &DL) { |
925 | SDVTList VTs = getVTList(MVT::Other, MVT::Glue); |
926 | SDValue Ops[] = { Chain, |
927 | getIntPtrConstant(InSize, DL, true), |
928 | getIntPtrConstant(OutSize, DL, true) }; |
929 | return getNode(ISD::CALLSEQ_START, DL, VTs, Ops); |
930 | } |
931 | |
932 | /// Return a new CALLSEQ_END node, which always must have a |
933 | /// glue result (to ensure it's not CSE'd). |
934 | /// CALLSEQ_END does not have a useful SDLoc. |
935 | SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, |
936 | SDValue InGlue, const SDLoc &DL) { |
937 | SDVTList NodeTys = getVTList(MVT::Other, MVT::Glue); |
938 | SmallVector<SDValue, 4> Ops; |
939 | Ops.push_back(Chain); |
940 | Ops.push_back(Op1); |
941 | Ops.push_back(Op2); |
942 | if (InGlue.getNode()) |
943 | Ops.push_back(InGlue); |
944 | return getNode(ISD::CALLSEQ_END, DL, NodeTys, Ops); |
945 | } |
946 | |
947 | /// Return true if the result of this operation is always undefined. |
948 | bool isUndef(unsigned Opcode, ArrayRef<SDValue> Ops); |
949 | |
950 | /// Return an UNDEF node. UNDEF does not have a useful SDLoc. |
951 | SDValue getUNDEF(EVT VT) { |
952 | return getNode(ISD::UNDEF, SDLoc(), VT); |
953 | } |
954 | |
955 | /// Return a node that represents the runtime scaling 'MulImm * RuntimeVL'. |
956 | SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm) { |
957 | assert(MulImm.getMinSignedBits() <= VT.getSizeInBits() &&(static_cast <bool> (MulImm.getMinSignedBits() <= VT .getSizeInBits() && "Immediate does not fit VT") ? void (0) : __assert_fail ("MulImm.getMinSignedBits() <= VT.getSizeInBits() && \"Immediate does not fit VT\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 958, __extension__ __PRETTY_FUNCTION__)) |
958 | "Immediate does not fit VT")(static_cast <bool> (MulImm.getMinSignedBits() <= VT .getSizeInBits() && "Immediate does not fit VT") ? void (0) : __assert_fail ("MulImm.getMinSignedBits() <= VT.getSizeInBits() && \"Immediate does not fit VT\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 958, __extension__ __PRETTY_FUNCTION__)); |
959 | return getNode(ISD::VSCALE, DL, VT, |
960 | getConstant(MulImm.sextOrTrunc(VT.getSizeInBits()), DL, VT)); |
961 | } |
962 | |
963 | /// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc. |
964 | SDValue getGLOBAL_OFFSET_TABLE(EVT VT) { |
965 | return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT); |
966 | } |
967 | |
968 | /// Gets or creates the specified node. |
969 | /// |
970 | SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
971 | ArrayRef<SDUse> Ops); |
972 | SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
973 | ArrayRef<SDValue> Ops, const SDNodeFlags Flags); |
974 | SDValue getNode(unsigned Opcode, const SDLoc &DL, ArrayRef<EVT> ResultTys, |
975 | ArrayRef<SDValue> Ops); |
976 | SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
977 | ArrayRef<SDValue> Ops, const SDNodeFlags Flags); |
978 | |
979 | // Use flags from current flag inserter. |
980 | SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
981 | ArrayRef<SDValue> Ops); |
982 | SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
983 | ArrayRef<SDValue> Ops); |
984 | SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand); |
985 | SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, |
986 | SDValue N2); |
987 | SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, |
988 | SDValue N2, SDValue N3); |
989 | |
990 | // Specialize based on number of operands. |
991 | SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT); |
992 | SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand, |
993 | const SDNodeFlags Flags); |
994 | SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, |
995 | SDValue N2, const SDNodeFlags Flags); |
996 | SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, |
997 | SDValue N2, SDValue N3, const SDNodeFlags Flags); |
998 | SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, |
999 | SDValue N2, SDValue N3, SDValue N4); |
1000 | SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, |
1001 | SDValue N2, SDValue N3, SDValue N4, SDValue N5); |
1002 | |
1003 | // Specialize again based on number of operands for nodes with a VTList |
1004 | // rather than a single VT. |
1005 | SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList); |
1006 | SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N); |
1007 | SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, |
1008 | SDValue N2); |
1009 | SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, |
1010 | SDValue N2, SDValue N3); |
1011 | SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, |
1012 | SDValue N2, SDValue N3, SDValue N4); |
1013 | SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1, |
1014 | SDValue N2, SDValue N3, SDValue N4, SDValue N5); |
1015 | |
1016 | /// Compute a TokenFactor to force all the incoming stack arguments to be |
1017 | /// loaded from the stack. This is used in tail call lowering to protect |
1018 | /// stack arguments from being clobbered. |
1019 | SDValue getStackArgumentTokenFactor(SDValue Chain); |
1020 | |
1021 | SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, |
1022 | SDValue Size, Align Alignment, bool isVol, |
1023 | bool AlwaysInline, bool isTailCall, |
1024 | MachinePointerInfo DstPtrInfo, |
1025 | MachinePointerInfo SrcPtrInfo, |
1026 | const AAMDNodes &AAInfo = AAMDNodes()); |
1027 | |
1028 | SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, |
1029 | SDValue Size, Align Alignment, bool isVol, bool isTailCall, |
1030 | MachinePointerInfo DstPtrInfo, |
1031 | MachinePointerInfo SrcPtrInfo, |
1032 | const AAMDNodes &AAInfo = AAMDNodes()); |
1033 | |
1034 | SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, |
1035 | SDValue Size, Align Alignment, bool isVol, bool isTailCall, |
1036 | MachinePointerInfo DstPtrInfo, |
1037 | const AAMDNodes &AAInfo = AAMDNodes()); |
1038 | |
1039 | SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, |
1040 | unsigned DstAlign, SDValue Src, unsigned SrcAlign, |
1041 | SDValue Size, Type *SizeTy, unsigned ElemSz, |
1042 | bool isTailCall, MachinePointerInfo DstPtrInfo, |
1043 | MachinePointerInfo SrcPtrInfo); |
1044 | |
1045 | SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, |
1046 | unsigned DstAlign, SDValue Src, unsigned SrcAlign, |
1047 | SDValue Size, Type *SizeTy, unsigned ElemSz, |
1048 | bool isTailCall, MachinePointerInfo DstPtrInfo, |
1049 | MachinePointerInfo SrcPtrInfo); |
1050 | |
1051 | SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, |
1052 | unsigned DstAlign, SDValue Value, SDValue Size, |
1053 | Type *SizeTy, unsigned ElemSz, bool isTailCall, |
1054 | MachinePointerInfo DstPtrInfo); |
1055 | |
1056 | /// Helper function to make it easier to build SetCC's if you just have an |
1057 | /// ISD::CondCode instead of an SDValue. |
1058 | SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, |
1059 | ISD::CondCode Cond, SDValue Chain = SDValue(), |
1060 | bool IsSignaling = false) { |
1061 | assert(LHS.getValueType().isVector() == RHS.getValueType().isVector() &&(static_cast <bool> (LHS.getValueType().isVector() == RHS .getValueType().isVector() && "Cannot compare scalars to vectors" ) ? void (0) : __assert_fail ("LHS.getValueType().isVector() == RHS.getValueType().isVector() && \"Cannot compare scalars to vectors\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 1062, __extension__ __PRETTY_FUNCTION__)) |
1062 | "Cannot compare scalars to vectors")(static_cast <bool> (LHS.getValueType().isVector() == RHS .getValueType().isVector() && "Cannot compare scalars to vectors" ) ? void (0) : __assert_fail ("LHS.getValueType().isVector() == RHS.getValueType().isVector() && \"Cannot compare scalars to vectors\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 1062, __extension__ __PRETTY_FUNCTION__)); |
1063 | assert(LHS.getValueType().isVector() == VT.isVector() &&(static_cast <bool> (LHS.getValueType().isVector() == VT .isVector() && "Cannot compare scalars to vectors") ? void (0) : __assert_fail ("LHS.getValueType().isVector() == VT.isVector() && \"Cannot compare scalars to vectors\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 1064, __extension__ __PRETTY_FUNCTION__)) |
1064 | "Cannot compare scalars to vectors")(static_cast <bool> (LHS.getValueType().isVector() == VT .isVector() && "Cannot compare scalars to vectors") ? void (0) : __assert_fail ("LHS.getValueType().isVector() == VT.isVector() && \"Cannot compare scalars to vectors\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 1064, __extension__ __PRETTY_FUNCTION__)); |
1065 | assert(Cond != ISD::SETCC_INVALID &&(static_cast <bool> (Cond != ISD::SETCC_INVALID && "Cannot create a setCC of an invalid node.") ? void (0) : __assert_fail ("Cond != ISD::SETCC_INVALID && \"Cannot create a setCC of an invalid node.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 1066, __extension__ __PRETTY_FUNCTION__)) |
1066 | "Cannot create a setCC of an invalid node.")(static_cast <bool> (Cond != ISD::SETCC_INVALID && "Cannot create a setCC of an invalid node.") ? void (0) : __assert_fail ("Cond != ISD::SETCC_INVALID && \"Cannot create a setCC of an invalid node.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 1066, __extension__ __PRETTY_FUNCTION__)); |
1067 | if (Chain) |
1068 | return getNode(IsSignaling ? ISD::STRICT_FSETCCS : ISD::STRICT_FSETCC, DL, |
1069 | {VT, MVT::Other}, {Chain, LHS, RHS, getCondCode(Cond)}); |
1070 | return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond)); |
1071 | } |
1072 | |
1073 | /// Helper function to make it easier to build Select's if you just have |
1074 | /// operands and don't want to check for vector. |
1075 | SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, |
1076 | SDValue RHS) { |
1077 | assert(LHS.getValueType() == RHS.getValueType() &&(static_cast <bool> (LHS.getValueType() == RHS.getValueType () && "Cannot use select on differing types") ? void ( 0) : __assert_fail ("LHS.getValueType() == RHS.getValueType() && \"Cannot use select on differing types\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 1078, __extension__ __PRETTY_FUNCTION__)) |
1078 | "Cannot use select on differing types")(static_cast <bool> (LHS.getValueType() == RHS.getValueType () && "Cannot use select on differing types") ? void ( 0) : __assert_fail ("LHS.getValueType() == RHS.getValueType() && \"Cannot use select on differing types\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 1078, __extension__ __PRETTY_FUNCTION__)); |
1079 | assert(VT.isVector() == LHS.getValueType().isVector() &&(static_cast <bool> (VT.isVector() == LHS.getValueType( ).isVector() && "Cannot mix vectors and scalars") ? void (0) : __assert_fail ("VT.isVector() == LHS.getValueType().isVector() && \"Cannot mix vectors and scalars\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 1080, __extension__ __PRETTY_FUNCTION__)) |
1080 | "Cannot mix vectors and scalars")(static_cast <bool> (VT.isVector() == LHS.getValueType( ).isVector() && "Cannot mix vectors and scalars") ? void (0) : __assert_fail ("VT.isVector() == LHS.getValueType().isVector() && \"Cannot mix vectors and scalars\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 1080, __extension__ __PRETTY_FUNCTION__)); |
1081 | auto Opcode = Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT; |
1082 | return getNode(Opcode, DL, VT, Cond, LHS, RHS); |
1083 | } |
1084 | |
1085 | /// Helper function to make it easier to build SelectCC's if you just have an |
1086 | /// ISD::CondCode instead of an SDValue. |
1087 | SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, |
1088 | SDValue False, ISD::CondCode Cond) { |
1089 | return getNode(ISD::SELECT_CC, DL, True.getValueType(), LHS, RHS, True, |
1090 | False, getCondCode(Cond)); |
1091 | } |
1092 | |
1093 | /// Try to simplify a select/vselect into 1 of its operands or a constant. |
1094 | SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal); |
1095 | |
1096 | /// Try to simplify a shift into 1 of its operands or a constant. |
1097 | SDValue simplifyShift(SDValue X, SDValue Y); |
1098 | |
1099 | /// Try to simplify a floating-point binary operation into 1 of its operands |
1100 | /// or a constant. |
1101 | SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, |
1102 | SDNodeFlags Flags); |
1103 | |
1104 | /// VAArg produces a result and token chain, and takes a pointer |
1105 | /// and a source value as input. |
1106 | SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, |
1107 | SDValue SV, unsigned Align); |
1108 | |
1109 | /// Gets a node for an atomic cmpxchg op. There are two |
1110 | /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces the value loaded and a |
1111 | /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded, |
1112 | /// a success flag (initially i1), and a chain. |
1113 | SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
1114 | SDVTList VTs, SDValue Chain, SDValue Ptr, |
1115 | SDValue Cmp, SDValue Swp, MachineMemOperand *MMO); |
1116 | |
1117 | /// Gets a node for an atomic op, produces result (if relevant) |
1118 | /// and chain and takes 2 operands. |
1119 | SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, |
1120 | SDValue Ptr, SDValue Val, MachineMemOperand *MMO); |
1121 | |
1122 | /// Gets a node for an atomic op, produces result and chain and |
1123 | /// takes 1 operand. |
1124 | SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, EVT VT, |
1125 | SDValue Chain, SDValue Ptr, MachineMemOperand *MMO); |
1126 | |
1127 | /// Gets a node for an atomic op, produces result and chain and takes N |
1128 | /// operands. |
1129 | SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
1130 | SDVTList VTList, ArrayRef<SDValue> Ops, |
1131 | MachineMemOperand *MMO); |
1132 | |
1133 | /// Creates a MemIntrinsicNode that may produce a |
1134 | /// result and takes a list of operands. Opcode may be INTRINSIC_VOID, |
1135 | /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not |
1136 | /// less than FIRST_TARGET_MEMORY_OPCODE. |
1137 | SDValue getMemIntrinsicNode( |
1138 | unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops, |
1139 | EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, |
1140 | MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad | |
1141 | MachineMemOperand::MOStore, |
1142 | uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()); |
1143 | |
1144 | inline SDValue getMemIntrinsicNode( |
1145 | unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops, |
1146 | EVT MemVT, MachinePointerInfo PtrInfo, MaybeAlign Alignment = None, |
1147 | MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad | |
1148 | MachineMemOperand::MOStore, |
1149 | uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()) { |
1150 | // Ensure that codegen never sees alignment 0 |
1151 | return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, PtrInfo, |
1152 | Alignment.getValueOr(getEVTAlign(MemVT)), Flags, |
1153 | Size, AAInfo); |
1154 | } |
1155 | |
1156 | SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, |
1157 | ArrayRef<SDValue> Ops, EVT MemVT, |
1158 | MachineMemOperand *MMO); |
1159 | |
1160 | /// Creates a LifetimeSDNode that starts (`IsStart==true`) or ends |
1161 | /// (`IsStart==false`) the lifetime of the portion of `FrameIndex` between |
1162 | /// offsets `Offset` and `Offset + Size`. |
1163 | SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, |
1164 | int FrameIndex, int64_t Size, int64_t Offset = -1); |
1165 | |
1166 | /// Creates a PseudoProbeSDNode with function GUID `Guid` and |
1167 | /// the index of the block `Index` it is probing, as well as the attributes |
1168 | /// `attr` of the probe. |
1169 | SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, |
1170 | uint64_t Index, uint32_t Attr); |
1171 | |
1172 | /// Create a MERGE_VALUES node from the given operands. |
1173 | SDValue getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl); |
1174 | |
1175 | /// Loads are not normal binary operators: their result type is not |
1176 | /// determined by their operands, and they produce a value AND a token chain. |
1177 | /// |
1178 | /// This function will set the MOLoad flag on MMOFlags, but you can set it if |
1179 | /// you want. The MOStore flag must not be set. |
1180 | SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, |
1181 | MachinePointerInfo PtrInfo, |
1182 | MaybeAlign Alignment = MaybeAlign(), |
1183 | MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, |
1184 | const AAMDNodes &AAInfo = AAMDNodes(), |
1185 | const MDNode *Ranges = nullptr); |
1186 | /// FIXME: Remove once transition to Align is over. |
1187 | inline SDValue |
1188 | getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, |
1189 | MachinePointerInfo PtrInfo, unsigned Alignment, |
1190 | MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, |
1191 | const AAMDNodes &AAInfo = AAMDNodes(), |
1192 | const MDNode *Ranges = nullptr) { |
1193 | return getLoad(VT, dl, Chain, Ptr, PtrInfo, MaybeAlign(Alignment), MMOFlags, |
1194 | AAInfo, Ranges); |
1195 | } |
1196 | SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, |
1197 | MachineMemOperand *MMO); |
1198 | SDValue |
1199 | getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, |
1200 | SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, |
1201 | MaybeAlign Alignment = MaybeAlign(), |
1202 | MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, |
1203 | const AAMDNodes &AAInfo = AAMDNodes()); |
1204 | /// FIXME: Remove once transition to Align is over. |
1205 | inline SDValue |
1206 | getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, |
1207 | SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, |
1208 | unsigned Alignment, |
1209 | MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, |
1210 | const AAMDNodes &AAInfo = AAMDNodes()) { |
1211 | return getExtLoad(ExtType, dl, VT, Chain, Ptr, PtrInfo, MemVT, |
1212 | MaybeAlign(Alignment), MMOFlags, AAInfo); |
1213 | } |
1214 | SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, |
1215 | SDValue Chain, SDValue Ptr, EVT MemVT, |
1216 | MachineMemOperand *MMO); |
1217 | SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, |
1218 | SDValue Offset, ISD::MemIndexedMode AM); |
1219 | SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, |
1220 | const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, |
1221 | MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, |
1222 | MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, |
1223 | const AAMDNodes &AAInfo = AAMDNodes(), |
1224 | const MDNode *Ranges = nullptr); |
1225 | inline SDValue getLoad( |
1226 | ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, |
1227 | SDValue Chain, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, |
1228 | EVT MemVT, MaybeAlign Alignment = MaybeAlign(), |
1229 | MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, |
1230 | const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr) { |
1231 | // Ensures that codegen never sees a None Alignment. |
1232 | return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT, |
1233 | Alignment.getValueOr(getEVTAlign(MemVT)), MMOFlags, AAInfo, |
1234 | Ranges); |
1235 | } |
1236 | /// FIXME: Remove once transition to Align is over. |
1237 | inline SDValue |
1238 | getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, |
1239 | const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, |
1240 | MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment, |
1241 | MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, |
1242 | const AAMDNodes &AAInfo = AAMDNodes(), |
1243 | const MDNode *Ranges = nullptr) { |
1244 | return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT, |
1245 | MaybeAlign(Alignment), MMOFlags, AAInfo, Ranges); |
1246 | } |
1247 | SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, |
1248 | const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, |
1249 | EVT MemVT, MachineMemOperand *MMO); |
1250 | |
1251 | /// Helper function to build ISD::STORE nodes. |
1252 | /// |
1253 | /// This function will set the MOStore flag on MMOFlags, but you can set it if |
1254 | /// you want. The MOLoad and MOInvariant flags must not be set. |
1255 | |
1256 | SDValue |
1257 | getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, |
1258 | MachinePointerInfo PtrInfo, Align Alignment, |
1259 | MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, |
1260 | const AAMDNodes &AAInfo = AAMDNodes()); |
1261 | inline SDValue |
1262 | getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, |
1263 | MachinePointerInfo PtrInfo, MaybeAlign Alignment = MaybeAlign(), |
1264 | MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, |
1265 | const AAMDNodes &AAInfo = AAMDNodes()) { |
1266 | return getStore(Chain, dl, Val, Ptr, PtrInfo, |
1267 | Alignment.getValueOr(getEVTAlign(Val.getValueType())), |
1268 | MMOFlags, AAInfo); |
1269 | } |
1270 | /// FIXME: Remove once transition to Align is over. |
1271 | inline SDValue |
1272 | getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, |
1273 | MachinePointerInfo PtrInfo, unsigned Alignment, |
1274 | MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, |
1275 | const AAMDNodes &AAInfo = AAMDNodes()) { |
1276 | return getStore(Chain, dl, Val, Ptr, PtrInfo, MaybeAlign(Alignment), |
1277 | MMOFlags, AAInfo); |
1278 | } |
1279 | SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, |
1280 | MachineMemOperand *MMO); |
1281 | SDValue |
1282 | getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, |
1283 | MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, |
1284 | MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, |
1285 | const AAMDNodes &AAInfo = AAMDNodes()); |
1286 | inline SDValue |
1287 | getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, |
1288 | MachinePointerInfo PtrInfo, EVT SVT, |
1289 | MaybeAlign Alignment = MaybeAlign(), |
1290 | MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, |
1291 | const AAMDNodes &AAInfo = AAMDNodes()) { |
1292 | return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT, |
1293 | Alignment.getValueOr(getEVTAlign(SVT)), MMOFlags, |
1294 | AAInfo); |
1295 | } |
1296 | /// FIXME: Remove once transition to Align is over. |
1297 | inline SDValue |
1298 | getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, |
1299 | MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment, |
1300 | MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone, |
1301 | const AAMDNodes &AAInfo = AAMDNodes()) { |
1302 | return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT, |
1303 | MaybeAlign(Alignment), MMOFlags, AAInfo); |
1304 | } |
1305 | SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
1306 | SDValue Ptr, EVT SVT, MachineMemOperand *MMO); |
1307 | SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, |
1308 | SDValue Offset, ISD::MemIndexedMode AM); |
1309 | |
1310 | SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, |
1311 | SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, |
1312 | MachineMemOperand *MMO, ISD::MemIndexedMode AM, |
1313 | ISD::LoadExtType, bool IsExpanding = false); |
1314 | SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, |
1315 | SDValue Offset, ISD::MemIndexedMode AM); |
1316 | SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
1317 | SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, |
1318 | MachineMemOperand *MMO, ISD::MemIndexedMode AM, |
1319 | bool IsTruncating = false, bool IsCompressing = false); |
1320 | SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, |
1321 | SDValue Base, SDValue Offset, |
1322 | ISD::MemIndexedMode AM); |
1323 | SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, |
1324 | ArrayRef<SDValue> Ops, MachineMemOperand *MMO, |
1325 | ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy); |
1326 | SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, |
1327 | ArrayRef<SDValue> Ops, MachineMemOperand *MMO, |
1328 | ISD::MemIndexType IndexType, |
1329 | bool IsTruncating = false); |
1330 | |
1331 | /// Construct a node to track a Value* through the backend. |
1332 | SDValue getSrcValue(const Value *v); |
1333 | |
1334 | /// Return an MDNodeSDNode which holds an MDNode. |
1335 | SDValue getMDNode(const MDNode *MD); |
1336 | |
1337 | /// Return a bitcast using the SDLoc of the value operand, and casting to the |
1338 | /// provided type. Use getNode to set a custom SDLoc. |
1339 | SDValue getBitcast(EVT VT, SDValue V); |
1340 | |
1341 | /// Return an AddrSpaceCastSDNode. |
1342 | SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, |
1343 | unsigned DestAS); |
1344 | |
1345 | /// Return a freeze using the SDLoc of the value operand. |
1346 | SDValue getFreeze(SDValue V); |
1347 | |
1348 | /// Return an AssertAlignSDNode. |
1349 | SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A); |
1350 | |
1351 | /// Return the specified value casted to |
1352 | /// the target's desired shift amount type. |
1353 | SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op); |
1354 | |
1355 | /// Expand the specified \c ISD::VAARG node as the Legalize pass would. |
1356 | SDValue expandVAArg(SDNode *Node); |
1357 | |
1358 | /// Expand the specified \c ISD::VACOPY node as the Legalize pass would. |
1359 | SDValue expandVACopy(SDNode *Node); |
1360 | |
1361 | /// Returs an GlobalAddress of the function from the current module with |
1362 | /// name matching the given ExternalSymbol. Additionally can provide the |
1363 | /// matched function. |
1364 | /// Panics the function doesn't exists. |
1365 | SDValue getSymbolFunctionGlobalAddress(SDValue Op, |
1366 | Function **TargetFunction = nullptr); |
1367 | |
1368 | /// *Mutate* the specified node in-place to have the |
1369 | /// specified operands. If the resultant node already exists in the DAG, |
1370 | /// this does not modify the specified node, instead it returns the node that |
1371 | /// already exists. If the resultant node does not exist in the DAG, the |
1372 | /// input node is returned. As a degenerate case, if you specify the same |
1373 | /// input operands as the node already has, the input node is returned. |
1374 | SDNode *UpdateNodeOperands(SDNode *N, SDValue Op); |
1375 | SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2); |
1376 | SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, |
1377 | SDValue Op3); |
1378 | SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, |
1379 | SDValue Op3, SDValue Op4); |
1380 | SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, |
1381 | SDValue Op3, SDValue Op4, SDValue Op5); |
1382 | SDNode *UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops); |
1383 | |
1384 | /// Creates a new TokenFactor containing \p Vals. If \p Vals contains 64k |
1385 | /// values or more, move values into new TokenFactors in 64k-1 blocks, until |
1386 | /// the final TokenFactor has less than 64k operands. |
1387 | SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl<SDValue> &Vals); |
1388 | |
1389 | /// *Mutate* the specified machine node's memory references to the provided |
1390 | /// list. |
1391 | void setNodeMemRefs(MachineSDNode *N, |
1392 | ArrayRef<MachineMemOperand *> NewMemRefs); |
1393 | |
1394 | // Calculate divergence of node \p N based on its operands. |
1395 | bool calculateDivergence(SDNode *N); |
1396 | |
1397 | // Propagates the change in divergence to users |
1398 | void updateDivergence(SDNode * N); |
1399 | |
1400 | /// These are used for target selectors to *mutate* the |
1401 | /// specified node to have the specified return type, Target opcode, and |
1402 | /// operands. Note that target opcodes are stored as |
1403 | /// ~TargetOpcode in the node opcode field. The resultant node is returned. |
1404 | SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT); |
1405 | SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, SDValue Op1); |
1406 | SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, |
1407 | SDValue Op1, SDValue Op2); |
1408 | SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, |
1409 | SDValue Op1, SDValue Op2, SDValue Op3); |
1410 | SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, |
1411 | ArrayRef<SDValue> Ops); |
1412 | SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, EVT VT2); |
1413 | SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, |
1414 | EVT VT2, ArrayRef<SDValue> Ops); |
1415 | SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, |
1416 | EVT VT2, EVT VT3, ArrayRef<SDValue> Ops); |
1417 | SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, |
1418 | EVT VT2, SDValue Op1, SDValue Op2); |
1419 | SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, SDVTList VTs, |
1420 | ArrayRef<SDValue> Ops); |
1421 | |
1422 | /// This *mutates* the specified node to have the specified |
1423 | /// return type, opcode, and operands. |
1424 | SDNode *MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, |
1425 | ArrayRef<SDValue> Ops); |
1426 | |
1427 | /// Mutate the specified strict FP node to its non-strict equivalent, |
1428 | /// unlinking the node from its chain and dropping the metadata arguments. |
1429 | /// The node must be a strict FP node. |
1430 | SDNode *mutateStrictFPToFP(SDNode *Node); |
1431 | |
1432 | /// These are used for target selectors to create a new node |
1433 | /// with specified return type(s), MachineInstr opcode, and operands. |
1434 | /// |
1435 | /// Note that getMachineNode returns the resultant node. If there is already |
1436 | /// a node of the specified opcode and operands, it returns that node instead |
1437 | /// of the current one. |
1438 | MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT); |
1439 | MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, |
1440 | SDValue Op1); |
1441 | MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, |
1442 | SDValue Op1, SDValue Op2); |
1443 | MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, |
1444 | SDValue Op1, SDValue Op2, SDValue Op3); |
1445 | MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT, |
1446 | ArrayRef<SDValue> Ops); |
1447 | MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, |
1448 | EVT VT2, SDValue Op1, SDValue Op2); |
1449 | MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, |
1450 | EVT VT2, SDValue Op1, SDValue Op2, SDValue Op3); |
1451 | MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, |
1452 | EVT VT2, ArrayRef<SDValue> Ops); |
1453 | MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, |
1454 | EVT VT2, EVT VT3, SDValue Op1, SDValue Op2); |
1455 | MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, |
1456 | EVT VT2, EVT VT3, SDValue Op1, SDValue Op2, |
1457 | SDValue Op3); |
1458 | MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1, |
1459 | EVT VT2, EVT VT3, ArrayRef<SDValue> Ops); |
1460 | MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, |
1461 | ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops); |
1462 | MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, SDVTList VTs, |
1463 | ArrayRef<SDValue> Ops); |
1464 | |
1465 | /// A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes. |
1466 | SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, |
1467 | SDValue Operand); |
1468 | |
1469 | /// A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes. |
1470 | SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, |
1471 | SDValue Operand, SDValue Subreg); |
1472 | |
1473 | /// Get the specified node if it's already available, or else return NULL. |
1474 | SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList, |
1475 | ArrayRef<SDValue> Ops, const SDNodeFlags Flags); |
1476 | SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList, |
1477 | ArrayRef<SDValue> Ops); |
1478 | |
1479 | /// Check if a node exists without modifying its flags. |
1480 | bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef<SDValue> Ops); |
1481 | |
1482 | /// Creates a SDDbgValue node. |
1483 | SDDbgValue *getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, |
1484 | unsigned R, bool IsIndirect, const DebugLoc &DL, |
1485 | unsigned O); |
1486 | |
1487 | /// Creates a constant SDDbgValue node. |
1488 | SDDbgValue *getConstantDbgValue(DIVariable *Var, DIExpression *Expr, |
1489 | const Value *C, const DebugLoc &DL, |
1490 | unsigned O); |
1491 | |
1492 | /// Creates a FrameIndex SDDbgValue node. |
1493 | SDDbgValue *getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, |
1494 | unsigned FI, bool IsIndirect, |
1495 | const DebugLoc &DL, unsigned O); |
1496 | |
1497 | /// Creates a FrameIndex SDDbgValue node. |
1498 | SDDbgValue *getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, |
1499 | unsigned FI, |
1500 | ArrayRef<SDNode *> Dependencies, |
1501 | bool IsIndirect, const DebugLoc &DL, |
1502 | unsigned O); |
1503 | |
1504 | /// Creates a VReg SDDbgValue node. |
1505 | SDDbgValue *getVRegDbgValue(DIVariable *Var, DIExpression *Expr, |
1506 | unsigned VReg, bool IsIndirect, |
1507 | const DebugLoc &DL, unsigned O); |
1508 | |
1509 | /// Creates a SDDbgValue node from a list of locations. |
1510 | SDDbgValue *getDbgValueList(DIVariable *Var, DIExpression *Expr, |
1511 | ArrayRef<SDDbgOperand> Locs, |
1512 | ArrayRef<SDNode *> Dependencies, bool IsIndirect, |
1513 | const DebugLoc &DL, unsigned O, bool IsVariadic); |
1514 | |
1515 | /// Creates a SDDbgLabel node. |
1516 | SDDbgLabel *getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O); |
1517 | |
1518 | /// Transfer debug values from one node to another, while optionally |
1519 | /// generating fragment expressions for split-up values. If \p InvalidateDbg |
1520 | /// is set, debug values are invalidated after they are transferred. |
1521 | void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits = 0, |
1522 | unsigned SizeInBits = 0, bool InvalidateDbg = true); |
1523 | |
1524 | /// Remove the specified node from the system. If any of its |
1525 | /// operands then becomes dead, remove them as well. Inform UpdateListener |
1526 | /// for each node deleted. |
1527 | void RemoveDeadNode(SDNode *N); |
1528 | |
1529 | /// This method deletes the unreachable nodes in the |
1530 | /// given list, and any nodes that become unreachable as a result. |
1531 | void RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes); |
1532 | |
1533 | /// Modify anything using 'From' to use 'To' instead. |
1534 | /// This can cause recursive merging of nodes in the DAG. Use the first |
1535 | /// version if 'From' is known to have a single result, use the second |
1536 | /// if you have two nodes with identical results (or if 'To' has a superset |
1537 | /// of the results of 'From'), use the third otherwise. |
1538 | /// |
1539 | /// These methods all take an optional UpdateListener, which (if not null) is |
1540 | /// informed about nodes that are deleted and modified due to recursive |
1541 | /// changes in the dag. |
1542 | /// |
1543 | /// These functions only replace all existing uses. It's possible that as |
1544 | /// these replacements are being performed, CSE may cause the From node |
1545 | /// to be given new uses. These new uses of From are left in place, and |
1546 | /// not automatically transferred to To. |
1547 | /// |
1548 | void ReplaceAllUsesWith(SDValue From, SDValue To); |
1549 | void ReplaceAllUsesWith(SDNode *From, SDNode *To); |
1550 | void ReplaceAllUsesWith(SDNode *From, const SDValue *To); |
1551 | |
1552 | /// Replace any uses of From with To, leaving |
1553 | /// uses of other values produced by From.getNode() alone. |
1554 | void ReplaceAllUsesOfValueWith(SDValue From, SDValue To); |
1555 | |
1556 | /// Like ReplaceAllUsesOfValueWith, but for multiple values at once. |
1557 | /// This correctly handles the case where |
1558 | /// there is an overlap between the From values and the To values. |
1559 | void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, |
1560 | unsigned Num); |
1561 | |
1562 | /// If an existing load has uses of its chain, create a token factor node with |
1563 | /// that chain and the new memory node's chain and update users of the old |
1564 | /// chain to the token factor. This ensures that the new memory node will have |
1565 | /// the same relative memory dependency position as the old load. Returns the |
1566 | /// new merged load chain. |
1567 | SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain); |
1568 | |
1569 | /// If an existing load has uses of its chain, create a token factor node with |
1570 | /// that chain and the new memory node's chain and update users of the old |
1571 | /// chain to the token factor. This ensures that the new memory node will have |
1572 | /// the same relative memory dependency position as the old load. Returns the |
1573 | /// new merged load chain. |
1574 | SDValue makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, SDValue NewMemOp); |
1575 | |
1576 | /// Topological-sort the AllNodes list and a |
1577 | /// assign a unique node id for each node in the DAG based on their |
1578 | /// topological order. Returns the number of nodes. |
1579 | unsigned AssignTopologicalOrder(); |
1580 | |
1581 | /// Move node N in the AllNodes list to be immediately |
1582 | /// before the given iterator Position. This may be used to update the |
1583 | /// topological ordering when the list of nodes is modified. |
1584 | void RepositionNode(allnodes_iterator Position, SDNode *N) { |
1585 | AllNodes.insert(Position, AllNodes.remove(N)); |
1586 | } |
1587 | |
1588 | /// Returns an APFloat semantics tag appropriate for the given type. If VT is |
1589 | /// a vector type, the element semantics are returned. |
1590 | static const fltSemantics &EVTToAPFloatSemantics(EVT VT) { |
1591 | switch (VT.getScalarType().getSimpleVT().SimpleTy) { |
1592 | default: llvm_unreachable("Unknown FP format")::llvm::llvm_unreachable_internal("Unknown FP format", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h" , 1592); |
1593 | case MVT::f16: return APFloat::IEEEhalf(); |
1594 | case MVT::bf16: return APFloat::BFloat(); |
1595 | case MVT::f32: return APFloat::IEEEsingle(); |
1596 | case MVT::f64: return APFloat::IEEEdouble(); |
1597 | case MVT::f80: return APFloat::x87DoubleExtended(); |
1598 | case MVT::f128: return APFloat::IEEEquad(); |
1599 | case MVT::ppcf128: return APFloat::PPCDoubleDouble(); |
1600 | } |
1601 | } |
1602 | |
1603 | /// Add a dbg_value SDNode. If SD is non-null that means the |
1604 | /// value is produced by SD. |
1605 | void AddDbgValue(SDDbgValue *DB, bool isParameter); |
1606 | |
1607 | /// Add a dbg_label SDNode. |
1608 | void AddDbgLabel(SDDbgLabel *DB); |
1609 | |
1610 | /// Get the debug values which reference the given SDNode. |
1611 | ArrayRef<SDDbgValue*> GetDbgValues(const SDNode* SD) const { |
1612 | return DbgInfo->getSDDbgValues(SD); |
1613 | } |
1614 | |
1615 | public: |
1616 | /// Return true if there are any SDDbgValue nodes associated |
1617 | /// with this SelectionDAG. |
1618 | bool hasDebugValues() const { return !DbgInfo->empty(); } |
1619 | |
1620 | SDDbgInfo::DbgIterator DbgBegin() const { return DbgInfo->DbgBegin(); } |
1621 | SDDbgInfo::DbgIterator DbgEnd() const { return DbgInfo->DbgEnd(); } |
1622 | |
1623 | SDDbgInfo::DbgIterator ByvalParmDbgBegin() const { |
1624 | return DbgInfo->ByvalParmDbgBegin(); |
1625 | } |
1626 | SDDbgInfo::DbgIterator ByvalParmDbgEnd() const { |
1627 | return DbgInfo->ByvalParmDbgEnd(); |
1628 | } |
1629 | |
1630 | SDDbgInfo::DbgLabelIterator DbgLabelBegin() const { |
1631 | return DbgInfo->DbgLabelBegin(); |
1632 | } |
1633 | SDDbgInfo::DbgLabelIterator DbgLabelEnd() const { |
1634 | return DbgInfo->DbgLabelEnd(); |
1635 | } |
1636 | |
1637 | /// To be invoked on an SDNode that is slated to be erased. This |
1638 | /// function mirrors \c llvm::salvageDebugInfo. |
1639 | void salvageDebugInfo(SDNode &N); |
1640 | |
1641 | void dump() const; |
1642 | |
1643 | /// In most cases this function returns the ABI alignment for a given type, |
1644 | /// except for illegal vector types where the alignment exceeds that of the |
1645 | /// stack. In such cases we attempt to break the vector down to a legal type |
1646 | /// and return the ABI alignment for that instead. |
1647 | Align getReducedAlign(EVT VT, bool UseABI); |
1648 | |
1649 | /// Create a stack temporary based on the size in bytes and the alignment |
1650 | SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment); |
1651 | |
1652 | /// Create a stack temporary, suitable for holding the specified value type. |
1653 | /// If minAlign is specified, the slot size will have at least that alignment. |
1654 | SDValue CreateStackTemporary(EVT VT, unsigned minAlign = 1); |
1655 | |
1656 | /// Create a stack temporary suitable for holding either of the specified |
1657 | /// value types. |
1658 | SDValue CreateStackTemporary(EVT VT1, EVT VT2); |
1659 | |
1660 | SDValue FoldSymbolOffset(unsigned Opcode, EVT VT, |
1661 | const GlobalAddressSDNode *GA, |
1662 | const SDNode *N2); |
1663 | |
1664 | SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, |
1665 | ArrayRef<SDValue> Ops); |
1666 | |
1667 | SDValue FoldConstantVectorArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, |
1668 | ArrayRef<SDValue> Ops, |
1669 | const SDNodeFlags Flags = SDNodeFlags()); |
1670 | |
1671 | /// Fold floating-point operations with 2 operands when both operands are |
1672 | /// constants and/or undefined. |
1673 | SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT, |
1674 | SDValue N1, SDValue N2); |
1675 | |
1676 | /// Constant fold a setcc to true or false. |
1677 | SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, |
1678 | const SDLoc &dl); |
1679 | |
1680 | /// See if the specified operand can be simplified with the knowledge that |
1681 | /// only the bits specified by DemandedBits are used. If so, return the |
1682 | /// simpler operand, otherwise return a null SDValue. |
1683 | /// |
1684 | /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can |
1685 | /// simplify nodes with multiple uses more aggressively.) |
1686 | SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits); |
1687 | |
1688 | /// See if the specified operand can be simplified with the knowledge that |
1689 | /// only the bits specified by DemandedBits are used in the elements specified |
1690 | /// by DemandedElts. If so, return the simpler operand, otherwise return a |
1691 | /// null SDValue. |
1692 | /// |
1693 | /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can |
1694 | /// simplify nodes with multiple uses more aggressively.) |
1695 | SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits, |
1696 | const APInt &DemandedElts); |
1697 | |
1698 | /// Return true if the sign bit of Op is known to be zero. |
1699 | /// We use this predicate to simplify operations downstream. |
1700 | bool SignBitIsZero(SDValue Op, unsigned Depth = 0) const; |
1701 | |
1702 | /// Return true if 'Op & Mask' is known to be zero. We |
1703 | /// use this predicate to simplify operations downstream. Op and Mask are |
1704 | /// known to be the same type. |
1705 | bool MaskedValueIsZero(SDValue Op, const APInt &Mask, |
1706 | unsigned Depth = 0) const; |
1707 | |
1708 | /// Return true if 'Op & Mask' is known to be zero in DemandedElts. We |
1709 | /// use this predicate to simplify operations downstream. Op and Mask are |
1710 | /// known to be the same type. |
1711 | bool MaskedValueIsZero(SDValue Op, const APInt &Mask, |
1712 | const APInt &DemandedElts, unsigned Depth = 0) const; |
1713 | |
1714 | /// Return true if '(Op & Mask) == Mask'. |
1715 | /// Op and Mask are known to be the same type. |
1716 | bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, |
1717 | unsigned Depth = 0) const; |
1718 | |
1719 | /// Determine which bits of Op are known to be either zero or one and return |
1720 | /// them in Known. For vectors, the known bits are those that are shared by |
1721 | /// every vector element. |
1722 | /// Targets can implement the computeKnownBitsForTargetNode method in the |
1723 | /// TargetLowering class to allow target nodes to be understood. |
1724 | KnownBits computeKnownBits(SDValue Op, unsigned Depth = 0) const; |
1725 | |
1726 | /// Determine which bits of Op are known to be either zero or one and return |
1727 | /// them in Known. The DemandedElts argument allows us to only collect the |
1728 | /// known bits that are shared by the requested vector elements. |
1729 | /// Targets can implement the computeKnownBitsForTargetNode method in the |
1730 | /// TargetLowering class to allow target nodes to be understood. |
1731 | KnownBits computeKnownBits(SDValue Op, const APInt &DemandedElts, |
1732 | unsigned Depth = 0) const; |
1733 | |
1734 | /// Used to represent the possible overflow behavior of an operation. |
1735 | /// Never: the operation cannot overflow. |
1736 | /// Always: the operation will always overflow. |
1737 | /// Sometime: the operation may or may not overflow. |
1738 | enum OverflowKind { |
1739 | OFK_Never, |
1740 | OFK_Sometime, |
1741 | OFK_Always, |
1742 | }; |
1743 | |
1744 | /// Determine if the result of the addition of 2 node can overflow. |
1745 | OverflowKind computeOverflowKind(SDValue N0, SDValue N1) const; |
1746 | |
1747 | /// Test if the given value is known to have exactly one bit set. This differs |
1748 | /// from computeKnownBits in that it doesn't necessarily determine which bit |
1749 | /// is set. |
1750 | bool isKnownToBeAPowerOfTwo(SDValue Val) const; |
1751 | |
1752 | /// Return the number of times the sign bit of the register is replicated into |
1753 | /// the other bits. We know that at least 1 bit is always equal to the sign |
1754 | /// bit (itself), but other cases can give us information. For example, |
1755 | /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal |
1756 | /// to each other, so we return 3. Targets can implement the |
1757 | /// ComputeNumSignBitsForTarget method in the TargetLowering class to allow |
1758 | /// target nodes to be understood. |
1759 | unsigned ComputeNumSignBits(SDValue Op, unsigned Depth = 0) const; |
1760 | |
1761 | /// Return the number of times the sign bit of the register is replicated into |
1762 | /// the other bits. We know that at least 1 bit is always equal to the sign |
1763 | /// bit (itself), but other cases can give us information. For example, |
1764 | /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal |
1765 | /// to each other, so we return 3. The DemandedElts argument allows |
1766 | /// us to only collect the minimum sign bits of the requested vector elements. |
1767 | /// Targets can implement the ComputeNumSignBitsForTarget method in the |
1768 | /// TargetLowering class to allow target nodes to be understood. |
1769 | unsigned ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, |
1770 | unsigned Depth = 0) const; |
1771 | |
1772 | /// Return true if this function can prove that \p Op is never poison |
1773 | /// and, if \p PoisonOnly is false, does not have undef bits. |
1774 | bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly = false, |
1775 | unsigned Depth = 0) const; |
1776 | |
1777 | /// Return true if this function can prove that \p Op is never poison |
1778 | /// and, if \p PoisonOnly is false, does not have undef bits. The DemandedElts |
1779 | /// argument limits the check to the requested vector elements. |
1780 | bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, const APInt &DemandedElts, |
1781 | bool PoisonOnly = false, |
1782 | unsigned Depth = 0) const; |
1783 | |
1784 | /// Return true if this function can prove that \p Op is never poison. |
1785 | bool isGuaranteedNotToBePoison(SDValue Op, unsigned Depth = 0) const { |
1786 | return isGuaranteedNotToBeUndefOrPoison(Op, /*PoisonOnly*/ true, Depth); |
1787 | } |
1788 | |
1789 | /// Return true if this function can prove that \p Op is never poison. The |
1790 | /// DemandedElts argument limits the check to the requested vector elements. |
1791 | bool isGuaranteedNotToBePoison(SDValue Op, const APInt &DemandedElts, |
1792 | unsigned Depth = 0) const { |
1793 | return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, |
1794 | /*PoisonOnly*/ true, Depth); |
1795 | } |
1796 | |
1797 | /// Return true if the specified operand is an ISD::ADD with a ConstantSDNode |
1798 | /// on the right-hand side, or if it is an ISD::OR with a ConstantSDNode that |
1799 | /// is guaranteed to have the same semantics as an ADD. This handles the |
1800 | /// equivalence: |
1801 | /// X|Cst == X+Cst iff X&Cst = 0. |
1802 | bool isBaseWithConstantOffset(SDValue Op) const; |
1803 | |
1804 | /// Test whether the given SDValue is known to never be NaN. If \p SNaN is |
1805 | /// true, returns if \p Op is known to never be a signaling NaN (it may still |
1806 | /// be a qNaN). |
1807 | bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const; |
1808 | |
1809 | /// \returns true if \p Op is known to never be a signaling NaN. |
1810 | bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const { |
1811 | return isKnownNeverNaN(Op, true, Depth); |
1812 | } |
1813 | |
1814 | /// Test whether the given floating point SDValue is known to never be |
1815 | /// positive or negative zero. |
1816 | bool isKnownNeverZeroFloat(SDValue Op) const; |
1817 | |
1818 | /// Test whether the given SDValue is known to contain non-zero value(s). |
1819 | bool isKnownNeverZero(SDValue Op) const; |
1820 | |
1821 | /// Test whether two SDValues are known to compare equal. This |
1822 | /// is true if they are the same value, or if one is negative zero and the |
1823 | /// other positive zero. |
1824 | bool isEqualTo(SDValue A, SDValue B) const; |
1825 | |
1826 | /// Return true if A and B have no common bits set. As an example, this can |
1827 | /// allow an 'add' to be transformed into an 'or'. |
1828 | bool haveNoCommonBitsSet(SDValue A, SDValue B) const; |
1829 | |
1830 | /// Test whether \p V has a splatted value for all the demanded elements. |
1831 | /// |
1832 | /// On success \p UndefElts will indicate the elements that have UNDEF |
1833 | /// values instead of the splat value, this is only guaranteed to be correct |
1834 | /// for \p DemandedElts. |
1835 | /// |
1836 | /// NOTE: The function will return true for a demanded splat of UNDEF values. |
1837 | bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, |
1838 | unsigned Depth = 0); |
1839 | |
1840 | /// Test whether \p V has a splatted value. |
1841 | bool isSplatValue(SDValue V, bool AllowUndefs = false); |
1842 | |
1843 | /// If V is a splatted value, return the source vector and its splat index. |
1844 | SDValue getSplatSourceVector(SDValue V, int &SplatIndex); |
1845 | |
1846 | /// If V is a splat vector, return its scalar source operand by extracting |
1847 | /// that element from the source vector. If LegalTypes is true, this method |
1848 | /// may only return a legally-typed splat value. If it cannot legalize the |
1849 | /// splatted value it will return SDValue(). |
1850 | SDValue getSplatValue(SDValue V, bool LegalTypes = false); |
1851 | |
1852 | /// If a SHL/SRA/SRL node \p V has a constant or splat constant shift amount |
1853 | /// that is less than the element bit-width of the shift node, return it. |
1854 | const APInt *getValidShiftAmountConstant(SDValue V, |
1855 | const APInt &DemandedElts) const; |
1856 | |
1857 | /// If a SHL/SRA/SRL node \p V has constant shift amounts that are all less |
1858 | /// than the element bit-width of the shift node, return the minimum value. |
1859 | const APInt * |
1860 | getValidMinimumShiftAmountConstant(SDValue V, |
1861 | const APInt &DemandedElts) const; |
1862 | |
1863 | /// If a SHL/SRA/SRL node \p V has constant shift amounts that are all less |
1864 | /// than the element bit-width of the shift node, return the maximum value. |
1865 | const APInt * |
1866 | getValidMaximumShiftAmountConstant(SDValue V, |
1867 | const APInt &DemandedElts) const; |
1868 | |
1869 | /// Match a binop + shuffle pyramid that represents a horizontal reduction |
1870 | /// over the elements of a vector starting from the EXTRACT_VECTOR_ELT node /p |
1871 | /// Extract. The reduction must use one of the opcodes listed in /p |
1872 | /// CandidateBinOps and on success /p BinOp will contain the matching opcode. |
1873 | /// Returns the vector that is being reduced on, or SDValue() if a reduction |
1874 | /// was not matched. If \p AllowPartials is set then in the case of a |
1875 | /// reduction pattern that only matches the first few stages, the extracted |
1876 | /// subvector of the start of the reduction is returned. |
1877 | SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, |
1878 | ArrayRef<ISD::NodeType> CandidateBinOps, |
1879 | bool AllowPartials = false); |
1880 | |
1881 | /// Utility function used by legalize and lowering to |
1882 | /// "unroll" a vector operation by splitting out the scalars and operating |
1883 | /// on each element individually. If the ResNE is 0, fully unroll the vector |
1884 | /// op. If ResNE is less than the width of the vector op, unroll up to ResNE. |
1885 | /// If the ResNE is greater than the width of the vector op, unroll the |
1886 | /// vector op and fill the end of the resulting vector with UNDEFS. |
1887 | SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0); |
1888 | |
1889 | /// Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes. |
1890 | /// This is a separate function because those opcodes have two results. |
1891 | std::pair<SDValue, SDValue> UnrollVectorOverflowOp(SDNode *N, |
1892 | unsigned ResNE = 0); |
1893 | |
1894 | /// Return true if loads are next to each other and can be |
1895 | /// merged. Check that both are nonvolatile and if LD is loading |
1896 | /// 'Bytes' bytes from a location that is 'Dist' units away from the |
1897 | /// location that the 'Base' load is loading from. |
1898 | bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, |
1899 | unsigned Bytes, int Dist) const; |
1900 | |
1901 | /// Infer alignment of a load / store address. Return None if it cannot be |
1902 | /// inferred. |
1903 | MaybeAlign InferPtrAlign(SDValue Ptr) const; |
1904 | |
1905 | /// Compute the VTs needed for the low/hi parts of a type |
1906 | /// which is split (or expanded) into two not necessarily identical pieces. |
1907 | std::pair<EVT, EVT> GetSplitDestVTs(const EVT &VT) const; |
1908 | |
1909 | /// Compute the VTs needed for the low/hi parts of a type, dependent on an |
1910 | /// enveloping VT that has been split into two identical pieces. Sets the |
1911 | /// HisIsEmpty flag when hi type has zero storage size. |
1912 | std::pair<EVT, EVT> GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, |
1913 | bool *HiIsEmpty) const; |
1914 | |
1915 | /// Split the vector with EXTRACT_SUBVECTOR using the provides |
1916 | /// VTs and return the low/high part. |
1917 | std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL, |
1918 | const EVT &LoVT, const EVT &HiVT); |
1919 | |
1920 | /// Split the vector with EXTRACT_SUBVECTOR and return the low/high part. |
1921 | std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) { |
1922 | EVT LoVT, HiVT; |
1923 | std::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType()); |
1924 | return SplitVector(N, DL, LoVT, HiVT); |
1925 | } |
1926 | |
1927 | /// Split the node's operand with EXTRACT_SUBVECTOR and |
1928 | /// return the low/high part. |
1929 | std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo) |
1930 | { |
1931 | return SplitVector(N->getOperand(OpNo), SDLoc(N)); |
1932 | } |
1933 | |
1934 | /// Widen the vector up to the next power of two using INSERT_SUBVECTOR. |
1935 | SDValue WidenVector(const SDValue &N, const SDLoc &DL); |
1936 | |
1937 | /// Append the extracted elements from Start to Count out of the vector Op in |
1938 | /// Args. If Count is 0, all of the elements will be extracted. The extracted |
1939 | /// elements will have type EVT if it is provided, and otherwise their type |
1940 | /// will be Op's element type. |
1941 | void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args, |
1942 | unsigned Start = 0, unsigned Count = 0, |
1943 | EVT EltVT = EVT()); |
1944 | |
1945 | /// Compute the default alignment value for the given type. |
1946 | Align getEVTAlign(EVT MemoryVT) const; |
1947 | /// Compute the default alignment value for the given type. |
1948 | /// FIXME: Remove once transition to Align is over. |
1949 | inline unsigned getEVTAlignment(EVT MemoryVT) const { |
1950 | return getEVTAlign(MemoryVT).value(); |
1951 | } |
1952 | |
1953 | /// Test whether the given value is a constant int or similar node. |
1954 | SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N) const; |
1955 | |
1956 | /// Test whether the given value is a constant FP or similar node. |
1957 | SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) const ; |
1958 | |
1959 | /// \returns true if \p N is any kind of constant or build_vector of |
1960 | /// constants, int or float. If a vector, it may not necessarily be a splat. |
1961 | inline bool isConstantValueOfAnyType(SDValue N) const { |
1962 | return isConstantIntBuildVectorOrConstantInt(N) || |
1963 | isConstantFPBuildVectorOrConstantFP(N); |
1964 | } |
1965 | |
1966 | void addCallSiteInfo(const SDNode *CallNode, CallSiteInfoImpl &&CallInfo) { |
1967 | SDCallSiteDbgInfo[CallNode].CSInfo = std::move(CallInfo); |
1968 | } |
1969 | |
1970 | CallSiteInfo getSDCallSiteInfo(const SDNode *CallNode) { |
1971 | auto I = SDCallSiteDbgInfo.find(CallNode); |
1972 | if (I != SDCallSiteDbgInfo.end()) |
1973 | return std::move(I->second).CSInfo; |
1974 | return CallSiteInfo(); |
1975 | } |
1976 | |
1977 | void addHeapAllocSite(const SDNode *Node, MDNode *MD) { |
1978 | SDCallSiteDbgInfo[Node].HeapAllocSite = MD; |
1979 | } |
1980 | |
1981 | /// Return the HeapAllocSite type associated with the SDNode, if it exists. |
1982 | MDNode *getHeapAllocSite(const SDNode *Node) { |
1983 | auto It = SDCallSiteDbgInfo.find(Node); |
1984 | if (It == SDCallSiteDbgInfo.end()) |
1985 | return nullptr; |
1986 | return It->second.HeapAllocSite; |
1987 | } |
1988 | |
1989 | void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge) { |
1990 | if (NoMerge) |
1991 | SDCallSiteDbgInfo[Node].NoMerge = NoMerge; |
1992 | } |
1993 | |
1994 | bool getNoMergeSiteInfo(const SDNode *Node) { |
1995 | auto I = SDCallSiteDbgInfo.find(Node); |
1996 | if (I == SDCallSiteDbgInfo.end()) |
1997 | return false; |
1998 | return I->second.NoMerge; |
1999 | } |
2000 | |
2001 | /// Return the current function's default denormal handling kind for the given |
2002 | /// floating point type. |
2003 | DenormalMode getDenormalMode(EVT VT) const { |
2004 | return MF->getDenormalMode(EVTToAPFloatSemantics(VT)); |
2005 | } |
2006 | |
2007 | bool shouldOptForSize() const; |
2008 | |
2009 | /// Get the (commutative) neutral element for the given opcode, if it exists. |
2010 | SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, |
2011 | SDNodeFlags Flags); |
2012 | |
2013 | private: |
2014 | void InsertNode(SDNode *N); |
2015 | bool RemoveNodeFromCSEMaps(SDNode *N); |
2016 | void AddModifiedNodeToCSEMaps(SDNode *N); |
2017 | SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op, void *&InsertPos); |
2018 | SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2, |
2019 | void *&InsertPos); |
2020 | SDNode *FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops, |
2021 | void *&InsertPos); |
2022 | SDNode *UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &loc); |
2023 | |
2024 | void DeleteNodeNotInCSEMaps(SDNode *N); |
2025 | void DeallocateNode(SDNode *N); |
2026 | |
2027 | void allnodes_clear(); |
2028 | |
2029 | /// Look up the node specified by ID in CSEMap. If it exists, return it. If |
2030 | /// not, return the insertion token that will make insertion faster. This |
2031 | /// overload is for nodes other than Constant or ConstantFP, use the other one |
2032 | /// for those. |
2033 | SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, void *&InsertPos); |
2034 | |
2035 | /// Look up the node specified by ID in CSEMap. If it exists, return it. If |
2036 | /// not, return the insertion token that will make insertion faster. Performs |
2037 | /// additional processing for constant nodes. |
2038 | SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, const SDLoc &DL, |
2039 | void *&InsertPos); |
2040 | |
2041 | /// List of non-single value types. |
2042 | FoldingSet<SDVTListNode> VTListMap; |
2043 | |
2044 | /// Maps to auto-CSE operations. |
2045 | std::vector<CondCodeSDNode*> CondCodeNodes; |
2046 | |
2047 | std::vector<SDNode*> ValueTypeNodes; |
2048 | std::map<EVT, SDNode*, EVT::compareRawBits> ExtendedValueTypeNodes; |
2049 | StringMap<SDNode*> ExternalSymbols; |
2050 | |
2051 | std::map<std::pair<std::string, unsigned>, SDNode *> TargetExternalSymbols; |
2052 | DenseMap<MCSymbol *, SDNode *> MCSymbols; |
2053 | |
2054 | FlagInserter *Inserter = nullptr; |
2055 | }; |
2056 | |
2057 | template <> struct GraphTraits<SelectionDAG*> : public GraphTraits<SDNode*> { |
2058 | using nodes_iterator = pointer_iterator<SelectionDAG::allnodes_iterator>; |
2059 | |
2060 | static nodes_iterator nodes_begin(SelectionDAG *G) { |
2061 | return nodes_iterator(G->allnodes_begin()); |
2062 | } |
2063 | |
2064 | static nodes_iterator nodes_end(SelectionDAG *G) { |
2065 | return nodes_iterator(G->allnodes_end()); |
2066 | } |
2067 | }; |
2068 | |
2069 | } // end namespace llvm |
2070 | |
2071 | #endif // LLVM_CODEGEN_SELECTIONDAG_H |
1 | //===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // This file declares the SDNode class and derived classes, which are used to | |||
10 | // represent the nodes and operations present in a SelectionDAG. These nodes | |||
11 | // and operations are machine code level operations, with some similarities to | |||
12 | // the GCC RTL representation. | |||
13 | // | |||
14 | // Clients should include the SelectionDAG.h file instead of this file directly. | |||
15 | // | |||
16 | //===----------------------------------------------------------------------===// | |||
17 | ||||
18 | #ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H | |||
19 | #define LLVM_CODEGEN_SELECTIONDAGNODES_H | |||
20 | ||||
21 | #include "llvm/ADT/APFloat.h" | |||
22 | #include "llvm/ADT/ArrayRef.h" | |||
23 | #include "llvm/ADT/BitVector.h" | |||
24 | #include "llvm/ADT/FoldingSet.h" | |||
25 | #include "llvm/ADT/GraphTraits.h" | |||
26 | #include "llvm/ADT/SmallPtrSet.h" | |||
27 | #include "llvm/ADT/SmallVector.h" | |||
28 | #include "llvm/ADT/ilist_node.h" | |||
29 | #include "llvm/ADT/iterator.h" | |||
30 | #include "llvm/ADT/iterator_range.h" | |||
31 | #include "llvm/CodeGen/ISDOpcodes.h" | |||
32 | #include "llvm/CodeGen/MachineMemOperand.h" | |||
33 | #include "llvm/CodeGen/Register.h" | |||
34 | #include "llvm/CodeGen/ValueTypes.h" | |||
35 | #include "llvm/IR/Constants.h" | |||
36 | #include "llvm/IR/DebugLoc.h" | |||
37 | #include "llvm/IR/Instruction.h" | |||
38 | #include "llvm/IR/Instructions.h" | |||
39 | #include "llvm/IR/Metadata.h" | |||
40 | #include "llvm/IR/Operator.h" | |||
41 | #include "llvm/Support/AlignOf.h" | |||
42 | #include "llvm/Support/AtomicOrdering.h" | |||
43 | #include "llvm/Support/Casting.h" | |||
44 | #include "llvm/Support/ErrorHandling.h" | |||
45 | #include "llvm/Support/MachineValueType.h" | |||
46 | #include "llvm/Support/TypeSize.h" | |||
47 | #include <algorithm> | |||
48 | #include <cassert> | |||
49 | #include <climits> | |||
50 | #include <cstddef> | |||
51 | #include <cstdint> | |||
52 | #include <cstring> | |||
53 | #include <iterator> | |||
54 | #include <string> | |||
55 | #include <tuple> | |||
56 | ||||
57 | namespace llvm { | |||
58 | ||||
59 | class APInt; | |||
60 | class Constant; | |||
61 | template <typename T> struct DenseMapInfo; | |||
62 | class GlobalValue; | |||
63 | class MachineBasicBlock; | |||
64 | class MachineConstantPoolValue; | |||
65 | class MCSymbol; | |||
66 | class raw_ostream; | |||
67 | class SDNode; | |||
68 | class SelectionDAG; | |||
69 | class Type; | |||
70 | class Value; | |||
71 | ||||
72 | void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr, | |||
73 | bool force = false); | |||
74 | ||||
75 | /// This represents a list of ValueType's that has been intern'd by | |||
76 | /// a SelectionDAG. Instances of this simple value class are returned by | |||
77 | /// SelectionDAG::getVTList(...). | |||
78 | /// | |||
79 | struct SDVTList { | |||
80 | const EVT *VTs; | |||
81 | unsigned int NumVTs; | |||
82 | }; | |||
83 | ||||
84 | namespace ISD { | |||
85 | ||||
86 | /// Node predicates | |||
87 | ||||
88 | /// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the | |||
89 | /// same constant or undefined, return true and return the constant value in | |||
90 | /// \p SplatValue. | |||
91 | bool isConstantSplatVector(const SDNode *N, APInt &SplatValue); | |||
92 | ||||
93 | /// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where | |||
94 | /// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to | |||
95 | /// true, it only checks BUILD_VECTOR. | |||
96 | bool isConstantSplatVectorAllOnes(const SDNode *N, | |||
97 | bool BuildVectorOnly = false); | |||
98 | ||||
99 | /// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where | |||
100 | /// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it | |||
101 | /// only checks BUILD_VECTOR. | |||
102 | bool isConstantSplatVectorAllZeros(const SDNode *N, | |||
103 | bool BuildVectorOnly = false); | |||
104 | ||||
105 | /// Return true if the specified node is a BUILD_VECTOR where all of the | |||
106 | /// elements are ~0 or undef. | |||
107 | bool isBuildVectorAllOnes(const SDNode *N); | |||
108 | ||||
109 | /// Return true if the specified node is a BUILD_VECTOR where all of the | |||
110 | /// elements are 0 or undef. | |||
111 | bool isBuildVectorAllZeros(const SDNode *N); | |||
112 | ||||
113 | /// Return true if the specified node is a BUILD_VECTOR node of all | |||
114 | /// ConstantSDNode or undef. | |||
115 | bool isBuildVectorOfConstantSDNodes(const SDNode *N); | |||
116 | ||||
117 | /// Return true if the specified node is a BUILD_VECTOR node of all | |||
118 | /// ConstantFPSDNode or undef. | |||
119 | bool isBuildVectorOfConstantFPSDNodes(const SDNode *N); | |||
120 | ||||
121 | /// Return true if the node has at least one operand and all operands of the | |||
122 | /// specified node are ISD::UNDEF. | |||
123 | bool allOperandsUndef(const SDNode *N); | |||
124 | ||||
125 | } // end namespace ISD | |||
126 | ||||
127 | //===----------------------------------------------------------------------===// | |||
128 | /// Unlike LLVM values, Selection DAG nodes may return multiple | |||
129 | /// values as the result of a computation. Many nodes return multiple values, | |||
130 | /// from loads (which define a token and a return value) to ADDC (which returns | |||
131 | /// a result and a carry value), to calls (which may return an arbitrary number | |||
132 | /// of values). | |||
133 | /// | |||
134 | /// As such, each use of a SelectionDAG computation must indicate the node that | |||
135 | /// computes it as well as which return value to use from that node. This pair | |||
136 | /// of information is represented with the SDValue value type. | |||
137 | /// | |||
138 | class SDValue { | |||
139 | friend struct DenseMapInfo<SDValue>; | |||
140 | ||||
141 | SDNode *Node = nullptr; // The node defining the value we are using. | |||
142 | unsigned ResNo = 0; // Which return value of the node we are using. | |||
143 | ||||
144 | public: | |||
145 | SDValue() = default; | |||
146 | SDValue(SDNode *node, unsigned resno); | |||
147 | ||||
148 | /// get the index which selects a specific result in the SDNode | |||
149 | unsigned getResNo() const { return ResNo; } | |||
150 | ||||
151 | /// get the SDNode which holds the desired result | |||
152 | SDNode *getNode() const { return Node; } | |||
153 | ||||
154 | /// set the SDNode | |||
155 | void setNode(SDNode *N) { Node = N; } | |||
156 | ||||
157 | inline SDNode *operator->() const { return Node; } | |||
158 | ||||
159 | bool operator==(const SDValue &O) const { | |||
160 | return Node == O.Node && ResNo == O.ResNo; | |||
161 | } | |||
162 | bool operator!=(const SDValue &O) const { | |||
163 | return !operator==(O); | |||
164 | } | |||
165 | bool operator<(const SDValue &O) const { | |||
166 | return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo); | |||
167 | } | |||
168 | explicit operator bool() const { | |||
169 | return Node != nullptr; | |||
170 | } | |||
171 | ||||
172 | SDValue getValue(unsigned R) const { | |||
173 | return SDValue(Node, R); | |||
174 | } | |||
175 | ||||
176 | /// Return true if this node is an operand of N. | |||
177 | bool isOperandOf(const SDNode *N) const; | |||
178 | ||||
179 | /// Return the ValueType of the referenced return value. | |||
180 | inline EVT getValueType() const; | |||
181 | ||||
182 | /// Return the simple ValueType of the referenced return value. | |||
183 | MVT getSimpleValueType() const { | |||
184 | return getValueType().getSimpleVT(); | |||
185 | } | |||
186 | ||||
187 | /// Returns the size of the value in bits. | |||
188 | /// | |||
189 | /// If the value type is a scalable vector type, the scalable property will | |||
190 | /// be set and the runtime size will be a positive integer multiple of the | |||
191 | /// base size. | |||
192 | TypeSize getValueSizeInBits() const { | |||
193 | return getValueType().getSizeInBits(); | |||
194 | } | |||
195 | ||||
196 | uint64_t getScalarValueSizeInBits() const { | |||
197 | return getValueType().getScalarType().getFixedSizeInBits(); | |||
198 | } | |||
199 | ||||
200 | // Forwarding methods - These forward to the corresponding methods in SDNode. | |||
201 | inline unsigned getOpcode() const; | |||
202 | inline unsigned getNumOperands() const; | |||
203 | inline const SDValue &getOperand(unsigned i) const; | |||
204 | inline uint64_t getConstantOperandVal(unsigned i) const; | |||
205 | inline const APInt &getConstantOperandAPInt(unsigned i) const; | |||
206 | inline bool isTargetMemoryOpcode() const; | |||
207 | inline bool isTargetOpcode() const; | |||
208 | inline bool isMachineOpcode() const; | |||
209 | inline bool isUndef() const; | |||
210 | inline unsigned getMachineOpcode() const; | |||
211 | inline const DebugLoc &getDebugLoc() const; | |||
212 | inline void dump() const; | |||
213 | inline void dump(const SelectionDAG *G) const; | |||
214 | inline void dumpr() const; | |||
215 | inline void dumpr(const SelectionDAG *G) const; | |||
216 | ||||
217 | /// Return true if this operand (which must be a chain) reaches the | |||
218 | /// specified operand without crossing any side-effecting instructions. | |||
219 | /// In practice, this looks through token factors and non-volatile loads. | |||
220 | /// In order to remain efficient, this only | |||
221 | /// looks a couple of nodes in, it does not do an exhaustive search. | |||
222 | bool reachesChainWithoutSideEffects(SDValue Dest, | |||
223 | unsigned Depth = 2) const; | |||
224 | ||||
225 | /// Return true if there are no nodes using value ResNo of Node. | |||
226 | inline bool use_empty() const; | |||
227 | ||||
228 | /// Return true if there is exactly one node using value ResNo of Node. | |||
229 | inline bool hasOneUse() const; | |||
230 | }; | |||
231 | ||||
232 | template<> struct DenseMapInfo<SDValue> { | |||
233 | static inline SDValue getEmptyKey() { | |||
234 | SDValue V; | |||
235 | V.ResNo = -1U; | |||
236 | return V; | |||
237 | } | |||
238 | ||||
239 | static inline SDValue getTombstoneKey() { | |||
240 | SDValue V; | |||
241 | V.ResNo = -2U; | |||
242 | return V; | |||
243 | } | |||
244 | ||||
245 | static unsigned getHashValue(const SDValue &Val) { | |||
246 | return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^ | |||
247 | (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo(); | |||
248 | } | |||
249 | ||||
250 | static bool isEqual(const SDValue &LHS, const SDValue &RHS) { | |||
251 | return LHS == RHS; | |||
252 | } | |||
253 | }; | |||
254 | ||||
255 | /// Allow casting operators to work directly on | |||
256 | /// SDValues as if they were SDNode*'s. | |||
257 | template<> struct simplify_type<SDValue> { | |||
258 | using SimpleType = SDNode *; | |||
259 | ||||
260 | static SimpleType getSimplifiedValue(SDValue &Val) { | |||
261 | return Val.getNode(); | |||
262 | } | |||
263 | }; | |||
264 | template<> struct simplify_type<const SDValue> { | |||
265 | using SimpleType = /*const*/ SDNode *; | |||
266 | ||||
267 | static SimpleType getSimplifiedValue(const SDValue &Val) { | |||
268 | return Val.getNode(); | |||
269 | } | |||
270 | }; | |||
271 | ||||
272 | /// Represents a use of a SDNode. This class holds an SDValue, | |||
273 | /// which records the SDNode being used and the result number, a | |||
274 | /// pointer to the SDNode using the value, and Next and Prev pointers, | |||
275 | /// which link together all the uses of an SDNode. | |||
276 | /// | |||
277 | class SDUse { | |||
278 | /// Val - The value being used. | |||
279 | SDValue Val; | |||
280 | /// User - The user of this value. | |||
281 | SDNode *User = nullptr; | |||
282 | /// Prev, Next - Pointers to the uses list of the SDNode referred by | |||
283 | /// this operand. | |||
284 | SDUse **Prev = nullptr; | |||
285 | SDUse *Next = nullptr; | |||
286 | ||||
287 | public: | |||
288 | SDUse() = default; | |||
289 | SDUse(const SDUse &U) = delete; | |||
290 | SDUse &operator=(const SDUse &) = delete; | |||
291 | ||||
292 | /// Normally SDUse will just implicitly convert to an SDValue that it holds. | |||
293 | operator const SDValue&() const { return Val; } | |||
294 | ||||
295 | /// If implicit conversion to SDValue doesn't work, the get() method returns | |||
296 | /// the SDValue. | |||
297 | const SDValue &get() const { return Val; } | |||
298 | ||||
299 | /// This returns the SDNode that contains this Use. | |||
300 | SDNode *getUser() { return User; } | |||
301 | ||||
302 | /// Get the next SDUse in the use list. | |||
303 | SDUse *getNext() const { return Next; } | |||
304 | ||||
305 | /// Convenience function for get().getNode(). | |||
306 | SDNode *getNode() const { return Val.getNode(); } | |||
307 | /// Convenience function for get().getResNo(). | |||
308 | unsigned getResNo() const { return Val.getResNo(); } | |||
309 | /// Convenience function for get().getValueType(). | |||
310 | EVT getValueType() const { return Val.getValueType(); } | |||
311 | ||||
312 | /// Convenience function for get().operator== | |||
313 | bool operator==(const SDValue &V) const { | |||
314 | return Val == V; | |||
315 | } | |||
316 | ||||
317 | /// Convenience function for get().operator!= | |||
318 | bool operator!=(const SDValue &V) const { | |||
319 | return Val != V; | |||
320 | } | |||
321 | ||||
322 | /// Convenience function for get().operator< | |||
323 | bool operator<(const SDValue &V) const { | |||
324 | return Val < V; | |||
325 | } | |||
326 | ||||
327 | private: | |||
328 | friend class SelectionDAG; | |||
329 | friend class SDNode; | |||
330 | // TODO: unfriend HandleSDNode once we fix its operand handling. | |||
331 | friend class HandleSDNode; | |||
332 | ||||
333 | void setUser(SDNode *p) { User = p; } | |||
334 | ||||
335 | /// Remove this use from its existing use list, assign it the | |||
336 | /// given value, and add it to the new value's node's use list. | |||
337 | inline void set(const SDValue &V); | |||
338 | /// Like set, but only supports initializing a newly-allocated | |||
339 | /// SDUse with a non-null value. | |||
340 | inline void setInitial(const SDValue &V); | |||
341 | /// Like set, but only sets the Node portion of the value, | |||
342 | /// leaving the ResNo portion unmodified. | |||
343 | inline void setNode(SDNode *N); | |||
344 | ||||
345 | void addToList(SDUse **List) { | |||
346 | Next = *List; | |||
347 | if (Next) Next->Prev = &Next; | |||
348 | Prev = List; | |||
349 | *List = this; | |||
350 | } | |||
351 | ||||
352 | void removeFromList() { | |||
353 | *Prev = Next; | |||
354 | if (Next) Next->Prev = Prev; | |||
355 | } | |||
356 | }; | |||
357 | ||||
358 | /// simplify_type specializations - Allow casting operators to work directly on | |||
359 | /// SDValues as if they were SDNode*'s. | |||
360 | template<> struct simplify_type<SDUse> { | |||
361 | using SimpleType = SDNode *; | |||
362 | ||||
363 | static SimpleType getSimplifiedValue(SDUse &Val) { | |||
364 | return Val.getNode(); | |||
365 | } | |||
366 | }; | |||
367 | ||||
368 | /// These are IR-level optimization flags that may be propagated to SDNodes. | |||
369 | /// TODO: This data structure should be shared by the IR optimizer and the | |||
370 | /// the backend. | |||
371 | struct SDNodeFlags { | |||
372 | private: | |||
373 | bool NoUnsignedWrap : 1; | |||
374 | bool NoSignedWrap : 1; | |||
375 | bool Exact : 1; | |||
376 | bool NoNaNs : 1; | |||
377 | bool NoInfs : 1; | |||
378 | bool NoSignedZeros : 1; | |||
379 | bool AllowReciprocal : 1; | |||
380 | bool AllowContract : 1; | |||
381 | bool ApproximateFuncs : 1; | |||
382 | bool AllowReassociation : 1; | |||
383 | ||||
384 | // We assume instructions do not raise floating-point exceptions by default, | |||
385 | // and only those marked explicitly may do so. We could choose to represent | |||
386 | // this via a positive "FPExcept" flags like on the MI level, but having a | |||
387 | // negative "NoFPExcept" flag here (that defaults to true) makes the flag | |||
388 | // intersection logic more straightforward. | |||
389 | bool NoFPExcept : 1; | |||
390 | ||||
391 | public: | |||
392 | /// Default constructor turns off all optimization flags. | |||
393 | SDNodeFlags() | |||
394 | : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false), | |||
395 | NoInfs(false), NoSignedZeros(false), AllowReciprocal(false), | |||
396 | AllowContract(false), ApproximateFuncs(false), | |||
397 | AllowReassociation(false), NoFPExcept(false) {} | |||
398 | ||||
399 | /// Propagate the fast-math-flags from an IR FPMathOperator. | |||
400 | void copyFMF(const FPMathOperator &FPMO) { | |||
401 | setNoNaNs(FPMO.hasNoNaNs()); | |||
402 | setNoInfs(FPMO.hasNoInfs()); | |||
403 | setNoSignedZeros(FPMO.hasNoSignedZeros()); | |||
404 | setAllowReciprocal(FPMO.hasAllowReciprocal()); | |||
405 | setAllowContract(FPMO.hasAllowContract()); | |||
406 | setApproximateFuncs(FPMO.hasApproxFunc()); | |||
407 | setAllowReassociation(FPMO.hasAllowReassoc()); | |||
408 | } | |||
409 | ||||
410 | // These are mutators for each flag. | |||
411 | void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; } | |||
412 | void setNoSignedWrap(bool b) { NoSignedWrap = b; } | |||
413 | void setExact(bool b) { Exact = b; } | |||
414 | void setNoNaNs(bool b) { NoNaNs = b; } | |||
415 | void setNoInfs(bool b) { NoInfs = b; } | |||
416 | void setNoSignedZeros(bool b) { NoSignedZeros = b; } | |||
417 | void setAllowReciprocal(bool b) { AllowReciprocal = b; } | |||
418 | void setAllowContract(bool b) { AllowContract = b; } | |||
419 | void setApproximateFuncs(bool b) { ApproximateFuncs = b; } | |||
420 | void setAllowReassociation(bool b) { AllowReassociation = b; } | |||
421 | void setNoFPExcept(bool b) { NoFPExcept = b; } | |||
422 | ||||
423 | // These are accessors for each flag. | |||
424 | bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } | |||
425 | bool hasNoSignedWrap() const { return NoSignedWrap; } | |||
426 | bool hasExact() const { return Exact; } | |||
427 | bool hasNoNaNs() const { return NoNaNs; } | |||
428 | bool hasNoInfs() const { return NoInfs; } | |||
429 | bool hasNoSignedZeros() const { return NoSignedZeros; } | |||
430 | bool hasAllowReciprocal() const { return AllowReciprocal; } | |||
431 | bool hasAllowContract() const { return AllowContract; } | |||
432 | bool hasApproximateFuncs() const { return ApproximateFuncs; } | |||
433 | bool hasAllowReassociation() const { return AllowReassociation; } | |||
434 | bool hasNoFPExcept() const { return NoFPExcept; } | |||
435 | ||||
436 | /// Clear any flags in this flag set that aren't also set in Flags. All | |||
437 | /// flags will be cleared if Flags are undefined. | |||
438 | void intersectWith(const SDNodeFlags Flags) { | |||
439 | NoUnsignedWrap &= Flags.NoUnsignedWrap; | |||
440 | NoSignedWrap &= Flags.NoSignedWrap; | |||
441 | Exact &= Flags.Exact; | |||
442 | NoNaNs &= Flags.NoNaNs; | |||
443 | NoInfs &= Flags.NoInfs; | |||
444 | NoSignedZeros &= Flags.NoSignedZeros; | |||
445 | AllowReciprocal &= Flags.AllowReciprocal; | |||
446 | AllowContract &= Flags.AllowContract; | |||
447 | ApproximateFuncs &= Flags.ApproximateFuncs; | |||
448 | AllowReassociation &= Flags.AllowReassociation; | |||
449 | NoFPExcept &= Flags.NoFPExcept; | |||
450 | } | |||
451 | }; | |||
452 | ||||
453 | /// Represents one node in the SelectionDAG. | |||
454 | /// | |||
455 | class SDNode : public FoldingSetNode, public ilist_node<SDNode> { | |||
456 | private: | |||
457 | /// The operation that this node performs. | |||
458 | int16_t NodeType; | |||
459 | ||||
460 | protected: | |||
461 | // We define a set of mini-helper classes to help us interpret the bits in our | |||
462 | // SubclassData. These are designed to fit within a uint16_t so they pack | |||
463 | // with NodeType. | |||
464 | ||||
465 | #if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1)) | |||
466 | // Except for GCC; by default, AIX compilers store bit-fields in 4-byte words | |||
467 | // and give the `pack` pragma push semantics. | |||
468 | #define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2) | |||
469 | #define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop) | |||
470 | #else | |||
471 | #define BEGIN_TWO_BYTE_PACK() | |||
472 | #define END_TWO_BYTE_PACK() | |||
473 | #endif | |||
474 | ||||
475 | BEGIN_TWO_BYTE_PACK() | |||
476 | class SDNodeBitfields { | |||
477 | friend class SDNode; | |||
478 | friend class MemIntrinsicSDNode; | |||
479 | friend class MemSDNode; | |||
480 | friend class SelectionDAG; | |||
481 | ||||
482 | uint16_t HasDebugValue : 1; | |||
483 | uint16_t IsMemIntrinsic : 1; | |||
484 | uint16_t IsDivergent : 1; | |||
485 | }; | |||
486 | enum { NumSDNodeBits = 3 }; | |||
487 | ||||
488 | class ConstantSDNodeBitfields { | |||
489 | friend class ConstantSDNode; | |||
490 | ||||
491 | uint16_t : NumSDNodeBits; | |||
492 | ||||
493 | uint16_t IsOpaque : 1; | |||
494 | }; | |||
495 | ||||
496 | class MemSDNodeBitfields { | |||
497 | friend class MemSDNode; | |||
498 | friend class MemIntrinsicSDNode; | |||
499 | friend class AtomicSDNode; | |||
500 | ||||
501 | uint16_t : NumSDNodeBits; | |||
502 | ||||
503 | uint16_t IsVolatile : 1; | |||
504 | uint16_t IsNonTemporal : 1; | |||
505 | uint16_t IsDereferenceable : 1; | |||
506 | uint16_t IsInvariant : 1; | |||
507 | }; | |||
508 | enum { NumMemSDNodeBits = NumSDNodeBits + 4 }; | |||
509 | ||||
510 | class LSBaseSDNodeBitfields { | |||
511 | friend class LSBaseSDNode; | |||
512 | friend class MaskedLoadStoreSDNode; | |||
513 | friend class MaskedGatherScatterSDNode; | |||
514 | ||||
515 | uint16_t : NumMemSDNodeBits; | |||
516 | ||||
517 | // This storage is shared between disparate class hierarchies to hold an | |||
518 | // enumeration specific to the class hierarchy in use. | |||
519 | // LSBaseSDNode => enum ISD::MemIndexedMode | |||
520 | // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode | |||
521 | // MaskedGatherScatterSDNode => enum ISD::MemIndexType | |||
522 | uint16_t AddressingMode : 3; | |||
523 | }; | |||
524 | enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 }; | |||
525 | ||||
526 | class LoadSDNodeBitfields { | |||
527 | friend class LoadSDNode; | |||
528 | friend class MaskedLoadSDNode; | |||
529 | friend class MaskedGatherSDNode; | |||
530 | ||||
531 | uint16_t : NumLSBaseSDNodeBits; | |||
532 | ||||
533 | uint16_t ExtTy : 2; // enum ISD::LoadExtType | |||
534 | uint16_t IsExpanding : 1; | |||
535 | }; | |||
536 | ||||
537 | class StoreSDNodeBitfields { | |||
538 | friend class StoreSDNode; | |||
539 | friend class MaskedStoreSDNode; | |||
540 | friend class MaskedScatterSDNode; | |||
541 | ||||
542 | uint16_t : NumLSBaseSDNodeBits; | |||
543 | ||||
544 | uint16_t IsTruncating : 1; | |||
545 | uint16_t IsCompressing : 1; | |||
546 | }; | |||
547 | ||||
548 | union { | |||
549 | char RawSDNodeBits[sizeof(uint16_t)]; | |||
550 | SDNodeBitfields SDNodeBits; | |||
551 | ConstantSDNodeBitfields ConstantSDNodeBits; | |||
552 | MemSDNodeBitfields MemSDNodeBits; | |||
553 | LSBaseSDNodeBitfields LSBaseSDNodeBits; | |||
554 | LoadSDNodeBitfields LoadSDNodeBits; | |||
555 | StoreSDNodeBitfields StoreSDNodeBits; | |||
556 | }; | |||
557 | END_TWO_BYTE_PACK() | |||
558 | #undef BEGIN_TWO_BYTE_PACK | |||
559 | #undef END_TWO_BYTE_PACK | |||
560 | ||||
561 | // RawSDNodeBits must cover the entirety of the union. This means that all of | |||
562 | // the union's members must have size <= RawSDNodeBits. We write the RHS as | |||
563 | // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter. | |||
564 | static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide"); | |||
565 | static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide"); | |||
566 | static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide"); | |||
567 | static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide"); | |||
568 | static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide"); | |||
569 | static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide"); | |||
570 | ||||
571 | private: | |||
572 | friend class SelectionDAG; | |||
573 | // TODO: unfriend HandleSDNode once we fix its operand handling. | |||
574 | friend class HandleSDNode; | |||
575 | ||||
576 | /// Unique id per SDNode in the DAG. | |||
577 | int NodeId = -1; | |||
578 | ||||
579 | /// The values that are used by this operation. | |||
580 | SDUse *OperandList = nullptr; | |||
581 | ||||
582 | /// The types of the values this node defines. SDNode's may | |||
583 | /// define multiple values simultaneously. | |||
584 | const EVT *ValueList; | |||
585 | ||||
586 | /// List of uses for this SDNode. | |||
587 | SDUse *UseList = nullptr; | |||
588 | ||||
589 | /// The number of entries in the Operand/Value list. | |||
590 | unsigned short NumOperands = 0; | |||
591 | unsigned short NumValues; | |||
592 | ||||
593 | // The ordering of the SDNodes. It roughly corresponds to the ordering of the | |||
594 | // original LLVM instructions. | |||
595 | // This is used for turning off scheduling, because we'll forgo | |||
596 | // the normal scheduling algorithms and output the instructions according to | |||
597 | // this ordering. | |||
598 | unsigned IROrder; | |||
599 | ||||
600 | /// Source line information. | |||
601 | DebugLoc debugLoc; | |||
602 | ||||
603 | /// Return a pointer to the specified value type. | |||
604 | static const EVT *getValueTypeList(EVT VT); | |||
605 | ||||
606 | SDNodeFlags Flags; | |||
607 | ||||
608 | public: | |||
609 | /// Unique and persistent id per SDNode in the DAG. | |||
610 | /// Used for debug printing. | |||
611 | uint16_t PersistentId; | |||
612 | ||||
613 | //===--------------------------------------------------------------------===// | |||
614 | // Accessors | |||
615 | // | |||
616 | ||||
617 | /// Return the SelectionDAG opcode value for this node. For | |||
618 | /// pre-isel nodes (those for which isMachineOpcode returns false), these | |||
619 | /// are the opcode values in the ISD and <target>ISD namespaces. For | |||
620 | /// post-isel opcodes, see getMachineOpcode. | |||
621 | unsigned getOpcode() const { return (unsigned short)NodeType; } | |||
622 | ||||
623 | /// Test if this node has a target-specific opcode (in the | |||
624 | /// \<target\>ISD namespace). | |||
625 | bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; } | |||
626 | ||||
627 | /// Test if this node has a target-specific opcode that may raise | |||
628 | /// FP exceptions (in the \<target\>ISD namespace and greater than | |||
629 | /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory | |||
630 | /// opcode are currently automatically considered to possibly raise | |||
631 | /// FP exceptions as well. | |||
632 | bool isTargetStrictFPOpcode() const { | |||
633 | return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE; | |||
634 | } | |||
635 | ||||
636 | /// Test if this node has a target-specific | |||
637 | /// memory-referencing opcode (in the \<target\>ISD namespace and | |||
638 | /// greater than FIRST_TARGET_MEMORY_OPCODE). | |||
639 | bool isTargetMemoryOpcode() const { | |||
640 | return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE; | |||
641 | } | |||
642 | ||||
643 | /// Return true if the type of the node type undefined. | |||
644 | bool isUndef() const { return NodeType == ISD::UNDEF; } | |||
645 | ||||
646 | /// Test if this node is a memory intrinsic (with valid pointer information). | |||
647 | /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for | |||
648 | /// non-memory intrinsics (with chains) that are not really instances of | |||
649 | /// MemSDNode. For such nodes, we need some extra state to determine the | |||
650 | /// proper classof relationship. | |||
651 | bool isMemIntrinsic() const { | |||
652 | return (NodeType == ISD::INTRINSIC_W_CHAIN || | |||
653 | NodeType == ISD::INTRINSIC_VOID) && | |||
654 | SDNodeBits.IsMemIntrinsic; | |||
655 | } | |||
656 | ||||
657 | /// Test if this node is a strict floating point pseudo-op. | |||
658 | bool isStrictFPOpcode() { | |||
659 | switch (NodeType) { | |||
660 | default: | |||
661 | return false; | |||
662 | case ISD::STRICT_FP16_TO_FP: | |||
663 | case ISD::STRICT_FP_TO_FP16: | |||
664 | #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ | |||
665 | case ISD::STRICT_##DAGN: | |||
666 | #include "llvm/IR/ConstrainedOps.def" | |||
667 | return true; | |||
668 | } | |||
669 | } | |||
670 | ||||
671 | /// Test if this node has a post-isel opcode, directly | |||
672 | /// corresponding to a MachineInstr opcode. | |||
673 | bool isMachineOpcode() const { return NodeType < 0; } | |||
674 | ||||
675 | /// This may only be called if isMachineOpcode returns | |||
676 | /// true. It returns the MachineInstr opcode value that the node's opcode | |||
677 | /// corresponds to. | |||
678 | unsigned getMachineOpcode() const { | |||
679 | assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!" ) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 679, __extension__ __PRETTY_FUNCTION__)); | |||
680 | return ~NodeType; | |||
681 | } | |||
682 | ||||
683 | bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; } | |||
684 | void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; } | |||
685 | ||||
686 | bool isDivergent() const { return SDNodeBits.IsDivergent; } | |||
687 | ||||
688 | /// Return true if there are no uses of this node. | |||
689 | bool use_empty() const { return UseList == nullptr; } | |||
690 | ||||
691 | /// Return true if there is exactly one use of this node. | |||
692 | bool hasOneUse() const { return hasSingleElement(uses()); } | |||
693 | ||||
694 | /// Return the number of uses of this node. This method takes | |||
695 | /// time proportional to the number of uses. | |||
696 | size_t use_size() const { return std::distance(use_begin(), use_end()); } | |||
697 | ||||
698 | /// Return the unique node id. | |||
699 | int getNodeId() const { return NodeId; } | |||
700 | ||||
701 | /// Set unique node id. | |||
702 | void setNodeId(int Id) { NodeId = Id; } | |||
703 | ||||
704 | /// Return the node ordering. | |||
705 | unsigned getIROrder() const { return IROrder; } | |||
706 | ||||
707 | /// Set the node ordering. | |||
708 | void setIROrder(unsigned Order) { IROrder = Order; } | |||
709 | ||||
710 | /// Return the source location info. | |||
711 | const DebugLoc &getDebugLoc() const { return debugLoc; } | |||
712 | ||||
713 | /// Set source location info. Try to avoid this, putting | |||
714 | /// it in the constructor is preferable. | |||
715 | void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); } | |||
716 | ||||
717 | /// This class provides iterator support for SDUse | |||
718 | /// operands that use a specific SDNode. | |||
719 | class use_iterator { | |||
720 | friend class SDNode; | |||
721 | ||||
722 | SDUse *Op = nullptr; | |||
723 | ||||
724 | explicit use_iterator(SDUse *op) : Op(op) {} | |||
725 | ||||
726 | public: | |||
727 | using iterator_category = std::forward_iterator_tag; | |||
728 | using value_type = SDUse; | |||
729 | using difference_type = std::ptrdiff_t; | |||
730 | using pointer = value_type *; | |||
731 | using reference = value_type &; | |||
732 | ||||
733 | use_iterator() = default; | |||
734 | use_iterator(const use_iterator &I) : Op(I.Op) {} | |||
735 | ||||
736 | bool operator==(const use_iterator &x) const { | |||
737 | return Op == x.Op; | |||
738 | } | |||
739 | bool operator!=(const use_iterator &x) const { | |||
740 | return !operator==(x); | |||
741 | } | |||
742 | ||||
743 | /// Return true if this iterator is at the end of uses list. | |||
744 | bool atEnd() const { return Op == nullptr; } | |||
745 | ||||
746 | // Iterator traversal: forward iteration only. | |||
747 | use_iterator &operator++() { // Preincrement | |||
748 | assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!" ) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 748, __extension__ __PRETTY_FUNCTION__)); | |||
749 | Op = Op->getNext(); | |||
750 | return *this; | |||
751 | } | |||
752 | ||||
753 | use_iterator operator++(int) { // Postincrement | |||
754 | use_iterator tmp = *this; ++*this; return tmp; | |||
755 | } | |||
756 | ||||
757 | /// Retrieve a pointer to the current user node. | |||
758 | SDNode *operator*() const { | |||
759 | assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!" ) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 759, __extension__ __PRETTY_FUNCTION__)); | |||
760 | return Op->getUser(); | |||
761 | } | |||
762 | ||||
763 | SDNode *operator->() const { return operator*(); } | |||
764 | ||||
765 | SDUse &getUse() const { return *Op; } | |||
766 | ||||
767 | /// Retrieve the operand # of this use in its user. | |||
768 | unsigned getOperandNo() const { | |||
769 | assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!" ) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 769, __extension__ __PRETTY_FUNCTION__)); | |||
770 | return (unsigned)(Op - Op->getUser()->OperandList); | |||
771 | } | |||
772 | }; | |||
773 | ||||
774 | /// Provide iteration support to walk over all uses of an SDNode. | |||
775 | use_iterator use_begin() const { | |||
776 | return use_iterator(UseList); | |||
777 | } | |||
778 | ||||
779 | static use_iterator use_end() { return use_iterator(nullptr); } | |||
780 | ||||
781 | inline iterator_range<use_iterator> uses() { | |||
782 | return make_range(use_begin(), use_end()); | |||
783 | } | |||
784 | inline iterator_range<use_iterator> uses() const { | |||
785 | return make_range(use_begin(), use_end()); | |||
786 | } | |||
787 | ||||
788 | /// Return true if there are exactly NUSES uses of the indicated value. | |||
789 | /// This method ignores uses of other values defined by this operation. | |||
790 | bool hasNUsesOfValue(unsigned NUses, unsigned Value) const; | |||
791 | ||||
792 | /// Return true if there are any use of the indicated value. | |||
793 | /// This method ignores uses of other values defined by this operation. | |||
794 | bool hasAnyUseOfValue(unsigned Value) const; | |||
795 | ||||
796 | /// Return true if this node is the only use of N. | |||
797 | bool isOnlyUserOf(const SDNode *N) const; | |||
798 | ||||
799 | /// Return true if this node is an operand of N. | |||
800 | bool isOperandOf(const SDNode *N) const; | |||
801 | ||||
802 | /// Return true if this node is a predecessor of N. | |||
803 | /// NOTE: Implemented on top of hasPredecessor and every bit as | |||
804 | /// expensive. Use carefully. | |||
805 | bool isPredecessorOf(const SDNode *N) const { | |||
806 | return N->hasPredecessor(this); | |||
807 | } | |||
808 | ||||
809 | /// Return true if N is a predecessor of this node. | |||
810 | /// N is either an operand of this node, or can be reached by recursively | |||
811 | /// traversing up the operands. | |||
812 | /// NOTE: This is an expensive method. Use it carefully. | |||
813 | bool hasPredecessor(const SDNode *N) const; | |||
814 | ||||
815 | /// Returns true if N is a predecessor of any node in Worklist. This | |||
816 | /// helper keeps Visited and Worklist sets externally to allow unions | |||
817 | /// searches to be performed in parallel, caching of results across | |||
818 | /// queries and incremental addition to Worklist. Stops early if N is | |||
819 | /// found but will resume. Remember to clear Visited and Worklists | |||
820 | /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before | |||
821 | /// giving up. The TopologicalPrune flag signals that positive NodeIds are | |||
822 | /// topologically ordered (Operands have strictly smaller node id) and search | |||
823 | /// can be pruned leveraging this. | |||
824 | static bool hasPredecessorHelper(const SDNode *N, | |||
825 | SmallPtrSetImpl<const SDNode *> &Visited, | |||
826 | SmallVectorImpl<const SDNode *> &Worklist, | |||
827 | unsigned int MaxSteps = 0, | |||
828 | bool TopologicalPrune = false) { | |||
829 | SmallVector<const SDNode *, 8> DeferredNodes; | |||
830 | if (Visited.count(N)) | |||
831 | return true; | |||
832 | ||||
833 | // Node Id's are assigned in three places: As a topological | |||
834 | // ordering (> 0), during legalization (results in values set to | |||
835 | // 0), new nodes (set to -1). If N has a topolgical id then we | |||
836 | // know that all nodes with ids smaller than it cannot be | |||
837 | // successors and we need not check them. Filter out all node | |||
838 | // that can't be matches. We add them to the worklist before exit | |||
839 | // in case of multiple calls. Note that during selection the topological id | |||
840 | // may be violated if a node's predecessor is selected before it. We mark | |||
841 | // this at selection negating the id of unselected successors and | |||
842 | // restricting topological pruning to positive ids. | |||
843 | ||||
844 | int NId = N->getNodeId(); | |||
845 | // If we Invalidated the Id, reconstruct original NId. | |||
846 | if (NId < -1) | |||
847 | NId = -(NId + 1); | |||
848 | ||||
849 | bool Found = false; | |||
850 | while (!Worklist.empty()) { | |||
851 | const SDNode *M = Worklist.pop_back_val(); | |||
852 | int MId = M->getNodeId(); | |||
853 | if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) && | |||
854 | (MId > 0) && (MId < NId)) { | |||
855 | DeferredNodes.push_back(M); | |||
856 | continue; | |||
857 | } | |||
858 | for (const SDValue &OpV : M->op_values()) { | |||
859 | SDNode *Op = OpV.getNode(); | |||
860 | if (Visited.insert(Op).second) | |||
861 | Worklist.push_back(Op); | |||
862 | if (Op == N) | |||
863 | Found = true; | |||
864 | } | |||
865 | if (Found) | |||
866 | break; | |||
867 | if (MaxSteps != 0 && Visited.size() >= MaxSteps) | |||
868 | break; | |||
869 | } | |||
870 | // Push deferred nodes back on worklist. | |||
871 | Worklist.append(DeferredNodes.begin(), DeferredNodes.end()); | |||
872 | // If we bailed early, conservatively return found. | |||
873 | if (MaxSteps != 0 && Visited.size() >= MaxSteps) | |||
874 | return true; | |||
875 | return Found; | |||
876 | } | |||
877 | ||||
878 | /// Return true if all the users of N are contained in Nodes. | |||
879 | /// NOTE: Requires at least one match, but doesn't require them all. | |||
880 | static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N); | |||
881 | ||||
882 | /// Return the number of values used by this operation. | |||
883 | unsigned getNumOperands() const { return NumOperands; } | |||
884 | ||||
885 | /// Return the maximum number of operands that a SDNode can hold. | |||
886 | static constexpr size_t getMaxNumOperands() { | |||
887 | return std::numeric_limits<decltype(SDNode::NumOperands)>::max(); | |||
888 | } | |||
889 | ||||
890 | /// Helper method returns the integer value of a ConstantSDNode operand. | |||
891 | inline uint64_t getConstantOperandVal(unsigned Num) const; | |||
892 | ||||
893 | /// Helper method returns the APInt of a ConstantSDNode operand. | |||
894 | inline const APInt &getConstantOperandAPInt(unsigned Num) const; | |||
895 | ||||
896 | const SDValue &getOperand(unsigned Num) const { | |||
897 | assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!" ) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 897, __extension__ __PRETTY_FUNCTION__)); | |||
898 | return OperandList[Num]; | |||
899 | } | |||
900 | ||||
901 | using op_iterator = SDUse *; | |||
902 | ||||
903 | op_iterator op_begin() const { return OperandList; } | |||
904 | op_iterator op_end() const { return OperandList+NumOperands; } | |||
905 | ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); } | |||
906 | ||||
907 | /// Iterator for directly iterating over the operand SDValue's. | |||
908 | struct value_op_iterator | |||
909 | : iterator_adaptor_base<value_op_iterator, op_iterator, | |||
910 | std::random_access_iterator_tag, SDValue, | |||
911 | ptrdiff_t, value_op_iterator *, | |||
912 | value_op_iterator *> { | |||
913 | explicit value_op_iterator(SDUse *U = nullptr) | |||
914 | : iterator_adaptor_base(U) {} | |||
915 | ||||
916 | const SDValue &operator*() const { return I->get(); } | |||
917 | }; | |||
918 | ||||
919 | iterator_range<value_op_iterator> op_values() const { | |||
920 | return make_range(value_op_iterator(op_begin()), | |||
921 | value_op_iterator(op_end())); | |||
922 | } | |||
923 | ||||
924 | SDVTList getVTList() const { | |||
925 | SDVTList X = { ValueList, NumValues }; | |||
926 | return X; | |||
927 | } | |||
928 | ||||
929 | /// If this node has a glue operand, return the node | |||
930 | /// to which the glue operand points. Otherwise return NULL. | |||
931 | SDNode *getGluedNode() const { | |||
932 | if (getNumOperands() != 0 && | |||
933 | getOperand(getNumOperands()-1).getValueType() == MVT::Glue) | |||
934 | return getOperand(getNumOperands()-1).getNode(); | |||
935 | return nullptr; | |||
936 | } | |||
937 | ||||
938 | /// If this node has a glue value with a user, return | |||
939 | /// the user (there is at most one). Otherwise return NULL. | |||
940 | SDNode *getGluedUser() const { | |||
941 | for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI) | |||
942 | if (UI.getUse().get().getValueType() == MVT::Glue) | |||
943 | return *UI; | |||
944 | return nullptr; | |||
945 | } | |||
946 | ||||
947 | SDNodeFlags getFlags() const { return Flags; } | |||
948 | void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; } | |||
949 | ||||
950 | /// Clear any flags in this node that aren't also set in Flags. | |||
951 | /// If Flags is not in a defined state then this has no effect. | |||
952 | void intersectFlagsWith(const SDNodeFlags Flags); | |||
953 | ||||
954 | /// Return the number of values defined/returned by this operator. | |||
955 | unsigned getNumValues() const { return NumValues; } | |||
956 | ||||
957 | /// Return the type of a specified result. | |||
958 | EVT getValueType(unsigned ResNo) const { | |||
959 | assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!" ) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 959, __extension__ __PRETTY_FUNCTION__)); | |||
960 | return ValueList[ResNo]; | |||
961 | } | |||
962 | ||||
963 | /// Return the type of a specified result as a simple type. | |||
964 | MVT getSimpleValueType(unsigned ResNo) const { | |||
965 | return getValueType(ResNo).getSimpleVT(); | |||
966 | } | |||
967 | ||||
968 | /// Returns MVT::getSizeInBits(getValueType(ResNo)). | |||
969 | /// | |||
970 | /// If the value type is a scalable vector type, the scalable property will | |||
971 | /// be set and the runtime size will be a positive integer multiple of the | |||
972 | /// base size. | |||
973 | TypeSize getValueSizeInBits(unsigned ResNo) const { | |||
974 | return getValueType(ResNo).getSizeInBits(); | |||
975 | } | |||
976 | ||||
977 | using value_iterator = const EVT *; | |||
978 | ||||
979 | value_iterator value_begin() const { return ValueList; } | |||
980 | value_iterator value_end() const { return ValueList+NumValues; } | |||
981 | iterator_range<value_iterator> values() const { | |||
982 | return llvm::make_range(value_begin(), value_end()); | |||
983 | } | |||
984 | ||||
985 | /// Return the opcode of this operation for printing. | |||
986 | std::string getOperationName(const SelectionDAG *G = nullptr) const; | |||
987 | static const char* getIndexedModeName(ISD::MemIndexedMode AM); | |||
988 | void print_types(raw_ostream &OS, const SelectionDAG *G) const; | |||
989 | void print_details(raw_ostream &OS, const SelectionDAG *G) const; | |||
990 | void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const; | |||
991 | void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const; | |||
992 | ||||
993 | /// Print a SelectionDAG node and all children down to | |||
994 | /// the leaves. The given SelectionDAG allows target-specific nodes | |||
995 | /// to be printed in human-readable form. Unlike printr, this will | |||
996 | /// print the whole DAG, including children that appear multiple | |||
997 | /// times. | |||
998 | /// | |||
999 | void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const; | |||
1000 | ||||
1001 | /// Print a SelectionDAG node and children up to | |||
1002 | /// depth "depth." The given SelectionDAG allows target-specific | |||
1003 | /// nodes to be printed in human-readable form. Unlike printr, this | |||
1004 | /// will print children that appear multiple times wherever they are | |||
1005 | /// used. | |||
1006 | /// | |||
1007 | void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr, | |||
1008 | unsigned depth = 100) const; | |||
1009 | ||||
1010 | /// Dump this node, for debugging. | |||
1011 | void dump() const; | |||
1012 | ||||
1013 | /// Dump (recursively) this node and its use-def subgraph. | |||
1014 | void dumpr() const; | |||
1015 | ||||
1016 | /// Dump this node, for debugging. | |||
1017 | /// The given SelectionDAG allows target-specific nodes to be printed | |||
1018 | /// in human-readable form. | |||
1019 | void dump(const SelectionDAG *G) const; | |||
1020 | ||||
1021 | /// Dump (recursively) this node and its use-def subgraph. | |||
1022 | /// The given SelectionDAG allows target-specific nodes to be printed | |||
1023 | /// in human-readable form. | |||
1024 | void dumpr(const SelectionDAG *G) const; | |||
1025 | ||||
1026 | /// printrFull to dbgs(). The given SelectionDAG allows | |||
1027 | /// target-specific nodes to be printed in human-readable form. | |||
1028 | /// Unlike dumpr, this will print the whole DAG, including children | |||
1029 | /// that appear multiple times. | |||
1030 | void dumprFull(const SelectionDAG *G = nullptr) const; | |||
1031 | ||||
1032 | /// printrWithDepth to dbgs(). The given | |||
1033 | /// SelectionDAG allows target-specific nodes to be printed in | |||
1034 | /// human-readable form. Unlike dumpr, this will print children | |||
1035 | /// that appear multiple times wherever they are used. | |||
1036 | /// | |||
1037 | void dumprWithDepth(const SelectionDAG *G = nullptr, | |||
1038 | unsigned depth = 100) const; | |||
1039 | ||||
1040 | /// Gather unique data for the node. | |||
1041 | void Profile(FoldingSetNodeID &ID) const; | |||
1042 | ||||
1043 | /// This method should only be used by the SDUse class. | |||
1044 | void addUse(SDUse &U) { U.addToList(&UseList); } | |||
1045 | ||||
1046 | protected: | |||
1047 | static SDVTList getSDVTList(EVT VT) { | |||
1048 | SDVTList Ret = { getValueTypeList(VT), 1 }; | |||
1049 | return Ret; | |||
1050 | } | |||
1051 | ||||
1052 | /// Create an SDNode. | |||
1053 | /// | |||
1054 | /// SDNodes are created without any operands, and never own the operand | |||
1055 | /// storage. To add operands, see SelectionDAG::createOperands. | |||
1056 | SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) | |||
1057 | : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs), | |||
1058 | IROrder(Order), debugLoc(std::move(dl)) { | |||
1059 | memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits)); | |||
1060 | assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() && "Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1060, __extension__ __PRETTY_FUNCTION__)); | |||
1061 | assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!") ? void (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1062, __extension__ __PRETTY_FUNCTION__)) | |||
1062 | "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!") ? void (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1062, __extension__ __PRETTY_FUNCTION__)); | |||
1063 | } | |||
1064 | ||||
1065 | /// Release the operands and set this node to have zero operands. | |||
1066 | void DropOperands(); | |||
1067 | }; | |||
1068 | ||||
1069 | /// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed | |||
1070 | /// into SDNode creation functions. | |||
1071 | /// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted | |||
1072 | /// from the original Instruction, and IROrder is the ordinal position of | |||
1073 | /// the instruction. | |||
1074 | /// When an SDNode is created after the DAG is being built, both DebugLoc and | |||
1075 | /// the IROrder are propagated from the original SDNode. | |||
1076 | /// So SDLoc class provides two constructors besides the default one, one to | |||
1077 | /// be used by the DAGBuilder, the other to be used by others. | |||
1078 | class SDLoc { | |||
1079 | private: | |||
1080 | DebugLoc DL; | |||
1081 | int IROrder = 0; | |||
1082 | ||||
1083 | public: | |||
1084 | SDLoc() = default; | |||
1085 | SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {} | |||
1086 | SDLoc(const SDValue V) : SDLoc(V.getNode()) {} | |||
1087 | SDLoc(const Instruction *I, int Order) : IROrder(Order) { | |||
1088 | assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder" ) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1088, __extension__ __PRETTY_FUNCTION__)); | |||
1089 | if (I) | |||
1090 | DL = I->getDebugLoc(); | |||
1091 | } | |||
1092 | ||||
1093 | unsigned getIROrder() const { return IROrder; } | |||
1094 | const DebugLoc &getDebugLoc() const { return DL; } | |||
1095 | }; | |||
1096 | ||||
1097 | // Define inline functions from the SDValue class. | |||
1098 | ||||
1099 | inline SDValue::SDValue(SDNode *node, unsigned resno) | |||
1100 | : Node(node), ResNo(resno) { | |||
1101 | // Explicitly check for !ResNo to avoid use-after-free, because there are | |||
1102 | // callers that use SDValue(N, 0) with a deleted N to indicate successful | |||
1103 | // combines. | |||
1104 | assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node ->getNumValues()) && "Invalid result number for the given node!" ) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1105, __extension__ __PRETTY_FUNCTION__)) | |||
1105 | "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node ->getNumValues()) && "Invalid result number for the given node!" ) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1105, __extension__ __PRETTY_FUNCTION__)); | |||
1106 | assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps." ) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1106, __extension__ __PRETTY_FUNCTION__)); | |||
1107 | } | |||
1108 | ||||
1109 | inline unsigned SDValue::getOpcode() const { | |||
1110 | return Node->getOpcode(); | |||
1111 | } | |||
1112 | ||||
1113 | inline EVT SDValue::getValueType() const { | |||
1114 | return Node->getValueType(ResNo); | |||
| ||||
1115 | } | |||
1116 | ||||
1117 | inline unsigned SDValue::getNumOperands() const { | |||
1118 | return Node->getNumOperands(); | |||
1119 | } | |||
1120 | ||||
1121 | inline const SDValue &SDValue::getOperand(unsigned i) const { | |||
1122 | return Node->getOperand(i); | |||
1123 | } | |||
1124 | ||||
1125 | inline uint64_t SDValue::getConstantOperandVal(unsigned i) const { | |||
1126 | return Node->getConstantOperandVal(i); | |||
1127 | } | |||
1128 | ||||
1129 | inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const { | |||
1130 | return Node->getConstantOperandAPInt(i); | |||
1131 | } | |||
1132 | ||||
1133 | inline bool SDValue::isTargetOpcode() const { | |||
1134 | return Node->isTargetOpcode(); | |||
1135 | } | |||
1136 | ||||
1137 | inline bool SDValue::isTargetMemoryOpcode() const { | |||
1138 | return Node->isTargetMemoryOpcode(); | |||
1139 | } | |||
1140 | ||||
1141 | inline bool SDValue::isMachineOpcode() const { | |||
1142 | return Node->isMachineOpcode(); | |||
1143 | } | |||
1144 | ||||
1145 | inline unsigned SDValue::getMachineOpcode() const { | |||
1146 | return Node->getMachineOpcode(); | |||
1147 | } | |||
1148 | ||||
1149 | inline bool SDValue::isUndef() const { | |||
1150 | return Node->isUndef(); | |||
1151 | } | |||
1152 | ||||
1153 | inline bool SDValue::use_empty() const { | |||
1154 | return !Node->hasAnyUseOfValue(ResNo); | |||
1155 | } | |||
1156 | ||||
1157 | inline bool SDValue::hasOneUse() const { | |||
1158 | return Node->hasNUsesOfValue(1, ResNo); | |||
1159 | } | |||
1160 | ||||
1161 | inline const DebugLoc &SDValue::getDebugLoc() const { | |||
1162 | return Node->getDebugLoc(); | |||
1163 | } | |||
1164 | ||||
1165 | inline void SDValue::dump() const { | |||
1166 | return Node->dump(); | |||
1167 | } | |||
1168 | ||||
1169 | inline void SDValue::dump(const SelectionDAG *G) const { | |||
1170 | return Node->dump(G); | |||
1171 | } | |||
1172 | ||||
1173 | inline void SDValue::dumpr() const { | |||
1174 | return Node->dumpr(); | |||
1175 | } | |||
1176 | ||||
1177 | inline void SDValue::dumpr(const SelectionDAG *G) const { | |||
1178 | return Node->dumpr(G); | |||
1179 | } | |||
1180 | ||||
1181 | // Define inline functions from the SDUse class. | |||
1182 | ||||
1183 | inline void SDUse::set(const SDValue &V) { | |||
1184 | if (Val.getNode()) removeFromList(); | |||
1185 | Val = V; | |||
1186 | if (V.getNode()) V.getNode()->addUse(*this); | |||
1187 | } | |||
1188 | ||||
1189 | inline void SDUse::setInitial(const SDValue &V) { | |||
1190 | Val = V; | |||
1191 | V.getNode()->addUse(*this); | |||
1192 | } | |||
1193 | ||||
1194 | inline void SDUse::setNode(SDNode *N) { | |||
1195 | if (Val.getNode()) removeFromList(); | |||
1196 | Val.setNode(N); | |||
1197 | if (N) N->addUse(*this); | |||
1198 | } | |||
1199 | ||||
1200 | /// This class is used to form a handle around another node that | |||
1201 | /// is persistent and is updated across invocations of replaceAllUsesWith on its | |||
1202 | /// operand. This node should be directly created by end-users and not added to | |||
1203 | /// the AllNodes list. | |||
1204 | class HandleSDNode : public SDNode { | |||
1205 | SDUse Op; | |||
1206 | ||||
1207 | public: | |||
1208 | explicit HandleSDNode(SDValue X) | |||
1209 | : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) { | |||
1210 | // HandleSDNodes are never inserted into the DAG, so they won't be | |||
1211 | // auto-numbered. Use ID 65535 as a sentinel. | |||
1212 | PersistentId = 0xffff; | |||
1213 | ||||
1214 | // Manually set up the operand list. This node type is special in that it's | |||
1215 | // always stack allocated and SelectionDAG does not manage its operands. | |||
1216 | // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not | |||
1217 | // be so special. | |||
1218 | Op.setUser(this); | |||
1219 | Op.setInitial(X); | |||
1220 | NumOperands = 1; | |||
1221 | OperandList = &Op; | |||
1222 | } | |||
1223 | ~HandleSDNode(); | |||
1224 | ||||
1225 | const SDValue &getValue() const { return Op; } | |||
1226 | }; | |||
1227 | ||||
1228 | class AddrSpaceCastSDNode : public SDNode { | |||
1229 | private: | |||
1230 | unsigned SrcAddrSpace; | |||
1231 | unsigned DestAddrSpace; | |||
1232 | ||||
1233 | public: | |||
1234 | AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT, | |||
1235 | unsigned SrcAS, unsigned DestAS); | |||
1236 | ||||
1237 | unsigned getSrcAddressSpace() const { return SrcAddrSpace; } | |||
1238 | unsigned getDestAddressSpace() const { return DestAddrSpace; } | |||
1239 | ||||
1240 | static bool classof(const SDNode *N) { | |||
1241 | return N->getOpcode() == ISD::ADDRSPACECAST; | |||
1242 | } | |||
1243 | }; | |||
1244 | ||||
1245 | /// This is an abstract virtual class for memory operations. | |||
1246 | class MemSDNode : public SDNode { | |||
1247 | private: | |||
1248 | // VT of in-memory value. | |||
1249 | EVT MemoryVT; | |||
1250 | ||||
1251 | protected: | |||
1252 | /// Memory reference information. | |||
1253 | MachineMemOperand *MMO; | |||
1254 | ||||
1255 | public: | |||
1256 | MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
1257 | EVT memvt, MachineMemOperand *MMO); | |||
1258 | ||||
1259 | bool readMem() const { return MMO->isLoad(); } | |||
1260 | bool writeMem() const { return MMO->isStore(); } | |||
1261 | ||||
1262 | /// Returns alignment and volatility of the memory access | |||
1263 | Align getOriginalAlign() const { return MMO->getBaseAlign(); } | |||
1264 | Align getAlign() const { return MMO->getAlign(); } | |||
1265 | // FIXME: Remove once transition to getAlign is over. | |||
1266 | unsigned getAlignment() const { return MMO->getAlign().value(); } | |||
1267 | ||||
1268 | /// Return the SubclassData value, without HasDebugValue. This contains an | |||
1269 | /// encoding of the volatile flag, as well as bits used by subclasses. This | |||
1270 | /// function should only be used to compute a FoldingSetNodeID value. | |||
1271 | /// The HasDebugValue bit is masked out because CSE map needs to match | |||
1272 | /// nodes with debug info with nodes without debug info. Same is about | |||
1273 | /// isDivergent bit. | |||
1274 | unsigned getRawSubclassData() const { | |||
1275 | uint16_t Data; | |||
1276 | union { | |||
1277 | char RawSDNodeBits[sizeof(uint16_t)]; | |||
1278 | SDNodeBitfields SDNodeBits; | |||
1279 | }; | |||
1280 | memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits)); | |||
1281 | SDNodeBits.HasDebugValue = 0; | |||
1282 | SDNodeBits.IsDivergent = false; | |||
1283 | memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits)); | |||
1284 | return Data; | |||
1285 | } | |||
1286 | ||||
1287 | bool isVolatile() const { return MemSDNodeBits.IsVolatile; } | |||
1288 | bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; } | |||
1289 | bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; } | |||
1290 | bool isInvariant() const { return MemSDNodeBits.IsInvariant; } | |||
1291 | ||||
1292 | // Returns the offset from the location of the access. | |||
1293 | int64_t getSrcValueOffset() const { return MMO->getOffset(); } | |||
1294 | ||||
1295 | /// Returns the AA info that describes the dereference. | |||
1296 | AAMDNodes getAAInfo() const { return MMO->getAAInfo(); } | |||
1297 | ||||
1298 | /// Returns the Ranges that describes the dereference. | |||
1299 | const MDNode *getRanges() const { return MMO->getRanges(); } | |||
1300 | ||||
1301 | /// Returns the synchronization scope ID for this memory operation. | |||
1302 | SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); } | |||
1303 | ||||
1304 | /// Return the atomic ordering requirements for this memory operation. For | |||
1305 | /// cmpxchg atomic operations, return the atomic ordering requirements when | |||
1306 | /// store occurs. | |||
1307 | AtomicOrdering getSuccessOrdering() const { | |||
1308 | return MMO->getSuccessOrdering(); | |||
1309 | } | |||
1310 | ||||
1311 | /// Return a single atomic ordering that is at least as strong as both the | |||
1312 | /// success and failure orderings for an atomic operation. (For operations | |||
1313 | /// other than cmpxchg, this is equivalent to getSuccessOrdering().) | |||
1314 | AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); } | |||
1315 | ||||
1316 | /// Return true if the memory operation ordering is Unordered or higher. | |||
1317 | bool isAtomic() const { return MMO->isAtomic(); } | |||
1318 | ||||
1319 | /// Returns true if the memory operation doesn't imply any ordering | |||
1320 | /// constraints on surrounding memory operations beyond the normal memory | |||
1321 | /// aliasing rules. | |||
1322 | bool isUnordered() const { return MMO->isUnordered(); } | |||
1323 | ||||
1324 | /// Returns true if the memory operation is neither atomic or volatile. | |||
1325 | bool isSimple() const { return !isAtomic() && !isVolatile(); } | |||
1326 | ||||
1327 | /// Return the type of the in-memory value. | |||
1328 | EVT getMemoryVT() const { return MemoryVT; } | |||
1329 | ||||
1330 | /// Return a MachineMemOperand object describing the memory | |||
1331 | /// reference performed by operation. | |||
1332 | MachineMemOperand *getMemOperand() const { return MMO; } | |||
1333 | ||||
1334 | const MachinePointerInfo &getPointerInfo() const { | |||
1335 | return MMO->getPointerInfo(); | |||
1336 | } | |||
1337 | ||||
1338 | /// Return the address space for the associated pointer | |||
1339 | unsigned getAddressSpace() const { | |||
1340 | return getPointerInfo().getAddrSpace(); | |||
1341 | } | |||
1342 | ||||
1343 | /// Update this MemSDNode's MachineMemOperand information | |||
1344 | /// to reflect the alignment of NewMMO, if it has a greater alignment. | |||
1345 | /// This must only be used when the new alignment applies to all users of | |||
1346 | /// this MachineMemOperand. | |||
1347 | void refineAlignment(const MachineMemOperand *NewMMO) { | |||
1348 | MMO->refineAlignment(NewMMO); | |||
1349 | } | |||
1350 | ||||
1351 | const SDValue &getChain() const { return getOperand(0); } | |||
1352 | ||||
1353 | const SDValue &getBasePtr() const { | |||
1354 | switch (getOpcode()) { | |||
1355 | case ISD::STORE: | |||
1356 | case ISD::MSTORE: | |||
1357 | return getOperand(2); | |||
1358 | case ISD::MGATHER: | |||
1359 | case ISD::MSCATTER: | |||
1360 | return getOperand(3); | |||
1361 | default: | |||
1362 | return getOperand(1); | |||
1363 | } | |||
1364 | } | |||
1365 | ||||
1366 | // Methods to support isa and dyn_cast | |||
1367 | static bool classof(const SDNode *N) { | |||
1368 | // For some targets, we lower some target intrinsics to a MemIntrinsicNode | |||
1369 | // with either an intrinsic or a target opcode. | |||
1370 | switch (N->getOpcode()) { | |||
1371 | case ISD::LOAD: | |||
1372 | case ISD::STORE: | |||
1373 | case ISD::PREFETCH: | |||
1374 | case ISD::ATOMIC_CMP_SWAP: | |||
1375 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: | |||
1376 | case ISD::ATOMIC_SWAP: | |||
1377 | case ISD::ATOMIC_LOAD_ADD: | |||
1378 | case ISD::ATOMIC_LOAD_SUB: | |||
1379 | case ISD::ATOMIC_LOAD_AND: | |||
1380 | case ISD::ATOMIC_LOAD_CLR: | |||
1381 | case ISD::ATOMIC_LOAD_OR: | |||
1382 | case ISD::ATOMIC_LOAD_XOR: | |||
1383 | case ISD::ATOMIC_LOAD_NAND: | |||
1384 | case ISD::ATOMIC_LOAD_MIN: | |||
1385 | case ISD::ATOMIC_LOAD_MAX: | |||
1386 | case ISD::ATOMIC_LOAD_UMIN: | |||
1387 | case ISD::ATOMIC_LOAD_UMAX: | |||
1388 | case ISD::ATOMIC_LOAD_FADD: | |||
1389 | case ISD::ATOMIC_LOAD_FSUB: | |||
1390 | case ISD::ATOMIC_LOAD: | |||
1391 | case ISD::ATOMIC_STORE: | |||
1392 | case ISD::MLOAD: | |||
1393 | case ISD::MSTORE: | |||
1394 | case ISD::MGATHER: | |||
1395 | case ISD::MSCATTER: | |||
1396 | return true; | |||
1397 | default: | |||
1398 | return N->isMemIntrinsic() || N->isTargetMemoryOpcode(); | |||
1399 | } | |||
1400 | } | |||
1401 | }; | |||
1402 | ||||
1403 | /// This is an SDNode representing atomic operations. | |||
1404 | class AtomicSDNode : public MemSDNode { | |||
1405 | public: | |||
1406 | AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL, | |||
1407 | EVT MemVT, MachineMemOperand *MMO) | |||
1408 | : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { | |||
1409 | assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?" ) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1410, __extension__ __PRETTY_FUNCTION__)) | |||
1410 | MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?" ) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1410, __extension__ __PRETTY_FUNCTION__)); | |||
1411 | } | |||
1412 | ||||
1413 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
1414 | const SDValue &getVal() const { return getOperand(2); } | |||
1415 | ||||
1416 | /// Returns true if this SDNode represents cmpxchg atomic operation, false | |||
1417 | /// otherwise. | |||
1418 | bool isCompareAndSwap() const { | |||
1419 | unsigned Op = getOpcode(); | |||
1420 | return Op == ISD::ATOMIC_CMP_SWAP || | |||
1421 | Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS; | |||
1422 | } | |||
1423 | ||||
1424 | /// For cmpxchg atomic operations, return the atomic ordering requirements | |||
1425 | /// when store does not occur. | |||
1426 | AtomicOrdering getFailureOrdering() const { | |||
1427 | assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation" ) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1427, __extension__ __PRETTY_FUNCTION__)); | |||
1428 | return MMO->getFailureOrdering(); | |||
1429 | } | |||
1430 | ||||
1431 | // Methods to support isa and dyn_cast | |||
1432 | static bool classof(const SDNode *N) { | |||
1433 | return N->getOpcode() == ISD::ATOMIC_CMP_SWAP || | |||
1434 | N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || | |||
1435 | N->getOpcode() == ISD::ATOMIC_SWAP || | |||
1436 | N->getOpcode() == ISD::ATOMIC_LOAD_ADD || | |||
1437 | N->getOpcode() == ISD::ATOMIC_LOAD_SUB || | |||
1438 | N->getOpcode() == ISD::ATOMIC_LOAD_AND || | |||
1439 | N->getOpcode() == ISD::ATOMIC_LOAD_CLR || | |||
1440 | N->getOpcode() == ISD::ATOMIC_LOAD_OR || | |||
1441 | N->getOpcode() == ISD::ATOMIC_LOAD_XOR || | |||
1442 | N->getOpcode() == ISD::ATOMIC_LOAD_NAND || | |||
1443 | N->getOpcode() == ISD::ATOMIC_LOAD_MIN || | |||
1444 | N->getOpcode() == ISD::ATOMIC_LOAD_MAX || | |||
1445 | N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || | |||
1446 | N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || | |||
1447 | N->getOpcode() == ISD::ATOMIC_LOAD_FADD || | |||
1448 | N->getOpcode() == ISD::ATOMIC_LOAD_FSUB || | |||
1449 | N->getOpcode() == ISD::ATOMIC_LOAD || | |||
1450 | N->getOpcode() == ISD::ATOMIC_STORE; | |||
1451 | } | |||
1452 | }; | |||
1453 | ||||
1454 | /// This SDNode is used for target intrinsics that touch | |||
1455 | /// memory and need an associated MachineMemOperand. Its opcode may be | |||
1456 | /// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode | |||
1457 | /// with a value not less than FIRST_TARGET_MEMORY_OPCODE. | |||
1458 | class MemIntrinsicSDNode : public MemSDNode { | |||
1459 | public: | |||
1460 | MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, | |||
1461 | SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO) | |||
1462 | : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) { | |||
1463 | SDNodeBits.IsMemIntrinsic = true; | |||
1464 | } | |||
1465 | ||||
1466 | // Methods to support isa and dyn_cast | |||
1467 | static bool classof(const SDNode *N) { | |||
1468 | // We lower some target intrinsics to their target opcode | |||
1469 | // early a node with a target opcode can be of this class | |||
1470 | return N->isMemIntrinsic() || | |||
1471 | N->getOpcode() == ISD::PREFETCH || | |||
1472 | N->isTargetMemoryOpcode(); | |||
1473 | } | |||
1474 | }; | |||
1475 | ||||
1476 | /// This SDNode is used to implement the code generator | |||
1477 | /// support for the llvm IR shufflevector instruction. It combines elements | |||
1478 | /// from two input vectors into a new input vector, with the selection and | |||
1479 | /// ordering of elements determined by an array of integers, referred to as | |||
1480 | /// the shuffle mask. For input vectors of width N, mask indices of 0..N-1 | |||
1481 | /// refer to elements from the LHS input, and indices from N to 2N-1 the RHS. | |||
1482 | /// An index of -1 is treated as undef, such that the code generator may put | |||
1483 | /// any value in the corresponding element of the result. | |||
1484 | class ShuffleVectorSDNode : public SDNode { | |||
1485 | // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and | |||
1486 | // is freed when the SelectionDAG object is destroyed. | |||
1487 | const int *Mask; | |||
1488 | ||||
1489 | protected: | |||
1490 | friend class SelectionDAG; | |||
1491 | ||||
1492 | ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M) | |||
1493 | : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {} | |||
1494 | ||||
1495 | public: | |||
1496 | ArrayRef<int> getMask() const { | |||
1497 | EVT VT = getValueType(0); | |||
1498 | return makeArrayRef(Mask, VT.getVectorNumElements()); | |||
1499 | } | |||
1500 | ||||
1501 | int getMaskElt(unsigned Idx) const { | |||
1502 | assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements () && "Idx out of range!") ? void (0) : __assert_fail ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1502, __extension__ __PRETTY_FUNCTION__)); | |||
1503 | return Mask[Idx]; | |||
1504 | } | |||
1505 | ||||
1506 | bool isSplat() const { return isSplatMask(Mask, getValueType(0)); } | |||
1507 | ||||
1508 | int getSplatIndex() const { | |||
1509 | assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!" ) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1509, __extension__ __PRETTY_FUNCTION__)); | |||
1510 | EVT VT = getValueType(0); | |||
1511 | for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) | |||
1512 | if (Mask[i] >= 0) | |||
1513 | return Mask[i]; | |||
1514 | ||||
1515 | // We can choose any index value here and be correct because all elements | |||
1516 | // are undefined. Return 0 for better potential for callers to simplify. | |||
1517 | return 0; | |||
1518 | } | |||
1519 | ||||
1520 | static bool isSplatMask(const int *Mask, EVT VT); | |||
1521 | ||||
1522 | /// Change values in a shuffle permute mask assuming | |||
1523 | /// the two vector operands have swapped position. | |||
1524 | static void commuteMask(MutableArrayRef<int> Mask) { | |||
1525 | unsigned NumElems = Mask.size(); | |||
1526 | for (unsigned i = 0; i != NumElems; ++i) { | |||
1527 | int idx = Mask[i]; | |||
1528 | if (idx < 0) | |||
1529 | continue; | |||
1530 | else if (idx < (int)NumElems) | |||
1531 | Mask[i] = idx + NumElems; | |||
1532 | else | |||
1533 | Mask[i] = idx - NumElems; | |||
1534 | } | |||
1535 | } | |||
1536 | ||||
1537 | static bool classof(const SDNode *N) { | |||
1538 | return N->getOpcode() == ISD::VECTOR_SHUFFLE; | |||
1539 | } | |||
1540 | }; | |||
1541 | ||||
1542 | class ConstantSDNode : public SDNode { | |||
1543 | friend class SelectionDAG; | |||
1544 | ||||
1545 | const ConstantInt *Value; | |||
1546 | ||||
1547 | ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT) | |||
1548 | : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(), | |||
1549 | getSDVTList(VT)), | |||
1550 | Value(val) { | |||
1551 | ConstantSDNodeBits.IsOpaque = isOpaque; | |||
1552 | } | |||
1553 | ||||
1554 | public: | |||
1555 | const ConstantInt *getConstantIntValue() const { return Value; } | |||
1556 | const APInt &getAPIntValue() const { return Value->getValue(); } | |||
1557 | uint64_t getZExtValue() const { return Value->getZExtValue(); } | |||
1558 | int64_t getSExtValue() const { return Value->getSExtValue(); } | |||
1559 | uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) { | |||
1560 | return Value->getLimitedValue(Limit); | |||
1561 | } | |||
1562 | MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); } | |||
1563 | Align getAlignValue() const { return Value->getAlignValue(); } | |||
1564 | ||||
1565 | bool isOne() const { return Value->isOne(); } | |||
1566 | bool isNullValue() const { return Value->isZero(); } | |||
1567 | bool isAllOnesValue() const { return Value->isMinusOne(); } | |||
1568 | bool isMaxSignedValue() const { return Value->isMaxValue(true); } | |||
1569 | bool isMinSignedValue() const { return Value->isMinValue(true); } | |||
1570 | ||||
1571 | bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; } | |||
1572 | ||||
1573 | static bool classof(const SDNode *N) { | |||
1574 | return N->getOpcode() == ISD::Constant || | |||
1575 | N->getOpcode() == ISD::TargetConstant; | |||
1576 | } | |||
1577 | }; | |||
1578 | ||||
1579 | uint64_t SDNode::getConstantOperandVal(unsigned Num) const { | |||
1580 | return cast<ConstantSDNode>(getOperand(Num))->getZExtValue(); | |||
1581 | } | |||
1582 | ||||
1583 | const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const { | |||
1584 | return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue(); | |||
1585 | } | |||
1586 | ||||
1587 | class ConstantFPSDNode : public SDNode { | |||
1588 | friend class SelectionDAG; | |||
1589 | ||||
1590 | const ConstantFP *Value; | |||
1591 | ||||
1592 | ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT) | |||
1593 | : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0, | |||
1594 | DebugLoc(), getSDVTList(VT)), | |||
1595 | Value(val) {} | |||
1596 | ||||
1597 | public: | |||
1598 | const APFloat& getValueAPF() const { return Value->getValueAPF(); } | |||
1599 | const ConstantFP *getConstantFPValue() const { return Value; } | |||
1600 | ||||
1601 | /// Return true if the value is positive or negative zero. | |||
1602 | bool isZero() const { return Value->isZero(); } | |||
1603 | ||||
1604 | /// Return true if the value is a NaN. | |||
1605 | bool isNaN() const { return Value->isNaN(); } | |||
1606 | ||||
1607 | /// Return true if the value is an infinity | |||
1608 | bool isInfinity() const { return Value->isInfinity(); } | |||
1609 | ||||
1610 | /// Return true if the value is negative. | |||
1611 | bool isNegative() const { return Value->isNegative(); } | |||
1612 | ||||
1613 | /// We don't rely on operator== working on double values, as | |||
1614 | /// it returns true for things that are clearly not equal, like -0.0 and 0.0. | |||
1615 | /// As such, this method can be used to do an exact bit-for-bit comparison of | |||
1616 | /// two floating point values. | |||
1617 | ||||
1618 | /// We leave the version with the double argument here because it's just so | |||
1619 | /// convenient to write "2.0" and the like. Without this function we'd | |||
1620 | /// have to duplicate its logic everywhere it's called. | |||
1621 | bool isExactlyValue(double V) const { | |||
1622 | return Value->getValueAPF().isExactlyValue(V); | |||
1623 | } | |||
1624 | bool isExactlyValue(const APFloat& V) const; | |||
1625 | ||||
1626 | static bool isValueValidForType(EVT VT, const APFloat& Val); | |||
1627 | ||||
1628 | static bool classof(const SDNode *N) { | |||
1629 | return N->getOpcode() == ISD::ConstantFP || | |||
1630 | N->getOpcode() == ISD::TargetConstantFP; | |||
1631 | } | |||
1632 | }; | |||
1633 | ||||
1634 | /// Returns true if \p V is a constant integer zero. | |||
1635 | bool isNullConstant(SDValue V); | |||
1636 | ||||
1637 | /// Returns true if \p V is an FP constant with a value of positive zero. | |||
1638 | bool isNullFPConstant(SDValue V); | |||
1639 | ||||
1640 | /// Returns true if \p V is an integer constant with all bits set. | |||
1641 | bool isAllOnesConstant(SDValue V); | |||
1642 | ||||
1643 | /// Returns true if \p V is a constant integer one. | |||
1644 | bool isOneConstant(SDValue V); | |||
1645 | ||||
1646 | /// Return the non-bitcasted source operand of \p V if it exists. | |||
1647 | /// If \p V is not a bitcasted value, it is returned as-is. | |||
1648 | SDValue peekThroughBitcasts(SDValue V); | |||
1649 | ||||
1650 | /// Return the non-bitcasted and one-use source operand of \p V if it exists. | |||
1651 | /// If \p V is not a bitcasted one-use value, it is returned as-is. | |||
1652 | SDValue peekThroughOneUseBitcasts(SDValue V); | |||
1653 | ||||
1654 | /// Return the non-extracted vector source operand of \p V if it exists. | |||
1655 | /// If \p V is not an extracted subvector, it is returned as-is. | |||
1656 | SDValue peekThroughExtractSubvectors(SDValue V); | |||
1657 | ||||
1658 | /// Returns true if \p V is a bitwise not operation. Assumes that an all ones | |||
1659 | /// constant is canonicalized to be operand 1. | |||
1660 | bool isBitwiseNot(SDValue V, bool AllowUndefs = false); | |||
1661 | ||||
1662 | /// Returns the SDNode if it is a constant splat BuildVector or constant int. | |||
1663 | ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false, | |||
1664 | bool AllowTruncation = false); | |||
1665 | ||||
1666 | /// Returns the SDNode if it is a demanded constant splat BuildVector or | |||
1667 | /// constant int. | |||
1668 | ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts, | |||
1669 | bool AllowUndefs = false, | |||
1670 | bool AllowTruncation = false); | |||
1671 | ||||
1672 | /// Returns the SDNode if it is a constant splat BuildVector or constant float. | |||
1673 | ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false); | |||
1674 | ||||
1675 | /// Returns the SDNode if it is a demanded constant splat BuildVector or | |||
1676 | /// constant float. | |||
1677 | ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts, | |||
1678 | bool AllowUndefs = false); | |||
1679 | ||||
1680 | /// Return true if the value is a constant 0 integer or a splatted vector of | |||
1681 | /// a constant 0 integer (with no undefs by default). | |||
1682 | /// Build vector implicit truncation is not an issue for null values. | |||
1683 | bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false); | |||
1684 | ||||
1685 | /// Return true if the value is a constant 1 integer or a splatted vector of a | |||
1686 | /// constant 1 integer (with no undefs). | |||
1687 | /// Does not permit build vector implicit truncation. | |||
1688 | bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false); | |||
1689 | ||||
1690 | /// Return true if the value is a constant -1 integer or a splatted vector of a | |||
1691 | /// constant -1 integer (with no undefs). | |||
1692 | /// Does not permit build vector implicit truncation. | |||
1693 | bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false); | |||
1694 | ||||
1695 | /// Return true if \p V is either a integer or FP constant. | |||
1696 | inline bool isIntOrFPConstant(SDValue V) { | |||
1697 | return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V); | |||
1698 | } | |||
1699 | ||||
1700 | class GlobalAddressSDNode : public SDNode { | |||
1701 | friend class SelectionDAG; | |||
1702 | ||||
1703 | const GlobalValue *TheGlobal; | |||
1704 | int64_t Offset; | |||
1705 | unsigned TargetFlags; | |||
1706 | ||||
1707 | GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, | |||
1708 | const GlobalValue *GA, EVT VT, int64_t o, | |||
1709 | unsigned TF); | |||
1710 | ||||
1711 | public: | |||
1712 | const GlobalValue *getGlobal() const { return TheGlobal; } | |||
1713 | int64_t getOffset() const { return Offset; } | |||
1714 | unsigned getTargetFlags() const { return TargetFlags; } | |||
1715 | // Return the address space this GlobalAddress belongs to. | |||
1716 | unsigned getAddressSpace() const; | |||
1717 | ||||
1718 | static bool classof(const SDNode *N) { | |||
1719 | return N->getOpcode() == ISD::GlobalAddress || | |||
1720 | N->getOpcode() == ISD::TargetGlobalAddress || | |||
1721 | N->getOpcode() == ISD::GlobalTLSAddress || | |||
1722 | N->getOpcode() == ISD::TargetGlobalTLSAddress; | |||
1723 | } | |||
1724 | }; | |||
1725 | ||||
1726 | class FrameIndexSDNode : public SDNode { | |||
1727 | friend class SelectionDAG; | |||
1728 | ||||
1729 | int FI; | |||
1730 | ||||
1731 | FrameIndexSDNode(int fi, EVT VT, bool isTarg) | |||
1732 | : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex, | |||
1733 | 0, DebugLoc(), getSDVTList(VT)), FI(fi) { | |||
1734 | } | |||
1735 | ||||
1736 | public: | |||
1737 | int getIndex() const { return FI; } | |||
1738 | ||||
1739 | static bool classof(const SDNode *N) { | |||
1740 | return N->getOpcode() == ISD::FrameIndex || | |||
1741 | N->getOpcode() == ISD::TargetFrameIndex; | |||
1742 | } | |||
1743 | }; | |||
1744 | ||||
1745 | /// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate | |||
1746 | /// the offet and size that are started/ended in the underlying FrameIndex. | |||
1747 | class LifetimeSDNode : public SDNode { | |||
1748 | friend class SelectionDAG; | |||
1749 | int64_t Size; | |||
1750 | int64_t Offset; // -1 if offset is unknown. | |||
1751 | ||||
1752 | LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, | |||
1753 | SDVTList VTs, int64_t Size, int64_t Offset) | |||
1754 | : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {} | |||
1755 | public: | |||
1756 | int64_t getFrameIndex() const { | |||
1757 | return cast<FrameIndexSDNode>(getOperand(1))->getIndex(); | |||
1758 | } | |||
1759 | ||||
1760 | bool hasOffset() const { return Offset >= 0; } | |||
1761 | int64_t getOffset() const { | |||
1762 | assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown" ) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1762, __extension__ __PRETTY_FUNCTION__)); | |||
1763 | return Offset; | |||
1764 | } | |||
1765 | int64_t getSize() const { | |||
1766 | assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown" ) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1766, __extension__ __PRETTY_FUNCTION__)); | |||
1767 | return Size; | |||
1768 | } | |||
1769 | ||||
1770 | // Methods to support isa and dyn_cast | |||
1771 | static bool classof(const SDNode *N) { | |||
1772 | return N->getOpcode() == ISD::LIFETIME_START || | |||
1773 | N->getOpcode() == ISD::LIFETIME_END; | |||
1774 | } | |||
1775 | }; | |||
1776 | ||||
1777 | /// This SDNode is used for PSEUDO_PROBE values, which are the function guid and | |||
1778 | /// the index of the basic block being probed. A pseudo probe serves as a place | |||
1779 | /// holder and will be removed at the end of compilation. It does not have any | |||
1780 | /// operand because we do not want the instruction selection to deal with any. | |||
1781 | class PseudoProbeSDNode : public SDNode { | |||
1782 | friend class SelectionDAG; | |||
1783 | uint64_t Guid; | |||
1784 | uint64_t Index; | |||
1785 | uint32_t Attributes; | |||
1786 | ||||
1787 | PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl, | |||
1788 | SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr) | |||
1789 | : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index), | |||
1790 | Attributes(Attr) {} | |||
1791 | ||||
1792 | public: | |||
1793 | uint64_t getGuid() const { return Guid; } | |||
1794 | uint64_t getIndex() const { return Index; } | |||
1795 | uint32_t getAttributes() const { return Attributes; } | |||
1796 | ||||
1797 | // Methods to support isa and dyn_cast | |||
1798 | static bool classof(const SDNode *N) { | |||
1799 | return N->getOpcode() == ISD::PSEUDO_PROBE; | |||
1800 | } | |||
1801 | }; | |||
1802 | ||||
1803 | class JumpTableSDNode : public SDNode { | |||
1804 | friend class SelectionDAG; | |||
1805 | ||||
1806 | int JTI; | |||
1807 | unsigned TargetFlags; | |||
1808 | ||||
1809 | JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF) | |||
1810 | : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable, | |||
1811 | 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) { | |||
1812 | } | |||
1813 | ||||
1814 | public: | |||
1815 | int getIndex() const { return JTI; } | |||
1816 | unsigned getTargetFlags() const { return TargetFlags; } | |||
1817 | ||||
1818 | static bool classof(const SDNode *N) { | |||
1819 | return N->getOpcode() == ISD::JumpTable || | |||
1820 | N->getOpcode() == ISD::TargetJumpTable; | |||
1821 | } | |||
1822 | }; | |||
1823 | ||||
1824 | class ConstantPoolSDNode : public SDNode { | |||
1825 | friend class SelectionDAG; | |||
1826 | ||||
1827 | union { | |||
1828 | const Constant *ConstVal; | |||
1829 | MachineConstantPoolValue *MachineCPVal; | |||
1830 | } Val; | |||
1831 | int Offset; // It's a MachineConstantPoolValue if top bit is set. | |||
1832 | Align Alignment; // Minimum alignment requirement of CP. | |||
1833 | unsigned TargetFlags; | |||
1834 | ||||
1835 | ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o, | |||
1836 | Align Alignment, unsigned TF) | |||
1837 | : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, | |||
1838 | DebugLoc(), getSDVTList(VT)), | |||
1839 | Offset(o), Alignment(Alignment), TargetFlags(TF) { | |||
1840 | assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large" ) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1840, __extension__ __PRETTY_FUNCTION__)); | |||
1841 | Val.ConstVal = c; | |||
1842 | } | |||
1843 | ||||
1844 | ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o, | |||
1845 | Align Alignment, unsigned TF) | |||
1846 | : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, | |||
1847 | DebugLoc(), getSDVTList(VT)), | |||
1848 | Offset(o), Alignment(Alignment), TargetFlags(TF) { | |||
1849 | assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large" ) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1849, __extension__ __PRETTY_FUNCTION__)); | |||
1850 | Val.MachineCPVal = v; | |||
1851 | Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1); | |||
1852 | } | |||
1853 | ||||
1854 | public: | |||
1855 | bool isMachineConstantPoolEntry() const { | |||
1856 | return Offset < 0; | |||
1857 | } | |||
1858 | ||||
1859 | const Constant *getConstVal() const { | |||
1860 | assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() && "Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1860, __extension__ __PRETTY_FUNCTION__)); | |||
1861 | return Val.ConstVal; | |||
1862 | } | |||
1863 | ||||
1864 | MachineConstantPoolValue *getMachineCPVal() const { | |||
1865 | assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() && "Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1865, __extension__ __PRETTY_FUNCTION__)); | |||
1866 | return Val.MachineCPVal; | |||
1867 | } | |||
1868 | ||||
1869 | int getOffset() const { | |||
1870 | return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1)); | |||
1871 | } | |||
1872 | ||||
1873 | // Return the alignment of this constant pool object, which is either 0 (for | |||
1874 | // default alignment) or the desired value. | |||
1875 | Align getAlign() const { return Alignment; } | |||
1876 | unsigned getTargetFlags() const { return TargetFlags; } | |||
1877 | ||||
1878 | Type *getType() const; | |||
1879 | ||||
1880 | static bool classof(const SDNode *N) { | |||
1881 | return N->getOpcode() == ISD::ConstantPool || | |||
1882 | N->getOpcode() == ISD::TargetConstantPool; | |||
1883 | } | |||
1884 | }; | |||
1885 | ||||
1886 | /// Completely target-dependent object reference. | |||
1887 | class TargetIndexSDNode : public SDNode { | |||
1888 | friend class SelectionDAG; | |||
1889 | ||||
1890 | unsigned TargetFlags; | |||
1891 | int Index; | |||
1892 | int64_t Offset; | |||
1893 | ||||
1894 | public: | |||
1895 | TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF) | |||
1896 | : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)), | |||
1897 | TargetFlags(TF), Index(Idx), Offset(Ofs) {} | |||
1898 | ||||
1899 | unsigned getTargetFlags() const { return TargetFlags; } | |||
1900 | int getIndex() const { return Index; } | |||
1901 | int64_t getOffset() const { return Offset; } | |||
1902 | ||||
1903 | static bool classof(const SDNode *N) { | |||
1904 | return N->getOpcode() == ISD::TargetIndex; | |||
1905 | } | |||
1906 | }; | |||
1907 | ||||
1908 | class BasicBlockSDNode : public SDNode { | |||
1909 | friend class SelectionDAG; | |||
1910 | ||||
1911 | MachineBasicBlock *MBB; | |||
1912 | ||||
1913 | /// Debug info is meaningful and potentially useful here, but we create | |||
1914 | /// blocks out of order when they're jumped to, which makes it a bit | |||
1915 | /// harder. Let's see if we need it first. | |||
1916 | explicit BasicBlockSDNode(MachineBasicBlock *mbb) | |||
1917 | : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb) | |||
1918 | {} | |||
1919 | ||||
1920 | public: | |||
1921 | MachineBasicBlock *getBasicBlock() const { return MBB; } | |||
1922 | ||||
1923 | static bool classof(const SDNode *N) { | |||
1924 | return N->getOpcode() == ISD::BasicBlock; | |||
1925 | } | |||
1926 | }; | |||
1927 | ||||
1928 | /// A "pseudo-class" with methods for operating on BUILD_VECTORs. | |||
1929 | class BuildVectorSDNode : public SDNode { | |||
1930 | public: | |||
1931 | // These are constructed as SDNodes and then cast to BuildVectorSDNodes. | |||
1932 | explicit BuildVectorSDNode() = delete; | |||
1933 | ||||
1934 | /// Check if this is a constant splat, and if so, find the | |||
1935 | /// smallest element size that splats the vector. If MinSplatBits is | |||
1936 | /// nonzero, the element size must be at least that large. Note that the | |||
1937 | /// splat element may be the entire vector (i.e., a one element vector). | |||
1938 | /// Returns the splat element value in SplatValue. Any undefined bits in | |||
1939 | /// that value are zero, and the corresponding bits in the SplatUndef mask | |||
1940 | /// are set. The SplatBitSize value is set to the splat element size in | |||
1941 | /// bits. HasAnyUndefs is set to true if any bits in the vector are | |||
1942 | /// undefined. isBigEndian describes the endianness of the target. | |||
1943 | bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, | |||
1944 | unsigned &SplatBitSize, bool &HasAnyUndefs, | |||
1945 | unsigned MinSplatBits = 0, | |||
1946 | bool isBigEndian = false) const; | |||
1947 | ||||
1948 | /// Returns the demanded splatted value or a null value if this is not a | |||
1949 | /// splat. | |||
1950 | /// | |||
1951 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
1952 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
1953 | /// the vector width and set the bits where elements are undef. | |||
1954 | SDValue getSplatValue(const APInt &DemandedElts, | |||
1955 | BitVector *UndefElements = nullptr) const; | |||
1956 | ||||
1957 | /// Returns the splatted value or a null value if this is not a splat. | |||
1958 | /// | |||
1959 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
1960 | /// the vector width and set the bits where elements are undef. | |||
1961 | SDValue getSplatValue(BitVector *UndefElements = nullptr) const; | |||
1962 | ||||
1963 | /// Find the shortest repeating sequence of values in the build vector. | |||
1964 | /// | |||
1965 | /// e.g. { u, X, u, X, u, u, X, u } -> { X } | |||
1966 | /// { X, Y, u, Y, u, u, X, u } -> { X, Y } | |||
1967 | /// | |||
1968 | /// Currently this must be a power-of-2 build vector. | |||
1969 | /// The DemandedElts mask indicates the elements that must be present, | |||
1970 | /// undemanded elements in Sequence may be null (SDValue()). If passed a | |||
1971 | /// non-null UndefElements bitvector, it will resize it to match the original | |||
1972 | /// vector width and set the bits where elements are undef. If result is | |||
1973 | /// false, Sequence will be empty. | |||
1974 | bool getRepeatedSequence(const APInt &DemandedElts, | |||
1975 | SmallVectorImpl<SDValue> &Sequence, | |||
1976 | BitVector *UndefElements = nullptr) const; | |||
1977 | ||||
1978 | /// Find the shortest repeating sequence of values in the build vector. | |||
1979 | /// | |||
1980 | /// e.g. { u, X, u, X, u, u, X, u } -> { X } | |||
1981 | /// { X, Y, u, Y, u, u, X, u } -> { X, Y } | |||
1982 | /// | |||
1983 | /// Currently this must be a power-of-2 build vector. | |||
1984 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
1985 | /// the original vector width and set the bits where elements are undef. | |||
1986 | /// If result is false, Sequence will be empty. | |||
1987 | bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence, | |||
1988 | BitVector *UndefElements = nullptr) const; | |||
1989 | ||||
1990 | /// Returns the demanded splatted constant or null if this is not a constant | |||
1991 | /// splat. | |||
1992 | /// | |||
1993 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
1994 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
1995 | /// the vector width and set the bits where elements are undef. | |||
1996 | ConstantSDNode * | |||
1997 | getConstantSplatNode(const APInt &DemandedElts, | |||
1998 | BitVector *UndefElements = nullptr) const; | |||
1999 | ||||
2000 | /// Returns the splatted constant or null if this is not a constant | |||
2001 | /// splat. | |||
2002 | /// | |||
2003 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
2004 | /// the vector width and set the bits where elements are undef. | |||
2005 | ConstantSDNode * | |||
2006 | getConstantSplatNode(BitVector *UndefElements = nullptr) const; | |||
2007 | ||||
2008 | /// Returns the demanded splatted constant FP or null if this is not a | |||
2009 | /// constant FP splat. | |||
2010 | /// | |||
2011 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
2012 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
2013 | /// the vector width and set the bits where elements are undef. | |||
2014 | ConstantFPSDNode * | |||
2015 | getConstantFPSplatNode(const APInt &DemandedElts, | |||
2016 | BitVector *UndefElements = nullptr) const; | |||
2017 | ||||
2018 | /// Returns the splatted constant FP or null if this is not a constant | |||
2019 | /// FP splat. | |||
2020 | /// | |||
2021 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
2022 | /// the vector width and set the bits where elements are undef. | |||
2023 | ConstantFPSDNode * | |||
2024 | getConstantFPSplatNode(BitVector *UndefElements = nullptr) const; | |||
2025 | ||||
2026 | /// If this is a constant FP splat and the splatted constant FP is an | |||
2027 | /// exact power or 2, return the log base 2 integer value. Otherwise, | |||
2028 | /// return -1. | |||
2029 | /// | |||
2030 | /// The BitWidth specifies the necessary bit precision. | |||
2031 | int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, | |||
2032 | uint32_t BitWidth) const; | |||
2033 | ||||
2034 | bool isConstant() const; | |||
2035 | ||||
2036 | static bool classof(const SDNode *N) { | |||
2037 | return N->getOpcode() == ISD::BUILD_VECTOR; | |||
2038 | } | |||
2039 | }; | |||
2040 | ||||
2041 | /// An SDNode that holds an arbitrary LLVM IR Value. This is | |||
2042 | /// used when the SelectionDAG needs to make a simple reference to something | |||
2043 | /// in the LLVM IR representation. | |||
2044 | /// | |||
2045 | class SrcValueSDNode : public SDNode { | |||
2046 | friend class SelectionDAG; | |||
2047 | ||||
2048 | const Value *V; | |||
2049 | ||||
2050 | /// Create a SrcValue for a general value. | |||
2051 | explicit SrcValueSDNode(const Value *v) | |||
2052 | : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {} | |||
2053 | ||||
2054 | public: | |||
2055 | /// Return the contained Value. | |||
2056 | const Value *getValue() const { return V; } | |||
2057 | ||||
2058 | static bool classof(const SDNode *N) { | |||
2059 | return N->getOpcode() == ISD::SRCVALUE; | |||
2060 | } | |||
2061 | }; | |||
2062 | ||||
2063 | class MDNodeSDNode : public SDNode { | |||
2064 | friend class SelectionDAG; | |||
2065 | ||||
2066 | const MDNode *MD; | |||
2067 | ||||
2068 | explicit MDNodeSDNode(const MDNode *md) | |||
2069 | : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md) | |||
2070 | {} | |||
2071 | ||||
2072 | public: | |||
2073 | const MDNode *getMD() const { return MD; } | |||
2074 | ||||
2075 | static bool classof(const SDNode *N) { | |||
2076 | return N->getOpcode() == ISD::MDNODE_SDNODE; | |||
2077 | } | |||
2078 | }; | |||
2079 | ||||
2080 | class RegisterSDNode : public SDNode { | |||
2081 | friend class SelectionDAG; | |||
2082 | ||||
2083 | Register Reg; | |||
2084 | ||||
2085 | RegisterSDNode(Register reg, EVT VT) | |||
2086 | : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {} | |||
2087 | ||||
2088 | public: | |||
2089 | Register getReg() const { return Reg; } | |||
2090 | ||||
2091 | static bool classof(const SDNode *N) { | |||
2092 | return N->getOpcode() == ISD::Register; | |||
2093 | } | |||
2094 | }; | |||
2095 | ||||
2096 | class RegisterMaskSDNode : public SDNode { | |||
2097 | friend class SelectionDAG; | |||
2098 | ||||
2099 | // The memory for RegMask is not owned by the node. | |||
2100 | const uint32_t *RegMask; | |||
2101 | ||||
2102 | RegisterMaskSDNode(const uint32_t *mask) | |||
2103 | : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)), | |||
2104 | RegMask(mask) {} | |||
2105 | ||||
2106 | public: | |||
2107 | const uint32_t *getRegMask() const { return RegMask; } | |||
2108 | ||||
2109 | static bool classof(const SDNode *N) { | |||
2110 | return N->getOpcode() == ISD::RegisterMask; | |||
2111 | } | |||
2112 | }; | |||
2113 | ||||
2114 | class BlockAddressSDNode : public SDNode { | |||
2115 | friend class SelectionDAG; | |||
2116 | ||||
2117 | const BlockAddress *BA; | |||
2118 | int64_t Offset; | |||
2119 | unsigned TargetFlags; | |||
2120 | ||||
2121 | BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba, | |||
2122 | int64_t o, unsigned Flags) | |||
2123 | : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)), | |||
2124 | BA(ba), Offset(o), TargetFlags(Flags) {} | |||
2125 | ||||
2126 | public: | |||
2127 | const BlockAddress *getBlockAddress() const { return BA; } | |||
2128 | int64_t getOffset() const { return Offset; } | |||
2129 | unsigned getTargetFlags() const { return TargetFlags; } | |||
2130 | ||||
2131 | static bool classof(const SDNode *N) { | |||
2132 | return N->getOpcode() == ISD::BlockAddress || | |||
2133 | N->getOpcode() == ISD::TargetBlockAddress; | |||
2134 | } | |||
2135 | }; | |||
2136 | ||||
2137 | class LabelSDNode : public SDNode { | |||
2138 | friend class SelectionDAG; | |||
2139 | ||||
2140 | MCSymbol *Label; | |||
2141 | ||||
2142 | LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L) | |||
2143 | : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) { | |||
2144 | assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) && "not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2144, __extension__ __PRETTY_FUNCTION__)); | |||
2145 | } | |||
2146 | ||||
2147 | public: | |||
2148 | MCSymbol *getLabel() const { return Label; } | |||
2149 | ||||
2150 | static bool classof(const SDNode *N) { | |||
2151 | return N->getOpcode() == ISD::EH_LABEL || | |||
2152 | N->getOpcode() == ISD::ANNOTATION_LABEL; | |||
2153 | } | |||
2154 | }; | |||
2155 | ||||
2156 | class ExternalSymbolSDNode : public SDNode { | |||
2157 | friend class SelectionDAG; | |||
2158 | ||||
2159 | const char *Symbol; | |||
2160 | unsigned TargetFlags; | |||
2161 | ||||
2162 | ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT) | |||
2163 | : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0, | |||
2164 | DebugLoc(), getSDVTList(VT)), | |||
2165 | Symbol(Sym), TargetFlags(TF) {} | |||
2166 | ||||
2167 | public: | |||
2168 | const char *getSymbol() const { return Symbol; } | |||
2169 | unsigned getTargetFlags() const { return TargetFlags; } | |||
2170 | ||||
2171 | static bool classof(const SDNode *N) { | |||
2172 | return N->getOpcode() == ISD::ExternalSymbol || | |||
2173 | N->getOpcode() == ISD::TargetExternalSymbol; | |||
2174 | } | |||
2175 | }; | |||
2176 | ||||
2177 | class MCSymbolSDNode : public SDNode { | |||
2178 | friend class SelectionDAG; | |||
2179 | ||||
2180 | MCSymbol *Symbol; | |||
2181 | ||||
2182 | MCSymbolSDNode(MCSymbol *Symbol, EVT VT) | |||
2183 | : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {} | |||
2184 | ||||
2185 | public: | |||
2186 | MCSymbol *getMCSymbol() const { return Symbol; } | |||
2187 | ||||
2188 | static bool classof(const SDNode *N) { | |||
2189 | return N->getOpcode() == ISD::MCSymbol; | |||
2190 | } | |||
2191 | }; | |||
2192 | ||||
2193 | class CondCodeSDNode : public SDNode { | |||
2194 | friend class SelectionDAG; | |||
2195 | ||||
2196 | ISD::CondCode Condition; | |||
2197 | ||||
2198 | explicit CondCodeSDNode(ISD::CondCode Cond) | |||
2199 | : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)), | |||
2200 | Condition(Cond) {} | |||
2201 | ||||
2202 | public: | |||
2203 | ISD::CondCode get() const { return Condition; } | |||
2204 | ||||
2205 | static bool classof(const SDNode *N) { | |||
2206 | return N->getOpcode() == ISD::CONDCODE; | |||
2207 | } | |||
2208 | }; | |||
2209 | ||||
2210 | /// This class is used to represent EVT's, which are used | |||
2211 | /// to parameterize some operations. | |||
2212 | class VTSDNode : public SDNode { | |||
2213 | friend class SelectionDAG; | |||
2214 | ||||
2215 | EVT ValueType; | |||
2216 | ||||
2217 | explicit VTSDNode(EVT VT) | |||
2218 | : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)), | |||
2219 | ValueType(VT) {} | |||
2220 | ||||
2221 | public: | |||
2222 | EVT getVT() const { return ValueType; } | |||
2223 | ||||
2224 | static bool classof(const SDNode *N) { | |||
2225 | return N->getOpcode() == ISD::VALUETYPE; | |||
2226 | } | |||
2227 | }; | |||
2228 | ||||
2229 | /// Base class for LoadSDNode and StoreSDNode | |||
2230 | class LSBaseSDNode : public MemSDNode { | |||
2231 | public: | |||
2232 | LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl, | |||
2233 | SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT, | |||
2234 | MachineMemOperand *MMO) | |||
2235 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { | |||
2236 | LSBaseSDNodeBits.AddressingMode = AM; | |||
2237 | assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM && "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2237, __extension__ __PRETTY_FUNCTION__)); | |||
2238 | } | |||
2239 | ||||
2240 | const SDValue &getOffset() const { | |||
2241 | return getOperand(getOpcode() == ISD::LOAD ? 2 : 3); | |||
2242 | } | |||
2243 | ||||
2244 | /// Return the addressing mode for this load or store: | |||
2245 | /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. | |||
2246 | ISD::MemIndexedMode getAddressingMode() const { | |||
2247 | return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode); | |||
2248 | } | |||
2249 | ||||
2250 | /// Return true if this is a pre/post inc/dec load/store. | |||
2251 | bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } | |||
2252 | ||||
2253 | /// Return true if this is NOT a pre/post inc/dec load/store. | |||
2254 | bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } | |||
2255 | ||||
2256 | static bool classof(const SDNode *N) { | |||
2257 | return N->getOpcode() == ISD::LOAD || | |||
2258 | N->getOpcode() == ISD::STORE; | |||
2259 | } | |||
2260 | }; | |||
2261 | ||||
2262 | /// This class is used to represent ISD::LOAD nodes. | |||
2263 | class LoadSDNode : public LSBaseSDNode { | |||
2264 | friend class SelectionDAG; | |||
2265 | ||||
2266 | LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
2267 | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT, | |||
2268 | MachineMemOperand *MMO) | |||
2269 | : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) { | |||
2270 | LoadSDNodeBits.ExtTy = ETy; | |||
2271 | assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!" ) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2271, __extension__ __PRETTY_FUNCTION__)); | |||
2272 | assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!" ) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2272, __extension__ __PRETTY_FUNCTION__)); | |||
2273 | } | |||
2274 | ||||
2275 | public: | |||
2276 | /// Return whether this is a plain node, | |||
2277 | /// or one of the varieties of value-extending loads. | |||
2278 | ISD::LoadExtType getExtensionType() const { | |||
2279 | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); | |||
2280 | } | |||
2281 | ||||
2282 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
2283 | const SDValue &getOffset() const { return getOperand(2); } | |||
2284 | ||||
2285 | static bool classof(const SDNode *N) { | |||
2286 | return N->getOpcode() == ISD::LOAD; | |||
2287 | } | |||
2288 | }; | |||
2289 | ||||
2290 | /// This class is used to represent ISD::STORE nodes. | |||
2291 | class StoreSDNode : public LSBaseSDNode { | |||
2292 | friend class SelectionDAG; | |||
2293 | ||||
2294 | StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
2295 | ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT, | |||
2296 | MachineMemOperand *MMO) | |||
2297 | : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) { | |||
2298 | StoreSDNodeBits.IsTruncating = isTrunc; | |||
2299 | assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <bool> (!readMem() && "Store MachineMemOperand is a load!" ) ? void (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2299, __extension__ __PRETTY_FUNCTION__)); | |||
2300 | assert(writeMem() && "Store MachineMemOperand is not a store!")(static_cast <bool> (writeMem() && "Store MachineMemOperand is not a store!" ) ? void (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2300, __extension__ __PRETTY_FUNCTION__)); | |||
2301 | } | |||
2302 | ||||
2303 | public: | |||
2304 | /// Return true if the op does a truncation before store. | |||
2305 | /// For integers this is the same as doing a TRUNCATE and storing the result. | |||
2306 | /// For floats, it is the same as doing an FP_ROUND and storing the result. | |||
2307 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } | |||
2308 | void setTruncatingStore(bool Truncating) { | |||
2309 | StoreSDNodeBits.IsTruncating = Truncating; | |||
2310 | } | |||
2311 | ||||
2312 | const SDValue &getValue() const { return getOperand(1); } | |||
2313 | const SDValue &getBasePtr() const { return getOperand(2); } | |||
2314 | const SDValue &getOffset() const { return getOperand(3); } | |||
2315 | ||||
2316 | static bool classof(const SDNode *N) { | |||
2317 | return N->getOpcode() == ISD::STORE; | |||
2318 | } | |||
2319 | }; | |||
2320 | ||||
2321 | /// This base class is used to represent MLOAD and MSTORE nodes | |||
2322 | class MaskedLoadStoreSDNode : public MemSDNode { | |||
2323 | public: | |||
2324 | friend class SelectionDAG; | |||
2325 | ||||
2326 | MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, | |||
2327 | const DebugLoc &dl, SDVTList VTs, | |||
2328 | ISD::MemIndexedMode AM, EVT MemVT, | |||
2329 | MachineMemOperand *MMO) | |||
2330 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { | |||
2331 | LSBaseSDNodeBits.AddressingMode = AM; | |||
2332 | assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM && "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2332, __extension__ __PRETTY_FUNCTION__)); | |||
2333 | } | |||
2334 | ||||
2335 | // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru) | |||
2336 | // MaskedStoreSDNode (Chain, data, ptr, offset, mask) | |||
2337 | // Mask is a vector of i1 elements | |||
2338 | const SDValue &getOffset() const { | |||
2339 | return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3); | |||
2340 | } | |||
2341 | const SDValue &getMask() const { | |||
2342 | return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4); | |||
2343 | } | |||
2344 | ||||
2345 | /// Return the addressing mode for this load or store: | |||
2346 | /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. | |||
2347 | ISD::MemIndexedMode getAddressingMode() const { | |||
2348 | return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode); | |||
2349 | } | |||
2350 | ||||
2351 | /// Return true if this is a pre/post inc/dec load/store. | |||
2352 | bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } | |||
2353 | ||||
2354 | /// Return true if this is NOT a pre/post inc/dec load/store. | |||
2355 | bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } | |||
2356 | ||||
2357 | static bool classof(const SDNode *N) { | |||
2358 | return N->getOpcode() == ISD::MLOAD || | |||
2359 | N->getOpcode() == ISD::MSTORE; | |||
2360 | } | |||
2361 | }; | |||
2362 | ||||
2363 | /// This class is used to represent an MLOAD node | |||
2364 | class MaskedLoadSDNode : public MaskedLoadStoreSDNode { | |||
2365 | public: | |||
2366 | friend class SelectionDAG; | |||
2367 | ||||
2368 | MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
2369 | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, | |||
2370 | bool IsExpanding, EVT MemVT, MachineMemOperand *MMO) | |||
2371 | : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) { | |||
2372 | LoadSDNodeBits.ExtTy = ETy; | |||
2373 | LoadSDNodeBits.IsExpanding = IsExpanding; | |||
2374 | } | |||
2375 | ||||
2376 | ISD::LoadExtType getExtensionType() const { | |||
2377 | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); | |||
2378 | } | |||
2379 | ||||
2380 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
2381 | const SDValue &getOffset() const { return getOperand(2); } | |||
2382 | const SDValue &getMask() const { return getOperand(3); } | |||
2383 | const SDValue &getPassThru() const { return getOperand(4); } | |||
2384 | ||||
2385 | static bool classof(const SDNode *N) { | |||
2386 | return N->getOpcode() == ISD::MLOAD; | |||
2387 | } | |||
2388 | ||||
2389 | bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; } | |||
2390 | }; | |||
2391 | ||||
2392 | /// This class is used to represent an MSTORE node | |||
2393 | class MaskedStoreSDNode : public MaskedLoadStoreSDNode { | |||
2394 | public: | |||
2395 | friend class SelectionDAG; | |||
2396 | ||||
2397 | MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
2398 | ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing, | |||
2399 | EVT MemVT, MachineMemOperand *MMO) | |||
2400 | : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) { | |||
2401 | StoreSDNodeBits.IsTruncating = isTrunc; | |||
2402 | StoreSDNodeBits.IsCompressing = isCompressing; | |||
2403 | } | |||
2404 | ||||
2405 | /// Return true if the op does a truncation before store. | |||
2406 | /// For integers this is the same as doing a TRUNCATE and storing the result. | |||
2407 | /// For floats, it is the same as doing an FP_ROUND and storing the result. | |||
2408 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } | |||
2409 | ||||
2410 | /// Returns true if the op does a compression to the vector before storing. | |||
2411 | /// The node contiguously stores the active elements (integers or floats) | |||
2412 | /// in src (those with their respective bit set in writemask k) to unaligned | |||
2413 | /// memory at base_addr. | |||
2414 | bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; } | |||
2415 | ||||
2416 | const SDValue &getValue() const { return getOperand(1); } | |||
2417 | const SDValue &getBasePtr() const { return getOperand(2); } | |||
2418 | const SDValue &getOffset() const { return getOperand(3); } | |||
2419 | const SDValue &getMask() const { return getOperand(4); } | |||
2420 | ||||
2421 | static bool classof(const SDNode *N) { | |||
2422 | return N->getOpcode() == ISD::MSTORE; | |||
2423 | } | |||
2424 | }; | |||
2425 | ||||
2426 | /// This is a base class used to represent | |||
2427 | /// MGATHER and MSCATTER nodes | |||
2428 | /// | |||
2429 | class MaskedGatherScatterSDNode : public MemSDNode { | |||
2430 | public: | |||
2431 | friend class SelectionDAG; | |||
2432 | ||||
2433 | MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, | |||
2434 | const DebugLoc &dl, SDVTList VTs, EVT MemVT, | |||
2435 | MachineMemOperand *MMO, ISD::MemIndexType IndexType) | |||
2436 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { | |||
2437 | LSBaseSDNodeBits.AddressingMode = IndexType; | |||
2438 | assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType && "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2438, __extension__ __PRETTY_FUNCTION__)); | |||
2439 | } | |||
2440 | ||||
2441 | /// How is Index applied to BasePtr when computing addresses. | |||
2442 | ISD::MemIndexType getIndexType() const { | |||
2443 | return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode); | |||
2444 | } | |||
2445 | void setIndexType(ISD::MemIndexType IndexType) { | |||
2446 | LSBaseSDNodeBits.AddressingMode = IndexType; | |||
2447 | } | |||
2448 | bool isIndexScaled() const { | |||
2449 | return (getIndexType() == ISD::SIGNED_SCALED) || | |||
2450 | (getIndexType() == ISD::UNSIGNED_SCALED); | |||
2451 | } | |||
2452 | bool isIndexSigned() const { | |||
2453 | return (getIndexType() == ISD::SIGNED_SCALED) || | |||
2454 | (getIndexType() == ISD::SIGNED_UNSCALED); | |||
2455 | } | |||
2456 | ||||
2457 | // In the both nodes address is Op1, mask is Op2: | |||
2458 | // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale) | |||
2459 | // MaskedScatterSDNode (Chain, value, mask, base, index, scale) | |||
2460 | // Mask is a vector of i1 elements | |||
2461 | const SDValue &getBasePtr() const { return getOperand(3); } | |||
2462 | const SDValue &getIndex() const { return getOperand(4); } | |||
2463 | const SDValue &getMask() const { return getOperand(2); } | |||
2464 | const SDValue &getScale() const { return getOperand(5); } | |||
2465 | ||||
2466 | static bool classof(const SDNode *N) { | |||
2467 | return N->getOpcode() == ISD::MGATHER || | |||
2468 | N->getOpcode() == ISD::MSCATTER; | |||
2469 | } | |||
2470 | }; | |||
2471 | ||||
2472 | /// This class is used to represent an MGATHER node | |||
2473 | /// | |||
2474 | class MaskedGatherSDNode : public MaskedGatherScatterSDNode { | |||
2475 | public: | |||
2476 | friend class SelectionDAG; | |||
2477 | ||||
2478 | MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
2479 | EVT MemVT, MachineMemOperand *MMO, | |||
2480 | ISD::MemIndexType IndexType, ISD::LoadExtType ETy) | |||
2481 | : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO, | |||
2482 | IndexType) { | |||
2483 | LoadSDNodeBits.ExtTy = ETy; | |||
2484 | } | |||
2485 | ||||
2486 | const SDValue &getPassThru() const { return getOperand(1); } | |||
2487 | ||||
2488 | ISD::LoadExtType getExtensionType() const { | |||
2489 | return ISD::LoadExtType(LoadSDNodeBits.ExtTy); | |||
2490 | } | |||
2491 | ||||
2492 | static bool classof(const SDNode *N) { | |||
2493 | return N->getOpcode() == ISD::MGATHER; | |||
2494 | } | |||
2495 | }; | |||
2496 | ||||
2497 | /// This class is used to represent an MSCATTER node | |||
2498 | /// | |||
2499 | class MaskedScatterSDNode : public MaskedGatherScatterSDNode { | |||
2500 | public: | |||
2501 | friend class SelectionDAG; | |||
2502 | ||||
2503 | MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
2504 | EVT MemVT, MachineMemOperand *MMO, | |||
2505 | ISD::MemIndexType IndexType, bool IsTrunc) | |||
2506 | : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO, | |||
2507 | IndexType) { | |||
2508 | StoreSDNodeBits.IsTruncating = IsTrunc; | |||
2509 | } | |||
2510 | ||||
2511 | /// Return true if the op does a truncation before store. | |||
2512 | /// For integers this is the same as doing a TRUNCATE and storing the result. | |||
2513 | /// For floats, it is the same as doing an FP_ROUND and storing the result. | |||
2514 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } | |||
2515 | ||||
2516 | const SDValue &getValue() const { return getOperand(1); } | |||
2517 | ||||
2518 | static bool classof(const SDNode *N) { | |||
2519 | return N->getOpcode() == ISD::MSCATTER; | |||
2520 | } | |||
2521 | }; | |||
2522 | ||||
2523 | /// An SDNode that represents everything that will be needed | |||
2524 | /// to construct a MachineInstr. These nodes are created during the | |||
2525 | /// instruction selection proper phase. | |||
2526 | /// | |||
2527 | /// Note that the only supported way to set the `memoperands` is by calling the | |||
2528 | /// `SelectionDAG::setNodeMemRefs` function as the memory management happens | |||
2529 | /// inside the DAG rather than in the node. | |||
2530 | class MachineSDNode : public SDNode { | |||
2531 | private: | |||
2532 | friend class SelectionDAG; | |||
2533 | ||||
2534 | MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs) | |||
2535 | : SDNode(Opc, Order, DL, VTs) {} | |||
2536 | ||||
2537 | // We use a pointer union between a single `MachineMemOperand` pointer and | |||
2538 | // a pointer to an array of `MachineMemOperand` pointers. This is null when | |||
2539 | // the number of these is zero, the single pointer variant used when the | |||
2540 | // number is one, and the array is used for larger numbers. | |||
2541 | // | |||
2542 | // The array is allocated via the `SelectionDAG`'s allocator and so will | |||
2543 | // always live until the DAG is cleaned up and doesn't require ownership here. | |||
2544 | // | |||
2545 | // We can't use something simpler like `TinyPtrVector` here because `SDNode` | |||
2546 | // subclasses aren't managed in a conforming C++ manner. See the comments on | |||
2547 | // `SelectionDAG::MorphNodeTo` which details what all goes on, but the | |||
2548 | // constraint here is that these don't manage memory with their constructor or | |||
2549 | // destructor and can be initialized to a good state even if they start off | |||
2550 | // uninitialized. | |||
2551 | PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {}; | |||
2552 | ||||
2553 | // Note that this could be folded into the above `MemRefs` member if doing so | |||
2554 | // is advantageous at some point. We don't need to store this in most cases. | |||
2555 | // However, at the moment this doesn't appear to make the allocation any | |||
2556 | // smaller and makes the code somewhat simpler to read. | |||
2557 | int NumMemRefs = 0; | |||
2558 | ||||
2559 | public: | |||
2560 | using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator; | |||
2561 | ||||
2562 | ArrayRef<MachineMemOperand *> memoperands() const { | |||
2563 | // Special case the common cases. | |||
2564 | if (NumMemRefs == 0) | |||
2565 | return {}; | |||
2566 | if (NumMemRefs == 1) | |||
2567 | return makeArrayRef(MemRefs.getAddrOfPtr1(), 1); | |||
2568 | ||||
2569 | // Otherwise we have an actual array. | |||
2570 | return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs); | |||
2571 | } | |||
2572 | mmo_iterator memoperands_begin() const { return memoperands().begin(); } | |||
2573 | mmo_iterator memoperands_end() const { return memoperands().end(); } | |||
2574 | bool memoperands_empty() const { return memoperands().empty(); } | |||
2575 | ||||
2576 | /// Clear out the memory reference descriptor list. | |||
2577 | void clearMemRefs() { | |||
2578 | MemRefs = nullptr; | |||
2579 | NumMemRefs = 0; | |||
2580 | } | |||
2581 | ||||
2582 | static bool classof(const SDNode *N) { | |||
2583 | return N->isMachineOpcode(); | |||
2584 | } | |||
2585 | }; | |||
2586 | ||||
2587 | /// An SDNode that records if a register contains a value that is guaranteed to | |||
2588 | /// be aligned accordingly. | |||
2589 | class AssertAlignSDNode : public SDNode { | |||
2590 | Align Alignment; | |||
2591 | ||||
2592 | public: | |||
2593 | AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A) | |||
2594 | : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {} | |||
2595 | ||||
2596 | Align getAlign() const { return Alignment; } | |||
2597 | ||||
2598 | static bool classof(const SDNode *N) { | |||
2599 | return N->getOpcode() == ISD::AssertAlign; | |||
2600 | } | |||
2601 | }; | |||
2602 | ||||
2603 | class SDNodeIterator { | |||
2604 | const SDNode *Node; | |||
2605 | unsigned Operand; | |||
2606 | ||||
2607 | SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {} | |||
2608 | ||||
2609 | public: | |||
2610 | using iterator_category = std::forward_iterator_tag; | |||
2611 | using value_type = SDNode; | |||
2612 | using difference_type = std::ptrdiff_t; | |||
2613 | using pointer = value_type *; | |||
2614 | using reference = value_type &; | |||
2615 | ||||
2616 | bool operator==(const SDNodeIterator& x) const { | |||
2617 | return Operand == x.Operand; | |||
2618 | } | |||
2619 | bool operator!=(const SDNodeIterator& x) const { return !operator==(x); } | |||
2620 | ||||
2621 | pointer operator*() const { | |||
2622 | return Node->getOperand(Operand).getNode(); | |||
2623 | } | |||
2624 | pointer operator->() const { return operator*(); } | |||
2625 | ||||
2626 | SDNodeIterator& operator++() { // Preincrement | |||
2627 | ++Operand; | |||
2628 | return *this; | |||
2629 | } | |||
2630 | SDNodeIterator operator++(int) { // Postincrement | |||
2631 | SDNodeIterator tmp = *this; ++*this; return tmp; | |||
2632 | } | |||
2633 | size_t operator-(SDNodeIterator Other) const { | |||
2634 | assert(Node == Other.Node &&(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!" ) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2635, __extension__ __PRETTY_FUNCTION__)) | |||
2635 | "Cannot compare iterators of two different nodes!")(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!" ) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2635, __extension__ __PRETTY_FUNCTION__)); | |||
2636 | return Operand - Other.Operand; | |||
2637 | } | |||
2638 | ||||
2639 | static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); } | |||
2640 | static SDNodeIterator end (const SDNode *N) { | |||
2641 | return SDNodeIterator(N, N->getNumOperands()); | |||
2642 | } | |||
2643 | ||||
2644 | unsigned getOperand() const { return Operand; } | |||
2645 | const SDNode *getNode() const { return Node; } | |||
2646 | }; | |||
2647 | ||||
2648 | template <> struct GraphTraits<SDNode*> { | |||
2649 | using NodeRef = SDNode *; | |||
2650 | using ChildIteratorType = SDNodeIterator; | |||
2651 | ||||
2652 | static NodeRef getEntryNode(SDNode *N) { return N; } | |||
2653 | ||||
2654 | static ChildIteratorType child_begin(NodeRef N) { | |||
2655 | return SDNodeIterator::begin(N); | |||
2656 | } | |||
2657 | ||||
2658 | static ChildIteratorType child_end(NodeRef N) { | |||
2659 | return SDNodeIterator::end(N); | |||
2660 | } | |||
2661 | }; | |||
2662 | ||||
2663 | /// A representation of the largest SDNode, for use in sizeof(). | |||
2664 | /// | |||
2665 | /// This needs to be a union because the largest node differs on 32 bit systems | |||
2666 | /// with 4 and 8 byte pointer alignment, respectively. | |||
2667 | using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode, | |||
2668 | BlockAddressSDNode, | |||
2669 | GlobalAddressSDNode, | |||
2670 | PseudoProbeSDNode>; | |||
2671 | ||||
2672 | /// The SDNode class with the greatest alignment requirement. | |||
2673 | using MostAlignedSDNode = GlobalAddressSDNode; | |||
2674 | ||||
2675 | namespace ISD { | |||
2676 | ||||
2677 | /// Returns true if the specified node is a non-extending and unindexed load. | |||
2678 | inline bool isNormalLoad(const SDNode *N) { | |||
2679 | const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N); | |||
2680 | return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD && | |||
2681 | Ld->getAddressingMode() == ISD::UNINDEXED; | |||
2682 | } | |||
2683 | ||||
2684 | /// Returns true if the specified node is a non-extending load. | |||
2685 | inline bool isNON_EXTLoad(const SDNode *N) { | |||
2686 | return isa<LoadSDNode>(N) && | |||
2687 | cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD; | |||
2688 | } | |||
2689 | ||||
2690 | /// Returns true if the specified node is a EXTLOAD. | |||
2691 | inline bool isEXTLoad(const SDNode *N) { | |||
2692 | return isa<LoadSDNode>(N) && | |||
2693 | cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD; | |||
2694 | } | |||
2695 | ||||
2696 | /// Returns true if the specified node is a SEXTLOAD. | |||
2697 | inline bool isSEXTLoad(const SDNode *N) { | |||
2698 | return isa<LoadSDNode>(N) && | |||
2699 | cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD; | |||
2700 | } | |||
2701 | ||||
2702 | /// Returns true if the specified node is a ZEXTLOAD. | |||
2703 | inline bool isZEXTLoad(const SDNode *N) { | |||
2704 | return isa<LoadSDNode>(N) && | |||
2705 | cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD; | |||
2706 | } | |||
2707 | ||||
2708 | /// Returns true if the specified node is an unindexed load. | |||
2709 | inline bool isUNINDEXEDLoad(const SDNode *N) { | |||
2710 | return isa<LoadSDNode>(N) && | |||
2711 | cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; | |||
2712 | } | |||
2713 | ||||
2714 | /// Returns true if the specified node is a non-truncating | |||
2715 | /// and unindexed store. | |||
2716 | inline bool isNormalStore(const SDNode *N) { | |||
2717 | const StoreSDNode *St = dyn_cast<StoreSDNode>(N); | |||
2718 | return St && !St->isTruncatingStore() && | |||
2719 | St->getAddressingMode() == ISD::UNINDEXED; | |||
2720 | } | |||
2721 | ||||
2722 | /// Returns true if the specified node is an unindexed store. | |||
2723 | inline bool isUNINDEXEDStore(const SDNode *N) { | |||
2724 | return isa<StoreSDNode>(N) && | |||
2725 | cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; | |||
2726 | } | |||
2727 | ||||
2728 | /// Attempt to match a unary predicate against a scalar/splat constant or | |||
2729 | /// every element of a constant BUILD_VECTOR. | |||
2730 | /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. | |||
2731 | bool matchUnaryPredicate(SDValue Op, | |||
2732 | std::function<bool(ConstantSDNode *)> Match, | |||
2733 | bool AllowUndefs = false); | |||
2734 | ||||
2735 | /// Attempt to match a binary predicate against a pair of scalar/splat | |||
2736 | /// constants or every element of a pair of constant BUILD_VECTORs. | |||
2737 | /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. | |||
2738 | /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match. | |||
2739 | bool matchBinaryPredicate( | |||
2740 | SDValue LHS, SDValue RHS, | |||
2741 | std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match, | |||
2742 | bool AllowUndefs = false, bool AllowTypeMismatch = false); | |||
2743 | ||||
2744 | /// Returns true if the specified value is the overflow result from one | |||
2745 | /// of the overflow intrinsic nodes. | |||
2746 | inline bool isOverflowIntrOpRes(SDValue Op) { | |||
2747 | unsigned Opc = Op.getOpcode(); | |||
2748 | return (Op.getResNo() == 1 && | |||
2749 | (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || | |||
2750 | Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)); | |||
2751 | } | |||
2752 | ||||
2753 | } // end namespace ISD | |||
2754 | ||||
2755 | } // end namespace llvm | |||
2756 | ||||
2757 | #endif // LLVM_CODEGEN_SELECTIONDAGNODES_H |