clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizeDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/CodeGen/SelectionDAG -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/include -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/include -D NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/CodeGen/SelectionDAG -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-09-04-040900-46481-1 -x c++ /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
1 | |
2 | |
3 | |
4 | |
5 | |
6 | |
7 | |
8 | |
9 | |
10 | |
11 | |
12 | |
13 | #include "llvm/ADT/APFloat.h" |
14 | #include "llvm/ADT/APInt.h" |
15 | #include "llvm/ADT/ArrayRef.h" |
16 | #include "llvm/ADT/SetVector.h" |
17 | #include "llvm/ADT/SmallPtrSet.h" |
18 | #include "llvm/ADT/SmallSet.h" |
19 | #include "llvm/ADT/SmallVector.h" |
20 | #include "llvm/Analysis/TargetLibraryInfo.h" |
21 | #include "llvm/CodeGen/ISDOpcodes.h" |
22 | #include "llvm/CodeGen/MachineFunction.h" |
23 | #include "llvm/CodeGen/MachineJumpTableInfo.h" |
24 | #include "llvm/CodeGen/MachineMemOperand.h" |
25 | #include "llvm/CodeGen/RuntimeLibcalls.h" |
26 | #include "llvm/CodeGen/SelectionDAG.h" |
27 | #include "llvm/CodeGen/SelectionDAGNodes.h" |
28 | #include "llvm/CodeGen/TargetFrameLowering.h" |
29 | #include "llvm/CodeGen/TargetLowering.h" |
30 | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
31 | #include "llvm/CodeGen/ValueTypes.h" |
32 | #include "llvm/IR/CallingConv.h" |
33 | #include "llvm/IR/Constants.h" |
34 | #include "llvm/IR/DataLayout.h" |
35 | #include "llvm/IR/DerivedTypes.h" |
36 | #include "llvm/IR/Function.h" |
37 | #include "llvm/IR/Metadata.h" |
38 | #include "llvm/IR/Type.h" |
39 | #include "llvm/Support/Casting.h" |
40 | #include "llvm/Support/Compiler.h" |
41 | #include "llvm/Support/Debug.h" |
42 | #include "llvm/Support/ErrorHandling.h" |
43 | #include "llvm/Support/MachineValueType.h" |
44 | #include "llvm/Support/MathExtras.h" |
45 | #include "llvm/Support/raw_ostream.h" |
46 | #include "llvm/Target/TargetMachine.h" |
47 | #include "llvm/Target/TargetOptions.h" |
48 | #include <algorithm> |
49 | #include <cassert> |
50 | #include <cstdint> |
51 | #include <tuple> |
52 | #include <utility> |
53 | |
54 | using namespace llvm; |
55 | |
56 | #define DEBUG_TYPE "legalizedag" |
57 | |
58 | namespace { |
59 | |
60 | |
61 | |
62 | struct FloatSignAsInt { |
63 | EVT FloatVT; |
64 | SDValue Chain; |
65 | SDValue FloatPtr; |
66 | SDValue IntPtr; |
67 | MachinePointerInfo IntPointerInfo; |
68 | MachinePointerInfo FloatPointerInfo; |
69 | SDValue IntValue; |
70 | APInt SignMask; |
71 | uint8_t SignBit; |
72 | }; |
73 | |
74 | |
75 | |
76 | |
77 | |
78 | |
79 | |
80 | |
81 | |
82 | |
83 | |
84 | |
85 | class SelectionDAGLegalize { |
86 | const TargetMachine &TM; |
87 | const TargetLowering &TLI; |
88 | SelectionDAG &DAG; |
89 | |
90 | |
91 | |
92 | SmallPtrSetImpl<SDNode *> &LegalizedNodes; |
93 | |
94 | |
95 | SmallSetVector<SDNode *, 16> *UpdatedNodes; |
96 | |
97 | EVT getSetCCResultType(EVT VT) const { |
98 | return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); |
99 | } |
100 | |
101 | |
102 | |
103 | public: |
104 | SelectionDAGLegalize(SelectionDAG &DAG, |
105 | SmallPtrSetImpl<SDNode *> &LegalizedNodes, |
106 | SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr) |
107 | : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG), |
108 | LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {} |
109 | |
110 | |
111 | void LegalizeOp(SDNode *Node); |
112 | |
113 | private: |
114 | SDValue OptimizeFloatStore(StoreSDNode *ST); |
115 | |
116 | void LegalizeLoadOps(SDNode *Node); |
117 | void LegalizeStoreOps(SDNode *Node); |
118 | |
119 | |
120 | |
121 | |
122 | |
123 | SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, |
124 | const SDLoc &dl); |
125 | SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, |
126 | const SDLoc &dl); |
127 | |
128 | |
129 | |
130 | |
131 | |
132 | SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl, |
133 | SDValue N1, SDValue N2, |
134 | ArrayRef<int> Mask) const; |
135 | |
136 | SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); |
137 | |
138 | void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC, |
139 | SmallVectorImpl<SDValue> &Results); |
140 | void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, |
141 | RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, |
142 | RTLIB::Libcall Call_F128, |
143 | RTLIB::Libcall Call_PPCF128, |
144 | SmallVectorImpl<SDValue> &Results); |
145 | SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, |
146 | RTLIB::Libcall Call_I8, |
147 | RTLIB::Libcall Call_I16, |
148 | RTLIB::Libcall Call_I32, |
149 | RTLIB::Libcall Call_I64, |
150 | RTLIB::Libcall Call_I128); |
151 | void ExpandArgFPLibCall(SDNode *Node, |
152 | RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, |
153 | RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, |
154 | RTLIB::Libcall Call_PPCF128, |
155 | SmallVectorImpl<SDValue> &Results); |
156 | void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); |
157 | void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); |
158 | |
159 | SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, |
160 | const SDLoc &dl); |
161 | SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, |
162 | const SDLoc &dl, SDValue ChainIn); |
163 | SDValue ExpandBUILD_VECTOR(SDNode *Node); |
164 | SDValue ExpandSPLAT_VECTOR(SDNode *Node); |
165 | SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); |
166 | void ExpandDYNAMIC_STACKALLOC(SDNode *Node, |
167 | SmallVectorImpl<SDValue> &Results); |
168 | void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL, |
169 | SDValue Value) const; |
170 | SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL, |
171 | SDValue NewIntValue) const; |
172 | SDValue ExpandFCOPYSIGN(SDNode *Node) const; |
173 | SDValue ExpandFABS(SDNode *Node) const; |
174 | SDValue ExpandFNEG(SDNode *Node) const; |
175 | SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain); |
176 | void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl, |
177 | SmallVectorImpl<SDValue> &Results); |
178 | void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl, |
179 | SmallVectorImpl<SDValue> &Results); |
180 | SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl); |
181 | |
182 | SDValue ExpandPARITY(SDValue Op, const SDLoc &dl); |
183 | |
184 | SDValue ExpandExtractFromVectorThroughStack(SDValue Op); |
185 | SDValue ExpandInsertToVectorThroughStack(SDValue Op); |
186 | SDValue ExpandVectorBuildThroughStack(SDNode* Node); |
187 | |
188 | SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP); |
189 | SDValue ExpandConstant(ConstantSDNode *CP); |
190 | |
191 | |
192 | bool ExpandNode(SDNode *Node); |
193 | void ConvertNodeToLibcall(SDNode *Node); |
194 | void PromoteNode(SDNode *Node); |
195 | |
196 | public: |
197 | |
198 | |
199 | void ReplacedNode(SDNode *N) { |
200 | LegalizedNodes.erase(N); |
201 | if (UpdatedNodes) |
202 | UpdatedNodes->insert(N); |
203 | } |
204 | |
205 | void ReplaceNode(SDNode *Old, SDNode *New) { |
206 | LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); |
207 | dbgs() << " with: "; New->dump(&DAG)); |
208 | |
209 | assert(Old->getNumValues() == New->getNumValues() && |
210 | "Replacing one node with another that produces a different number " |
211 | "of values!"); |
212 | DAG.ReplaceAllUsesWith(Old, New); |
213 | if (UpdatedNodes) |
214 | UpdatedNodes->insert(New); |
215 | ReplacedNode(Old); |
216 | } |
217 | |
218 | void ReplaceNode(SDValue Old, SDValue New) { |
219 | LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); |
220 | dbgs() << " with: "; New->dump(&DAG)); |
221 | |
222 | DAG.ReplaceAllUsesWith(Old, New); |
223 | if (UpdatedNodes) |
224 | UpdatedNodes->insert(New.getNode()); |
225 | ReplacedNode(Old.getNode()); |
226 | } |
227 | |
228 | void ReplaceNode(SDNode *Old, const SDValue *New) { |
229 | LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG)); |
230 | |
231 | DAG.ReplaceAllUsesWith(Old, New); |
232 | for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) { |
233 | LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: "); |
234 | New[i]->dump(&DAG)); |
235 | if (UpdatedNodes) |
236 | UpdatedNodes->insert(New[i].getNode()); |
237 | } |
238 | ReplacedNode(Old); |
239 | } |
240 | |
241 | void ReplaceNodeWithValue(SDValue Old, SDValue New) { |
242 | LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); |
243 | dbgs() << " with: "; New->dump(&DAG)); |
244 | |
245 | DAG.ReplaceAllUsesOfValueWith(Old, New); |
246 | if (UpdatedNodes) |
247 | UpdatedNodes->insert(New.getNode()); |
248 | ReplacedNode(Old.getNode()); |
249 | } |
250 | }; |
251 | |
252 | } |
253 | |
254 | |
255 | |
256 | |
257 | |
258 | SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType( |
259 | EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, |
260 | ArrayRef<int> Mask) const { |
261 | unsigned NumMaskElts = VT.getVectorNumElements(); |
262 | unsigned NumDestElts = NVT.getVectorNumElements(); |
263 | unsigned NumEltsGrowth = NumDestElts / NumMaskElts; |
264 | |
265 | assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); |
266 | |
267 | if (NumEltsGrowth == 1) |
268 | return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask); |
269 | |
270 | SmallVector<int, 8> NewMask; |
271 | for (unsigned i = 0; i != NumMaskElts; ++i) { |
272 | int Idx = Mask[i]; |
273 | for (unsigned j = 0; j != NumEltsGrowth; ++j) { |
274 | if (Idx < 0) |
275 | NewMask.push_back(-1); |
276 | else |
277 | NewMask.push_back(Idx * NumEltsGrowth + j); |
278 | } |
279 | } |
280 | assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); |
281 | assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"); |
282 | return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask); |
283 | } |
284 | |
285 | |
286 | |
287 | SDValue |
288 | SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { |
289 | bool Extend = false; |
290 | SDLoc dl(CFP); |
291 | |
292 | |
293 | |
294 | |
295 | |
296 | |
297 | |
298 | EVT VT = CFP->getValueType(0); |
299 | ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); |
300 | if (!UseCP) { |
301 | assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"); |
302 | return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl, |
303 | (VT == MVT::f64) ? MVT::i64 : MVT::i32); |
304 | } |
305 | |
306 | APFloat APF = CFP->getValueAPF(); |
307 | EVT OrigVT = VT; |
308 | EVT SVT = VT; |
309 | |
310 | |
311 | |
312 | if (!APF.isSignaling()) { |
313 | while (SVT != MVT::f32 && SVT != MVT::f16) { |
314 | SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); |
315 | if (ConstantFPSDNode::isValueValidForType(SVT, APF) && |
316 | |
317 | |
318 | TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) && |
319 | TLI.ShouldShrinkFPConstant(OrigVT)) { |
320 | Type *SType = SVT.getTypeForEVT(*DAG.getContext()); |
321 | LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); |
322 | VT = SVT; |
323 | Extend = true; |
324 | } |
325 | } |
326 | } |
327 | |
328 | SDValue CPIdx = |
329 | DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout())); |
330 | Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); |
331 | if (Extend) { |
332 | SDValue Result = DAG.getExtLoad( |
333 | ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx, |
334 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT, |
335 | Alignment); |
336 | return Result; |
337 | } |
338 | SDValue Result = DAG.getLoad( |
339 | OrigVT, dl, DAG.getEntryNode(), CPIdx, |
340 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); |
341 | return Result; |
342 | } |
343 | |
344 | |
345 | SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) { |
346 | SDLoc dl(CP); |
347 | EVT VT = CP->getValueType(0); |
348 | SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(), |
349 | TLI.getPointerTy(DAG.getDataLayout())); |
350 | Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); |
351 | SDValue Result = DAG.getLoad( |
352 | VT, dl, DAG.getEntryNode(), CPIdx, |
353 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); |
354 | return Result; |
355 | } |
356 | |
357 | |
358 | |
359 | |
360 | |
361 | SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec, |
362 | SDValue Val, |
363 | SDValue Idx, |
364 | const SDLoc &dl) { |
365 | SDValue Tmp1 = Vec; |
366 | SDValue Tmp2 = Val; |
367 | SDValue Tmp3 = Idx; |
368 | |
369 | |
370 | |
371 | |
372 | |
373 | |
374 | |
375 | EVT VT = Tmp1.getValueType(); |
376 | EVT EltVT = VT.getVectorElementType(); |
377 | SDValue StackPtr = DAG.CreateStackTemporary(VT); |
378 | |
379 | int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); |
380 | |
381 | |
382 | SDValue Ch = DAG.getStore( |
383 | DAG.getEntryNode(), dl, Tmp1, StackPtr, |
384 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); |
385 | |
386 | SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3); |
387 | |
388 | |
389 | Ch = DAG.getTruncStore( |
390 | Ch, dl, Tmp2, StackPtr2, |
391 | MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT); |
392 | |
393 | return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack( |
394 | DAG.getMachineFunction(), SPFI)); |
395 | } |
396 | |
397 | SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, |
398 | SDValue Idx, |
399 | const SDLoc &dl) { |
400 | if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) { |
401 | |
402 | |
403 | |
404 | EVT EltVT = Vec.getValueType().getVectorElementType(); |
405 | if (Val.getValueType() == EltVT || |
406 | (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) { |
407 | SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, |
408 | Vec.getValueType(), Val); |
409 | |
410 | unsigned NumElts = Vec.getValueType().getVectorNumElements(); |
411 | |
412 | |
413 | |
414 | SmallVector<int, 8> ShufOps; |
415 | for (unsigned i = 0; i != NumElts; ++i) |
416 | ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); |
417 | |
418 | return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps); |
419 | } |
420 | } |
421 | return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl); |
422 | } |
423 | |
424 | SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { |
425 | if (!ISD::isNormalStore(ST)) |
426 | return SDValue(); |
427 | |
428 | LLVM_DEBUG(dbgs() << "Optimizing float store operations\n"); |
429 | |
430 | |
431 | |
432 | |
433 | |
434 | |
435 | SDValue Chain = ST->getChain(); |
436 | SDValue Ptr = ST->getBasePtr(); |
437 | SDValue Value = ST->getValue(); |
438 | MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); |
439 | AAMDNodes AAInfo = ST->getAAInfo(); |
440 | SDLoc dl(ST); |
441 | |
442 | |
443 | if (Value.getOpcode() == ISD::TargetConstantFP) |
444 | return SDValue(); |
445 | |
446 | if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) { |
447 | if (CFP->getValueType(0) == MVT::f32 && |
448 | TLI.isTypeLegal(MVT::i32)) { |
449 | SDValue Con = DAG.getConstant(CFP->getValueAPF(). |
450 | bitcastToAPInt().zextOrTrunc(32), |
451 | SDLoc(CFP), MVT::i32); |
452 | return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), |
453 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
454 | } |
455 | |
456 | if (CFP->getValueType(0) == MVT::f64) { |
457 | |
458 | if (TLI.isTypeLegal(MVT::i64)) { |
459 | SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). |
460 | zextOrTrunc(64), SDLoc(CFP), MVT::i64); |
461 | return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), |
462 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
463 | } |
464 | |
465 | if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) { |
466 | |
467 | |
468 | |
469 | const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt(); |
470 | SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32); |
471 | SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32); |
472 | if (DAG.getDataLayout().isBigEndian()) |
473 | std::swap(Lo, Hi); |
474 | |
475 | Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), |
476 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
477 | Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(4), dl); |
478 | Hi = DAG.getStore(Chain, dl, Hi, Ptr, |
479 | ST->getPointerInfo().getWithOffset(4), |
480 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
481 | |
482 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
483 | } |
484 | } |
485 | } |
486 | return SDValue(); |
487 | } |
488 | |
489 | void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { |
490 | StoreSDNode *ST = cast<StoreSDNode>(Node); |
| 1 | 'Node' is a 'StoreSDNode' | |
|
491 | SDValue Chain = ST->getChain(); |
492 | SDValue Ptr = ST->getBasePtr(); |
493 | SDLoc dl(Node); |
494 | |
495 | MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); |
496 | AAMDNodes AAInfo = ST->getAAInfo(); |
497 | |
498 | if (!ST->isTruncatingStore()) { |
| 2 | | Assuming the condition is false | |
|
| |
499 | LLVM_DEBUG(dbgs() << "Legalizing store operation\n"); |
500 | if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) { |
501 | ReplaceNode(ST, OptStore); |
502 | return; |
503 | } |
504 | |
505 | SDValue Value = ST->getValue(); |
506 | MVT VT = Value.getSimpleValueType(); |
507 | switch (TLI.getOperationAction(ISD::STORE, VT)) { |
508 | default: llvm_unreachable("This action is not supported yet!"); |
509 | case TargetLowering::Legal: { |
510 | |
511 | |
512 | EVT MemVT = ST->getMemoryVT(); |
513 | const DataLayout &DL = DAG.getDataLayout(); |
514 | if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT, |
515 | *ST->getMemOperand())) { |
516 | LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n"); |
517 | SDValue Result = TLI.expandUnalignedStore(ST, DAG); |
518 | ReplaceNode(SDValue(ST, 0), Result); |
519 | } else |
520 | LLVM_DEBUG(dbgs() << "Legal store\n"); |
521 | break; |
522 | } |
523 | case TargetLowering::Custom: { |
524 | LLVM_DEBUG(dbgs() << "Trying custom lowering\n"); |
525 | SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); |
526 | if (Res && Res != SDValue(Node, 0)) |
527 | ReplaceNode(SDValue(Node, 0), Res); |
528 | return; |
529 | } |
530 | case TargetLowering::Promote: { |
531 | MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); |
532 | assert(NVT.getSizeInBits() == VT.getSizeInBits() && |
533 | "Can only promote stores to same size type"); |
534 | Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); |
535 | SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), |
536 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
537 | ReplaceNode(SDValue(Node, 0), Result); |
538 | break; |
539 | } |
540 | } |
541 | return; |
542 | } |
543 | |
544 | LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n"); |
| 4 | | Loop condition is false. Exiting loop | |
|
545 | SDValue Value = ST->getValue(); |
546 | EVT StVT = ST->getMemoryVT(); |
547 | TypeSize StWidth = StVT.getSizeInBits(); |
548 | TypeSize StSize = StVT.getStoreSizeInBits(); |
549 | auto &DL = DAG.getDataLayout(); |
550 | |
551 | if (StWidth != StSize) { |
| 5 | | Calling 'UnivariateLinearPolyBase::operator!=' | |
|
| 12 | | Returning from 'UnivariateLinearPolyBase::operator!=' | |
|
552 | |
553 | |
554 | |
555 | EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedSize()); |
556 | Value = DAG.getZeroExtendInReg(Value, dl, StVT); |
557 | SDValue Result = |
558 | DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT, |
559 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
560 | ReplaceNode(SDValue(Node, 0), Result); |
561 | } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedSize())) { |
| 13 | | Calling 'EVT::isVector' | |
|
| 16 | | Returning from 'EVT::isVector' | |
|
| 17 | | Assuming the condition is true | |
|
| 18 | | Calling 'isPowerOf2_64' | |
|
| 21 | | Returning from 'isPowerOf2_64' | |
|
| |
562 | |
563 | assert(!StVT.isVector() && "Unsupported truncstore!"); |
564 | unsigned StWidthBits = StWidth.getFixedSize(); |
565 | unsigned LogStWidth = Log2_32(StWidthBits); |
| |
| 25 | | Returning from 'Log2_32' | |
|
| 26 | | 'LogStWidth' initialized to 4294967295 | |
|
566 | assert(LogStWidth < 32); |
567 | unsigned RoundWidth = 1 << LogStWidth; |
| 27 | | The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int' |
|
568 | assert(RoundWidth < StWidthBits); |
569 | unsigned ExtraWidth = StWidthBits - RoundWidth; |
570 | assert(ExtraWidth < RoundWidth); |
571 | assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && |
572 | "Store size not an integral number of bytes!"); |
573 | EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); |
574 | EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); |
575 | SDValue Lo, Hi; |
576 | unsigned IncrementSize; |
577 | |
578 | if (DL.isLittleEndian()) { |
579 | |
580 | |
581 | Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), |
582 | RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo); |
583 | |
584 | |
585 | IncrementSize = RoundWidth / 8; |
586 | Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); |
587 | Hi = DAG.getNode( |
588 | ISD::SRL, dl, Value.getValueType(), Value, |
589 | DAG.getConstant(RoundWidth, dl, |
590 | TLI.getShiftAmountTy(Value.getValueType(), DL))); |
591 | Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, |
592 | ST->getPointerInfo().getWithOffset(IncrementSize), |
593 | ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo); |
594 | } else { |
595 | |
596 | |
597 | |
598 | Hi = DAG.getNode( |
599 | ISD::SRL, dl, Value.getValueType(), Value, |
600 | DAG.getConstant(ExtraWidth, dl, |
601 | TLI.getShiftAmountTy(Value.getValueType(), DL))); |
602 | Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT, |
603 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
604 | |
605 | |
606 | IncrementSize = RoundWidth / 8; |
607 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
608 | DAG.getConstant(IncrementSize, dl, |
609 | Ptr.getValueType())); |
610 | Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, |
611 | ST->getPointerInfo().getWithOffset(IncrementSize), |
612 | ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo); |
613 | } |
614 | |
615 | |
616 | SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
617 | ReplaceNode(SDValue(Node, 0), Result); |
618 | } else { |
619 | switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) { |
620 | default: llvm_unreachable("This action is not supported yet!"); |
621 | case TargetLowering::Legal: { |
622 | EVT MemVT = ST->getMemoryVT(); |
623 | |
624 | |
625 | if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT, |
626 | *ST->getMemOperand())) { |
627 | SDValue Result = TLI.expandUnalignedStore(ST, DAG); |
628 | ReplaceNode(SDValue(ST, 0), Result); |
629 | } |
630 | break; |
631 | } |
632 | case TargetLowering::Custom: { |
633 | SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); |
634 | if (Res && Res != SDValue(Node, 0)) |
635 | ReplaceNode(SDValue(Node, 0), Res); |
636 | return; |
637 | } |
638 | case TargetLowering::Expand: |
639 | assert(!StVT.isVector() && |
640 | "Vector Stores are handled in LegalizeVectorOps"); |
641 | |
642 | SDValue Result; |
643 | |
644 | |
645 | if (TLI.isTypeLegal(StVT)) { |
646 | Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); |
647 | Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), |
648 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
649 | } else { |
650 | |
651 | |
652 | Value = DAG.getNode(ISD::TRUNCATE, dl, |
653 | TLI.getTypeToTransformTo(*DAG.getContext(), StVT), |
654 | Value); |
655 | Result = |
656 | DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT, |
657 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
658 | } |
659 | |
660 | ReplaceNode(SDValue(Node, 0), Result); |
661 | break; |
662 | } |
663 | } |
664 | } |
665 | |
666 | void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { |
667 | LoadSDNode *LD = cast<LoadSDNode>(Node); |
668 | SDValue Chain = LD->getChain(); |
669 | SDValue Ptr = LD->getBasePtr(); |
670 | SDValue Value; |
671 | SDLoc dl(Node); |
672 | |
673 | ISD::LoadExtType ExtType = LD->getExtensionType(); |
674 | if (ExtType == ISD::NON_EXTLOAD) { |
675 | LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n"); |
676 | MVT VT = Node->getSimpleValueType(0); |
677 | SDValue RVal = SDValue(Node, 0); |
678 | SDValue RChain = SDValue(Node, 1); |
679 | |
680 | switch (TLI.getOperationAction(Node->getOpcode(), VT)) { |
681 | default: llvm_unreachable("This action is not supported yet!"); |
682 | case TargetLowering::Legal: { |
683 | EVT MemVT = LD->getMemoryVT(); |
684 | const DataLayout &DL = DAG.getDataLayout(); |
685 | |
686 | |
687 | if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT, |
688 | *LD->getMemOperand())) { |
689 | std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG); |
690 | } |
691 | break; |
692 | } |
693 | case TargetLowering::Custom: |
694 | if (SDValue Res = TLI.LowerOperation(RVal, DAG)) { |
695 | RVal = Res; |
696 | RChain = Res.getValue(1); |
697 | } |
698 | break; |
699 | |
700 | case TargetLowering::Promote: { |
701 | MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); |
702 | assert(NVT.getSizeInBits() == VT.getSizeInBits() && |
703 | "Can only promote loads to same size type"); |
704 | |
705 | SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand()); |
706 | RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); |
707 | RChain = Res.getValue(1); |
708 | break; |
709 | } |
710 | } |
711 | if (RChain.getNode() != Node) { |
712 | assert(RVal.getNode() != Node && "Load must be completely replaced"); |
713 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); |
714 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); |
715 | if (UpdatedNodes) { |
716 | UpdatedNodes->insert(RVal.getNode()); |
717 | UpdatedNodes->insert(RChain.getNode()); |
718 | } |
719 | ReplacedNode(Node); |
720 | } |
721 | return; |
722 | } |
723 | |
724 | LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n"); |
725 | EVT SrcVT = LD->getMemoryVT(); |
726 | TypeSize SrcWidth = SrcVT.getSizeInBits(); |
727 | MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); |
728 | AAMDNodes AAInfo = LD->getAAInfo(); |
729 | |
730 | if (SrcWidth != SrcVT.getStoreSizeInBits() && |
731 | |
732 | |
733 | |
734 | |
735 | |
736 | |
737 | |
738 | (SrcVT != MVT::i1 || |
739 | TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) == |
740 | TargetLowering::Promote)) { |
741 | |
742 | |
743 | unsigned NewWidth = SrcVT.getStoreSizeInBits(); |
744 | EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth); |
745 | SDValue Ch; |
746 | |
747 | |
748 | |
749 | |
750 | ISD::LoadExtType NewExtType = |
751 | ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; |
752 | |
753 | SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), |
754 | Chain, Ptr, LD->getPointerInfo(), NVT, |
755 | LD->getOriginalAlign(), MMOFlags, AAInfo); |
756 | |
757 | Ch = Result.getValue(1); |
758 | |
759 | if (ExtType == ISD::SEXTLOAD) |
760 | |
761 | Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, |
762 | Result.getValueType(), |
763 | Result, DAG.getValueType(SrcVT)); |
764 | else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) |
765 | |
766 | Result = DAG.getNode(ISD::AssertZext, dl, |
767 | Result.getValueType(), Result, |
768 | DAG.getValueType(SrcVT)); |
769 | |
770 | Value = Result; |
771 | Chain = Ch; |
772 | } else if (!isPowerOf2_64(SrcWidth.getKnownMinSize())) { |
773 | |
774 | assert(!SrcVT.isVector() && "Unsupported extload!"); |
775 | unsigned SrcWidthBits = SrcWidth.getFixedSize(); |
776 | unsigned LogSrcWidth = Log2_32(SrcWidthBits); |
777 | assert(LogSrcWidth < 32); |
778 | unsigned RoundWidth = 1 << LogSrcWidth; |
779 | assert(RoundWidth < SrcWidthBits); |
780 | unsigned ExtraWidth = SrcWidthBits - RoundWidth; |
781 | assert(ExtraWidth < RoundWidth); |
782 | assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && |
783 | "Load size not an integral number of bytes!"); |
784 | EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); |
785 | EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); |
786 | SDValue Lo, Hi, Ch; |
787 | unsigned IncrementSize; |
788 | auto &DL = DAG.getDataLayout(); |
789 | |
790 | if (DL.isLittleEndian()) { |
791 | |
792 | |
793 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, |
794 | LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(), |
795 | MMOFlags, AAInfo); |
796 | |
797 | |
798 | IncrementSize = RoundWidth / 8; |
799 | Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); |
800 | Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, |
801 | LD->getPointerInfo().getWithOffset(IncrementSize), |
802 | ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo); |
803 | |
804 | |
805 | |
806 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
807 | Hi.getValue(1)); |
808 | |
809 | |
810 | Hi = DAG.getNode( |
811 | ISD::SHL, dl, Hi.getValueType(), Hi, |
812 | DAG.getConstant(RoundWidth, dl, |
813 | TLI.getShiftAmountTy(Hi.getValueType(), DL))); |
814 | |
815 | |
816 | Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); |
817 | } else { |
818 | |
819 | |
820 | |
821 | Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, |
822 | LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(), |
823 | MMOFlags, AAInfo); |
824 | |
825 | |
826 | IncrementSize = RoundWidth / 8; |
827 | Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); |
828 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, |
829 | LD->getPointerInfo().getWithOffset(IncrementSize), |
830 | ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo); |
831 | |
832 | |
833 | |
834 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
835 | Hi.getValue(1)); |
836 | |
837 | |
838 | Hi = DAG.getNode( |
839 | ISD::SHL, dl, Hi.getValueType(), Hi, |
840 | DAG.getConstant(ExtraWidth, dl, |
841 | TLI.getShiftAmountTy(Hi.getValueType(), DL))); |
842 | |
843 | |
844 | Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); |
845 | } |
846 | |
847 | Chain = Ch; |
848 | } else { |
849 | bool isCustom = false; |
850 | switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0), |
851 | SrcVT.getSimpleVT())) { |
852 | default: llvm_unreachable("This action is not supported yet!"); |
853 | case TargetLowering::Custom: |
854 | isCustom = true; |
855 | LLVM_FALLTHROUGH; |
856 | case TargetLowering::Legal: |
857 | Value = SDValue(Node, 0); |
858 | Chain = SDValue(Node, 1); |
859 | |
860 | if (isCustom) { |
861 | if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { |
862 | Value = Res; |
863 | Chain = Res.getValue(1); |
864 | } |
865 | } else { |
866 | |
867 | |
868 | EVT MemVT = LD->getMemoryVT(); |
869 | const DataLayout &DL = DAG.getDataLayout(); |
870 | if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, |
871 | *LD->getMemOperand())) { |
872 | std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG); |
873 | } |
874 | } |
875 | break; |
876 | |
877 | case TargetLowering::Expand: { |
878 | EVT DestVT = Node->getValueType(0); |
879 | if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) { |
880 | |
881 | |
882 | EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT()); |
883 | if (TLI.isTypeLegal(SrcVT) || |
884 | TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) { |
885 | |
886 | |
887 | ISD::LoadExtType MidExtType = |
888 | (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType; |
889 | |
890 | SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr, |
891 | SrcVT, LD->getMemOperand()); |
892 | unsigned ExtendOp = |
893 | ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType); |
894 | Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); |
895 | Chain = Load.getValue(1); |
896 | break; |
897 | } |
898 | |
899 | |
900 | |
901 | |
902 | |
903 | if (SrcVT.getScalarType() == MVT::f16) { |
904 | EVT ISrcVT = SrcVT.changeTypeToInteger(); |
905 | EVT IDestVT = DestVT.changeTypeToInteger(); |
906 | EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT()); |
907 | |
908 | SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain, |
909 | Ptr, ISrcVT, LD->getMemOperand()); |
910 | Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result); |
911 | Chain = Result.getValue(1); |
912 | break; |
913 | } |
914 | } |
915 | |
916 | assert(!SrcVT.isVector() && |
917 | "Vector Loads are handled in LegalizeVectorOps"); |
918 | |
919 | |
920 | |
921 | |
922 | |
923 | assert(ExtType != ISD::EXTLOAD && |
924 | "EXTLOAD should always be supported!"); |
925 | |
926 | |
927 | SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, |
928 | Node->getValueType(0), |
929 | Chain, Ptr, SrcVT, |
930 | LD->getMemOperand()); |
931 | SDValue ValRes; |
932 | if (ExtType == ISD::SEXTLOAD) |
933 | ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, |
934 | Result.getValueType(), |
935 | Result, DAG.getValueType(SrcVT)); |
936 | else |
937 | ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT); |
938 | Value = ValRes; |
939 | Chain = Result.getValue(1); |
940 | break; |
941 | } |
942 | } |
943 | } |
944 | |
945 | |
946 | |
947 | if (Chain.getNode() != Node) { |
948 | assert(Value.getNode() != Node && "Load must be completely replaced"); |
949 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); |
950 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); |
951 | if (UpdatedNodes) { |
952 | UpdatedNodes->insert(Value.getNode()); |
953 | UpdatedNodes->insert(Chain.getNode()); |
954 | } |
955 | ReplacedNode(Node); |
956 | } |
957 | } |
958 | |
959 | |
960 | void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { |
961 | LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG)); |
962 | |
963 | |
964 | if (Node->getOpcode() == ISD::TargetConstant || |
965 | Node->getOpcode() == ISD::Register) |
966 | return; |
967 | |
968 | #ifndef NDEBUG |
969 | for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) |
970 | assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == |
971 | TargetLowering::TypeLegal && |
972 | "Unexpected illegal type!"); |
973 | |
974 | for (const SDValue &Op : Node->op_values()) |
975 | assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == |
976 | TargetLowering::TypeLegal || |
977 | Op.getOpcode() == ISD::TargetConstant || |
978 | Op.getOpcode() == ISD::Register) && |
979 | "Unexpected illegal type!"); |
980 | #endif |
981 | |
982 | |
983 | TargetLowering::LegalizeAction Action = TargetLowering::Legal; |
984 | bool SimpleFinishLegalizing = true; |
985 | switch (Node->getOpcode()) { |
986 | case ISD::INTRINSIC_W_CHAIN: |
987 | case ISD::INTRINSIC_WO_CHAIN: |
988 | case ISD::INTRINSIC_VOID: |
989 | case ISD::STACKSAVE: |
990 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); |
991 | break; |
992 | case ISD::GET_DYNAMIC_AREA_OFFSET: |
993 | Action = TLI.getOperationAction(Node->getOpcode(), |
994 | Node->getValueType(0)); |
995 | break; |
996 | case ISD::VAARG: |
997 | Action = TLI.getOperationAction(Node->getOpcode(), |
998 | Node->getValueType(0)); |
999 | if (Action != TargetLowering::Promote) |
1000 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); |
1001 | break; |
1002 | case ISD::FP_TO_FP16: |
1003 | case ISD::SINT_TO_FP: |
1004 | case ISD::UINT_TO_FP: |
1005 | case ISD::EXTRACT_VECTOR_ELT: |
1006 | case ISD::LROUND: |
1007 | case ISD::LLROUND: |
1008 | case ISD::LRINT: |
1009 | case ISD::LLRINT: |
1010 | Action = TLI.getOperationAction(Node->getOpcode(), |
1011 | Node->getOperand(0).getValueType()); |
1012 | break; |
1013 | case ISD::STRICT_FP_TO_FP16: |
1014 | case ISD::STRICT_SINT_TO_FP: |
1015 | case ISD::STRICT_UINT_TO_FP: |
1016 | case ISD::STRICT_LRINT: |
1017 | case ISD::STRICT_LLRINT: |
1018 | case ISD::STRICT_LROUND: |
1019 | case ISD::STRICT_LLROUND: |
1020 | |
1021 | |
1022 | |
1023 | Action = TLI.getOperationAction(Node->getOpcode(), |
1024 | Node->getOperand(1).getValueType()); |
1025 | break; |
1026 | case ISD::SIGN_EXTEND_INREG: { |
1027 | EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); |
1028 | Action = TLI.getOperationAction(Node->getOpcode(), InnerType); |
1029 | break; |
1030 | } |
1031 | case ISD::ATOMIC_STORE: |
1032 | Action = TLI.getOperationAction(Node->getOpcode(), |
1033 | Node->getOperand(2).getValueType()); |
1034 | break; |
1035 | case ISD::SELECT_CC: |
1036 | case ISD::STRICT_FSETCC: |
1037 | case ISD::STRICT_FSETCCS: |
1038 | case ISD::SETCC: |
1039 | case ISD::BR_CC: { |
1040 | unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : |
1041 | Node->getOpcode() == ISD::STRICT_FSETCC ? 3 : |
1042 | Node->getOpcode() == ISD::STRICT_FSETCCS ? 3 : |
1043 | Node->getOpcode() == ISD::SETCC ? 2 : 1; |
1044 | unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : |
1045 | Node->getOpcode() == ISD::STRICT_FSETCC ? 1 : |
1046 | Node->getOpcode() == ISD::STRICT_FSETCCS ? 1 : 0; |
1047 | MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); |
1048 | ISD::CondCode CCCode = |
1049 | cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); |
1050 | Action = TLI.getCondCodeAction(CCCode, OpVT); |
1051 | if (Action == TargetLowering::Legal) { |
1052 | if (Node->getOpcode() == ISD::SELECT_CC) |
1053 | Action = TLI.getOperationAction(Node->getOpcode(), |
1054 | Node->getValueType(0)); |
1055 | else |
1056 | Action = TLI.getOperationAction(Node->getOpcode(), OpVT); |
1057 | } |
1058 | break; |
1059 | } |
1060 | case ISD::LOAD: |
1061 | case ISD::STORE: |
1062 | |
1063 | |
1064 | SimpleFinishLegalizing = false; |
1065 | break; |
1066 | case ISD::CALLSEQ_START: |
1067 | case ISD::CALLSEQ_END: |
1068 | |
1069 | |
1070 | |
1071 | SimpleFinishLegalizing = false; |
1072 | break; |
1073 | case ISD::EXTRACT_ELEMENT: |
1074 | case ISD::FLT_ROUNDS_: |
1075 | case ISD::MERGE_VALUES: |
1076 | case ISD::EH_RETURN: |
1077 | case ISD::FRAME_TO_ARGS_OFFSET: |
1078 | case ISD::EH_DWARF_CFA: |
1079 | case ISD::EH_SJLJ_SETJMP: |
1080 | case ISD::EH_SJLJ_LONGJMP: |
1081 | case ISD::EH_SJLJ_SETUP_DISPATCH: |
1082 | |
1083 | |
1084 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1085 | if (Action == TargetLowering::Legal) |
1086 | Action = TargetLowering::Expand; |
1087 | break; |
1088 | case ISD::INIT_TRAMPOLINE: |
1089 | case ISD::ADJUST_TRAMPOLINE: |
1090 | case ISD::FRAMEADDR: |
1091 | case ISD::RETURNADDR: |
1092 | case ISD::ADDROFRETURNADDR: |
1093 | case ISD::SPONENTRY: |
1094 | |
1095 | |
1096 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1097 | if (Action == TargetLowering::Legal) |
1098 | Action = TargetLowering::Custom; |
1099 | break; |
1100 | case ISD::READCYCLECOUNTER: |
1101 | |
1102 | |
1103 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64); |
1104 | break; |
1105 | case ISD::READ_REGISTER: |
1106 | case ISD::WRITE_REGISTER: |
1107 | |
1108 | |
1109 | |
1110 | Action = TargetLowering::Legal; |
1111 | break; |
1112 | case ISD::UBSANTRAP: |
1113 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1114 | if (Action == TargetLowering::Expand) { |
1115 | |
1116 | SDValue NewVal; |
1117 | NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), |
1118 | Node->getOperand(0)); |
1119 | ReplaceNode(Node, NewVal.getNode()); |
1120 | LegalizeOp(NewVal.getNode()); |
1121 | return; |
1122 | } |
1123 | break; |
1124 | case ISD::DEBUGTRAP: |
1125 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1126 | if (Action == TargetLowering::Expand) { |
1127 | |
1128 | SDValue NewVal; |
1129 | NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), |
1130 | Node->getOperand(0)); |
1131 | ReplaceNode(Node, NewVal.getNode()); |
1132 | LegalizeOp(NewVal.getNode()); |
1133 | return; |
1134 | } |
1135 | break; |
1136 | case ISD::SADDSAT: |
1137 | case ISD::UADDSAT: |
1138 | case ISD::SSUBSAT: |
1139 | case ISD::USUBSAT: |
1140 | case ISD::SSHLSAT: |
1141 | case ISD::USHLSAT: |
1142 | case ISD::FP_TO_SINT_SAT: |
1143 | case ISD::FP_TO_UINT_SAT: |
1144 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1145 | break; |
1146 | case ISD::SMULFIX: |
1147 | case ISD::SMULFIXSAT: |
1148 | case ISD::UMULFIX: |
1149 | case ISD::UMULFIXSAT: |
1150 | case ISD::SDIVFIX: |
1151 | case ISD::SDIVFIXSAT: |
1152 | case ISD::UDIVFIX: |
1153 | case ISD::UDIVFIXSAT: { |
1154 | unsigned Scale = Node->getConstantOperandVal(2); |
1155 | Action = TLI.getFixedPointOperationAction(Node->getOpcode(), |
1156 | Node->getValueType(0), Scale); |
1157 | break; |
1158 | } |
1159 | case ISD::MSCATTER: |
1160 | Action = TLI.getOperationAction(Node->getOpcode(), |
1161 | cast<MaskedScatterSDNode>(Node)->getValue().getValueType()); |
1162 | break; |
1163 | case ISD::MSTORE: |
1164 | Action = TLI.getOperationAction(Node->getOpcode(), |
1165 | cast<MaskedStoreSDNode>(Node)->getValue().getValueType()); |
1166 | break; |
1167 | case ISD::VECREDUCE_FADD: |
1168 | case ISD::VECREDUCE_FMUL: |
1169 | case ISD::VECREDUCE_ADD: |
1170 | case ISD::VECREDUCE_MUL: |
1171 | case ISD::VECREDUCE_AND: |
1172 | case ISD::VECREDUCE_OR: |
1173 | case ISD::VECREDUCE_XOR: |
1174 | case ISD::VECREDUCE_SMAX: |
1175 | case ISD::VECREDUCE_SMIN: |
1176 | case ISD::VECREDUCE_UMAX: |
1177 | case ISD::VECREDUCE_UMIN: |
1178 | case ISD::VECREDUCE_FMAX: |
1179 | case ISD::VECREDUCE_FMIN: |
1180 | Action = TLI.getOperationAction( |
1181 | Node->getOpcode(), Node->getOperand(0).getValueType()); |
1182 | break; |
1183 | case ISD::VECREDUCE_SEQ_FADD: |
1184 | case ISD::VECREDUCE_SEQ_FMUL: |
1185 | Action = TLI.getOperationAction( |
1186 | Node->getOpcode(), Node->getOperand(1).getValueType()); |
1187 | break; |
1188 | default: |
1189 | if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { |
1190 | Action = TargetLowering::Legal; |
1191 | } else { |
1192 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1193 | } |
1194 | break; |
1195 | } |
1196 | |
1197 | if (SimpleFinishLegalizing) { |
1198 | SDNode *NewNode = Node; |
1199 | switch (Node->getOpcode()) { |
1200 | default: break; |
1201 | case ISD::SHL: |
1202 | case ISD::SRL: |
1203 | case ISD::SRA: |
1204 | case ISD::ROTL: |
1205 | case ISD::ROTR: { |
1206 | |
1207 | |
1208 | SDValue Op0 = Node->getOperand(0); |
1209 | SDValue Op1 = Node->getOperand(1); |
1210 | if (!Op1.getValueType().isVector()) { |
1211 | SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1); |
1212 | |
1213 | |
1214 | |
1215 | |
1216 | |
1217 | if (SAO != Op1) |
1218 | NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO); |
1219 | } |
1220 | } |
1221 | break; |
1222 | case ISD::FSHL: |
1223 | case ISD::FSHR: |
1224 | case ISD::SRL_PARTS: |
1225 | case ISD::SRA_PARTS: |
1226 | case ISD::SHL_PARTS: { |
1227 | |
1228 | |
1229 | SDValue Op0 = Node->getOperand(0); |
1230 | SDValue Op1 = Node->getOperand(1); |
1231 | SDValue Op2 = Node->getOperand(2); |
1232 | if (!Op2.getValueType().isVector()) { |
1233 | SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2); |
1234 | |
1235 | |
1236 | |
1237 | if (SAO != Op2) |
1238 | NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO); |
1239 | } |
1240 | break; |
1241 | } |
1242 | } |
1243 | |
1244 | if (NewNode != Node) { |
1245 | ReplaceNode(Node, NewNode); |
1246 | Node = NewNode; |
1247 | } |
1248 | switch (Action) { |
1249 | case TargetLowering::Legal: |
1250 | LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n"); |
1251 | return; |
1252 | case TargetLowering::Custom: |
1253 | LLVM_DEBUG(dbgs() << "Trying custom legalization\n"); |
1254 | |
1255 | |
1256 | if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { |
1257 | if (!(Res.getNode() != Node || Res.getResNo() != 0)) |
1258 | return; |
1259 | |
1260 | if (Node->getNumValues() == 1) { |
1261 | |
1262 | |
1263 | assert((Res.getValueType() == Node->getValueType(0) || |
1264 | Node->getValueType(0) == MVT::Glue) && |
1265 | "Type mismatch for custom legalized operation"); |
1266 | LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n"); |
1267 | |
1268 | ReplaceNode(SDValue(Node, 0), Res); |
1269 | return; |
1270 | } |
1271 | |
1272 | SmallVector<SDValue, 8> ResultVals; |
1273 | for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { |
1274 | |
1275 | |
1276 | assert((Res->getValueType(i) == Node->getValueType(i) || |
1277 | Node->getValueType(i) == MVT::Glue) && |
1278 | "Type mismatch for custom legalized operation"); |
1279 | ResultVals.push_back(Res.getValue(i)); |
1280 | } |
1281 | LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n"); |
1282 | ReplaceNode(Node, ResultVals.data()); |
1283 | return; |
1284 | } |
1285 | LLVM_DEBUG(dbgs() << "Could not custom legalize node\n"); |
1286 | LLVM_FALLTHROUGH; |
1287 | case TargetLowering::Expand: |
1288 | if (ExpandNode(Node)) |
1289 | return; |
1290 | LLVM_FALLTHROUGH; |
1291 | case TargetLowering::LibCall: |
1292 | ConvertNodeToLibcall(Node); |
1293 | return; |
1294 | case TargetLowering::Promote: |
1295 | PromoteNode(Node); |
1296 | return; |
1297 | } |
1298 | } |
1299 | |
1300 | switch (Node->getOpcode()) { |
1301 | default: |
1302 | #ifndef NDEBUG |
1303 | dbgs() << "NODE: "; |
1304 | Node->dump( &DAG); |
1305 | dbgs() << "\n"; |
1306 | #endif |
1307 | llvm_unreachable("Do not know how to legalize this operator!"); |
1308 | |
1309 | case ISD::CALLSEQ_START: |
1310 | case ISD::CALLSEQ_END: |
1311 | break; |
1312 | case ISD::LOAD: |
1313 | return LegalizeLoadOps(Node); |
1314 | case ISD::STORE: |
1315 | return LegalizeStoreOps(Node); |
1316 | } |
1317 | } |
1318 | |
1319 | SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { |
1320 | SDValue Vec = Op.getOperand(0); |
1321 | SDValue Idx = Op.getOperand(1); |
1322 | SDLoc dl(Op); |
1323 | |
1324 | |
1325 | |
1326 | |
1327 | |
1328 | |
1329 | |
1330 | |
1331 | |
1332 | SmallPtrSet<const SDNode *, 32> Visited; |
1333 | SmallVector<const SDNode *, 16> Worklist; |
1334 | Visited.insert(Op.getNode()); |
1335 | Worklist.push_back(Idx.getNode()); |
1336 | SDValue StackPtr, Ch; |
1337 | for (SDNode::use_iterator UI = Vec.getNode()->use_begin(), |
1338 | UE = Vec.getNode()->use_end(); UI != UE; ++UI) { |
1339 | SDNode *User = *UI; |
1340 | if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) { |
1341 | if (ST->isIndexed() || ST->isTruncatingStore() || |
1342 | ST->getValue() != Vec) |
1343 | continue; |
1344 | |
1345 | |
1346 | |
1347 | if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode())) |
1348 | continue; |
1349 | |
1350 | |
1351 | |
1352 | |
1353 | |
1354 | |
1355 | if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) || |
1356 | ST->hasPredecessor(Op.getNode())) |
1357 | continue; |
1358 | |
1359 | StackPtr = ST->getBasePtr(); |
1360 | Ch = SDValue(ST, 0); |
1361 | break; |
1362 | } |
1363 | } |
1364 | |
1365 | EVT VecVT = Vec.getValueType(); |
1366 | |
1367 | if (!Ch.getNode()) { |
1368 | |
1369 | StackPtr = DAG.CreateStackTemporary(VecVT); |
1370 | Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, |
1371 | MachinePointerInfo()); |
1372 | } |
1373 | |
1374 | SDValue NewLoad; |
1375 | |
1376 | if (Op.getValueType().isVector()) { |
1377 | StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, |
1378 | Op.getValueType(), Idx); |
1379 | NewLoad = |
1380 | DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo()); |
1381 | } else { |
1382 | StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); |
1383 | NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, |
1384 | MachinePointerInfo(), |
1385 | VecVT.getVectorElementType()); |
1386 | } |
1387 | |
1388 | |
1389 | DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1)); |
1390 | |
1391 | |
1392 | |
1393 | SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(), |
1394 | NewLoad->op_end()); |
1395 | NewLoadOperands[0] = Ch; |
1396 | NewLoad = |
1397 | SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0); |
1398 | return NewLoad; |
1399 | } |
1400 | |
1401 | SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { |
1402 | assert(Op.getValueType().isVector() && "Non-vector insert subvector!"); |
1403 | |
1404 | SDValue Vec = Op.getOperand(0); |
1405 | SDValue Part = Op.getOperand(1); |
1406 | SDValue Idx = Op.getOperand(2); |
1407 | SDLoc dl(Op); |
1408 | |
1409 | |
1410 | EVT VecVT = Vec.getValueType(); |
1411 | EVT SubVecVT = Part.getValueType(); |
1412 | SDValue StackPtr = DAG.CreateStackTemporary(VecVT); |
1413 | int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); |
1414 | MachinePointerInfo PtrInfo = |
1415 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); |
1416 | |
1417 | |
1418 | SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo); |
1419 | |
1420 | |
1421 | SDValue SubStackPtr = |
1422 | TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx); |
1423 | |
1424 | |
1425 | Ch = DAG.getStore( |
1426 | Ch, dl, Part, SubStackPtr, |
1427 | MachinePointerInfo::getUnknownStack(DAG.getMachineFunction())); |
1428 | |
1429 | |
1430 | return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo); |
1431 | } |
1432 | |
1433 | SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { |
1434 | assert((Node->getOpcode() == ISD::BUILD_VECTOR || |
1435 | Node->getOpcode() == ISD::CONCAT_VECTORS) && |
1436 | "Unexpected opcode!"); |
1437 | |
1438 | |
1439 | |
1440 | |
1441 | |
1442 | EVT VT = Node->getValueType(0); |
1443 | EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType() |
1444 | : Node->getOperand(0).getValueType(); |
1445 | SDLoc dl(Node); |
1446 | SDValue FIPtr = DAG.CreateStackTemporary(VT); |
1447 | int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); |
1448 | MachinePointerInfo PtrInfo = |
1449 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); |
1450 | |
1451 | |
1452 | SmallVector<SDValue, 8> Stores; |
1453 | unsigned TypeByteSize = MemVT.getSizeInBits() / 8; |
1454 | assert(TypeByteSize > 0 && "Vector element type too small for stack store!"); |
1455 | |
1456 | |
1457 | |
1458 | bool Truncate = isa<BuildVectorSDNode>(Node) && |
1459 | MemVT.bitsLT(Node->getOperand(0).getValueType()); |
1460 | |
1461 | |
1462 | for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { |
1463 | |
1464 | if (Node->getOperand(i).isUndef()) continue; |
1465 | |
1466 | unsigned Offset = TypeByteSize*i; |
1467 | |
1468 | SDValue Idx = DAG.getMemBasePlusOffset(FIPtr, TypeSize::Fixed(Offset), dl); |
1469 | |
1470 | if (Truncate) |
1471 | Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, |
1472 | Node->getOperand(i), Idx, |
1473 | PtrInfo.getWithOffset(Offset), MemVT)); |
1474 | else |
1475 | Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), |
1476 | Idx, PtrInfo.getWithOffset(Offset))); |
1477 | } |
1478 | |
1479 | SDValue StoreChain; |
1480 | if (!Stores.empty()) |
1481 | StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); |
1482 | else |
1483 | StoreChain = DAG.getEntryNode(); |
1484 | |
1485 | |
1486 | return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo); |
1487 | } |
1488 | |
1489 | |
1490 | |
1491 | |
1492 | void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State, |
1493 | const SDLoc &DL, |
1494 | SDValue Value) const { |
1495 | EVT FloatVT = Value.getValueType(); |
1496 | unsigned NumBits = FloatVT.getScalarSizeInBits(); |
1497 | State.FloatVT = FloatVT; |
1498 | EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits); |
1499 | |
1500 | if (TLI.isTypeLegal(IVT)) { |
1501 | State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value); |
1502 | State.SignMask = APInt::getSignMask(NumBits); |
1503 | State.SignBit = NumBits - 1; |
1504 | return; |
1505 | } |
1506 | |
1507 | auto &DataLayout = DAG.getDataLayout(); |
1508 | |
1509 | MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8); |
1510 | |
1511 | SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy); |
1512 | int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); |
1513 | |
1514 | State.FloatPtr = StackPtr; |
1515 | MachineFunction &MF = DAG.getMachineFunction(); |
1516 | State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI); |
1517 | State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr, |
1518 | State.FloatPointerInfo); |
1519 | |
1520 | SDValue IntPtr; |
1521 | if (DataLayout.isBigEndian()) { |
1522 | assert(FloatVT.isByteSized() && "Unsupported floating point type!"); |
1523 | |
1524 | IntPtr = StackPtr; |
1525 | State.IntPointerInfo = State.FloatPointerInfo; |
1526 | } else { |
1527 | |
1528 | unsigned ByteOffset = (NumBits / 8) - 1; |
1529 | IntPtr = |
1530 | DAG.getMemBasePlusOffset(StackPtr, TypeSize::Fixed(ByteOffset), DL); |
1531 | State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI, |
1532 | ByteOffset); |
1533 | } |
1534 | |
1535 | State.IntPtr = IntPtr; |
1536 | State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr, |
1537 | State.IntPointerInfo, MVT::i8); |
1538 | State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7); |
1539 | State.SignBit = 7; |
1540 | } |
1541 | |
1542 | |
1543 | |
1544 | SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State, |
1545 | const SDLoc &DL, |
1546 | SDValue NewIntValue) const { |
1547 | if (!State.Chain) |
1548 | return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue); |
1549 | |
1550 | |
1551 | SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr, |
1552 | State.IntPointerInfo, MVT::i8); |
1553 | return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr, |
1554 | State.FloatPointerInfo); |
1555 | } |
1556 | |
1557 | SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const { |
1558 | SDLoc DL(Node); |
1559 | SDValue Mag = Node->getOperand(0); |
1560 | SDValue Sign = Node->getOperand(1); |
1561 | |
1562 | |
1563 | FloatSignAsInt SignAsInt; |
1564 | getSignAsIntValue(SignAsInt, DL, Sign); |
1565 | |
1566 | EVT IntVT = SignAsInt.IntValue.getValueType(); |
1567 | SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT); |
1568 | SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue, |
1569 | SignMask); |
1570 | |
1571 | |
1572 | EVT FloatVT = Mag.getValueType(); |
1573 | if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) && |
1574 | TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) { |
1575 | SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag); |
1576 | SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue); |
1577 | SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit, |
1578 | DAG.getConstant(0, DL, IntVT), ISD::SETNE); |
1579 | return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue); |
1580 | } |
1581 | |
1582 | |
1583 | FloatSignAsInt MagAsInt; |
1584 | getSignAsIntValue(MagAsInt, DL, Mag); |
1585 | EVT MagVT = MagAsInt.IntValue.getValueType(); |
1586 | SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT); |
1587 | SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue, |
1588 | ClearSignMask); |
1589 | |
1590 | |
1591 | int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit; |
1592 | EVT ShiftVT = IntVT; |
1593 | if (SignBit.getScalarValueSizeInBits() < |
1594 | ClearedSign.getScalarValueSizeInBits()) { |
1595 | SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit); |
1596 | ShiftVT = MagVT; |
1597 | } |
1598 | if (ShiftAmount > 0) { |
1599 | SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT); |
1600 | SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst); |
1601 | } else if (ShiftAmount < 0) { |
1602 | SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT); |
1603 | SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst); |
1604 | } |
1605 | if (SignBit.getScalarValueSizeInBits() > |
1606 | ClearedSign.getScalarValueSizeInBits()) { |
1607 | SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit); |
1608 | } |
1609 | |
1610 | |
1611 | SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit); |
1612 | return modifySignAsInt(MagAsInt, DL, CopiedSign); |
1613 | } |
1614 | |
1615 | SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const { |
1616 | |
1617 | SDLoc DL(Node); |
1618 | FloatSignAsInt SignAsInt; |
1619 | getSignAsIntValue(SignAsInt, DL, Node->getOperand(0)); |
1620 | EVT IntVT = SignAsInt.IntValue.getValueType(); |
1621 | |
1622 | |
1623 | SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT); |
1624 | SDValue SignFlip = |
1625 | DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask); |
1626 | |
1627 | |
1628 | return modifySignAsInt(SignAsInt, DL, SignFlip); |
1629 | } |
1630 | |
1631 | SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const { |
1632 | SDLoc DL(Node); |
1633 | SDValue Value = Node->getOperand(0); |
1634 | |
1635 | |
1636 | EVT FloatVT = Value.getValueType(); |
1637 | if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) { |
1638 | SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT); |
1639 | return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero); |
1640 | } |
1641 | |
1642 | |
1643 | FloatSignAsInt ValueAsInt; |
1644 | getSignAsIntValue(ValueAsInt, DL, Value); |
1645 | EVT IntVT = ValueAsInt.IntValue.getValueType(); |
1646 | SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT); |
1647 | SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue, |
1648 | ClearSignMask); |
1649 | return modifySignAsInt(ValueAsInt, DL, ClearedSign); |
1650 | } |
1651 | |
1652 | void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, |
1653 | SmallVectorImpl<SDValue> &Results) { |
1654 | Register SPReg = TLI.getStackPointerRegisterToSaveRestore(); |
1655 | assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" |
1656 | " not tell us which reg is the stack pointer!"); |
1657 | SDLoc dl(Node); |
1658 | EVT VT = Node->getValueType(0); |
1659 | SDValue Tmp1 = SDValue(Node, 0); |
1660 | SDValue Tmp2 = SDValue(Node, 1); |
1661 | SDValue Tmp3 = Node->getOperand(2); |
1662 | SDValue Chain = Tmp1.getOperand(0); |
1663 | |
1664 | |
1665 | |
1666 | Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl); |
1667 | |
1668 | SDValue Size = Tmp2.getOperand(1); |
1669 | SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); |
1670 | Chain = SP.getValue(1); |
1671 | Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue(); |
1672 | const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering(); |
1673 | unsigned Opc = |
1674 | TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ? |
1675 | ISD::ADD : ISD::SUB; |
1676 | |
1677 | Align StackAlign = TFL->getStackAlign(); |
1678 | Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); |
1679 | if (Alignment > StackAlign) |
1680 | Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, |
1681 | DAG.getConstant(-Alignment.value(), dl, VT)); |
1682 | Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); |
1683 | |
1684 | Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true), |
1685 | DAG.getIntPtrConstant(0, dl, true), SDValue(), dl); |
1686 | |
1687 | Results.push_back(Tmp1); |
1688 | Results.push_back(Tmp2); |
1689 | } |
1690 | |
1691 | |
1692 | |
1693 | |
1694 | |
1695 | SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, |
1696 | EVT DestVT, const SDLoc &dl) { |
1697 | return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode()); |
1698 | } |
1699 | |
1700 | SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, |
1701 | EVT DestVT, const SDLoc &dl, |
1702 | SDValue Chain) { |
1703 | unsigned SrcSize = SrcOp.getValueSizeInBits(); |
1704 | unsigned SlotSize = SlotVT.getSizeInBits(); |
1705 | unsigned DestSize = DestVT.getSizeInBits(); |
1706 | Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); |
1707 | Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType); |
1708 | |
1709 | |
1710 | if ((SrcSize > SlotSize && |
1711 | !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) || |
1712 | (SlotSize < DestSize && |
1713 | !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT))) |
1714 | return SDValue(); |
1715 | |
1716 | |
1717 | Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign( |
1718 | SrcOp.getValueType().getTypeForEVT(*DAG.getContext())); |
1719 | SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign); |
1720 | |
1721 | FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); |
1722 | int SPFI = StackPtrFI->getIndex(); |
1723 | MachinePointerInfo PtrInfo = |
1724 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); |
1725 | |
1726 | |
1727 | |
1728 | SDValue Store; |
1729 | |
1730 | if (SrcSize > SlotSize) |
1731 | Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo, |
1732 | SlotVT, SrcAlign); |
1733 | else { |
1734 | assert(SrcSize == SlotSize && "Invalid store"); |
1735 | Store = |
1736 | DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign); |
1737 | } |
1738 | |
1739 | |
1740 | if (SlotSize == DestSize) |
1741 | return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign); |
1742 | |
1743 | assert(SlotSize < DestSize && "Unknown extension!"); |
1744 | return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT, |
1745 | DestAlign); |
1746 | } |
1747 | |
1748 | SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { |
1749 | SDLoc dl(Node); |
1750 | |
1751 | |
1752 | SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); |
1753 | |
1754 | FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); |
1755 | int SPFI = StackPtrFI->getIndex(); |
1756 | |
1757 | SDValue Ch = DAG.getTruncStore( |
1758 | DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr, |
1759 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI), |
1760 | Node->getValueType(0).getVectorElementType()); |
1761 | return DAG.getLoad( |
1762 | Node->getValueType(0), dl, Ch, StackPtr, |
1763 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); |
1764 | } |
1765 | |
1766 | static bool |
1767 | ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, |
1768 | const TargetLowering &TLI, SDValue &Res) { |
1769 | unsigned NumElems = Node->getNumOperands(); |
1770 | SDLoc dl(Node); |
1771 | EVT VT = Node->getValueType(0); |
1772 | |
1773 | |
1774 | |
1775 | |
1776 | |
1777 | |
1778 | |
1779 | |
1780 | for (int Phase = 0; Phase < 2; ++Phase) { |
1781 | SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals, |
1782 | NewIntermedVals; |
1783 | for (unsigned i = 0; i < NumElems; ++i) { |
1784 | SDValue V = Node->getOperand(i); |
1785 | if (V.isUndef()) |
1786 | continue; |
1787 | |
1788 | SDValue Vec; |
1789 | if (Phase) |
1790 | Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V); |
1791 | IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i))); |
1792 | } |
1793 | |
1794 | while (IntermedVals.size() > 2) { |
1795 | NewIntermedVals.clear(); |
1796 | for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) { |
1797 | |
1798 | |
1799 | SmallVector<int, 16> ShuffleVec(NumElems, -1); |
1800 | |
1801 | SmallVector<int, 16> FinalIndices; |
1802 | FinalIndices.reserve(IntermedVals[i].second.size() + |
1803 | IntermedVals[i+1].second.size()); |
1804 | |
1805 | int k = 0; |
1806 | for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f; |
1807 | ++j, ++k) { |
1808 | ShuffleVec[k] = j; |
1809 | FinalIndices.push_back(IntermedVals[i].second[j]); |
1810 | } |
1811 | for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f; |
1812 | ++j, ++k) { |
1813 | ShuffleVec[k] = NumElems + j; |
1814 | FinalIndices.push_back(IntermedVals[i+1].second[j]); |
1815 | } |
1816 | |
1817 | SDValue Shuffle; |
1818 | if (Phase) |
1819 | Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first, |
1820 | IntermedVals[i+1].first, |
1821 | ShuffleVec); |
1822 | else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) |
1823 | return false; |
1824 | NewIntermedVals.push_back( |
1825 | std::make_pair(Shuffle, std::move(FinalIndices))); |
1826 | } |
1827 | |
1828 | |
1829 | |
1830 | if ((IntermedVals.size() & 1) != 0) |
1831 | NewIntermedVals.push_back(IntermedVals.back()); |
1832 | |
1833 | IntermedVals.swap(NewIntermedVals); |
1834 | } |
1835 | |
1836 | assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 && |
1837 | "Invalid number of intermediate vectors"); |
1838 | SDValue Vec1 = IntermedVals[0].first; |
1839 | SDValue Vec2; |
1840 | if (IntermedVals.size() > 1) |
1841 | Vec2 = IntermedVals[1].first; |
1842 | else if (Phase) |
1843 | Vec2 = DAG.getUNDEF(VT); |
1844 | |
1845 | SmallVector<int, 16> ShuffleVec(NumElems, -1); |
1846 | for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i) |
1847 | ShuffleVec[IntermedVals[0].second[i]] = i; |
1848 | for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i) |
1849 | ShuffleVec[IntermedVals[1].second[i]] = NumElems + i; |
1850 | |
1851 | if (Phase) |
1852 | Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); |
1853 | else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) |
1854 | return false; |
1855 | } |
1856 | |
1857 | return true; |
1858 | } |
1859 | |
1860 | |
1861 | |
1862 | SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { |
1863 | unsigned NumElems = Node->getNumOperands(); |
1864 | SDValue Value1, Value2; |
1865 | SDLoc dl(Node); |
1866 | EVT VT = Node->getValueType(0); |
1867 | EVT OpVT = Node->getOperand(0).getValueType(); |
1868 | EVT EltVT = VT.getVectorElementType(); |
1869 | |
1870 | |
1871 | |
1872 | bool isOnlyLowElement = true; |
1873 | bool MoreThanTwoValues = false; |
1874 | bool isConstant = true; |
1875 | for (unsigned i = 0; i < NumElems; ++i) { |
1876 | SDValue V = Node->getOperand(i); |
1877 | if (V.isUndef()) |
1878 | continue; |
1879 | if (i > 0) |
1880 | isOnlyLowElement = false; |
1881 | if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)) |
1882 | isConstant = false; |
1883 | |
1884 | if (!Value1.getNode()) { |
1885 | Value1 = V; |
1886 | } else if (!Value2.getNode()) { |
1887 | if (V != Value1) |
1888 | Value2 = V; |
1889 | } else if (V != Value1 && V != Value2) { |
1890 | MoreThanTwoValues = true; |
1891 | } |
1892 | } |
1893 | |
1894 | if (!Value1.getNode()) |
1895 | return DAG.getUNDEF(VT); |
1896 | |
1897 | if (isOnlyLowElement) |
1898 | return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); |
1899 | |
1900 | |
1901 | if (isConstant) { |
1902 | SmallVector<Constant*, 16> CV; |
1903 | for (unsigned i = 0, e = NumElems; i != e; ++i) { |
1904 | if (ConstantFPSDNode *V = |
1905 | dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { |
1906 | CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); |
1907 | } else if (ConstantSDNode *V = |
1908 | dyn_cast<ConstantSDNode>(Node->getOperand(i))) { |
1909 | if (OpVT==EltVT) |
1910 | CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); |
1911 | else { |
1912 | |
1913 | |
1914 | |
1915 | const ConstantInt *CI = V->getConstantIntValue(); |
1916 | CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), |
1917 | CI->getZExtValue())); |
1918 | } |
1919 | } else { |
1920 | assert(Node->getOperand(i).isUndef()); |
1921 | Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); |
1922 | CV.push_back(UndefValue::get(OpNTy)); |
1923 | } |
1924 | } |
1925 | Constant *CP = ConstantVector::get(CV); |
1926 | SDValue CPIdx = |
1927 | DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout())); |
1928 | Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); |
1929 | return DAG.getLoad( |
1930 | VT, dl, DAG.getEntryNode(), CPIdx, |
1931 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), |
1932 | Alignment); |
1933 | } |
1934 | |
1935 | SmallSet<SDValue, 16> DefinedValues; |
1936 | for (unsigned i = 0; i < NumElems; ++i) { |
1937 | if (Node->getOperand(i).isUndef()) |
1938 | continue; |
1939 | DefinedValues.insert(Node->getOperand(i)); |
1940 | } |
1941 | |
1942 | if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) { |
1943 | if (!MoreThanTwoValues) { |
1944 | SmallVector<int, 8> ShuffleVec(NumElems, -1); |
1945 | for (unsigned i = 0; i < NumElems; ++i) { |
1946 | SDValue V = Node->getOperand(i); |
1947 | if (V.isUndef()) |
1948 | continue; |
1949 | ShuffleVec[i] = V == Value1 ? 0 : NumElems; |
1950 | } |
1951 | if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { |
1952 | |
1953 | SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); |
1954 | SDValue Vec2; |
1955 | if (Value2.getNode()) |
1956 | Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); |
1957 | else |
1958 | Vec2 = DAG.getUNDEF(VT); |
1959 | |
1960 | |
1961 | return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); |
1962 | } |
1963 | } else { |
1964 | SDValue Res; |
1965 | if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) |
1966 | return Res; |
1967 | } |
1968 | } |
1969 | |
1970 | |
1971 | return ExpandVectorBuildThroughStack(Node); |
1972 | } |
1973 | |
1974 | SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) { |
1975 | SDLoc DL(Node); |
1976 | EVT VT = Node->getValueType(0); |
1977 | SDValue SplatVal = Node->getOperand(0); |
1978 | |
1979 | return DAG.getSplatBuildVector(VT, DL, SplatVal); |
1980 | } |
1981 | |
1982 | |
1983 | |
1984 | |
1985 | |
1986 | SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, |
1987 | bool isSigned) { |
1988 | TargetLowering::ArgListTy Args; |
1989 | TargetLowering::ArgListEntry Entry; |
1990 | for (const SDValue &Op : Node->op_values()) { |
1991 | EVT ArgVT = Op.getValueType(); |
1992 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); |
1993 | Entry.Node = Op; |
1994 | Entry.Ty = ArgTy; |
1995 | Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned); |
1996 | Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned); |
1997 | Args.push_back(Entry); |
1998 | } |
1999 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
2000 | TLI.getPointerTy(DAG.getDataLayout())); |
2001 | |
2002 | EVT RetVT = Node->getValueType(0); |
2003 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); |
2004 | |
2005 | |
2006 | |
2007 | |
2008 | |
2009 | SDValue InChain = DAG.getEntryNode(); |
2010 | |
2011 | |
2012 | |
2013 | SDValue TCChain = InChain; |
2014 | const Function &F = DAG.getMachineFunction().getFunction(); |
2015 | bool isTailCall = |
2016 | TLI.isInTailCallPosition(DAG, Node, TCChain) && |
2017 | (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy()); |
2018 | if (isTailCall) |
2019 | InChain = TCChain; |
2020 | |
2021 | TargetLowering::CallLoweringInfo CLI(DAG); |
2022 | bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned); |
2023 | CLI.setDebugLoc(SDLoc(Node)) |
2024 | .setChain(InChain) |
2025 | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, |
2026 | std::move(Args)) |
2027 | .setTailCall(isTailCall) |
2028 | .setSExtResult(signExtend) |
2029 | .setZExtResult(!signExtend) |
2030 | .setIsPostTypeLegalization(true); |
2031 | |
2032 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2033 | |
2034 | if (!CallInfo.second.getNode()) { |
2035 | LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG)); |
2036 | |
2037 | return DAG.getRoot(); |
2038 | } |
2039 | |
2040 | LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG)); |
2041 | return CallInfo.first; |
2042 | } |
2043 | |
2044 | void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, |
2045 | RTLIB::Libcall LC, |
2046 | SmallVectorImpl<SDValue> &Results) { |
2047 | if (LC == RTLIB::UNKNOWN_LIBCALL) |
2048 | llvm_unreachable("Can't create an unknown libcall!"); |
2049 | |
2050 | if (Node->isStrictFPOpcode()) { |
2051 | EVT RetVT = Node->getValueType(0); |
2052 | SmallVector<SDValue, 4> Ops(drop_begin(Node->ops())); |
2053 | TargetLowering::MakeLibCallOptions CallOptions; |
2054 | |
2055 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, |
2056 | Ops, CallOptions, |
2057 | SDLoc(Node), |
2058 | Node->getOperand(0)); |
2059 | Results.push_back(Tmp.first); |
2060 | Results.push_back(Tmp.second); |
2061 | } else { |
2062 | SDValue Tmp = ExpandLibCall(LC, Node, false); |
2063 | Results.push_back(Tmp); |
2064 | } |
2065 | } |
2066 | |
2067 | |
2068 | void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, |
2069 | RTLIB::Libcall Call_F32, |
2070 | RTLIB::Libcall Call_F64, |
2071 | RTLIB::Libcall Call_F80, |
2072 | RTLIB::Libcall Call_F128, |
2073 | RTLIB::Libcall Call_PPCF128, |
2074 | SmallVectorImpl<SDValue> &Results) { |
2075 | RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0), |
2076 | Call_F32, Call_F64, Call_F80, |
2077 | Call_F128, Call_PPCF128); |
2078 | ExpandFPLibCall(Node, LC, Results); |
2079 | } |
2080 | |
2081 | SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, |
2082 | RTLIB::Libcall Call_I8, |
2083 | RTLIB::Libcall Call_I16, |
2084 | RTLIB::Libcall Call_I32, |
2085 | RTLIB::Libcall Call_I64, |
2086 | RTLIB::Libcall Call_I128) { |
2087 | RTLIB::Libcall LC; |
2088 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2089 | default: llvm_unreachable("Unexpected request for libcall!"); |
2090 | case MVT::i8: LC = Call_I8; break; |
2091 | case MVT::i16: LC = Call_I16; break; |
2092 | case MVT::i32: LC = Call_I32; break; |
2093 | case MVT::i64: LC = Call_I64; break; |
2094 | case MVT::i128: LC = Call_I128; break; |
2095 | } |
2096 | return ExpandLibCall(LC, Node, isSigned); |
2097 | } |
2098 | |
2099 | |
2100 | |
2101 | void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node, |
2102 | RTLIB::Libcall Call_F32, |
2103 | RTLIB::Libcall Call_F64, |
2104 | RTLIB::Libcall Call_F80, |
2105 | RTLIB::Libcall Call_F128, |
2106 | RTLIB::Libcall Call_PPCF128, |
2107 | SmallVectorImpl<SDValue> &Results) { |
2108 | EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType(); |
2109 | RTLIB::Libcall LC = RTLIB::getFPLibCall(InVT.getSimpleVT(), |
2110 | Call_F32, Call_F64, Call_F80, |
2111 | Call_F128, Call_PPCF128); |
2112 | ExpandFPLibCall(Node, LC, Results); |
2113 | } |
2114 | |
2115 | |
2116 | void |
2117 | SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, |
2118 | SmallVectorImpl<SDValue> &Results) { |
2119 | unsigned Opcode = Node->getOpcode(); |
2120 | bool isSigned = Opcode == ISD::SDIVREM; |
2121 | |
2122 | RTLIB::Libcall LC; |
2123 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2124 | default: llvm_unreachable("Unexpected request for libcall!"); |
2125 | case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; |
2126 | case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; |
2127 | case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; |
2128 | case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; |
2129 | case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; |
2130 | } |
2131 | |
2132 | |
2133 | |
2134 | |
2135 | SDValue InChain = DAG.getEntryNode(); |
2136 | |
2137 | EVT RetVT = Node->getValueType(0); |
2138 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); |
2139 | |
2140 | TargetLowering::ArgListTy Args; |
2141 | TargetLowering::ArgListEntry Entry; |
2142 | for (const SDValue &Op : Node->op_values()) { |
2143 | EVT ArgVT = Op.getValueType(); |
2144 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); |
2145 | Entry.Node = Op; |
2146 | Entry.Ty = ArgTy; |
2147 | Entry.IsSExt = isSigned; |
2148 | Entry.IsZExt = !isSigned; |
2149 | Args.push_back(Entry); |
2150 | } |
2151 | |
2152 | |
2153 | SDValue FIPtr = DAG.CreateStackTemporary(RetVT); |
2154 | Entry.Node = FIPtr; |
2155 | Entry.Ty = RetTy->getPointerTo(); |
2156 | Entry.IsSExt = isSigned; |
2157 | Entry.IsZExt = !isSigned; |
2158 | Args.push_back(Entry); |
2159 | |
2160 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
2161 | TLI.getPointerTy(DAG.getDataLayout())); |
2162 | |
2163 | SDLoc dl(Node); |
2164 | TargetLowering::CallLoweringInfo CLI(DAG); |
2165 | CLI.setDebugLoc(dl) |
2166 | .setChain(InChain) |
2167 | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, |
2168 | std::move(Args)) |
2169 | .setSExtResult(isSigned) |
2170 | .setZExtResult(!isSigned); |
2171 | |
2172 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2173 | |
2174 | |
2175 | SDValue Rem = |
2176 | DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo()); |
2177 | Results.push_back(CallInfo.first); |
2178 | Results.push_back(Rem); |
2179 | } |
2180 | |
2181 | |
2182 | static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { |
2183 | RTLIB::Libcall LC; |
2184 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2185 | default: llvm_unreachable("Unexpected request for libcall!"); |
2186 | case MVT::f32: LC = RTLIB::SINCOS_F32; break; |
2187 | case MVT::f64: LC = RTLIB::SINCOS_F64; break; |
2188 | case MVT::f80: LC = RTLIB::SINCOS_F80; break; |
2189 | case MVT::f128: LC = RTLIB::SINCOS_F128; break; |
2190 | case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; |
2191 | } |
2192 | return TLI.getLibcallName(LC) != nullptr; |
2193 | } |
2194 | |
2195 | |
2196 | static bool useSinCos(SDNode *Node) { |
2197 | unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN |
2198 | ? ISD::FCOS : ISD::FSIN; |
2199 | |
2200 | SDValue Op0 = Node->getOperand(0); |
2201 | for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), |
2202 | UE = Op0.getNode()->use_end(); UI != UE; ++UI) { |
2203 | SDNode *User = *UI; |
2204 | if (User == Node) |
2205 | continue; |
2206 | |
2207 | if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) |
2208 | return true; |
2209 | } |
2210 | return false; |
2211 | } |
2212 | |
2213 | |
2214 | void |
2215 | SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, |
2216 | SmallVectorImpl<SDValue> &Results) { |
2217 | RTLIB::Libcall LC; |
2218 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2219 | default: llvm_unreachable("Unexpected request for libcall!"); |
2220 | case MVT::f32: LC = RTLIB::SINCOS_F32; break; |
2221 | case MVT::f64: LC = RTLIB::SINCOS_F64; break; |
2222 | case MVT::f80: LC = RTLIB::SINCOS_F80; break; |
2223 | case MVT::f128: LC = RTLIB::SINCOS_F128; break; |
2224 | case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; |
2225 | } |
2226 | |
2227 | |
2228 | |
2229 | |
2230 | SDValue InChain = DAG.getEntryNode(); |
2231 | |
2232 | EVT RetVT = Node->getValueType(0); |
2233 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); |
2234 | |
2235 | TargetLowering::ArgListTy Args; |
2236 | TargetLowering::ArgListEntry Entry; |
2237 | |
2238 | |
2239 | Entry.Node = Node->getOperand(0); |
2240 | Entry.Ty = RetTy; |
2241 | Entry.IsSExt = false; |
2242 | Entry.IsZExt = false; |
2243 | Args.push_back(Entry); |
2244 | |
2245 | |
2246 | SDValue SinPtr = DAG.CreateStackTemporary(RetVT); |
2247 | Entry.Node = SinPtr; |
2248 | Entry.Ty = RetTy->getPointerTo(); |
2249 | Entry.IsSExt = false; |
2250 | Entry.IsZExt = false; |
2251 | Args.push_back(Entry); |
2252 | |
2253 | |
2254 | SDValue CosPtr = DAG.CreateStackTemporary(RetVT); |
2255 | Entry.Node = CosPtr; |
2256 | Entry.Ty = RetTy->getPointerTo(); |
2257 | Entry.IsSExt = false; |
2258 | Entry.IsZExt = false; |
2259 | Args.push_back(Entry); |
2260 | |
2261 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
2262 | TLI.getPointerTy(DAG.getDataLayout())); |
2263 | |
2264 | SDLoc dl(Node); |
2265 | TargetLowering::CallLoweringInfo CLI(DAG); |
2266 | CLI.setDebugLoc(dl).setChain(InChain).setLibCallee( |
2267 | TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee, |
2268 | std::move(Args)); |
2269 | |
2270 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2271 | |
2272 | Results.push_back( |
2273 | DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo())); |
2274 | Results.push_back( |
2275 | DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo())); |
2276 | } |
2277 | |
2278 | |
2279 | |
2280 | |
2281 | |
2282 | SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node, |
2283 | SDValue &Chain) { |
2284 | bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP || |
2285 | Node->getOpcode() == ISD::SINT_TO_FP); |
2286 | EVT DestVT = Node->getValueType(0); |
2287 | SDLoc dl(Node); |
2288 | unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0; |
2289 | SDValue Op0 = Node->getOperand(OpNo); |
2290 | EVT SrcVT = Op0.getValueType(); |
2291 | |
2292 | |
2293 | LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n"); |
2294 | if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) && |
2295 | (DestVT.bitsLE(MVT::f64) || |
2296 | TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND |
2297 | : ISD::FP_EXTEND, |
2298 | DestVT))) { |
2299 | LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double " |
2300 | "expansion\n"); |
2301 | |
2302 | |
2303 | SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); |
2304 | |
2305 | SDValue Lo = Op0; |
2306 | |
2307 | if (isSigned) { |
2308 | |
2309 | Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo, |
2310 | DAG.getConstant(0x80000000u, dl, MVT::i32)); |
2311 | } |
2312 | |
2313 | SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32); |
2314 | |
2315 | |
2316 | if (DAG.getDataLayout().isBigEndian()) |
2317 | std::swap(Lo, Hi); |
2318 | |
2319 | SDValue MemChain = DAG.getEntryNode(); |
2320 | |
2321 | |
2322 | SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot, |
2323 | MachinePointerInfo()); |
2324 | |
2325 | SDValue HiPtr = DAG.getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), dl); |
2326 | SDValue Store2 = |
2327 | DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo()); |
2328 | MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); |
2329 | |
2330 | |
2331 | SDValue Load = |
2332 | DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo()); |
2333 | |
2334 | SDValue Bias = DAG.getConstantFP(isSigned ? |
2335 | BitsToDouble(0x4330000080000000ULL) : |
2336 | BitsToDouble(0x4330000000000000ULL), |
2337 | dl, MVT::f64); |
2338 | |
2339 | SDValue Sub; |
2340 | SDValue Result; |
2341 | if (Node->isStrictFPOpcode()) { |
2342 | Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other}, |
2343 | {Node->getOperand(0), Load, Bias}); |
2344 | Chain = Sub.getValue(1); |
2345 | if (DestVT != Sub.getValueType()) { |
2346 | std::pair<SDValue, SDValue> ResultPair; |
2347 | ResultPair = |
2348 | DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT); |
2349 | Result = ResultPair.first; |
2350 | Chain = ResultPair.second; |
2351 | } |
2352 | else |
2353 | Result = Sub; |
2354 | } else { |
2355 | Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); |
2356 | Result = DAG.getFPExtendOrRound(Sub, dl, DestVT); |
2357 | } |
2358 | return Result; |
2359 | } |
2360 | |
2361 | if (isSigned) |
2362 | return SDValue(); |
2363 | |
2364 | |
2365 | if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) || |
2366 | (SrcVT == MVT::i64 && DestVT == MVT::f64)) { |
2367 | LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n"); |
2368 | |
2369 | |
2370 | |
2371 | |
2372 | |
2373 | |
2374 | |
2375 | |
2376 | |
2377 | |
2378 | |
2379 | |
2380 | |
2381 | |
2382 | EVT SetCCVT = getSetCCResultType(SrcVT); |
2383 | |
2384 | SDValue SignBitTest = DAG.getSetCC( |
2385 | dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT); |
2386 | |
2387 | EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout()); |
2388 | SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT); |
2389 | SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst); |
2390 | SDValue AndConst = DAG.getConstant(1, dl, SrcVT); |
2391 | SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst); |
2392 | SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr); |
2393 | |
2394 | SDValue Slow, Fast; |
2395 | if (Node->isStrictFPOpcode()) { |
2396 | |
2397 | |
2398 | SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0); |
2399 | Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, |
2400 | { Node->getOperand(0), InCvt }); |
2401 | Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, |
2402 | { Fast.getValue(1), Fast, Fast }); |
2403 | Chain = Slow.getValue(1); |
2404 | |
2405 | |
2406 | |
2407 | SDNodeFlags Flags; |
2408 | Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept()); |
2409 | Fast->setFlags(Flags); |
2410 | Flags.setNoFPExcept(true); |
2411 | Slow->setFlags(Flags); |
2412 | } else { |
2413 | SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or); |
2414 | Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt); |
2415 | Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); |
2416 | } |
2417 | |
2418 | return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast); |
2419 | } |
2420 | |
2421 | |
2422 | if (!TLI.isOperationLegalOrCustom( |
2423 | Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT)) |
2424 | return SDValue(); |
2425 | |
2426 | |
2427 | |
2428 | |
2429 | assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >= |
2430 | SrcVT.getSizeInBits() - 1 && |
2431 | "Cannot perform lossless SINT_TO_FP!"); |
2432 | |
2433 | SDValue Tmp1; |
2434 | if (Node->isStrictFPOpcode()) { |
2435 | Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, |
2436 | { Node->getOperand(0), Op0 }); |
2437 | } else |
2438 | Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); |
2439 | |
2440 | SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0, |
2441 | DAG.getConstant(0, dl, SrcVT), ISD::SETLT); |
2442 | SDValue Zero = DAG.getIntPtrConstant(0, dl), |
2443 | Four = DAG.getIntPtrConstant(4, dl); |
2444 | SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(), |
2445 | SignSet, Four, Zero); |
2446 | |
2447 | |
2448 | |
2449 | |
2450 | uint64_t FF; |
2451 | switch (SrcVT.getSimpleVT().SimpleTy) { |
2452 | default: |
2453 | return SDValue(); |
2454 | case MVT::i8 : FF = 0x43800000ULL; break; |
2455 | case MVT::i16: FF = 0x47800000ULL; break; |
2456 | case MVT::i32: FF = 0x4F800000ULL; break; |
2457 | case MVT::i64: FF = 0x5F800000ULL; break; |
2458 | } |
2459 | if (DAG.getDataLayout().isLittleEndian()) |
2460 | FF <<= 32; |
2461 | Constant *FudgeFactor = ConstantInt::get( |
2462 | Type::getInt64Ty(*DAG.getContext()), FF); |
2463 | |
2464 | SDValue CPIdx = |
2465 | DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout())); |
2466 | Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); |
2467 | CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); |
2468 | Alignment = commonAlignment(Alignment, 4); |
2469 | SDValue FudgeInReg; |
2470 | if (DestVT == MVT::f32) |
2471 | FudgeInReg = DAG.getLoad( |
2472 | MVT::f32, dl, DAG.getEntryNode(), CPIdx, |
2473 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), |
2474 | Alignment); |
2475 | else { |
2476 | SDValue Load = DAG.getExtLoad( |
2477 | ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx, |
2478 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, |
2479 | Alignment); |
2480 | HandleSDNode Handle(Load); |
2481 | LegalizeOp(Load.getNode()); |
2482 | FudgeInReg = Handle.getValue(); |
2483 | } |
2484 | |
2485 | if (Node->isStrictFPOpcode()) { |
2486 | SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, |
2487 | { Tmp1.getValue(1), Tmp1, FudgeInReg }); |
2488 | Chain = Result.getValue(1); |
2489 | return Result; |
2490 | } |
2491 | |
2492 | return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); |
2493 | } |
2494 | |
2495 | |
2496 | |
2497 | |
2498 | |
2499 | |
2500 | void SelectionDAGLegalize::PromoteLegalINT_TO_FP( |
2501 | SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) { |
2502 | bool IsStrict = N->isStrictFPOpcode(); |
2503 | bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP || |
2504 | N->getOpcode() == ISD::STRICT_SINT_TO_FP; |
2505 | EVT DestVT = N->getValueType(0); |
2506 | SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0); |
2507 | unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP; |
2508 | unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP; |
2509 | |
2510 | |
2511 | EVT NewInTy = LegalOp.getValueType(); |
2512 | |
2513 | unsigned OpToUse = 0; |
2514 | |
2515 | |
2516 | while (true) { |
2517 | NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1); |
2518 | assert(NewInTy.isInteger() && "Ran out of possibilities!"); |
2519 | |
2520 | |
2521 | if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) { |
2522 | OpToUse = SIntOp; |
2523 | break; |
2524 | } |
2525 | if (IsSigned) |
2526 | continue; |
2527 | |
2528 | |
2529 | if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) { |
2530 | OpToUse = UIntOp; |
2531 | break; |
2532 | } |
2533 | |
2534 | |
2535 | } |
2536 | |
2537 | |
2538 | |
2539 | if (IsStrict) { |
2540 | SDValue Res = |
2541 | DAG.getNode(OpToUse, dl, {DestVT, MVT::Other}, |
2542 | {N->getOperand(0), |
2543 | DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, |
2544 | dl, NewInTy, LegalOp)}); |
2545 | Results.push_back(Res); |
2546 | Results.push_back(Res.getValue(1)); |
2547 | return; |
2548 | } |
2549 | |
2550 | Results.push_back( |
2551 | DAG.getNode(OpToUse, dl, DestVT, |
2552 | DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, |
2553 | dl, NewInTy, LegalOp))); |
2554 | } |
2555 | |
2556 | |
2557 | |
2558 | |
2559 | |
2560 | |
2561 | void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl, |
2562 | SmallVectorImpl<SDValue> &Results) { |
2563 | bool IsStrict = N->isStrictFPOpcode(); |
2564 | bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || |
2565 | N->getOpcode() == ISD::STRICT_FP_TO_SINT; |
2566 | EVT DestVT = N->getValueType(0); |
2567 | SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0); |
2568 | |
2569 | EVT NewOutTy = DestVT; |
2570 | |
2571 | unsigned OpToUse = 0; |
2572 | |
2573 | |
2574 | while (true) { |
2575 | NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1); |
2576 | assert(NewOutTy.isInteger() && "Ran out of possibilities!"); |
2577 | |
2578 | |
2579 | |
2580 | OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT; |
2581 | if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy)) |
2582 | break; |
2583 | |
2584 | |
2585 | OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT; |
2586 | if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy)) |
2587 | break; |
2588 | |
2589 | |
2590 | } |
2591 | |
2592 | |
2593 | SDValue Operation; |
2594 | if (IsStrict) { |
2595 | SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other); |
2596 | Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp); |
2597 | } else |
2598 | Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); |
2599 | |
2600 | |
2601 | |
2602 | SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); |
2603 | Results.push_back(Trunc); |
2604 | if (IsStrict) |
2605 | Results.push_back(Operation.getValue(1)); |
2606 | } |
2607 | |
2608 | |
2609 | |
2610 | |
2611 | SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node, |
2612 | const SDLoc &dl) { |
2613 | unsigned Opcode = Node->getOpcode(); |
2614 | |
2615 | |
2616 | EVT NewOutTy = Node->getValueType(0); |
2617 | while (true) { |
2618 | NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1); |
2619 | assert(NewOutTy.isInteger() && "Ran out of possibilities!"); |
2620 | |
2621 | if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy)) |
2622 | break; |
2623 | } |
2624 | |
2625 | |
2626 | |
2627 | SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0), |
2628 | Node->getOperand(1)); |
2629 | return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); |
2630 | } |
2631 | |
2632 | |
2633 | SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) { |
2634 | EVT VT = Op.getValueType(); |
2635 | EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
2636 | unsigned Sz = VT.getScalarSizeInBits(); |
2637 | |
2638 | |
2639 | SDValue Result; |
2640 | if (TLI.isOperationLegal(ISD::CTPOP, VT)) { |
2641 | Result = DAG.getNode(ISD::CTPOP, dl, VT, Op); |
2642 | } else { |
2643 | Result = Op; |
2644 | for (unsigned i = Log2_32_Ceil(Sz); i != 0;) { |
2645 | SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result, |
2646 | DAG.getConstant(1ULL << (--i), dl, ShVT)); |
2647 | Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift); |
2648 | } |
2649 | } |
2650 | |
2651 | return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT)); |
2652 | } |
2653 | |
2654 | bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { |
2655 | LLVM_DEBUG(dbgs() << "Trying to expand node\n"); |
2656 | SmallVector<SDValue, 8> Results; |
2657 | SDLoc dl(Node); |
2658 | SDValue Tmp1, Tmp2, Tmp3, Tmp4; |
2659 | bool NeedInvert; |
2660 | switch (Node->getOpcode()) { |
2661 | case ISD::ABS: |
2662 | if (TLI.expandABS(Node, Tmp1, DAG)) |
2663 | Results.push_back(Tmp1); |
2664 | break; |
2665 | case ISD::CTPOP: |
2666 | if (TLI.expandCTPOP(Node, Tmp1, DAG)) |
2667 | Results.push_back(Tmp1); |
2668 | break; |
2669 | case ISD::CTLZ: |
2670 | case ISD::CTLZ_ZERO_UNDEF: |
2671 | if (TLI.expandCTLZ(Node, Tmp1, DAG)) |
2672 | Results.push_back(Tmp1); |
2673 | break; |
2674 | case ISD::CTTZ: |
2675 | case ISD::CTTZ_ZERO_UNDEF: |
2676 | if (TLI.expandCTTZ(Node, Tmp1, DAG)) |
2677 | Results.push_back(Tmp1); |
2678 | break; |
2679 | case ISD::BITREVERSE: |
2680 | if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG))) |
2681 | Results.push_back(Tmp1); |
2682 | break; |
2683 | case ISD::BSWAP: |
2684 | if ((Tmp1 = TLI.expandBSWAP(Node, DAG))) |
2685 | Results.push_back(Tmp1); |
2686 | break; |
2687 | case ISD::PARITY: |
2688 | Results.push_back(ExpandPARITY(Node->getOperand(0), dl)); |
2689 | break; |
2690 | case ISD::FRAMEADDR: |
2691 | case ISD::RETURNADDR: |
2692 | case ISD::FRAME_TO_ARGS_OFFSET: |
2693 | Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); |
2694 | break; |
2695 | case ISD::EH_DWARF_CFA: { |
2696 | SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl, |
2697 | TLI.getPointerTy(DAG.getDataLayout())); |
2698 | SDValue Offset = DAG.getNode(ISD::ADD, dl, |
2699 | CfaArg.getValueType(), |
2700 | DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl, |
2701 | CfaArg.getValueType()), |
2702 | CfaArg); |
2703 | SDValue FA = DAG.getNode( |
2704 | ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()), |
2705 | DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()))); |
2706 | Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(), |
2707 | FA, Offset)); |
2708 | break; |
2709 | } |
2710 | case ISD::FLT_ROUNDS_: |
2711 | Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0))); |
2712 | Results.push_back(Node->getOperand(0)); |
2713 | break; |
2714 | case ISD::EH_RETURN: |
2715 | case ISD::EH_LABEL: |
2716 | case ISD::PREFETCH: |
2717 | case ISD::VAEND: |
2718 | case ISD::EH_SJLJ_LONGJMP: |
2719 | |
2720 | |
2721 | Results.push_back(Node->getOperand(0)); |
2722 | break; |
2723 | case ISD::READCYCLECOUNTER: |
2724 | |
2725 | |
2726 | Results.append(Node->getNumValues() - 1, |
2727 | DAG.getConstant(0, dl, Node->getValueType(0))); |
2728 | Results.push_back(Node->getOperand(0)); |
2729 | break; |
2730 | case ISD::EH_SJLJ_SETJMP: |
2731 | |
2732 | |
2733 | Results.push_back(DAG.getConstant(0, dl, MVT::i32)); |
2734 | Results.push_back(Node->getOperand(0)); |
2735 | break; |
2736 | case ISD::ATOMIC_LOAD: { |
2737 | |
2738 | SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0)); |
2739 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); |
2740 | SDValue Swap = DAG.getAtomicCmpSwap( |
2741 | ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, |
2742 | Node->getOperand(0), Node->getOperand(1), Zero, Zero, |
2743 | cast<AtomicSDNode>(Node)->getMemOperand()); |
2744 | Results.push_back(Swap.getValue(0)); |
2745 | Results.push_back(Swap.getValue(1)); |
2746 | break; |
2747 | } |
2748 | case ISD::ATOMIC_STORE: { |
2749 | |
2750 | SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, |
2751 | cast<AtomicSDNode>(Node)->getMemoryVT(), |
2752 | Node->getOperand(0), |
2753 | Node->getOperand(1), Node->getOperand(2), |
2754 | cast<AtomicSDNode>(Node)->getMemOperand()); |
2755 | Results.push_back(Swap.getValue(1)); |
2756 | break; |
2757 | } |
2758 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { |
2759 | |
2760 | |
2761 | |
2762 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); |
2763 | SDValue Res = DAG.getAtomicCmpSwap( |
2764 | ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, |
2765 | Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), |
2766 | Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand()); |
2767 | |
2768 | SDValue ExtRes = Res; |
2769 | SDValue LHS = Res; |
2770 | SDValue RHS = Node->getOperand(1); |
2771 | |
2772 | EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT(); |
2773 | EVT OuterType = Node->getValueType(0); |
2774 | switch (TLI.getExtendForAtomicOps()) { |
2775 | case ISD::SIGN_EXTEND: |
2776 | LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res, |
2777 | DAG.getValueType(AtomicType)); |
2778 | RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType, |
2779 | Node->getOperand(2), DAG.getValueType(AtomicType)); |
2780 | ExtRes = LHS; |
2781 | break; |
2782 | case ISD::ZERO_EXTEND: |
2783 | LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res, |
2784 | DAG.getValueType(AtomicType)); |
2785 | RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); |
2786 | ExtRes = LHS; |
2787 | break; |
2788 | case ISD::ANY_EXTEND: |
2789 | LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType); |
2790 | RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); |
2791 | break; |
2792 | default: |
2793 | llvm_unreachable("Invalid atomic op extension"); |
2794 | } |
2795 | |
2796 | SDValue Success = |
2797 | DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); |
2798 | |
2799 | Results.push_back(ExtRes.getValue(0)); |
2800 | Results.push_back(Success); |
2801 | Results.push_back(Res.getValue(1)); |
2802 | break; |
2803 | } |
2804 | case ISD::DYNAMIC_STACKALLOC: |
2805 | ExpandDYNAMIC_STACKALLOC(Node, Results); |
2806 | break; |
2807 | case ISD::MERGE_VALUES: |
2808 | for (unsigned i = 0; i < Node->getNumValues(); i++) |
2809 | Results.push_back(Node->getOperand(i)); |
2810 | break; |
2811 | case ISD::UNDEF: { |
2812 | EVT VT = Node->getValueType(0); |
2813 | if (VT.isInteger()) |
2814 | Results.push_back(DAG.getConstant(0, dl, VT)); |
2815 | else { |
2816 | assert(VT.isFloatingPoint() && "Unknown value type!"); |
2817 | Results.push_back(DAG.getConstantFP(0, dl, VT)); |
2818 | } |
2819 | break; |
2820 | } |
2821 | case ISD::STRICT_FP_ROUND: |
2822 | |
2823 | |
2824 | if (TLI.isStrictFPEnabled()) |
2825 | break; |
2826 | |
2827 | |
2828 | if (TLI.getStrictFPOperationAction(Node->getOpcode(), |
2829 | Node->getValueType(0)) |
2830 | == TargetLowering::Legal) |
2831 | break; |
2832 | |
2833 | |
2834 | if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0), |
2835 | Node->getValueType(0), dl, |
2836 | Node->getOperand(0)))) { |
2837 | ReplaceNode(Node, Tmp1.getNode()); |
2838 | LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n"); |
2839 | return true; |
2840 | } |
2841 | break; |
2842 | case ISD::FP_ROUND: |
2843 | case ISD::BITCAST: |
2844 | if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), |
2845 | Node->getValueType(0), dl))) |
2846 | Results.push_back(Tmp1); |
2847 | break; |
2848 | case ISD::STRICT_FP_EXTEND: |
2849 | |
2850 | |
2851 | if (TLI.isStrictFPEnabled()) |
2852 | break; |
2853 | |
2854 | |
2855 | if (TLI.getStrictFPOperationAction(Node->getOpcode(), |
2856 | Node->getValueType(0)) |
2857 | == TargetLowering::Legal) |
2858 | break; |
2859 | |
2860 | |
2861 | if ((Tmp1 = EmitStackConvert( |
2862 | Node->getOperand(1), Node->getOperand(1).getValueType(), |
2863 | Node->getValueType(0), dl, Node->getOperand(0)))) { |
2864 | ReplaceNode(Node, Tmp1.getNode()); |
2865 | LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n"); |
2866 | return true; |
2867 | } |
2868 | break; |
2869 | case ISD::FP_EXTEND: |
2870 | if ((Tmp1 = EmitStackConvert(Node->getOperand(0), |
2871 | Node->getOperand(0).getValueType(), |
2872 | Node->getValueType(0), dl))) |
2873 | Results.push_back(Tmp1); |
2874 | break; |
2875 | case ISD::SIGN_EXTEND_INREG: { |
2876 | EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); |
2877 | EVT VT = Node->getValueType(0); |
2878 | |
2879 | |
2880 | |
2881 | |
2882 | |
2883 | |
2884 | |
2885 | |
2886 | if (ExtraVT.getSizeInBits() == 1) { |
2887 | SDValue One = DAG.getConstant(1, dl, VT); |
2888 | SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); |
2889 | SDValue Zero = DAG.getConstant(0, dl, VT); |
2890 | SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And); |
2891 | Results.push_back(Neg); |
2892 | break; |
2893 | } |
2894 | |
2895 | |
2896 | |
2897 | EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
2898 | unsigned BitsDiff = VT.getScalarSizeInBits() - |
2899 | ExtraVT.getScalarSizeInBits(); |
2900 | SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy); |
2901 | Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), |
2902 | Node->getOperand(0), ShiftCst); |
2903 | Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); |
2904 | Results.push_back(Tmp1); |
2905 | break; |
2906 | } |
2907 | case ISD::UINT_TO_FP: |
2908 | case ISD::STRICT_UINT_TO_FP: |
2909 | if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) { |
2910 | Results.push_back(Tmp1); |
2911 | if (Node->isStrictFPOpcode()) |
2912 | Results.push_back(Tmp2); |
2913 | break; |
2914 | } |
2915 | LLVM_FALLTHROUGH; |
2916 | case ISD::SINT_TO_FP: |
2917 | case ISD::STRICT_SINT_TO_FP: |
2918 | if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) { |
2919 | Results.push_back(Tmp1); |
2920 | if (Node->isStrictFPOpcode()) |
2921 | Results.push_back(Tmp2); |
2922 | } |
2923 | break; |
2924 | case ISD::FP_TO_SINT: |
2925 | if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) |
2926 | Results.push_back(Tmp1); |
2927 | break; |
2928 | case ISD::STRICT_FP_TO_SINT: |
2929 | if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) { |
2930 | ReplaceNode(Node, Tmp1.getNode()); |
2931 | LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n"); |
2932 | return true; |
2933 | } |
2934 | break; |
2935 | case ISD::FP_TO_UINT: |
2936 | if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) |
2937 | Results.push_back(Tmp1); |
2938 | break; |
2939 | case ISD::STRICT_FP_TO_UINT: |
2940 | if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) { |
2941 | |
2942 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2); |
2943 | |
2944 | ReplaceNodeWithValue(SDValue(Node, 0), Tmp1); |
2945 | LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n"); |
2946 | return true; |
2947 | } |
2948 | break; |
2949 | case ISD::FP_TO_SINT_SAT: |
2950 | case ISD::FP_TO_UINT_SAT: |
2951 | Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG)); |
2952 | break; |
2953 | case ISD::VAARG: |
2954 | Results.push_back(DAG.expandVAArg(Node)); |
2955 | Results.push_back(Results[0].getValue(1)); |
2956 | break; |
2957 | case ISD::VACOPY: |
2958 | Results.push_back(DAG.expandVACopy(Node)); |
2959 | break; |
2960 | case ISD::EXTRACT_VECTOR_ELT: |
2961 | if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) |
2962 | |
2963 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), |
2964 | Node->getOperand(0)); |
2965 | else |
2966 | Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); |
2967 | Results.push_back(Tmp1); |
2968 | break; |
2969 | case ISD::EXTRACT_SUBVECTOR: |
2970 | Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); |
2971 | break; |
2972 | case ISD::INSERT_SUBVECTOR: |
2973 | Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); |
2974 | break; |
2975 | case ISD::CONCAT_VECTORS: |
2976 | Results.push_back(ExpandVectorBuildThroughStack(Node)); |
2977 | break; |
2978 | case ISD::SCALAR_TO_VECTOR: |
2979 | Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); |
2980 | break; |
2981 | case ISD::INSERT_VECTOR_ELT: |
2982 | Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), |
2983 | Node->getOperand(1), |
2984 | Node->getOperand(2), dl)); |
2985 | break; |
2986 | case ISD::VECTOR_SHUFFLE: { |
2987 | SmallVector<int, 32> NewMask; |
2988 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); |
2989 | |
2990 | EVT VT = Node->getValueType(0); |
2991 | EVT EltVT = VT.getVectorElementType(); |
2992 | SDValue Op0 = Node->getOperand(0); |
2993 | SDValue Op1 = Node->getOperand(1); |
2994 | if (!TLI.isTypeLegal(EltVT)) { |
2995 | EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); |
2996 | |
2997 | |
2998 | |
2999 | |
3000 | if (NewEltVT.bitsLT(EltVT)) { |
3001 | |
3002 | |
3003 | |
3004 | |
3005 | |
3006 | EVT NewVT = |
3007 | EVT::getVectorVT(*DAG.getContext(), NewEltVT, |
3008 | VT.getSizeInBits() / NewEltVT.getSizeInBits()); |
3009 | assert(NewVT.bitsEq(VT)); |
3010 | |
3011 | |
3012 | Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); |
3013 | Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); |
3014 | |
3015 | |
3016 | unsigned int factor = |
3017 | NewVT.getVectorNumElements()/VT.getVectorNumElements(); |
3018 | |
3019 | |
3020 | assert(factor > 0); |
3021 | |
3022 | for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { |
3023 | if (Mask[i] < 0) { |
3024 | for (unsigned fi = 0; fi < factor; ++fi) |
3025 | NewMask.push_back(Mask[i]); |
3026 | } |
3027 | else { |
3028 | for (unsigned fi = 0; fi < factor; ++fi) |
3029 | NewMask.push_back(Mask[i]*factor+fi); |
3030 | } |
3031 | } |
3032 | Mask = NewMask; |
3033 | VT = NewVT; |
3034 | } |
3035 | EltVT = NewEltVT; |
3036 | } |
3037 | unsigned NumElems = VT.getVectorNumElements(); |
3038 | SmallVector<SDValue, 16> Ops; |
3039 | for (unsigned i = 0; i != NumElems; ++i) { |
3040 | if (Mask[i] < 0) { |
3041 | Ops.push_back(DAG.getUNDEF(EltVT)); |
3042 | continue; |
3043 | } |
3044 | unsigned Idx = Mask[i]; |
3045 | if (Idx < NumElems) |
3046 | Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, |
3047 | DAG.getVectorIdxConstant(Idx, dl))); |
3048 | else |
3049 | Ops.push_back( |
3050 | DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1, |
3051 | DAG.getVectorIdxConstant(Idx - NumElems, dl))); |
3052 | } |
3053 | |
3054 | Tmp1 = DAG.getBuildVector(VT, dl, Ops); |
3055 | |
3056 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); |
3057 | Results.push_back(Tmp1); |
3058 | break; |
3059 | } |
3060 | case ISD::VECTOR_SPLICE: { |
3061 | Results.push_back(TLI.expandVectorSplice(Node, DAG)); |
3062 | break; |
3063 | } |
3064 | case ISD::EXTRACT_ELEMENT: { |
3065 | EVT OpTy = Node->getOperand(0).getValueType(); |
3066 | if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { |
3067 | |
3068 | Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), |
3069 | DAG.getConstant(OpTy.getSizeInBits() / 2, dl, |
3070 | TLI.getShiftAmountTy( |
3071 | Node->getOperand(0).getValueType(), |
3072 | DAG.getDataLayout()))); |
3073 | Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); |
3074 | } else { |
3075 | |
3076 | Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), |
3077 | Node->getOperand(0)); |
3078 | } |
3079 | Results.push_back(Tmp1); |
3080 | break; |
3081 | } |
3082 | case ISD::STACKSAVE: |
3083 | |
3084 | |
3085 | if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) { |
3086 | Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, |
3087 | Node->getValueType(0))); |
3088 | Results.push_back(Results[0].getValue(1)); |
3089 | } else { |
3090 | Results.push_back(DAG.getUNDEF(Node->getValueType(0))); |
3091 | Results.push_back(Node->getOperand(0)); |
3092 | } |
3093 | break; |
3094 | case ISD::STACKRESTORE: |
3095 | |
3096 | |
3097 | if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) { |
3098 | Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, |
3099 | Node->getOperand(1))); |
3100 | } else { |
3101 | Results.push_back(Node->getOperand(0)); |
3102 | } |
3103 | break; |
3104 | case ISD::GET_DYNAMIC_AREA_OFFSET: |
3105 | Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); |
3106 | Results.push_back(Results[0].getValue(0)); |
3107 | break; |
3108 | case ISD::FCOPYSIGN: |
3109 | Results.push_back(ExpandFCOPYSIGN(Node)); |
3110 | break; |
3111 | case ISD::FNEG: |
3112 | Results.push_back(ExpandFNEG(Node)); |
3113 | break; |
3114 | case ISD::FABS: |
3115 | Results.push_back(ExpandFABS(Node)); |
3116 | break; |
3117 | case ISD::SMIN: |
3118 | case ISD::SMAX: |
3119 | case ISD::UMIN: |
3120 | case ISD::UMAX: { |
3121 | |
3122 | ISD::CondCode Pred; |
3123 | switch (Node->getOpcode()) { |
3124 | default: llvm_unreachable("How did we get here?"); |
3125 | case ISD::SMAX: Pred = ISD::SETGT; break; |
3126 | case ISD::SMIN: Pred = ISD::SETLT; break; |
3127 | case ISD::UMAX: Pred = ISD::SETUGT; break; |
3128 | case ISD::UMIN: Pred = ISD::SETULT; break; |
3129 | } |
3130 | Tmp1 = Node->getOperand(0); |
3131 | Tmp2 = Node->getOperand(1); |
3132 | Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred); |
3133 | Results.push_back(Tmp1); |
3134 | break; |
3135 | } |
3136 | case ISD::FMINNUM: |
3137 | case ISD::FMAXNUM: { |
3138 | if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) |
3139 | Results.push_back(Expanded); |
3140 | break; |
3141 | } |
3142 | case ISD::FSIN: |
3143 | case ISD::FCOS: { |
3144 | EVT VT = Node->getValueType(0); |
3145 | |
3146 | |
3147 | if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || |
3148 | isSinCosLibcallAvailable(Node, TLI)) |
3149 | && useSinCos(Node)) { |
3150 | SDVTList VTs = DAG.getVTList(VT, VT); |
3151 | Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); |
3152 | if (Node->getOpcode() == ISD::FCOS) |
3153 | Tmp1 = Tmp1.getValue(1); |
3154 | Results.push_back(Tmp1); |
3155 | } |
3156 | break; |
3157 | } |
3158 | case ISD::FMAD: |
3159 | llvm_unreachable("Illegal fmad should never be formed"); |
3160 | |
3161 | case ISD::FP16_TO_FP: |
3162 | if (Node->getValueType(0) != MVT::f32) { |
3163 | |
3164 | |
3165 | |
3166 | SDValue Res = |
3167 | DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); |
3168 | Results.push_back( |
3169 | DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); |
3170 | } |
3171 | break; |
3172 | case ISD::STRICT_FP16_TO_FP: |
3173 | if (Node->getValueType(0) != MVT::f32) { |
3174 | |
3175 | |
3176 | |
3177 | SDValue Res = |
3178 | DAG.getNode(ISD::STRICT_FP16_TO_FP, dl, {MVT::f32, MVT::Other}, |
3179 | {Node->getOperand(0), Node->getOperand(1)}); |
3180 | Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, |
3181 | {Node->getValueType(0), MVT::Other}, |
3182 | {Res.getValue(1), Res}); |
3183 | Results.push_back(Res); |
3184 | Results.push_back(Res.getValue(1)); |
3185 | } |
3186 | break; |
3187 | case ISD::FP_TO_FP16: |
3188 | LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n"); |
3189 | if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) { |
3190 | SDValue Op = Node->getOperand(0); |
3191 | MVT SVT = Op.getSimpleValueType(); |
3192 | if ((SVT == MVT::f64 || SVT == MVT::f80) && |
3193 | TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) { |
3194 | |
3195 | |
3196 | SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, |
3197 | DAG.getIntPtrConstant(0, dl)); |
3198 | Results.push_back( |
3199 | DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); |
3200 | } |
3201 | } |
3202 | break; |
3203 | case ISD::ConstantFP: { |
3204 | ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); |
3205 | |
3206 | |
3207 | if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0), |
3208 | DAG.shouldOptForSize())) |
3209 | Results.push_back(ExpandConstantFP(CFP, true)); |
3210 | break; |
3211 | } |
3212 | case ISD::Constant: { |
3213 | ConstantSDNode *CP = cast<ConstantSDNode>(Node); |
3214 | Results.push_back(ExpandConstant(CP)); |
3215 | break; |
3216 | } |
3217 | case ISD::FSUB: { |
3218 | EVT VT = Node->getValueType(0); |
3219 | if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) && |
3220 | TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) { |
3221 | const SDNodeFlags Flags = Node->getFlags(); |
3222 | Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); |
3223 | Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); |
3224 | Results.push_back(Tmp1); |
3225 | } |
3226 | break; |
3227 | } |
3228 | case ISD::SUB: { |
3229 | EVT VT = Node->getValueType(0); |
3230 | assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && |
3231 | TLI.isOperationLegalOrCustom(ISD::XOR, VT) && |
3232 | "Don't know how to expand this subtraction!"); |
3233 | Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), |
3234 | DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl, |
3235 | VT)); |
3236 | Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT)); |
3237 | Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); |
3238 | break; |
3239 | } |
3240 | case ISD::UREM: |
3241 | case ISD::SREM: |
3242 | if (TLI.expandREM(Node, Tmp1, DAG)) |
3243 | Results.push_back(Tmp1); |
3244 | break; |
3245 | case ISD::UDIV: |
3246 | case ISD::SDIV: { |
3247 | bool isSigned = Node->getOpcode() == ISD::SDIV; |
3248 | unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; |
3249 | EVT VT = Node->getValueType(0); |
3250 | if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) { |
3251 | SDVTList VTs = DAG.getVTList(VT, VT); |
3252 | Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), |
3253 | Node->getOperand(1)); |
3254 | Results.push_back(Tmp1); |
3255 | } |
3256 | break; |
3257 | } |
3258 | case ISD::MULHU: |
3259 | case ISD::MULHS: { |
3260 | unsigned ExpandOpcode = |
3261 | Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; |
3262 | EVT VT = Node->getValueType(0); |
3263 | SDVTList VTs = DAG.getVTList(VT, VT); |
3264 | |
3265 | Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), |
3266 | Node->getOperand(1)); |
3267 | Results.push_back(Tmp1.getValue(1)); |
3268 | break; |
3269 | } |
3270 | case ISD::UMUL_LOHI: |
3271 | case ISD::SMUL_LOHI: { |
3272 | SDValue LHS = Node->getOperand(0); |
3273 | SDValue RHS = Node->getOperand(1); |
3274 | MVT VT = LHS.getSimpleValueType(); |
3275 | unsigned MULHOpcode = |
3276 | Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; |
3277 | |
3278 | if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) { |
3279 | Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS)); |
3280 | Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS)); |
3281 | break; |
3282 | } |
3283 | |
3284 | SmallVector<SDValue, 4> Halves; |
3285 | EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext()); |
3286 | assert(TLI.isTypeLegal(HalfType)); |
3287 | if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves, |
3288 | HalfType, DAG, |
3289 | TargetLowering::MulExpansionKind::Always)) { |
3290 | for (unsigned i = 0; i < 2; ++i) { |
3291 | SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]); |
3292 | SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]); |
3293 | SDValue Shift = DAG.getConstant( |
3294 | HalfType.getScalarSizeInBits(), dl, |
3295 | TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); |
3296 | Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); |
3297 | Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); |
3298 | } |
3299 | break; |
3300 | } |
3301 | break; |
3302 | } |
3303 | case ISD::MUL: { |
3304 | EVT VT = Node->getValueType(0); |
3305 | SDVTList VTs = DAG.getVTList(VT, VT); |
3306 | |
3307 | |
3308 | |
3309 | |
3310 | |
3311 | bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); |
3312 | bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); |
3313 | bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); |
3314 | bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); |
3315 | unsigned OpToUse = 0; |
3316 | if (HasSMUL_LOHI && !HasMULHS) { |
3317 | OpToUse = ISD::SMUL_LOHI; |
3318 | } else if (HasUMUL_LOHI && !HasMULHU) { |
3319 | OpToUse = ISD::UMUL_LOHI; |
3320 | } else if (HasSMUL_LOHI) { |
3321 | OpToUse = ISD::SMUL_LOHI; |
3322 | } else if (HasUMUL_LOHI) { |
3323 | OpToUse = ISD::UMUL_LOHI; |
3324 | } |
3325 | if (OpToUse) { |
3326 | Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), |
3327 | Node->getOperand(1))); |
3328 | break; |
3329 | } |
3330 | |
3331 | SDValue Lo, Hi; |
3332 | EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext()); |
3333 | if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) && |
3334 | TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) && |
3335 | TLI.isOperationLegalOrCustom(ISD::SHL, VT) && |
3336 | TLI.isOperationLegalOrCustom(ISD::OR, VT) && |
3337 | TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, |
3338 | TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) { |
3339 | Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); |
3340 | Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); |
3341 | SDValue Shift = |
3342 | DAG.getConstant(HalfType.getSizeInBits(), dl, |
3343 | TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); |
3344 | Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); |
3345 | Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); |
3346 | } |
3347 | break; |
3348 | } |
3349 | case ISD::FSHL: |
3350 | case ISD::FSHR: |
3351 | if (TLI.expandFunnelShift(Node, Tmp1, DAG)) |
3352 | Results.push_back(Tmp1); |
3353 | break; |
3354 | case ISD::ROTL: |
3355 | case ISD::ROTR: |
3356 | if (TLI.expandROT(Node, true , Tmp1, DAG)) |
3357 | Results.push_back(Tmp1); |
3358 | break; |
3359 | case ISD::SADDSAT: |
3360 | case ISD::UADDSAT: |
3361 | case ISD::SSUBSAT: |
3362 | case ISD::USUBSAT: |
3363 | Results.push_back(TLI.expandAddSubSat(Node, DAG)); |
3364 | break; |
3365 | case ISD::SSHLSAT: |
3366 | case ISD::USHLSAT: |
3367 | Results.push_back(TLI.expandShlSat(Node, DAG)); |
3368 | break; |
3369 | case ISD::SMULFIX: |
3370 | case ISD::SMULFIXSAT: |
3371 | case ISD::UMULFIX: |
3372 | case ISD::UMULFIXSAT: |
3373 | Results.push_back(TLI.expandFixedPointMul(Node, DAG)); |
3374 | break; |
3375 | case ISD::SDIVFIX: |
3376 | case ISD::SDIVFIXSAT: |
3377 | case ISD::UDIVFIX: |
3378 | case ISD::UDIVFIXSAT: |
3379 | if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node), |
3380 | Node->getOperand(0), |
3381 | Node->getOperand(1), |
3382 | Node->getConstantOperandVal(2), |
3383 | DAG)) { |
3384 | Results.push_back(V); |
3385 | break; |
3386 | } |
3387 | |
3388 | |
3389 | |
3390 | |
3391 | |
3392 | |
3393 | |
3394 | llvm_unreachable("Cannot expand DIVFIX!"); |
3395 | case ISD::ADDCARRY: |
3396 | case ISD::SUBCARRY: { |
3397 | SDValue LHS = Node->getOperand(0); |
3398 | SDValue RHS = Node->getOperand(1); |
3399 | SDValue Carry = Node->getOperand(2); |
3400 | |
3401 | bool IsAdd = Node->getOpcode() == ISD::ADDCARRY; |
3402 | |
3403 | |
3404 | unsigned Op = IsAdd ? ISD::ADD : ISD::SUB; |
3405 | EVT VT = LHS.getValueType(); |
3406 | SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS); |
3407 | |
3408 | |
3409 | EVT CarryType = Node->getValueType(1); |
3410 | EVT SetCCType = getSetCCResultType(Node->getValueType(0)); |
3411 | ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT; |
3412 | SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC); |
3413 | |
3414 | |
3415 | SDValue One = DAG.getConstant(1, dl, VT); |
3416 | SDValue CarryExt = |
3417 | DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One); |
3418 | SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt); |
3419 | |
3420 | |
3421 | |
3422 | |
3423 | |
3424 | SDValue Zero = DAG.getConstant(0, dl, VT); |
3425 | SDValue Overflow2 = |
3426 | IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ) |
3427 | : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ); |
3428 | Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2, |
3429 | DAG.getZExtOrTrunc(Carry, dl, SetCCType)); |
3430 | |
3431 | SDValue ResultCarry = |
3432 | DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2); |
3433 | |
3434 | Results.push_back(Sum2); |
3435 | Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT)); |
3436 | break; |
3437 | } |
3438 | case ISD::SADDO: |
3439 | case ISD::SSUBO: { |
3440 | SDValue Result, Overflow; |
3441 | TLI.expandSADDSUBO(Node, Result, Overflow, DAG); |
3442 | Results.push_back(Result); |
3443 | Results.push_back(Overflow); |
3444 | break; |
3445 | } |
3446 | case ISD::UADDO: |
3447 | case ISD::USUBO: { |
3448 | SDValue Result, Overflow; |
3449 | TLI.expandUADDSUBO(Node, Result, Overflow, DAG); |
3450 | Results.push_back(Result); |
3451 | Results.push_back(Overflow); |
3452 | break; |
3453 | } |
3454 | case ISD::UMULO: |
3455 | case ISD::SMULO: { |
3456 | SDValue Result, Overflow; |
3457 | if (TLI.expandMULO(Node, Result, Overflow, DAG)) { |
3458 | Results.push_back(Result); |
3459 | Results.push_back(Overflow); |
3460 | } |
3461 | break; |
3462 | } |
3463 | case ISD::BUILD_PAIR: { |
3464 | EVT PairTy = Node->getValueType(0); |
3465 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); |
3466 | Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); |
3467 | Tmp2 = DAG.getNode( |
3468 | ISD::SHL, dl, PairTy, Tmp2, |
3469 | DAG.getConstant(PairTy.getSizeInBits() / 2, dl, |
3470 | TLI.getShiftAmountTy(PairTy, DAG.getDataLayout()))); |
3471 | Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); |
3472 | break; |
3473 | } |
3474 | case ISD::SELECT: |
3475 | Tmp1 = Node->getOperand(0); |
3476 | Tmp2 = Node->getOperand(1); |
3477 | Tmp3 = Node->getOperand(2); |
3478 | if (Tmp1.getOpcode() == ISD::SETCC) { |
3479 | Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), |
3480 | Tmp2, Tmp3, |
3481 | cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); |
3482 | } else { |
3483 | Tmp1 = DAG.getSelectCC(dl, Tmp1, |
3484 | DAG.getConstant(0, dl, Tmp1.getValueType()), |
3485 | Tmp2, Tmp3, ISD::SETNE); |
3486 | } |
3487 | Tmp1->setFlags(Node->getFlags()); |
3488 | Results.push_back(Tmp1); |
3489 | break; |
3490 | case ISD::BR_JT: { |
3491 | SDValue Chain = Node->getOperand(0); |
3492 | SDValue Table = Node->getOperand(1); |
3493 | SDValue Index = Node->getOperand(2); |
3494 | |
3495 | const DataLayout &TD = DAG.getDataLayout(); |
3496 | EVT PTy = TLI.getPointerTy(TD); |
3497 | |
3498 | unsigned EntrySize = |
3499 | DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); |
3500 | |
3501 | |
3502 | |
3503 | |
3504 | |
3505 | if (llvm::isPowerOf2_32(EntrySize)) |
3506 | Index = DAG.getNode( |
3507 | ISD::SHL, dl, Index.getValueType(), Index, |
3508 | DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType())); |
3509 | else |
3510 | Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index, |
3511 | DAG.getConstant(EntrySize, dl, Index.getValueType())); |
3512 | SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), |
3513 | Index, Table); |
3514 | |
3515 | EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8); |
3516 | SDValue LD = DAG.getExtLoad( |
3517 | ISD::SEXTLOAD, dl, PTy, Chain, Addr, |
3518 | MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT); |
3519 | Addr = LD; |
3520 | if (TLI.isJumpTableRelative()) { |
3521 | |
3522 | |
3523 | |
3524 | Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, |
3525 | TLI.getPICJumpTableRelocBase(Table, DAG)); |
3526 | } |
3527 | |
3528 | Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG); |
3529 | Results.push_back(Tmp1); |
3530 | break; |
3531 | } |
3532 | case ISD::BRCOND: |
3533 | |
3534 | |
3535 | Tmp1 = Node->getOperand(0); |
3536 | Tmp2 = Node->getOperand(1); |
3537 | if (Tmp2.getOpcode() == ISD::SETCC) { |
3538 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, |
3539 | Tmp1, Tmp2.getOperand(2), |
3540 | Tmp2.getOperand(0), Tmp2.getOperand(1), |
3541 | Node->getOperand(2)); |
3542 | } else { |
3543 | |
3544 | if (Tmp2.isUndef() || |
3545 | (Tmp2.getOpcode() == ISD::AND && |
3546 | isa<ConstantSDNode>(Tmp2.getOperand(1)) && |
3547 | cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1)) |
3548 | Tmp3 = Tmp2; |
3549 | else |
3550 | Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, |
3551 | DAG.getConstant(1, dl, Tmp2.getValueType())); |
3552 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, |
3553 | DAG.getCondCode(ISD::SETNE), Tmp3, |
3554 | DAG.getConstant(0, dl, Tmp3.getValueType()), |
3555 | Node->getOperand(2)); |
3556 | } |
3557 | Results.push_back(Tmp1); |
3558 | break; |
3559 | case ISD::SETCC: |
3560 | case ISD::STRICT_FSETCC: |
3561 | case ISD::STRICT_FSETCCS: { |
3562 | bool IsStrict = Node->getOpcode() != ISD::SETCC; |
3563 | bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; |
3564 | SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); |
3565 | unsigned Offset = IsStrict ? 1 : 0; |
3566 | Tmp1 = Node->getOperand(0 + Offset); |
3567 | Tmp2 = Node->getOperand(1 + Offset); |
3568 | Tmp3 = Node->getOperand(2 + Offset); |
3569 | bool Legalized = |
3570 | TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, |
3571 | NeedInvert, dl, Chain, IsSignaling); |
3572 | |
3573 | if (Legalized) { |
3574 | |
3575 | |
3576 | if (Tmp3.getNode()) |
3577 | Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), |
3578 | Tmp1, Tmp2, Tmp3, Node->getFlags()); |
3579 | |
3580 | |
3581 | |
3582 | if (NeedInvert) |
3583 | Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0)); |
3584 | |
3585 | Results.push_back(Tmp1); |
3586 | if (IsStrict) |
3587 | Results.push_back(Chain); |
3588 | |
3589 | break; |
3590 | } |
3591 | |
3592 | |
3593 | |
3594 | assert(!IsStrict && "Don't know how to expand for strict nodes."); |
3595 | |
3596 | |
3597 | |
3598 | EVT VT = Node->getValueType(0); |
3599 | int TrueValue; |
3600 | switch (TLI.getBooleanContents(Tmp1.getValueType())) { |
3601 | case TargetLowering::ZeroOrOneBooleanContent: |
3602 | case TargetLowering::UndefinedBooleanContent: |
3603 | TrueValue = 1; |
3604 | break; |
3605 | case TargetLowering::ZeroOrNegativeOneBooleanContent: |
3606 | TrueValue = -1; |
3607 | break; |
3608 | } |
3609 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, |
3610 | DAG.getConstant(TrueValue, dl, VT), |
3611 | DAG.getConstant(0, dl, VT), |
3612 | Tmp3); |
3613 | Tmp1->setFlags(Node->getFlags()); |
3614 | Results.push_back(Tmp1); |
3615 | break; |
3616 | } |
3617 | case ISD::SELECT_CC: { |
3618 | |
3619 | Tmp1 = Node->getOperand(0); |
3620 | Tmp2 = Node->getOperand(1); |
3621 | Tmp3 = Node->getOperand(2); |
3622 | Tmp4 = Node->getOperand(3); |
3623 | EVT VT = Node->getValueType(0); |
3624 | SDValue Chain; |
3625 | SDValue CC = Node->getOperand(4); |
3626 | ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get(); |
3627 | |
3628 | if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) { |
3629 | |
3630 | |
3631 | EVT CmpVT = Tmp1.getValueType(); |
3632 | assert(!TLI.isOperationExpand(ISD::SELECT, VT) && |
3633 | "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be " |
3634 | "expanded."); |
3635 | EVT CCVT = getSetCCResultType(CmpVT); |
3636 | SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); |
3637 | Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4)); |
3638 | break; |
3639 | } |
3640 | |
3641 | |
3642 | bool Legalized = false; |
3643 | |
3644 | |
3645 | |
3646 | ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType()); |
3647 | if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) { |
3648 | |
3649 | Legalized = true; |
3650 | Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC); |
3651 | Tmp1->setFlags(Node->getFlags()); |
3652 | } else { |
3653 | |
3654 | |
3655 | ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC); |
3656 | if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) { |
3657 | |
3658 | |
3659 | Legalized = true; |
3660 | Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC); |
3661 | Tmp1->setFlags(Node->getFlags()); |
3662 | } |
3663 | } |
3664 | |
3665 | if (!Legalized) { |
3666 | Legalized = TLI.LegalizeSetCCCondCode( |
3667 | DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, |
3668 | NeedInvert, dl, Chain); |
3669 | |
3670 | assert(Legalized && "Can't legalize SELECT_CC with legal condition!"); |
3671 | |
3672 | |
3673 | |
3674 | if (NeedInvert) |
3675 | std::swap(Tmp3, Tmp4); |
3676 | |
3677 | |
3678 | |
3679 | if (CC.getNode()) { |
3680 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), |
3681 | Tmp1, Tmp2, Tmp3, Tmp4, CC); |
3682 | } else { |
3683 | Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType()); |
3684 | CC = DAG.getCondCode(ISD::SETNE); |
3685 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, |
3686 | Tmp2, Tmp3, Tmp4, CC); |
3687 | } |
3688 | Tmp1->setFlags(Node->getFlags()); |
3689 | } |
3690 | Results.push_back(Tmp1); |
3691 | break; |
3692 | } |
3693 | case ISD::BR_CC: { |
3694 | |
3695 | SDValue Chain; |
3696 | Tmp1 = Node->getOperand(0); |
3697 | Tmp2 = Node->getOperand(2); |
3698 | Tmp3 = Node->getOperand(3); |
3699 | Tmp4 = Node->getOperand(1); |
3700 | |
3701 | bool Legalized = |
3702 | TLI.LegalizeSetCCCondCode(DAG, getSetCCResultType(Tmp2.getValueType()), |
3703 | Tmp2, Tmp3, Tmp4, NeedInvert, dl, Chain); |
3704 | (void)Legalized; |
3705 | assert(Legalized && "Can't legalize BR_CC with legal condition!"); |
3706 | |
3707 | |
3708 | |
3709 | if (Tmp4.getNode()) { |
3710 | assert(!NeedInvert && "Don't know how to invert BR_CC!"); |
3711 | |
3712 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, |
3713 | Tmp4, Tmp2, Tmp3, Node->getOperand(4)); |
3714 | } else { |
3715 | Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType()); |
3716 | Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE); |
3717 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, |
3718 | Tmp2, Tmp3, Node->getOperand(4)); |
3719 | } |
3720 | Results.push_back(Tmp1); |
3721 | break; |
3722 | } |
3723 | case ISD::BUILD_VECTOR: |
3724 | Results.push_back(ExpandBUILD_VECTOR(Node)); |
3725 | break; |
3726 | case ISD::SPLAT_VECTOR: |
3727 | Results.push_back(ExpandSPLAT_VECTOR(Node)); |
3728 | break; |
3729 | case ISD::SRA: |
3730 | case ISD::SRL: |
3731 | case ISD::SHL: { |
3732 | |
3733 | EVT VT = Node->getValueType(0); |
3734 | assert(VT.isVector() && "Unable to legalize non-vector shift"); |
3735 | assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"); |
3736 | unsigned NumElem = VT.getVectorNumElements(); |
3737 | |
3738 | SmallVector<SDValue, 8> Scalars; |
3739 | for (unsigned Idx = 0; Idx < NumElem; Idx++) { |
3740 | SDValue Ex = |
3741 | DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), |
3742 | Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl)); |
3743 | SDValue Sh = |
3744 | DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), |
3745 | Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl)); |
3746 | Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, |
3747 | VT.getScalarType(), Ex, Sh)); |
3748 | } |
3749 | |
3750 | SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars); |
3751 | Results.push_back(Result); |
3752 | break; |
3753 | } |
3754 | case ISD::VECREDUCE_FADD: |
3755 | case ISD::VECREDUCE_FMUL: |
3756 | case ISD::VECREDUCE_ADD: |
3757 | case ISD::VECREDUCE_MUL: |
3758 | case ISD::VECREDUCE_AND: |
3759 | case ISD::VECREDUCE_OR: |
3760 | case ISD::VECREDUCE_XOR: |
3761 | case ISD::VECREDUCE_SMAX: |
3762 | case ISD::VECREDUCE_SMIN: |
3763 | case ISD::VECREDUCE_UMAX: |
3764 | case ISD::VECREDUCE_UMIN: |
3765 | case ISD::VECREDUCE_FMAX: |
3766 | case ISD::VECREDUCE_FMIN: |
3767 | Results.push_back(TLI.expandVecReduce(Node, DAG)); |
3768 | break; |
3769 | case ISD::GLOBAL_OFFSET_TABLE: |
3770 | case ISD::GlobalAddress: |
3771 | case ISD::GlobalTLSAddress: |
3772 | case ISD::ExternalSymbol: |
3773 | case ISD::ConstantPool: |
3774 | case ISD::JumpTable: |
3775 | case ISD::INTRINSIC_W_CHAIN: |
3776 | case ISD::INTRINSIC_WO_CHAIN: |
3777 | case ISD::INTRINSIC_VOID: |
3778 | |
3779 | |
3780 | |
3781 | return true; |
3782 | } |
3783 | |
3784 | if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) { |
3785 | |
3786 | |
3787 | |
3788 | |
3789 | |
3790 | switch (Node->getOpcode()) { |
3791 | default: |
3792 | if (TLI.getStrictFPOperationAction(Node->getOpcode(), |
3793 | Node->getValueType(0)) |
3794 | == TargetLowering::Legal) |
3795 | return true; |
3796 | break; |
3797 | case ISD::STRICT_FSUB: { |
3798 | if (TLI.getStrictFPOperationAction( |
3799 | ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal) |
3800 | return true; |
3801 | if (TLI.getStrictFPOperationAction( |
3802 | ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal) |
3803 | break; |
3804 | |
3805 | EVT VT = Node->getValueType(0); |
3806 | const SDNodeFlags Flags = Node->getFlags(); |
3807 | SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags); |
3808 | SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(), |
3809 | {Node->getOperand(0), Node->getOperand(1), Neg}, |
3810 | Flags); |
3811 | |
3812 | Results.push_back(Fadd); |
3813 | Results.push_back(Fadd.getValue(1)); |
3814 | break; |
3815 | } |
3816 | case ISD::STRICT_SINT_TO_FP: |
3817 | case ISD::STRICT_UINT_TO_FP: |
3818 | case ISD::STRICT_LRINT: |
3819 | case ISD::STRICT_LLRINT: |
3820 | case ISD::STRICT_LROUND: |
3821 | case ISD::STRICT_LLROUND: |
3822 | |
3823 | |
3824 | if (TLI.getStrictFPOperationAction(Node->getOpcode(), |
3825 | Node->getOperand(1).getValueType()) |
3826 | == TargetLowering::Legal) |
3827 | return true; |
3828 | break; |
3829 | } |
3830 | } |
3831 | |
3832 | |
3833 | if (Results.empty()) { |
3834 | LLVM_DEBUG(dbgs() << "Cannot expand node\n"); |
3835 | return false; |
3836 | } |
3837 | |
3838 | LLVM_DEBUG(dbgs() << "Successfully expanded node\n"); |
3839 | ReplaceNode(Node, Results.data()); |
3840 | return true; |
3841 | } |
3842 | |
3843 | void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) { |
3844 | LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n"); |
3845 | SmallVector<SDValue, 8> Results; |
3846 | SDLoc dl(Node); |
3847 | |
3848 | unsigned Opc = Node->getOpcode(); |
3849 | switch (Opc) { |
3850 | case ISD::ATOMIC_FENCE: { |
3851 | |
3852 | |
3853 | TargetLowering::ArgListTy Args; |
3854 | |
3855 | TargetLowering::CallLoweringInfo CLI(DAG); |
3856 | CLI.setDebugLoc(dl) |
3857 | .setChain(Node->getOperand(0)) |
3858 | .setLibCallee( |
3859 | CallingConv::C, Type::getVoidTy(*DAG.getContext()), |
3860 | DAG.getExternalSymbol("__sync_synchronize", |
3861 | TLI.getPointerTy(DAG.getDataLayout())), |
3862 | std::move(Args)); |
3863 | |
3864 | std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); |
3865 | |
3866 | Results.push_back(CallResult.second); |
3867 | break; |
3868 | } |
3869 | |
3870 | |
3871 | |
3872 | case ISD::ATOMIC_SWAP: |
3873 | case ISD::ATOMIC_LOAD_ADD: |
3874 | case ISD::ATOMIC_LOAD_SUB: |
3875 | case ISD::ATOMIC_LOAD_AND: |
3876 | case ISD::ATOMIC_LOAD_CLR: |
3877 | case ISD::ATOMIC_LOAD_OR: |
3878 | case ISD::ATOMIC_LOAD_XOR: |
3879 | case ISD::ATOMIC_LOAD_NAND: |
3880 | case ISD::ATOMIC_LOAD_MIN: |
3881 | case ISD::ATOMIC_LOAD_MAX: |
3882 | case ISD::ATOMIC_LOAD_UMIN: |
3883 | case ISD::ATOMIC_LOAD_UMAX: |
3884 | case ISD::ATOMIC_CMP_SWAP: { |
3885 | MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); |
3886 | AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering(); |
3887 | RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT); |
3888 | EVT RetVT = Node->getValueType(0); |
3889 | TargetLowering::MakeLibCallOptions CallOptions; |
3890 | SmallVector<SDValue, 4> Ops; |
3891 | if (TLI.getLibcallName(LC)) { |
3892 | |
3893 | Ops.append(Node->op_begin() + 2, Node->op_end()); |
3894 | Ops.push_back(Node->getOperand(1)); |
3895 | |
3896 | } else { |
3897 | LC = RTLIB::getSYNC(Opc, VT); |
3898 | assert(LC != RTLIB::UNKNOWN_LIBCALL && |
3899 | "Unexpected atomic op or value type!"); |
3900 | |
3901 | Ops.append(Node->op_begin() + 1, Node->op_end()); |
3902 | } |
3903 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, |
3904 | Ops, CallOptions, |
3905 | SDLoc(Node), |
3906 | Node->getOperand(0)); |
3907 | Results.push_back(Tmp.first); |
3908 | Results.push_back(Tmp.second); |
3909 | break; |
3910 | } |
3911 | case ISD::TRAP: { |
3912 | |
3913 | TargetLowering::ArgListTy Args; |
3914 | TargetLowering::CallLoweringInfo CLI(DAG); |
3915 | CLI.setDebugLoc(dl) |
3916 | .setChain(Node->getOperand(0)) |
3917 | .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), |
3918 | DAG.getExternalSymbol( |
3919 | "abort", TLI.getPointerTy(DAG.getDataLayout())), |
3920 | std::move(Args)); |
3921 | std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); |
3922 | |
3923 | Results.push_back(CallResult.second); |
3924 | break; |
3925 | } |
3926 | case ISD::FMINNUM: |
3927 | case ISD::STRICT_FMINNUM: |
3928 | ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64, |
3929 | RTLIB::FMIN_F80, RTLIB::FMIN_F128, |
3930 | RTLIB::FMIN_PPCF128, Results); |
3931 | break; |
3932 | case ISD::FMAXNUM: |
3933 | case ISD::STRICT_FMAXNUM: |
3934 | ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64, |
3935 | RTLIB::FMAX_F80, RTLIB::FMAX_F128, |
3936 | RTLIB::FMAX_PPCF128, Results); |
3937 | break; |
3938 | case ISD::FSQRT: |
3939 | case ISD::STRICT_FSQRT: |
3940 | ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, |
3941 | RTLIB::SQRT_F80, RTLIB::SQRT_F128, |
3942 | RTLIB::SQRT_PPCF128, Results); |
3943 | break; |
3944 | case ISD::FCBRT: |
3945 | ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64, |
3946 | RTLIB::CBRT_F80, RTLIB::CBRT_F128, |
3947 | RTLIB::CBRT_PPCF128, Results); |
3948 | break; |
3949 | case ISD::FSIN: |
3950 | case ISD::STRICT_FSIN: |
3951 | ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, |
3952 | RTLIB::SIN_F80, RTLIB::SIN_F128, |
3953 | RTLIB::SIN_PPCF128, Results); |
3954 | break; |
3955 | case ISD::FCOS: |
3956 | case ISD::STRICT_FCOS: |
3957 | ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, |
3958 | RTLIB::COS_F80, RTLIB::COS_F128, |
3959 | RTLIB::COS_PPCF128, Results); |
3960 | break; |
3961 | case ISD::FSINCOS: |
3962 | |
3963 | ExpandSinCosLibCall(Node, Results); |
3964 | break; |
3965 | case ISD::FLOG: |
3966 | case ISD::STRICT_FLOG: |
3967 | ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80, |
3968 | RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results); |
3969 | break; |
3970 | case ISD::FLOG2: |
3971 | case ISD::STRICT_FLOG2: |
3972 | ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80, |
3973 | RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results); |
3974 | break; |
3975 | case ISD::FLOG10: |
3976 | case ISD::STRICT_FLOG10: |
3977 | ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80, |
3978 | RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results); |
3979 | break; |
3980 | case ISD::FEXP: |
3981 | case ISD::STRICT_FEXP: |
3982 | ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80, |
3983 | RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results); |
3984 | break; |
3985 | case ISD::FEXP2: |
3986 | case ISD::STRICT_FEXP2: |
3987 | ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80, |
3988 | RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results); |
3989 | break; |
3990 | case ISD::FTRUNC: |
3991 | case ISD::STRICT_FTRUNC: |
3992 | ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, |
3993 | RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, |
3994 | RTLIB::TRUNC_PPCF128, Results); |
3995 | break; |
3996 | case ISD::FFLOOR: |
3997 | case ISD::STRICT_FFLOOR: |
3998 | ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, |
3999 | RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, |
4000 | RTLIB::FLOOR_PPCF128, Results); |
4001 | break; |
4002 | case ISD::FCEIL: |
4003 | case ISD::STRICT_FCEIL: |
4004 | ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, |
4005 | RTLIB::CEIL_F80, RTLIB::CEIL_F128, |
4006 | RTLIB::CEIL_PPCF128, Results); |
4007 | break; |
4008 | case ISD::FRINT: |
4009 | case ISD::STRICT_FRINT: |
4010 | ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, |
4011 | RTLIB::RINT_F80, RTLIB::RINT_F128, |
4012 | RTLIB::RINT_PPCF128, Results); |
4013 | break; |
4014 | case ISD::FNEARBYINT: |
4015 | case ISD::STRICT_FNEARBYINT: |
4016 | ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, |
4017 | RTLIB::NEARBYINT_F64, |
4018 | RTLIB::NEARBYINT_F80, |
4019 | RTLIB::NEARBYINT_F128, |
4020 | RTLIB::NEARBYINT_PPCF128, Results); |
4021 | break; |
4022 | case ISD::FROUND: |
4023 | case ISD::STRICT_FROUND: |
4024 | ExpandFPLibCall(Node, RTLIB::ROUND_F32, |
4025 | RTLIB::ROUND_F64, |
4026 | RTLIB::ROUND_F80, |
4027 | RTLIB::ROUND_F128, |
4028 | RTLIB::ROUND_PPCF128, Results); |
4029 | break; |
4030 | case ISD::FROUNDEVEN: |
4031 | case ISD::STRICT_FROUNDEVEN: |
4032 | ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32, |
4033 | RTLIB::ROUNDEVEN_F64, |
4034 | RTLIB::ROUNDEVEN_F80, |
4035 | RTLIB::ROUNDEVEN_F128, |
4036 | RTLIB::ROUNDEVEN_PPCF128, Results); |
4037 | break; |
4038 | case ISD::FPOWI: |
4039 | case ISD::STRICT_FPOWI: { |
4040 | RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0)); |
4041 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi."); |
4042 | if (!TLI.getLibcallName(LC)) { |
4043 | |
4044 | SDValue Exponent = DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), |
4045 | Node->getValueType(0), |
4046 | Node->getOperand(1)); |
4047 | Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node), |
4048 | Node->getValueType(0), Node->getOperand(0), |
4049 | Exponent)); |
4050 | break; |
4051 | } |
4052 | unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0; |
4053 | bool ExponentHasSizeOfInt = |
4054 | DAG.getLibInfo().getIntSize() == |
4055 | Node->getOperand(1 + Offset).getValueType().getSizeInBits(); |
4056 | if (!ExponentHasSizeOfInt) { |
4057 | |
4058 | |
4059 | DAG.getContext()->emitError("POWI exponent does not match sizeof(int)"); |
4060 | Results.push_back(DAG.getUNDEF(Node->getValueType(0))); |
4061 | break; |
4062 | } |
4063 | ExpandFPLibCall(Node, LC, Results); |
4064 | break; |
4065 | } |
4066 | case ISD::FPOW: |
4067 | case ISD::STRICT_FPOW: |
4068 | ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, |
4069 | RTLIB::POW_F128, RTLIB::POW_PPCF128, Results); |
4070 | break; |
4071 | case ISD::LROUND: |
4072 | case ISD::STRICT_LROUND: |
4073 | ExpandArgFPLibCall(Node, RTLIB::LROUND_F32, |
4074 | RTLIB::LROUND_F64, RTLIB::LROUND_F80, |
4075 | RTLIB::LROUND_F128, |
4076 | RTLIB::LROUND_PPCF128, Results); |
4077 | break; |
4078 | case ISD::LLROUND: |
4079 | case ISD::STRICT_LLROUND: |
4080 | ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32, |
4081 | RTLIB::LLROUND_F64, RTLIB::LLROUND_F80, |
4082 | RTLIB::LLROUND_F128, |
4083 | RTLIB::LLROUND_PPCF128, Results); |
4084 | break; |
4085 | case ISD::LRINT: |
4086 | case ISD::STRICT_LRINT: |
4087 | ExpandArgFPLibCall(Node, RTLIB::LRINT_F32, |
4088 | RTLIB::LRINT_F64, RTLIB::LRINT_F80, |
4089 | RTLIB::LRINT_F128, |
4090 | RTLIB::LRINT_PPCF128, Results); |
4091 | break; |
4092 | case ISD::LLRINT: |
4093 | case ISD::STRICT_LLRINT: |
4094 | ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32, |
4095 | RTLIB::LLRINT_F64, RTLIB::LLRINT_F80, |
4096 | RTLIB::LLRINT_F128, |
4097 | RTLIB::LLRINT_PPCF128, Results); |
4098 | break; |
4099 | case ISD::FDIV: |
4100 | case ISD::STRICT_FDIV: |
4101 | ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, |
4102 | RTLIB::DIV_F80, RTLIB::DIV_F128, |
4103 | RTLIB::DIV_PPCF128, Results); |
4104 | break; |
4105 | case ISD::FREM: |
4106 | case ISD::STRICT_FREM: |
4107 | ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, |
4108 | RTLIB::REM_F80, RTLIB::REM_F128, |
4109 | RTLIB::REM_PPCF128, Results); |
4110 | break; |
4111 | case ISD::FMA: |
4112 | case ISD::STRICT_FMA: |
4113 | ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, |
4114 | RTLIB::FMA_F80, RTLIB::FMA_F128, |
4115 | RTLIB::FMA_PPCF128, Results); |
4116 | break; |
4117 | case ISD::FADD: |
4118 | case ISD::STRICT_FADD: |
4119 | ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64, |
4120 | RTLIB::ADD_F80, RTLIB::ADD_F128, |
4121 | RTLIB::ADD_PPCF128, Results); |
4122 | break; |
4123 | case ISD::FMUL: |
4124 | case ISD::STRICT_FMUL: |
4125 | ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64, |
4126 | RTLIB::MUL_F80, RTLIB::MUL_F128, |
4127 | RTLIB::MUL_PPCF128, Results); |
4128 | break; |
4129 | case ISD::FP16_TO_FP: |
4130 | if (Node->getValueType(0) == MVT::f32) { |
4131 | Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); |
4132 | } |
4133 | break; |
4134 | case ISD::STRICT_FP16_TO_FP: { |
4135 | if (Node->getValueType(0) == MVT::f32) { |
4136 | TargetLowering::MakeLibCallOptions CallOptions; |
4137 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall( |
4138 | DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions, |
4139 | SDLoc(Node), Node->getOperand(0)); |
4140 | Results.push_back(Tmp.first); |
4141 | Results.push_back(Tmp.second); |
4142 | } |
4143 | break; |
4144 | } |
4145 | case ISD::FP_TO_FP16: { |
4146 | RTLIB::Libcall LC = |
4147 | RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); |
4148 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16"); |
4149 | Results.push_back(ExpandLibCall(LC, Node, false)); |
4150 | break; |
4151 | } |
4152 | case ISD::STRICT_SINT_TO_FP: |
4153 | case ISD::STRICT_UINT_TO_FP: |
4154 | case ISD::SINT_TO_FP: |
4155 | case ISD::UINT_TO_FP: { |
4156 | |
4157 | bool IsStrict = Node->isStrictFPOpcode(); |
4158 | bool Signed = Node->getOpcode() == ISD::SINT_TO_FP || |
4159 | Node->getOpcode() == ISD::STRICT_SINT_TO_FP; |
4160 | EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType(); |
4161 | EVT RVT = Node->getValueType(0); |
4162 | EVT NVT = EVT(); |
4163 | SDLoc dl(Node); |
4164 | |
4165 | |
4166 | |
4167 | |
4168 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4169 | for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE; |
4170 | t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; |
4171 | ++t) { |
4172 | NVT = (MVT::SimpleValueType)t; |
4173 | |
4174 | if (NVT.bitsGE(SVT)) |
4175 | LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT) |
4176 | : RTLIB::getUINTTOFP(NVT, RVT); |
4177 | } |
4178 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall"); |
4179 | |
4180 | SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); |
4181 | |
4182 | SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, |
4183 | NVT, Node->getOperand(IsStrict ? 1 : 0)); |
4184 | TargetLowering::MakeLibCallOptions CallOptions; |
4185 | CallOptions.setSExt(Signed); |
4186 | std::pair<SDValue, SDValue> Tmp = |
4187 | TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain); |
4188 | Results.push_back(Tmp.first); |
4189 | if (IsStrict) |
4190 | Results.push_back(Tmp.second); |
4191 | break; |
4192 | } |
4193 | case ISD::FP_TO_SINT: |
4194 | case ISD::FP_TO_UINT: |
4195 | case ISD::STRICT_FP_TO_SINT: |
4196 | case ISD::STRICT_FP_TO_UINT: { |
4197 | |
4198 | bool IsStrict = Node->isStrictFPOpcode(); |
4199 | bool Signed = Node->getOpcode() == ISD::FP_TO_SINT || |
4200 | Node->getOpcode() == ISD::STRICT_FP_TO_SINT; |
4201 | |
4202 | SDValue Op = Node->getOperand(IsStrict ? 1 : 0); |
4203 | EVT SVT = Op.getValueType(); |
4204 | EVT RVT = Node->getValueType(0); |
4205 | EVT NVT = EVT(); |
4206 | SDLoc dl(Node); |
4207 | |
4208 | |
4209 | |
4210 | |
4211 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4212 | for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE; |
4213 | IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; |
4214 | ++IntVT) { |
4215 | NVT = (MVT::SimpleValueType)IntVT; |
4216 | |
4217 | if (NVT.bitsGE(RVT)) |
4218 | LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT) |
4219 | : RTLIB::getFPTOUINT(SVT, NVT); |
4220 | } |
4221 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall"); |
4222 | |
4223 | SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); |
4224 | TargetLowering::MakeLibCallOptions CallOptions; |
4225 | std::pair<SDValue, SDValue> Tmp = |
4226 | TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain); |
4227 | |
4228 | |
4229 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first)); |
4230 | if (IsStrict) |
4231 | Results.push_back(Tmp.second); |
4232 | break; |
4233 | } |
4234 | |
4235 | case ISD::FP_ROUND: |
4236 | case ISD::STRICT_FP_ROUND: { |
4237 | |
4238 | |
4239 | |
4240 | |
4241 | |
4242 | bool IsStrict = Node->isStrictFPOpcode(); |
4243 | SDValue Op = Node->getOperand(IsStrict ? 1 : 0); |
4244 | SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); |
4245 | EVT VT = Node->getValueType(0); |
4246 | assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1)) |
4247 | ->isNullValue() && |
4248 | "Unable to expand as libcall if it is not normal rounding"); |
4249 | |
4250 | RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), VT); |
4251 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall"); |
4252 | |
4253 | TargetLowering::MakeLibCallOptions CallOptions; |
4254 | std::pair<SDValue, SDValue> Tmp = |
4255 | TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain); |
4256 | Results.push_back(Tmp.first); |
4257 | if (IsStrict) |
4258 | Results.push_back(Tmp.second); |
4259 | break; |
4260 | } |
4261 | case ISD::FP_EXTEND: { |
4262 | Results.push_back( |
4263 | ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(), |
4264 | Node->getValueType(0)), |
4265 | Node, false)); |
4266 | break; |
4267 | } |
4268 | case ISD::STRICT_FP_EXTEND: |
4269 | case ISD::STRICT_FP_TO_FP16: { |
4270 | RTLIB::Libcall LC = |
4271 | Node->getOpcode() == ISD::STRICT_FP_TO_FP16 |
4272 | ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16) |
4273 | : RTLIB::getFPEXT(Node->getOperand(1).getValueType(), |
4274 | Node->getValueType(0)); |
4275 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall"); |
4276 | |
4277 | TargetLowering::MakeLibCallOptions CallOptions; |
4278 | std::pair<SDValue, SDValue> Tmp = |
4279 | TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1), |
4280 | CallOptions, SDLoc(Node), Node->getOperand(0)); |
4281 | Results.push_back(Tmp.first); |
4282 | Results.push_back(Tmp.second); |
4283 | break; |
4284 | } |
4285 | case ISD::FSUB: |
4286 | case ISD::STRICT_FSUB: |
4287 | ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64, |
4288 | RTLIB::SUB_F80, RTLIB::SUB_F128, |
4289 | RTLIB::SUB_PPCF128, Results); |
4290 | break; |
4291 | case ISD::SREM: |
4292 | Results.push_back(ExpandIntLibCall(Node, true, |
4293 | RTLIB::SREM_I8, |
4294 | RTLIB::SREM_I16, RTLIB::SREM_I32, |
4295 | RTLIB::SREM_I64, RTLIB::SREM_I128)); |
4296 | break; |
4297 | case ISD::UREM: |
4298 | Results.push_back(ExpandIntLibCall(Node, false, |
4299 | RTLIB::UREM_I8, |
4300 | RTLIB::UREM_I16, RTLIB::UREM_I32, |
4301 | RTLIB::UREM_I64, RTLIB::UREM_I128)); |
4302 | break; |
4303 | case ISD::SDIV: |
4304 | Results.push_back(ExpandIntLibCall(Node, true, |
4305 | RTLIB::SDIV_I8, |
4306 | RTLIB::SDIV_I16, RTLIB::SDIV_I32, |
4307 | RTLIB::SDIV_I64, RTLIB::SDIV_I128)); |
4308 | break; |
4309 | case ISD::UDIV: |
4310 | Results.push_back(ExpandIntLibCall(Node, false, |
4311 | RTLIB::UDIV_I8, |
4312 | RTLIB::UDIV_I16, RTLIB::UDIV_I32, |
4313 | RTLIB::UDIV_I64, RTLIB::UDIV_I128)); |
4314 | break; |
4315 | case ISD::SDIVREM: |
4316 | case ISD::UDIVREM: |
4317 | |
4318 | ExpandDivRemLibCall(Node, Results); |
4319 | break; |
4320 | case ISD::MUL: |
4321 | Results.push_back(ExpandIntLibCall(Node, false, |
4322 | RTLIB::MUL_I8, |
4323 | RTLIB::MUL_I16, RTLIB::MUL_I32, |
4324 | RTLIB::MUL_I64, RTLIB::MUL_I128)); |
4325 | break; |
4326 | case ISD::CTLZ_ZERO_UNDEF: |
4327 | switch (Node->getSimpleValueType(0).SimpleTy) { |
4328 | default: |
4329 | llvm_unreachable("LibCall explicitly requested, but not available"); |
4330 | case MVT::i32: |
4331 | Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false)); |
4332 | break; |
4333 | case MVT::i64: |
4334 | Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false)); |
4335 | break; |
4336 | case MVT::i128: |
4337 | Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false)); |
4338 | break; |
4339 | } |
4340 | break; |
4341 | } |
4342 | |
4343 | |
4344 | if (!Results.empty()) { |
4345 | LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n"); |
4346 | ReplaceNode(Node, Results.data()); |
4347 | } else |
4348 | LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n"); |
4349 | } |
4350 | |
4351 | |
4352 | |
4353 | static MVT getPromotedVectorElementType(const TargetLowering &TLI, |
4354 | MVT EltVT, MVT NewEltVT) { |
4355 | unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits(); |
4356 | MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt); |
4357 | assert(TLI.isTypeLegal(MidVT) && "unexpected"); |
4358 | return MidVT; |
4359 | } |
4360 | |
4361 | void SelectionDAGLegalize::PromoteNode(SDNode *Node) { |
4362 | LLVM_DEBUG(dbgs() << "Trying to promote node\n"); |
4363 | SmallVector<SDValue, 8> Results; |
4364 | MVT OVT = Node->getSimpleValueType(0); |
4365 | if (Node->getOpcode() == ISD::UINT_TO_FP || |
4366 | Node->getOpcode() == ISD::SINT_TO_FP || |
4367 | Node->getOpcode() == ISD::SETCC || |
4368 | Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || |
4369 | Node->getOpcode() == ISD::INSERT_VECTOR_ELT) { |
4370 | OVT = Node->getOperand(0).getSimpleValueType(); |
4371 | } |
4372 | if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP || |
4373 | Node->getOpcode() == ISD::STRICT_SINT_TO_FP || |
4374 | Node->getOpcode() == ISD::STRICT_FSETCC || |
4375 | Node->getOpcode() == ISD::STRICT_FSETCCS) |
4376 | OVT = Node->getOperand(1).getSimpleValueType(); |
4377 | if (Node->getOpcode() == ISD::BR_CC || |
4378 | Node->getOpcode() == ISD::SELECT_CC) |
4379 | OVT = Node->getOperand(2).getSimpleValueType(); |
4380 | MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); |
4381 | SDLoc dl(Node); |
4382 | SDValue Tmp1, Tmp2, Tmp3, Tmp4; |
4383 | switch (Node->getOpcode()) { |
4384 | case ISD::CTTZ: |
4385 | case ISD::CTTZ_ZERO_UNDEF: |
4386 | case ISD::CTLZ: |
4387 | case ISD::CTLZ_ZERO_UNDEF: |
4388 | case ISD::CTPOP: |
4389 | |
4390 | if (Node->getOpcode() == ISD::CTTZ || |
4391 | Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF) |
4392 | Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); |
4393 | else |
4394 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); |
4395 | |
4396 | if (Node->getOpcode() == ISD::CTTZ) { |
4397 | |
4398 | |
4399 | |
4400 | auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(), |
4401 | OVT.getSizeInBits()); |
4402 | Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1, |
4403 | DAG.getConstant(TopBit, dl, NVT)); |
4404 | } |
4405 | |
4406 | |
4407 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); |
4408 | if (Node->getOpcode() == ISD::CTLZ || |
4409 | Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { |
4410 | |
4411 | Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, |
4412 | DAG.getConstant(NVT.getSizeInBits() - |
4413 | OVT.getSizeInBits(), dl, NVT)); |
4414 | } |
4415 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); |
4416 | break; |
4417 | case ISD::BITREVERSE: |
4418 | case ISD::BSWAP: { |
4419 | unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); |
4420 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); |
4421 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); |
4422 | Tmp1 = DAG.getNode( |
4423 | ISD::SRL, dl, NVT, Tmp1, |
4424 | DAG.getConstant(DiffBits, dl, |
4425 | TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); |
4426 | |
4427 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); |
4428 | break; |
4429 | } |
4430 | case ISD::FP_TO_UINT: |
4431 | case ISD::STRICT_FP_TO_UINT: |
4432 | case ISD::FP_TO_SINT: |
4433 | case ISD::STRICT_FP_TO_SINT: |
4434 | PromoteLegalFP_TO_INT(Node, dl, Results); |
4435 | break; |
4436 | case ISD::FP_TO_UINT_SAT: |
4437 | case ISD::FP_TO_SINT_SAT: |
4438 | Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl)); |
4439 | break; |
4440 | case ISD::UINT_TO_FP: |
4441 | case ISD::STRICT_UINT_TO_FP: |
4442 | case ISD::SINT_TO_FP: |
4443 | case ISD::STRICT_SINT_TO_FP: |
4444 | PromoteLegalINT_TO_FP(Node, dl, Results); |
4445 | break; |
4446 | case ISD::VAARG: { |
4447 | SDValue Chain = Node->getOperand(0); |
4448 | SDValue Ptr = Node->getOperand(1); |
4449 | |
4450 | unsigned TruncOp; |
4451 | if (OVT.isVector()) { |
4452 | TruncOp = ISD::BITCAST; |
4453 | } else { |
4454 | assert(OVT.isInteger() |
4455 | && "VAARG promotion is supported only for vectors or integer types"); |
4456 | TruncOp = ISD::TRUNCATE; |
4457 | } |
4458 | |
4459 | |
4460 | Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), |
4461 | Node->getConstantOperandVal(3)); |
4462 | Chain = Tmp1.getValue(1); |
4463 | |
4464 | Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1); |
4465 | |
4466 | |
4467 | |
4468 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); |
4469 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); |
4470 | if (UpdatedNodes) { |
4471 | UpdatedNodes->insert(Tmp2.getNode()); |
4472 | UpdatedNodes->insert(Chain.getNode()); |
4473 | } |
4474 | ReplacedNode(Node); |
4475 | break; |
4476 | } |
4477 | case ISD::MUL: |
4478 | case ISD::SDIV: |
4479 | case ISD::SREM: |
4480 | case ISD::UDIV: |
4481 | case ISD::UREM: |
4482 | case ISD::AND: |
4483 | case ISD::OR: |
4484 | case ISD::XOR: { |
4485 | unsigned ExtOp, TruncOp; |
4486 | if (OVT.isVector()) { |
4487 | ExtOp = ISD::BITCAST; |
4488 | TruncOp = ISD::BITCAST; |
4489 | } else { |
4490 | assert(OVT.isInteger() && "Cannot promote logic operation"); |
4491 | |
4492 | switch (Node->getOpcode()) { |
4493 | default: |
4494 | ExtOp = ISD::ANY_EXTEND; |
4495 | break; |
4496 | case ISD::SDIV: |
4497 | case ISD::SREM: |
4498 | ExtOp = ISD::SIGN_EXTEND; |
4499 | break; |
4500 | case ISD::UDIV: |
4501 | case ISD::UREM: |
4502 | ExtOp = ISD::ZERO_EXTEND; |
4503 | break; |
4504 | } |
4505 | TruncOp = ISD::TRUNCATE; |
4506 | } |
4507 | |
4508 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); |
4509 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4510 | |
4511 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); |
4512 | Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); |
4513 | break; |
4514 | } |
4515 | case ISD::UMUL_LOHI: |
4516 | case ISD::SMUL_LOHI: { |
4517 | |
4518 | unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND |
4519 | : ISD::SIGN_EXTEND; |
4520 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); |
4521 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4522 | Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2); |
4523 | |
4524 | auto &DL = DAG.getDataLayout(); |
4525 | unsigned OriginalSize = OVT.getScalarSizeInBits(); |
4526 | Tmp2 = DAG.getNode( |
4527 | ISD::SRL, dl, NVT, Tmp1, |
4528 | DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT))); |
4529 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); |
4530 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); |
4531 | break; |
4532 | } |
4533 | case ISD::SELECT: { |
4534 | unsigned ExtOp, TruncOp; |
4535 | if (Node->getValueType(0).isVector() || |
4536 | Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) { |
4537 | ExtOp = ISD::BITCAST; |
4538 | TruncOp = ISD::BITCAST; |
4539 | } else if (Node->getValueType(0).isInteger()) { |
4540 | ExtOp = ISD::ANY_EXTEND; |
4541 | TruncOp = ISD::TRUNCATE; |
4542 | } else { |
4543 | ExtOp = ISD::FP_EXTEND; |
4544 | TruncOp = ISD::FP_ROUND; |
4545 | } |
4546 | Tmp1 = Node->getOperand(0); |
4547 | |
4548 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4549 | Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); |
4550 | |
4551 | Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3); |
4552 | Tmp1->setFlags(Node->getFlags()); |
4553 | if (TruncOp != ISD::FP_ROUND) |
4554 | Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); |
4555 | else |
4556 | Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, |
4557 | DAG.getIntPtrConstant(0, dl)); |
4558 | Results.push_back(Tmp1); |
4559 | break; |
4560 | } |
4561 | case ISD::VECTOR_SHUFFLE: { |
4562 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); |
4563 | |
4564 | |
4565 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); |
4566 | Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); |
4567 | |
4568 | |
4569 | Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask); |
4570 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); |
4571 | Results.push_back(Tmp1); |
4572 | break; |
4573 | } |
4574 | case ISD::VECTOR_SPLICE: { |
4575 | Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); |
4576 | Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1)); |
4577 | Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2, |
4578 | Node->getOperand(2)); |
4579 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3)); |
4580 | break; |
4581 | } |
4582 | case ISD::SELECT_CC: { |
4583 | SDValue Cond = Node->getOperand(4); |
4584 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get(); |
4585 | |
4586 | MVT CVT = Node->getSimpleValueType(0); |
4587 | assert(CVT == OVT && "not handled"); |
4588 | |
4589 | unsigned ExtOp = ISD::FP_EXTEND; |
4590 | if (NVT.isInteger()) { |
4591 | ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
4592 | } |
4593 | |
4594 | |
4595 | if (TLI.isCondCodeLegal(CCCode, CVT)) { |
4596 | Tmp1 = Node->getOperand(0); |
4597 | Tmp2 = Node->getOperand(1); |
4598 | } else { |
4599 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); |
4600 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4601 | } |
4602 | |
4603 | Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); |
4604 | Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); |
4605 | |
4606 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond}, |
4607 | Node->getFlags()); |
4608 | |
4609 | |
4610 | if (ExtOp != ISD::FP_EXTEND) |
4611 | Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1); |
4612 | else |
4613 | Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1, |
4614 | DAG.getIntPtrConstant(0, dl)); |
4615 | |
4616 | Results.push_back(Tmp1); |
4617 | break; |
4618 | } |
4619 | case ISD::SETCC: |
4620 | case ISD::STRICT_FSETCC: |
4621 | case ISD::STRICT_FSETCCS: { |
4622 | unsigned ExtOp = ISD::FP_EXTEND; |
4623 | if (NVT.isInteger()) { |
4624 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); |
4625 | ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
4626 | } |
4627 | if (Node->isStrictFPOpcode()) { |
4628 | SDValue InChain = Node->getOperand(0); |
4629 | std::tie(Tmp1, std::ignore) = |
4630 | DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT); |
4631 | std::tie(Tmp2, std::ignore) = |
4632 | DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT); |
4633 | SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(1), Tmp2.getValue(1)}; |
4634 | SDValue OutChain = DAG.getTokenFactor(dl, TmpChains); |
4635 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); |
4636 | Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs, |
4637 | {OutChain, Tmp1, Tmp2, Node->getOperand(3)}, |
4638 | Node->getFlags())); |
4639 | Results.push_back(Results.back().getValue(1)); |
4640 | break; |
4641 | } |
4642 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); |
4643 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4644 | Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, |
4645 | Tmp2, Node->getOperand(2), Node->getFlags())); |
4646 | break; |
4647 | } |
4648 | case ISD::BR_CC: { |
4649 | unsigned ExtOp = ISD::FP_EXTEND; |
4650 | if (NVT.isInteger()) { |
4651 | ISD::CondCode CCCode = |
4652 | cast<CondCodeSDNode>(Node->getOperand(1))->get(); |
4653 | ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
4654 | } |
4655 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); |
4656 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); |
4657 | Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), |
4658 | Node->getOperand(0), Node->getOperand(1), |
4659 | Tmp1, Tmp2, Node->getOperand(4))); |
4660 | break; |
4661 | } |
4662 | case ISD::FADD: |
4663 | case ISD::FSUB: |
4664 | case ISD::FMUL: |
4665 | case ISD::FDIV: |
4666 | case ISD::FREM: |
4667 | case ISD::FMINNUM: |
4668 | case ISD::FMAXNUM: |
4669 | case ISD::FPOW: |
4670 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4671 | Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); |
4672 | Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, |
4673 | Node->getFlags()); |
4674 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4675 | Tmp3, DAG.getIntPtrConstant(0, dl))); |
4676 | break; |
4677 | case ISD::STRICT_FREM: |
4678 | case ISD::STRICT_FPOW: |
4679 | Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, |
4680 | {Node->getOperand(0), Node->getOperand(1)}); |
4681 | Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, |
4682 | {Node->getOperand(0), Node->getOperand(2)}); |
4683 | Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1), |
4684 | Tmp2.getValue(1)); |
4685 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, |
4686 | {Tmp3, Tmp1, Tmp2}); |
4687 | Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, |
4688 | {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)}); |
4689 | Results.push_back(Tmp1); |
4690 | Results.push_back(Tmp1.getValue(1)); |
4691 | break; |
4692 | case ISD::FMA: |
4693 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4694 | Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); |
4695 | Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); |
4696 | Results.push_back( |
4697 | DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4698 | DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3), |
4699 | DAG.getIntPtrConstant(0, dl))); |
4700 | break; |
4701 | case ISD::FCOPYSIGN: |
4702 | case ISD::FPOWI: { |
4703 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4704 | Tmp2 = Node->getOperand(1); |
4705 | Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); |
4706 | |
4707 | |
4708 | |
4709 | |
4710 | |
4711 | |
4712 | const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); |
4713 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4714 | Tmp3, DAG.getIntPtrConstant(isTrunc, dl))); |
4715 | break; |
4716 | } |
4717 | case ISD::FFLOOR: |
4718 | case ISD::FCEIL: |
4719 | case ISD::FRINT: |
4720 | case ISD::FNEARBYINT: |
4721 | case ISD::FROUND: |
4722 | case ISD::FROUNDEVEN: |
4723 | case ISD::FTRUNC: |
4724 | case ISD::FNEG: |
4725 | case ISD::FSQRT: |
4726 | case ISD::FSIN: |
4727 | case ISD::FCOS: |
4728 | case ISD::FLOG: |
4729 | case ISD::FLOG2: |
4730 | case ISD::FLOG10: |
4731 | case ISD::FABS: |
4732 | case ISD::FEXP: |
4733 | case ISD::FEXP2: |
4734 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4735 | Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); |
4736 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4737 | Tmp2, DAG.getIntPtrConstant(0, dl))); |
4738 | break; |
4739 | case ISD::STRICT_FFLOOR: |
4740 | case ISD::STRICT_FCEIL: |
4741 | case ISD::STRICT_FSIN: |
4742 | case ISD::STRICT_FCOS: |
4743 | case ISD::STRICT_FLOG: |
4744 | case ISD::STRICT_FLOG10: |
4745 | case ISD::STRICT_FEXP: |
4746 | Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, |
4747 | {Node->getOperand(0), Node->getOperand(1)}); |
4748 | Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, |
4749 | {Tmp1.getValue(1), Tmp1}); |
4750 | Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, |
4751 | {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)}); |
4752 | Results.push_back(Tmp3); |
4753 | Results.push_back(Tmp3.getValue(1)); |
4754 | break; |
4755 | case ISD::BUILD_VECTOR: { |
4756 | MVT EltVT = OVT.getVectorElementType(); |
4757 | MVT NewEltVT = NVT.getVectorElementType(); |
4758 | |
4759 | |
4760 | |
4761 | |
4762 | |
4763 | |
4764 | |
4765 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
4766 | "Invalid promote type for build_vector"); |
4767 | assert(NewEltVT.bitsLT(EltVT) && "not handled"); |
4768 | |
4769 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4770 | |
4771 | SmallVector<SDValue, 8> NewOps; |
4772 | for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) { |
4773 | SDValue Op = Node->getOperand(I); |
4774 | NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op)); |
4775 | } |
4776 | |
4777 | SDLoc SL(Node); |
4778 | SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps); |
4779 | SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); |
4780 | Results.push_back(CvtVec); |
4781 | break; |
4782 | } |
4783 | case ISD::EXTRACT_VECTOR_ELT: { |
4784 | MVT EltVT = OVT.getVectorElementType(); |
4785 | MVT NewEltVT = NVT.getVectorElementType(); |
4786 | |
4787 | |
4788 | |
4789 | |
4790 | |
4791 | |
4792 | |
4793 | |
4794 | |
4795 | |
4796 | |
4797 | |
4798 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
4799 | "Invalid promote type for extract_vector_elt"); |
4800 | assert(NewEltVT.bitsLT(EltVT) && "not handled"); |
4801 | |
4802 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4803 | unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); |
4804 | |
4805 | SDValue Idx = Node->getOperand(1); |
4806 | EVT IdxVT = Idx.getValueType(); |
4807 | SDLoc SL(Node); |
4808 | SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT); |
4809 | SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); |
4810 | |
4811 | SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); |
4812 | |
4813 | SmallVector<SDValue, 8> NewOps; |
4814 | for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { |
4815 | SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); |
4816 | SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); |
4817 | |
4818 | SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, |
4819 | CastVec, TmpIdx); |
4820 | NewOps.push_back(Elt); |
4821 | } |
4822 | |
4823 | SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps); |
4824 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec)); |
4825 | break; |
4826 | } |
4827 | case ISD::INSERT_VECTOR_ELT: { |
4828 | MVT EltVT = OVT.getVectorElementType(); |
4829 | MVT NewEltVT = NVT.getVectorElementType(); |
4830 | |
4831 | |
4832 | |
4833 | |
4834 | |
4835 | |
4836 | |
4837 | |
4838 | |
4839 | |
4840 | |
4841 | |
4842 | |
4843 | |
4844 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
4845 | "Invalid promote type for insert_vector_elt"); |
4846 | assert(NewEltVT.bitsLT(EltVT) && "not handled"); |
4847 | |
4848 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4849 | unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); |
4850 | |
4851 | SDValue Val = Node->getOperand(1); |
4852 | SDValue Idx = Node->getOperand(2); |
4853 | EVT IdxVT = Idx.getValueType(); |
4854 | SDLoc SL(Node); |
4855 | |
4856 | SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT); |
4857 | SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); |
4858 | |
4859 | SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); |
4860 | SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); |
4861 | |
4862 | SDValue NewVec = CastVec; |
4863 | for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { |
4864 | SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); |
4865 | SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); |
4866 | |
4867 | SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, |
4868 | CastVal, IdxOffset); |
4869 | |
4870 | NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT, |
4871 | NewVec, Elt, InEltIdx); |
4872 | } |
4873 | |
4874 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec)); |
4875 | break; |
4876 | } |
4877 | case ISD::SCALAR_TO_VECTOR: { |
4878 | MVT EltVT = OVT.getVectorElementType(); |
4879 | MVT NewEltVT = NVT.getVectorElementType(); |
4880 | |
4881 | |
4882 | |
4883 | |
4884 | |
4885 | |
4886 | |
4887 | |
4888 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4889 | SDValue Val = Node->getOperand(0); |
4890 | SDLoc SL(Node); |
4891 | |
4892 | SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); |
4893 | SDValue Undef = DAG.getUNDEF(MidVT); |
4894 | |
4895 | SmallVector<SDValue, 8> NewElts; |
4896 | NewElts.push_back(CastVal); |
4897 | for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I) |
4898 | NewElts.push_back(Undef); |
4899 | |
4900 | SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts); |
4901 | SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); |
4902 | Results.push_back(CvtVec); |
4903 | break; |
4904 | } |
4905 | case ISD::ATOMIC_SWAP: { |
4906 | AtomicSDNode *AM = cast<AtomicSDNode>(Node); |
4907 | SDLoc SL(Node); |
4908 | SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal()); |
4909 | assert(NVT.getSizeInBits() == OVT.getSizeInBits() && |
4910 | "unexpected promotion type"); |
4911 | assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && |
4912 | "unexpected atomic_swap with illegal type"); |
4913 | |
4914 | SDValue NewAtomic |
4915 | = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT, |
4916 | DAG.getVTList(NVT, MVT::Other), |
4917 | { AM->getChain(), AM->getBasePtr(), CastVal }, |
4918 | AM->getMemOperand()); |
4919 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic)); |
4920 | Results.push_back(NewAtomic.getValue(1)); |
4921 | break; |
4922 | } |
4923 | } |
4924 | |
4925 | |
4926 | if (!Results.empty()) { |
4927 | LLVM_DEBUG(dbgs() << "Successfully promoted node\n"); |
4928 | ReplaceNode(Node, Results.data()); |
4929 | } else |
4930 | LLVM_DEBUG(dbgs() << "Could not promote node\n"); |
4931 | } |
4932 | |
4933 | |
4934 | void SelectionDAG::Legalize() { |
4935 | AssignTopologicalOrder(); |
4936 | |
4937 | SmallPtrSet<SDNode *, 16> LegalizedNodes; |
4938 | |
4939 | |
4940 | |
4941 | |
4942 | DAGNodeDeletedListener DeleteListener( |
4943 | *this, |
4944 | [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); }); |
4945 | |
4946 | SelectionDAGLegalize Legalizer(*this, LegalizedNodes); |
4947 | |
4948 | |
4949 | |
4950 | |
4951 | |
4952 | while (true) { |
4953 | bool AnyLegalized = false; |
4954 | for (auto NI = allnodes_end(); NI != allnodes_begin();) { |
4955 | --NI; |
4956 | |
4957 | SDNode *N = &*NI; |
4958 | if (N->use_empty() && N != getRoot().getNode()) { |
4959 | ++NI; |
4960 | DeleteNode(N); |
4961 | continue; |
4962 | } |
4963 | |
4964 | if (LegalizedNodes.insert(N).second) { |
4965 | AnyLegalized = true; |
4966 | Legalizer.LegalizeOp(N); |
4967 | |
4968 | if (N->use_empty() && N != getRoot().getNode()) { |
4969 | ++NI; |
4970 | DeleteNode(N); |
4971 | } |
4972 | } |
4973 | } |
4974 | if (!AnyLegalized) |
4975 | break; |
4976 | |
4977 | } |
4978 | |
4979 | |
4980 | RemoveDeadNodes(); |
4981 | } |
4982 | |
4983 | bool SelectionDAG::LegalizeOp(SDNode *N, |
4984 | SmallSetVector<SDNode *, 16> &UpdatedNodes) { |
4985 | SmallPtrSet<SDNode *, 16> LegalizedNodes; |
4986 | SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes); |
4987 | |
4988 | |
4989 | |
4990 | LegalizedNodes.insert(N); |
4991 | Legalizer.LegalizeOp(N); |
4992 | |
4993 | return LegalizedNodes.count(N); |
4994 | } |