Bug Summary

File:lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
Warning:line 714, column 29
The result of the '<<' expression is undefined

Annotated Source Code

1//===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the SelectionDAG::Legalize method.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ADT/SetVector.h"
15#include "llvm/ADT/SmallPtrSet.h"
16#include "llvm/ADT/SmallSet.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Triple.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineJumpTableInfo.h"
21#include "llvm/CodeGen/SelectionDAG.h"
22#include "llvm/CodeGen/SelectionDAGNodes.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DataLayout.h"
26#include "llvm/IR/DebugInfo.h"
27#include "llvm/IR/DerivedTypes.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/LLVMContext.h"
30#include "llvm/Support/Debug.h"
31#include "llvm/Support/ErrorHandling.h"
32#include "llvm/Support/MathExtras.h"
33#include "llvm/Support/raw_ostream.h"
34#include "llvm/Target/TargetFrameLowering.h"
35#include "llvm/Target/TargetLowering.h"
36#include "llvm/Target/TargetMachine.h"
37#include "llvm/Target/TargetSubtargetInfo.h"
38using namespace llvm;
39
40#define DEBUG_TYPE"legalizedag" "legalizedag"
41
42namespace {
43
44struct FloatSignAsInt;
45
46//===----------------------------------------------------------------------===//
47/// This takes an arbitrary SelectionDAG as input and
48/// hacks on it until the target machine can handle it. This involves
49/// eliminating value sizes the machine cannot handle (promoting small sizes to
50/// large sizes or splitting up large values into small values) as well as
51/// eliminating operations the machine cannot handle.
52///
53/// This code also does a small amount of optimization and recognition of idioms
54/// as part of its processing. For example, if a target does not support a
55/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
56/// will attempt merge setcc and brc instructions into brcc's.
57///
58class SelectionDAGLegalize {
59 const TargetMachine &TM;
60 const TargetLowering &TLI;
61 SelectionDAG &DAG;
62
63 /// \brief The set of nodes which have already been legalized. We hold a
64 /// reference to it in order to update as necessary on node deletion.
65 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
66
67 /// \brief A set of all the nodes updated during legalization.
68 SmallSetVector<SDNode *, 16> *UpdatedNodes;
69
70 EVT getSetCCResultType(EVT VT) const {
71 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
72 }
73
74 // Libcall insertion helpers.
75
76public:
77 SelectionDAGLegalize(SelectionDAG &DAG,
78 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
79 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
80 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
81 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
82
83 /// \brief Legalizes the given operation.
84 void LegalizeOp(SDNode *Node);
85
86private:
87 SDValue OptimizeFloatStore(StoreSDNode *ST);
88
89 void LegalizeLoadOps(SDNode *Node);
90 void LegalizeStoreOps(SDNode *Node);
91
92 /// Some targets cannot handle a variable
93 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
94 /// is necessary to spill the vector being inserted into to memory, perform
95 /// the insert there, and then read the result back.
96 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
97 const SDLoc &dl);
98 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
99 const SDLoc &dl);
100
101 /// Return a vector shuffle operation which
102 /// performs the same shuffe in terms of order or result bytes, but on a type
103 /// whose vector element type is narrower than the original shuffle type.
104 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
105 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
106 SDValue N1, SDValue N2,
107 ArrayRef<int> Mask) const;
108
109 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
110 bool &NeedInvert, const SDLoc &dl);
111
112 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
113 SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
114 unsigned NumOps, bool isSigned, const SDLoc &dl);
115
116 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
117 SDNode *Node, bool isSigned);
118 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
119 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
120 RTLIB::Libcall Call_F128,
121 RTLIB::Libcall Call_PPCF128);
122 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
123 RTLIB::Libcall Call_I8,
124 RTLIB::Libcall Call_I16,
125 RTLIB::Libcall Call_I32,
126 RTLIB::Libcall Call_I64,
127 RTLIB::Libcall Call_I128);
128 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
129 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
130
131 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
132 const SDLoc &dl);
133 SDValue ExpandBUILD_VECTOR(SDNode *Node);
134 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
135 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
136 SmallVectorImpl<SDValue> &Results);
137 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
138 SDValue Value) const;
139 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
140 SDValue NewIntValue) const;
141 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
142 SDValue ExpandFABS(SDNode *Node) const;
143 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT,
144 const SDLoc &dl);
145 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
146 const SDLoc &dl);
147 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
148 const SDLoc &dl);
149
150 SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
151 SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
152 SDValue ExpandBitCount(unsigned Opc, SDValue Op, const SDLoc &dl);
153
154 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
155 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
156 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
157
158 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
159 SDValue ExpandConstant(ConstantSDNode *CP);
160
161 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
162 bool ExpandNode(SDNode *Node);
163 void ConvertNodeToLibcall(SDNode *Node);
164 void PromoteNode(SDNode *Node);
165
166public:
167 // Node replacement helpers
168 void ReplacedNode(SDNode *N) {
169 LegalizedNodes.erase(N);
170 if (UpdatedNodes)
171 UpdatedNodes->insert(N);
172 }
173 void ReplaceNode(SDNode *Old, SDNode *New) {
174 DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
175 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
176
177 assert(Old->getNumValues() == New->getNumValues() &&((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 179, __PRETTY_FUNCTION__))
178 "Replacing one node with another that produces a different number "((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 179, __PRETTY_FUNCTION__))
179 "of values!")((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 179, __PRETTY_FUNCTION__))
;
180 DAG.ReplaceAllUsesWith(Old, New);
181 if (UpdatedNodes)
182 UpdatedNodes->insert(New);
183 ReplacedNode(Old);
184 }
185 void ReplaceNode(SDValue Old, SDValue New) {
186 DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
187 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
188
189 DAG.ReplaceAllUsesWith(Old, New);
190 if (UpdatedNodes)
191 UpdatedNodes->insert(New.getNode());
192 ReplacedNode(Old.getNode());
193 }
194 void ReplaceNode(SDNode *Old, const SDValue *New) {
195 DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); } } while (false)
;
196
197 DAG.ReplaceAllUsesWith(Old, New);
198 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
199 DEBUG(dbgs() << (i == 0 ? " with: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
200 : " and: ");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
201 New[i]->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
;
202 if (UpdatedNodes)
203 UpdatedNodes->insert(New[i].getNode());
204 }
205 ReplacedNode(Old);
206 }
207};
208}
209
210/// Return a vector shuffle operation which
211/// performs the same shuffe in terms of order or result bytes, but on a type
212/// whose vector element type is narrower than the original shuffle type.
213/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
214SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
215 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
216 ArrayRef<int> Mask) const {
217 unsigned NumMaskElts = VT.getVectorNumElements();
218 unsigned NumDestElts = NVT.getVectorNumElements();
219 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
220
221 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!")((NumEltsGrowth && "Cannot promote to vector type with fewer elts!"
) ? static_cast<void> (0) : __assert_fail ("NumEltsGrowth && \"Cannot promote to vector type with fewer elts!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 221, __PRETTY_FUNCTION__))
;
222
223 if (NumEltsGrowth == 1)
224 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
225
226 SmallVector<int, 8> NewMask;
227 for (unsigned i = 0; i != NumMaskElts; ++i) {
228 int Idx = Mask[i];
229 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
230 if (Idx < 0)
231 NewMask.push_back(-1);
232 else
233 NewMask.push_back(Idx * NumEltsGrowth + j);
234 }
235 }
236 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?")((NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"
) ? static_cast<void> (0) : __assert_fail ("NewMask.size() == NumDestElts && \"Non-integer NumEltsGrowth?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 236, __PRETTY_FUNCTION__))
;
237 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?")((TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"
) ? static_cast<void> (0) : __assert_fail ("TLI.isShuffleMaskLegal(NewMask, NVT) && \"Shuffle not legal?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 237, __PRETTY_FUNCTION__))
;
238 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
239}
240
241/// Expands the ConstantFP node to an integer constant or
242/// a load from the constant pool.
243SDValue
244SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
245 bool Extend = false;
246 SDLoc dl(CFP);
247
248 // If a FP immediate is precise when represented as a float and if the
249 // target can do an extending load from float to double, we put it into
250 // the constant pool as a float, even if it's is statically typed as a
251 // double. This shrinks FP constants and canonicalizes them for targets where
252 // an FP extending load is the same cost as a normal load (such as on the x87
253 // fp stack or PPC FP unit).
254 EVT VT = CFP->getValueType(0);
255 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
256 if (!UseCP) {
257 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion")(((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::f64 || VT == MVT::f32) && \"Invalid type expansion\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 257, __PRETTY_FUNCTION__))
;
258 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
259 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
260 }
261
262 APFloat APF = CFP->getValueAPF();
263 EVT OrigVT = VT;
264 EVT SVT = VT;
265
266 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
267 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
268 if (!APF.isSignaling()) {
269 while (SVT != MVT::f32 && SVT != MVT::f16) {
270 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
271 if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
272 // Only do this if the target has a native EXTLOAD instruction from
273 // smaller type.
274 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
275 TLI.ShouldShrinkFPConstant(OrigVT)) {
276 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
277 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
278 VT = SVT;
279 Extend = true;
280 }
281 }
282 }
283
284 SDValue CPIdx =
285 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
286 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
287 if (Extend) {
288 SDValue Result = DAG.getExtLoad(
289 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
290 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
291 Alignment);
292 return Result;
293 }
294 SDValue Result = DAG.getLoad(
295 OrigVT, dl, DAG.getEntryNode(), CPIdx,
296 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
297 return Result;
298}
299
300/// Expands the Constant node to a load from the constant pool.
301SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
302 SDLoc dl(CP);
303 EVT VT = CP->getValueType(0);
304 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
305 TLI.getPointerTy(DAG.getDataLayout()));
306 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
307 SDValue Result = DAG.getLoad(
308 VT, dl, DAG.getEntryNode(), CPIdx,
309 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
310 return Result;
311}
312
313/// Some target cannot handle a variable insertion index for the
314/// INSERT_VECTOR_ELT instruction. In this case, it
315/// is necessary to spill the vector being inserted into to memory, perform
316/// the insert there, and then read the result back.
317SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
318 SDValue Val,
319 SDValue Idx,
320 const SDLoc &dl) {
321 SDValue Tmp1 = Vec;
322 SDValue Tmp2 = Val;
323 SDValue Tmp3 = Idx;
324
325 // If the target doesn't support this, we have to spill the input vector
326 // to a temporary stack slot, update the element, then reload it. This is
327 // badness. We could also load the value into a vector register (either
328 // with a "move to register" or "extload into register" instruction, then
329 // permute it into place, if the idx is a constant and if the idx is
330 // supported by the target.
331 EVT VT = Tmp1.getValueType();
332 EVT EltVT = VT.getVectorElementType();
333 SDValue StackPtr = DAG.CreateStackTemporary(VT);
334
335 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
336
337 // Store the vector.
338 SDValue Ch = DAG.getStore(
339 DAG.getEntryNode(), dl, Tmp1, StackPtr,
340 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
341
342 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
343
344 // Store the scalar value.
345 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
346 // Load the updated vector.
347 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
348 DAG.getMachineFunction(), SPFI));
349}
350
351SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
352 SDValue Idx,
353 const SDLoc &dl) {
354 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
355 // SCALAR_TO_VECTOR requires that the type of the value being inserted
356 // match the element type of the vector being created, except for
357 // integers in which case the inserted value can be over width.
358 EVT EltVT = Vec.getValueType().getVectorElementType();
359 if (Val.getValueType() == EltVT ||
360 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
361 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
362 Vec.getValueType(), Val);
363
364 unsigned NumElts = Vec.getValueType().getVectorNumElements();
365 // We generate a shuffle of InVec and ScVec, so the shuffle mask
366 // should be 0,1,2,3,4,5... with the appropriate element replaced with
367 // elt 0 of the RHS.
368 SmallVector<int, 8> ShufOps;
369 for (unsigned i = 0; i != NumElts; ++i)
370 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
371
372 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
373 }
374 }
375 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
376}
377
378SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
379 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
380 // FIXME: We shouldn't do this for TargetConstantFP's.
381 // FIXME: move this to the DAG Combiner! Note that we can't regress due
382 // to phase ordering between legalized code and the dag combiner. This
383 // probably means that we need to integrate dag combiner and legalizer
384 // together.
385 // We generally can't do this one for long doubles.
386 SDValue Chain = ST->getChain();
387 SDValue Ptr = ST->getBasePtr();
388 unsigned Alignment = ST->getAlignment();
389 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
390 AAMDNodes AAInfo = ST->getAAInfo();
391 SDLoc dl(ST);
392 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
393 if (CFP->getValueType(0) == MVT::f32 &&
394 TLI.isTypeLegal(MVT::i32)) {
395 SDValue Con = DAG.getConstant(CFP->getValueAPF().
396 bitcastToAPInt().zextOrTrunc(32),
397 SDLoc(CFP), MVT::i32);
398 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
399 MMOFlags, AAInfo);
400 }
401
402 if (CFP->getValueType(0) == MVT::f64) {
403 // If this target supports 64-bit registers, do a single 64-bit store.
404 if (TLI.isTypeLegal(MVT::i64)) {
405 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
406 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
407 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
408 Alignment, MMOFlags, AAInfo);
409 }
410
411 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
412 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
413 // stores. If the target supports neither 32- nor 64-bits, this
414 // xform is certainly not worth it.
415 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
416 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
417 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
418 if (DAG.getDataLayout().isBigEndian())
419 std::swap(Lo, Hi);
420
421 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
422 MMOFlags, AAInfo);
423 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
424 DAG.getConstant(4, dl, Ptr.getValueType()));
425 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
426 ST->getPointerInfo().getWithOffset(4),
427 MinAlign(Alignment, 4U), MMOFlags, AAInfo);
428
429 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
430 }
431 }
432 }
433 return SDValue(nullptr, 0);
434}
435
436void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
437 StoreSDNode *ST = cast<StoreSDNode>(Node);
438 SDValue Chain = ST->getChain();
439 SDValue Ptr = ST->getBasePtr();
440 SDLoc dl(Node);
441
442 unsigned Alignment = ST->getAlignment();
443 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
444 AAMDNodes AAInfo = ST->getAAInfo();
445
446 if (!ST->isTruncatingStore()) {
447 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
448 ReplaceNode(ST, OptStore);
449 return;
450 }
451
452 {
453 SDValue Value = ST->getValue();
454 MVT VT = Value.getSimpleValueType();
455 switch (TLI.getOperationAction(ISD::STORE, VT)) {
456 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 456)
;
457 case TargetLowering::Legal: {
458 // If this is an unaligned store and the target doesn't support it,
459 // expand it.
460 EVT MemVT = ST->getMemoryVT();
461 unsigned AS = ST->getAddressSpace();
462 unsigned Align = ST->getAlignment();
463 const DataLayout &DL = DAG.getDataLayout();
464 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
465 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
466 ReplaceNode(SDValue(ST, 0), Result);
467 }
468 break;
469 }
470 case TargetLowering::Custom: {
471 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
472 if (Res && Res != SDValue(Node, 0))
473 ReplaceNode(SDValue(Node, 0), Res);
474 return;
475 }
476 case TargetLowering::Promote: {
477 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
478 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote stores to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 479, __PRETTY_FUNCTION__))
479 "Can only promote stores to same size type")((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote stores to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 479, __PRETTY_FUNCTION__))
;
480 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
481 SDValue Result =
482 DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
483 Alignment, MMOFlags, AAInfo);
484 ReplaceNode(SDValue(Node, 0), Result);
485 break;
486 }
487 }
488 return;
489 }
490 } else {
491 SDValue Value = ST->getValue();
492
493 EVT StVT = ST->getMemoryVT();
494 unsigned StWidth = StVT.getSizeInBits();
495 auto &DL = DAG.getDataLayout();
496
497 if (StWidth != StVT.getStoreSizeInBits()) {
498 // Promote to a byte-sized store with upper bits zero if not
499 // storing an integral number of bytes. For example, promote
500 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
501 EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
502 StVT.getStoreSizeInBits());
503 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
504 SDValue Result =
505 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
506 Alignment, MMOFlags, AAInfo);
507 ReplaceNode(SDValue(Node, 0), Result);
508 } else if (StWidth & (StWidth - 1)) {
509 // If not storing a power-of-2 number of bits, expand as two stores.
510 assert(!StVT.isVector() && "Unsupported truncstore!")((!StVT.isVector() && "Unsupported truncstore!") ? static_cast
<void> (0) : __assert_fail ("!StVT.isVector() && \"Unsupported truncstore!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 510, __PRETTY_FUNCTION__))
;
511 unsigned RoundWidth = 1 << Log2_32(StWidth);
512 assert(RoundWidth < StWidth)((RoundWidth < StWidth) ? static_cast<void> (0) : __assert_fail
("RoundWidth < StWidth", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 512, __PRETTY_FUNCTION__))
;
513 unsigned ExtraWidth = StWidth - RoundWidth;
514 assert(ExtraWidth < RoundWidth)((ExtraWidth < RoundWidth) ? static_cast<void> (0) :
__assert_fail ("ExtraWidth < RoundWidth", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 514, __PRETTY_FUNCTION__))
;
515 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Store size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 516, __PRETTY_FUNCTION__))
516 "Store size not an integral number of bytes!")((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Store size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 516, __PRETTY_FUNCTION__))
;
517 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
518 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
519 SDValue Lo, Hi;
520 unsigned IncrementSize;
521
522 if (DL.isLittleEndian()) {
523 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
524 // Store the bottom RoundWidth bits.
525 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
526 RoundVT, Alignment, MMOFlags, AAInfo);
527
528 // Store the remaining ExtraWidth bits.
529 IncrementSize = RoundWidth / 8;
530 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
531 DAG.getConstant(IncrementSize, dl,
532 Ptr.getValueType()));
533 Hi = DAG.getNode(
534 ISD::SRL, dl, Value.getValueType(), Value,
535 DAG.getConstant(RoundWidth, dl,
536 TLI.getShiftAmountTy(Value.getValueType(), DL)));
537 Hi = DAG.getTruncStore(
538 Chain, dl, Hi, Ptr,
539 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
540 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
541 } else {
542 // Big endian - avoid unaligned stores.
543 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
544 // Store the top RoundWidth bits.
545 Hi = DAG.getNode(
546 ISD::SRL, dl, Value.getValueType(), Value,
547 DAG.getConstant(ExtraWidth, dl,
548 TLI.getShiftAmountTy(Value.getValueType(), DL)));
549 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
550 RoundVT, Alignment, MMOFlags, AAInfo);
551
552 // Store the remaining ExtraWidth bits.
553 IncrementSize = RoundWidth / 8;
554 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
555 DAG.getConstant(IncrementSize, dl,
556 Ptr.getValueType()));
557 Lo = DAG.getTruncStore(
558 Chain, dl, Value, Ptr,
559 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
560 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
561 }
562
563 // The order of the stores doesn't matter.
564 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
565 ReplaceNode(SDValue(Node, 0), Result);
566 } else {
567 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
568 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 568)
;
569 case TargetLowering::Legal: {
570 EVT MemVT = ST->getMemoryVT();
571 unsigned AS = ST->getAddressSpace();
572 unsigned Align = ST->getAlignment();
573 // If this is an unaligned store and the target doesn't support it,
574 // expand it.
575 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
576 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
577 ReplaceNode(SDValue(ST, 0), Result);
578 }
579 break;
580 }
581 case TargetLowering::Custom: {
582 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
583 if (Res && Res != SDValue(Node, 0))
584 ReplaceNode(SDValue(Node, 0), Res);
585 return;
586 }
587 case TargetLowering::Expand:
588 assert(!StVT.isVector() &&((!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 589, __PRETTY_FUNCTION__))
589 "Vector Stores are handled in LegalizeVectorOps")((!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 589, __PRETTY_FUNCTION__))
;
590
591 // TRUNCSTORE:i16 i32 -> STORE i16
592 assert(TLI.isTypeLegal(StVT) &&((TLI.isTypeLegal(StVT) && "Do not know how to expand this store!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isTypeLegal(StVT) && \"Do not know how to expand this store!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 593, __PRETTY_FUNCTION__))
593 "Do not know how to expand this store!")((TLI.isTypeLegal(StVT) && "Do not know how to expand this store!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isTypeLegal(StVT) && \"Do not know how to expand this store!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 593, __PRETTY_FUNCTION__))
;
594 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
595 SDValue Result =
596 DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
597 Alignment, MMOFlags, AAInfo);
598 ReplaceNode(SDValue(Node, 0), Result);
599 break;
600 }
601 }
602 }
603}
604
605void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
606 LoadSDNode *LD = cast<LoadSDNode>(Node);
607 SDValue Chain = LD->getChain(); // The chain.
608 SDValue Ptr = LD->getBasePtr(); // The base pointer.
609 SDValue Value; // The value returned by the load op.
610 SDLoc dl(Node);
611
612 ISD::LoadExtType ExtType = LD->getExtensionType();
613 if (ExtType == ISD::NON_EXTLOAD) {
1
Assuming 'ExtType' is not equal to NON_EXTLOAD
2
Taking false branch
614 MVT VT = Node->getSimpleValueType(0);
615 SDValue RVal = SDValue(Node, 0);
616 SDValue RChain = SDValue(Node, 1);
617
618 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
619 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 619)
;
620 case TargetLowering::Legal: {
621 EVT MemVT = LD->getMemoryVT();
622 unsigned AS = LD->getAddressSpace();
623 unsigned Align = LD->getAlignment();
624 const DataLayout &DL = DAG.getDataLayout();
625 // If this is an unaligned load and the target doesn't support it,
626 // expand it.
627 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
628 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
629 }
630 break;
631 }
632 case TargetLowering::Custom: {
633 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
634 RVal = Res;
635 RChain = Res.getValue(1);
636 }
637 break;
638 }
639 case TargetLowering::Promote: {
640 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
641 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote loads to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 642, __PRETTY_FUNCTION__))
642 "Can only promote loads to same size type")((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote loads to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 642, __PRETTY_FUNCTION__))
;
643
644 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
645 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
646 RChain = Res.getValue(1);
647 break;
648 }
649 }
650 if (RChain.getNode() != Node) {
651 assert(RVal.getNode() != Node && "Load must be completely replaced")((RVal.getNode() != Node && "Load must be completely replaced"
) ? static_cast<void> (0) : __assert_fail ("RVal.getNode() != Node && \"Load must be completely replaced\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 651, __PRETTY_FUNCTION__))
;
652 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
653 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
654 if (UpdatedNodes) {
655 UpdatedNodes->insert(RVal.getNode());
656 UpdatedNodes->insert(RChain.getNode());
657 }
658 ReplacedNode(Node);
659 }
660 return;
661 }
662
663 EVT SrcVT = LD->getMemoryVT();
664 unsigned SrcWidth = SrcVT.getSizeInBits();
665 unsigned Alignment = LD->getAlignment();
666 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
667 AAMDNodes AAInfo = LD->getAAInfo();
668
669 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
3
Taking false branch
670 // Some targets pretend to have an i1 loading operation, and actually
671 // load an i8. This trick is correct for ZEXTLOAD because the top 7
672 // bits are guaranteed to be zero; it helps the optimizers understand
673 // that these bits are zero. It is also useful for EXTLOAD, since it
674 // tells the optimizers that those bits are undefined. It would be
675 // nice to have an effective generic way of getting these benefits...
676 // Until such a way is found, don't insist on promoting i1 here.
677 (SrcVT != MVT::i1 ||
678 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
679 TargetLowering::Promote)) {
680 // Promote to a byte-sized load if not loading an integral number of
681 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
682 unsigned NewWidth = SrcVT.getStoreSizeInBits();
683 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
684 SDValue Ch;
685
686 // The extra bits are guaranteed to be zero, since we stored them that
687 // way. A zext load from NVT thus automatically gives zext from SrcVT.
688
689 ISD::LoadExtType NewExtType =
690 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
691
692 SDValue Result =
693 DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
694 LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
695
696 Ch = Result.getValue(1); // The chain.
697
698 if (ExtType == ISD::SEXTLOAD)
699 // Having the top bits zero doesn't help when sign extending.
700 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
701 Result.getValueType(),
702 Result, DAG.getValueType(SrcVT));
703 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
704 // All the top bits are guaranteed to be zero - inform the optimizers.
705 Result = DAG.getNode(ISD::AssertZext, dl,
706 Result.getValueType(), Result,
707 DAG.getValueType(SrcVT));
708
709 Value = Result;
710 Chain = Ch;
711 } else if (SrcWidth & (SrcWidth - 1)) {
4
Taking true branch
712 // If not loading a power-of-2 number of bits, expand as two loads.
713 assert(!SrcVT.isVector() && "Unsupported extload!")((!SrcVT.isVector() && "Unsupported extload!") ? static_cast
<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Unsupported extload!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 713, __PRETTY_FUNCTION__))
;
714 unsigned RoundWidth = 1 << Log2_32(SrcWidth);
5
The result of the '<<' expression is undefined
715 assert(RoundWidth < SrcWidth)((RoundWidth < SrcWidth) ? static_cast<void> (0) : __assert_fail
("RoundWidth < SrcWidth", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 715, __PRETTY_FUNCTION__))
;
716 unsigned ExtraWidth = SrcWidth - RoundWidth;
717 assert(ExtraWidth < RoundWidth)((ExtraWidth < RoundWidth) ? static_cast<void> (0) :
__assert_fail ("ExtraWidth < RoundWidth", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 717, __PRETTY_FUNCTION__))
;
718 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Load size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 719, __PRETTY_FUNCTION__))
719 "Load size not an integral number of bytes!")((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Load size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 719, __PRETTY_FUNCTION__))
;
720 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
721 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
722 SDValue Lo, Hi, Ch;
723 unsigned IncrementSize;
724 auto &DL = DAG.getDataLayout();
725
726 if (DL.isLittleEndian()) {
727 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
728 // Load the bottom RoundWidth bits.
729 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
730 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
731 AAInfo);
732
733 // Load the remaining ExtraWidth bits.
734 IncrementSize = RoundWidth / 8;
735 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
736 DAG.getConstant(IncrementSize, dl,
737 Ptr.getValueType()));
738 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
739 LD->getPointerInfo().getWithOffset(IncrementSize),
740 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
741 AAInfo);
742
743 // Build a factor node to remember that this load is independent of
744 // the other one.
745 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
746 Hi.getValue(1));
747
748 // Move the top bits to the right place.
749 Hi = DAG.getNode(
750 ISD::SHL, dl, Hi.getValueType(), Hi,
751 DAG.getConstant(RoundWidth, dl,
752 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
753
754 // Join the hi and lo parts.
755 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
756 } else {
757 // Big endian - avoid unaligned loads.
758 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
759 // Load the top RoundWidth bits.
760 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
761 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
762 AAInfo);
763
764 // Load the remaining ExtraWidth bits.
765 IncrementSize = RoundWidth / 8;
766 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
767 DAG.getConstant(IncrementSize, dl,
768 Ptr.getValueType()));
769 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
770 LD->getPointerInfo().getWithOffset(IncrementSize),
771 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
772 AAInfo);
773
774 // Build a factor node to remember that this load is independent of
775 // the other one.
776 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
777 Hi.getValue(1));
778
779 // Move the top bits to the right place.
780 Hi = DAG.getNode(
781 ISD::SHL, dl, Hi.getValueType(), Hi,
782 DAG.getConstant(ExtraWidth, dl,
783 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
784
785 // Join the hi and lo parts.
786 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
787 }
788
789 Chain = Ch;
790 } else {
791 bool isCustom = false;
792 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
793 SrcVT.getSimpleVT())) {
794 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 794)
;
795 case TargetLowering::Custom:
796 isCustom = true;
797 LLVM_FALLTHROUGH[[clang::fallthrough]];
798 case TargetLowering::Legal: {
799 Value = SDValue(Node, 0);
800 Chain = SDValue(Node, 1);
801
802 if (isCustom) {
803 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
804 Value = Res;
805 Chain = Res.getValue(1);
806 }
807 } else {
808 // If this is an unaligned load and the target doesn't support it,
809 // expand it.
810 EVT MemVT = LD->getMemoryVT();
811 unsigned AS = LD->getAddressSpace();
812 unsigned Align = LD->getAlignment();
813 const DataLayout &DL = DAG.getDataLayout();
814 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
815 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
816 }
817 }
818 break;
819 }
820 case TargetLowering::Expand:
821 EVT DestVT = Node->getValueType(0);
822 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
823 // If the source type is not legal, see if there is a legal extload to
824 // an intermediate type that we can then extend further.
825 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
826 if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
827 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
828 // If we are loading a legal type, this is a non-extload followed by a
829 // full extend.
830 ISD::LoadExtType MidExtType =
831 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
832
833 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
834 SrcVT, LD->getMemOperand());
835 unsigned ExtendOp =
836 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
837 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
838 Chain = Load.getValue(1);
839 break;
840 }
841
842 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
843 // normal undefined upper bits behavior to allow using an in-reg extend
844 // with the illegal FP type, so load as an integer and do the
845 // from-integer conversion.
846 if (SrcVT.getScalarType() == MVT::f16) {
847 EVT ISrcVT = SrcVT.changeTypeToInteger();
848 EVT IDestVT = DestVT.changeTypeToInteger();
849 EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
850
851 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
852 Chain, Ptr, ISrcVT,
853 LD->getMemOperand());
854 Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
855 Chain = Result.getValue(1);
856 break;
857 }
858 }
859
860 assert(!SrcVT.isVector() &&((!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 861, __PRETTY_FUNCTION__))
861 "Vector Loads are handled in LegalizeVectorOps")((!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 861, __PRETTY_FUNCTION__))
;
862
863 // FIXME: This does not work for vectors on most targets. Sign-
864 // and zero-extend operations are currently folded into extending
865 // loads, whether they are legal or not, and then we end up here
866 // without any support for legalizing them.
867 assert(ExtType != ISD::EXTLOAD &&((ExtType != ISD::EXTLOAD && "EXTLOAD should always be supported!"
) ? static_cast<void> (0) : __assert_fail ("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 868, __PRETTY_FUNCTION__))
868 "EXTLOAD should always be supported!")((ExtType != ISD::EXTLOAD && "EXTLOAD should always be supported!"
) ? static_cast<void> (0) : __assert_fail ("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 868, __PRETTY_FUNCTION__))
;
869 // Turn the unsupported load into an EXTLOAD followed by an
870 // explicit zero/sign extend inreg.
871 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
872 Node->getValueType(0),
873 Chain, Ptr, SrcVT,
874 LD->getMemOperand());
875 SDValue ValRes;
876 if (ExtType == ISD::SEXTLOAD)
877 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
878 Result.getValueType(),
879 Result, DAG.getValueType(SrcVT));
880 else
881 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
882 Value = ValRes;
883 Chain = Result.getValue(1);
884 break;
885 }
886 }
887
888 // Since loads produce two values, make sure to remember that we legalized
889 // both of them.
890 if (Chain.getNode() != Node) {
891 assert(Value.getNode() != Node && "Load must be completely replaced")((Value.getNode() != Node && "Load must be completely replaced"
) ? static_cast<void> (0) : __assert_fail ("Value.getNode() != Node && \"Load must be completely replaced\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 891, __PRETTY_FUNCTION__))
;
892 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
893 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
894 if (UpdatedNodes) {
895 UpdatedNodes->insert(Value.getNode());
896 UpdatedNodes->insert(Chain.getNode());
897 }
898 ReplacedNode(Node);
899 }
900}
901
902static TargetLowering::LegalizeAction
903getStrictFPOpcodeAction(const TargetLowering &TLI, unsigned Opcode, EVT VT) {
904 unsigned EqOpc;
905 switch (Opcode) {
906 default: llvm_unreachable("Unexpected FP pseudo-opcode")::llvm::llvm_unreachable_internal("Unexpected FP pseudo-opcode"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 906)
;
907 case ISD::STRICT_FSQRT: EqOpc = ISD::FSQRT; break;
908 case ISD::STRICT_FPOW: EqOpc = ISD::FPOW; break;
909 case ISD::STRICT_FPOWI: EqOpc = ISD::FPOWI; break;
910 case ISD::STRICT_FSIN: EqOpc = ISD::FSIN; break;
911 case ISD::STRICT_FCOS: EqOpc = ISD::FCOS; break;
912 case ISD::STRICT_FEXP: EqOpc = ISD::FEXP; break;
913 case ISD::STRICT_FEXP2: EqOpc = ISD::FEXP2; break;
914 case ISD::STRICT_FLOG: EqOpc = ISD::FLOG; break;
915 case ISD::STRICT_FLOG10: EqOpc = ISD::FLOG10; break;
916 case ISD::STRICT_FLOG2: EqOpc = ISD::FLOG2; break;
917 case ISD::STRICT_FRINT: EqOpc = ISD::FRINT; break;
918 case ISD::STRICT_FNEARBYINT: EqOpc = ISD::FNEARBYINT; break;
919 }
920
921 auto Action = TLI.getOperationAction(EqOpc, VT);
922
923 // We don't currently handle Custom or Promote for strict FP pseudo-ops.
924 // For now, we just expand for those cases.
925 if (Action != TargetLowering::Legal)
926 Action = TargetLowering::Expand;
927
928 return Action;
929}
930
931/// Return a legal replacement for the given operation, with all legal operands.
932void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
933 DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "\nLegalizing: "; Node->
dump(&DAG); } } while (false)
;
934
935 if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes.
936 return;
937
938#ifndef NDEBUG
939 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
940 assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 943, __PRETTY_FUNCTION__))
941 TargetLowering::TypeLegal ||(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 943, __PRETTY_FUNCTION__))
942 TLI.isTypeLegal(Node->getValueType(i))) &&(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 943, __PRETTY_FUNCTION__))
943 "Unexpected illegal type!")(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 943, __PRETTY_FUNCTION__))
;
944
945 for (const SDValue &Op : Node->op_values())
946 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant) && "Unexpected illegal type!"
) ? static_cast<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant) && \"Unexpected illegal type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 950, __PRETTY_FUNCTION__))
947 TargetLowering::TypeLegal ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant) && "Unexpected illegal type!"
) ? static_cast<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant) && \"Unexpected illegal type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 950, __PRETTY_FUNCTION__))
948 TLI.isTypeLegal(Op.getValueType()) ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant) && "Unexpected illegal type!"
) ? static_cast<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant) && \"Unexpected illegal type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 950, __PRETTY_FUNCTION__))
949 Op.getOpcode() == ISD::TargetConstant) &&(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant) && "Unexpected illegal type!"
) ? static_cast<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant) && \"Unexpected illegal type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 950, __PRETTY_FUNCTION__))
950 "Unexpected illegal type!")(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant) && "Unexpected illegal type!"
) ? static_cast<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant) && \"Unexpected illegal type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 950, __PRETTY_FUNCTION__))
;
951#endif
952
953 // Figure out the correct action; the way to query this varies by opcode
954 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
955 bool SimpleFinishLegalizing = true;
956 switch (Node->getOpcode()) {
957 case ISD::INTRINSIC_W_CHAIN:
958 case ISD::INTRINSIC_WO_CHAIN:
959 case ISD::INTRINSIC_VOID:
960 case ISD::STACKSAVE:
961 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
962 break;
963 case ISD::GET_DYNAMIC_AREA_OFFSET:
964 Action = TLI.getOperationAction(Node->getOpcode(),
965 Node->getValueType(0));
966 break;
967 case ISD::VAARG:
968 Action = TLI.getOperationAction(Node->getOpcode(),
969 Node->getValueType(0));
970 if (Action != TargetLowering::Promote)
971 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
972 break;
973 case ISD::FP_TO_FP16:
974 case ISD::SINT_TO_FP:
975 case ISD::UINT_TO_FP:
976 case ISD::EXTRACT_VECTOR_ELT:
977 Action = TLI.getOperationAction(Node->getOpcode(),
978 Node->getOperand(0).getValueType());
979 break;
980 case ISD::FP_ROUND_INREG:
981 case ISD::SIGN_EXTEND_INREG: {
982 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
983 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
984 break;
985 }
986 case ISD::ATOMIC_STORE: {
987 Action = TLI.getOperationAction(Node->getOpcode(),
988 Node->getOperand(2).getValueType());
989 break;
990 }
991 case ISD::SELECT_CC:
992 case ISD::SETCC:
993 case ISD::BR_CC: {
994 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
995 Node->getOpcode() == ISD::SETCC ? 2 :
996 Node->getOpcode() == ISD::SETCCE ? 3 : 1;
997 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
998 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
999 ISD::CondCode CCCode =
1000 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1001 Action = TLI.getCondCodeAction(CCCode, OpVT);
1002 if (Action == TargetLowering::Legal) {
1003 if (Node->getOpcode() == ISD::SELECT_CC)
1004 Action = TLI.getOperationAction(Node->getOpcode(),
1005 Node->getValueType(0));
1006 else
1007 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1008 }
1009 break;
1010 }
1011 case ISD::LOAD:
1012 case ISD::STORE:
1013 // FIXME: Model these properly. LOAD and STORE are complicated, and
1014 // STORE expects the unlegalized operand in some cases.
1015 SimpleFinishLegalizing = false;
1016 break;
1017 case ISD::CALLSEQ_START:
1018 case ISD::CALLSEQ_END:
1019 // FIXME: This shouldn't be necessary. These nodes have special properties
1020 // dealing with the recursive nature of legalization. Removing this
1021 // special case should be done as part of making LegalizeDAG non-recursive.
1022 SimpleFinishLegalizing = false;
1023 break;
1024 case ISD::EXTRACT_ELEMENT:
1025 case ISD::FLT_ROUNDS_:
1026 case ISD::MERGE_VALUES:
1027 case ISD::EH_RETURN:
1028 case ISD::FRAME_TO_ARGS_OFFSET:
1029 case ISD::EH_DWARF_CFA:
1030 case ISD::EH_SJLJ_SETJMP:
1031 case ISD::EH_SJLJ_LONGJMP:
1032 case ISD::EH_SJLJ_SETUP_DISPATCH:
1033 // These operations lie about being legal: when they claim to be legal,
1034 // they should actually be expanded.
1035 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1036 if (Action == TargetLowering::Legal)
1037 Action = TargetLowering::Expand;
1038 break;
1039 case ISD::INIT_TRAMPOLINE:
1040 case ISD::ADJUST_TRAMPOLINE:
1041 case ISD::FRAMEADDR:
1042 case ISD::RETURNADDR:
1043 case ISD::ADDROFRETURNADDR:
1044 // These operations lie about being legal: when they claim to be legal,
1045 // they should actually be custom-lowered.
1046 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1047 if (Action == TargetLowering::Legal)
1048 Action = TargetLowering::Custom;
1049 break;
1050 case ISD::READCYCLECOUNTER:
1051 // READCYCLECOUNTER returns an i64, even if type legalization might have
1052 // expanded that to several smaller types.
1053 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1054 break;
1055 case ISD::READ_REGISTER:
1056 case ISD::WRITE_REGISTER:
1057 // Named register is legal in the DAG, but blocked by register name
1058 // selection if not implemented by target (to chose the correct register)
1059 // They'll be converted to Copy(To/From)Reg.
1060 Action = TargetLowering::Legal;
1061 break;
1062 case ISD::DEBUGTRAP:
1063 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1064 if (Action == TargetLowering::Expand) {
1065 // replace ISD::DEBUGTRAP with ISD::TRAP
1066 SDValue NewVal;
1067 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1068 Node->getOperand(0));
1069 ReplaceNode(Node, NewVal.getNode());
1070 LegalizeOp(NewVal.getNode());
1071 return;
1072 }
1073 break;
1074 case ISD::STRICT_FSQRT:
1075 case ISD::STRICT_FPOW:
1076 case ISD::STRICT_FPOWI:
1077 case ISD::STRICT_FSIN:
1078 case ISD::STRICT_FCOS:
1079 case ISD::STRICT_FEXP:
1080 case ISD::STRICT_FEXP2:
1081 case ISD::STRICT_FLOG:
1082 case ISD::STRICT_FLOG10:
1083 case ISD::STRICT_FLOG2:
1084 case ISD::STRICT_FRINT:
1085 case ISD::STRICT_FNEARBYINT:
1086 // These pseudo-ops get legalized as if they were their non-strict
1087 // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
1088 // is also legal, but if ISD::FSQRT requires expansion then so does
1089 // ISD::STRICT_FSQRT.
1090 Action = getStrictFPOpcodeAction(TLI, Node->getOpcode(),
1091 Node->getValueType(0));
1092 break;
1093
1094 default:
1095 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1096 Action = TargetLowering::Legal;
1097 } else {
1098 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1099 }
1100 break;
1101 }
1102
1103 if (SimpleFinishLegalizing) {
1104 SDNode *NewNode = Node;
1105 switch (Node->getOpcode()) {
1106 default: break;
1107 case ISD::SHL:
1108 case ISD::SRL:
1109 case ISD::SRA:
1110 case ISD::ROTL:
1111 case ISD::ROTR: {
1112 // Legalizing shifts/rotates requires adjusting the shift amount
1113 // to the appropriate width.
1114 SDValue Op0 = Node->getOperand(0);
1115 SDValue Op1 = Node->getOperand(1);
1116 if (!Op1.getValueType().isVector()) {
1117 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1118 // The getShiftAmountOperand() may create a new operand node or
1119 // return the existing one. If new operand is created we need
1120 // to update the parent node.
1121 // Do not try to legalize SAO here! It will be automatically legalized
1122 // in the next round.
1123 if (SAO != Op1)
1124 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1125 }
1126 }
1127 break;
1128 case ISD::SRL_PARTS:
1129 case ISD::SRA_PARTS:
1130 case ISD::SHL_PARTS: {
1131 // Legalizing shifts/rotates requires adjusting the shift amount
1132 // to the appropriate width.
1133 SDValue Op0 = Node->getOperand(0);
1134 SDValue Op1 = Node->getOperand(1);
1135 SDValue Op2 = Node->getOperand(2);
1136 if (!Op2.getValueType().isVector()) {
1137 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1138 // The getShiftAmountOperand() may create a new operand node or
1139 // return the existing one. If new operand is created we need
1140 // to update the parent node.
1141 if (SAO != Op2)
1142 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1143 }
1144 }
1145 break;
1146 }
1147
1148 if (NewNode != Node) {
1149 ReplaceNode(Node, NewNode);
1150 Node = NewNode;
1151 }
1152 switch (Action) {
1153 case TargetLowering::Legal:
1154 return;
1155 case TargetLowering::Custom: {
1156 // FIXME: The handling for custom lowering with multiple results is
1157 // a complete mess.
1158 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1159 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1160 return;
1161
1162 if (Node->getNumValues() == 1) {
1163 // We can just directly replace this node with the lowered value.
1164 ReplaceNode(SDValue(Node, 0), Res);
1165 return;
1166 }
1167
1168 SmallVector<SDValue, 8> ResultVals;
1169 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1170 ResultVals.push_back(Res.getValue(i));
1171 ReplaceNode(Node, ResultVals.data());
1172 return;
1173 }
1174 LLVM_FALLTHROUGH[[clang::fallthrough]];
1175 }
1176 case TargetLowering::Expand:
1177 if (ExpandNode(Node))
1178 return;
1179 LLVM_FALLTHROUGH[[clang::fallthrough]];
1180 case TargetLowering::LibCall:
1181 ConvertNodeToLibcall(Node);
1182 return;
1183 case TargetLowering::Promote:
1184 PromoteNode(Node);
1185 return;
1186 }
1187 }
1188
1189 switch (Node->getOpcode()) {
1190 default:
1191#ifndef NDEBUG
1192 dbgs() << "NODE: ";
1193 Node->dump( &DAG);
1194 dbgs() << "\n";
1195#endif
1196 llvm_unreachable("Do not know how to legalize this operator!")::llvm::llvm_unreachable_internal("Do not know how to legalize this operator!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1196)
;
1197
1198 case ISD::CALLSEQ_START:
1199 case ISD::CALLSEQ_END:
1200 break;
1201 case ISD::LOAD: {
1202 return LegalizeLoadOps(Node);
1203 }
1204 case ISD::STORE: {
1205 return LegalizeStoreOps(Node);
1206 }
1207 }
1208}
1209
1210SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1211 SDValue Vec = Op.getOperand(0);
1212 SDValue Idx = Op.getOperand(1);
1213 SDLoc dl(Op);
1214
1215 // Before we generate a new store to a temporary stack slot, see if there is
1216 // already one that we can use. There often is because when we scalarize
1217 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1218 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1219 // the vector. If all are expanded here, we don't want one store per vector
1220 // element.
1221
1222 // Caches for hasPredecessorHelper
1223 SmallPtrSet<const SDNode *, 32> Visited;
1224 SmallVector<const SDNode *, 16> Worklist;
1225 Worklist.push_back(Idx.getNode());
1226 SDValue StackPtr, Ch;
1227 for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1228 UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1229 SDNode *User = *UI;
1230 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1231 if (ST->isIndexed() || ST->isTruncatingStore() ||
1232 ST->getValue() != Vec)
1233 continue;
1234
1235 // Make sure that nothing else could have stored into the destination of
1236 // this store.
1237 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1238 continue;
1239
1240 // If the index is dependent on the store we will introduce a cycle when
1241 // creating the load (the load uses the index, and by replacing the chain
1242 // we will make the index dependent on the load). Also, the store might be
1243 // dependent on the extractelement and introduce a cycle when creating
1244 // the load.
1245 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1246 ST->hasPredecessor(Op.getNode()))
1247 continue;
1248
1249 StackPtr = ST->getBasePtr();
1250 Ch = SDValue(ST, 0);
1251 break;
1252 }
1253 }
1254
1255 EVT VecVT = Vec.getValueType();
1256
1257 if (!Ch.getNode()) {
1258 // Store the value to a temporary stack slot, then LOAD the returned part.
1259 StackPtr = DAG.CreateStackTemporary(VecVT);
1260 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1261 MachinePointerInfo());
1262 }
1263
1264 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1265
1266 SDValue NewLoad;
1267
1268 if (Op.getValueType().isVector())
1269 NewLoad =
1270 DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1271 else
1272 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1273 MachinePointerInfo(),
1274 VecVT.getVectorElementType());
1275
1276 // Replace the chain going out of the store, by the one out of the load.
1277 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1278
1279 // We introduced a cycle though, so update the loads operands, making sure
1280 // to use the original store's chain as an incoming chain.
1281 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1282 NewLoad->op_end());
1283 NewLoadOperands[0] = Ch;
1284 NewLoad =
1285 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1286 return NewLoad;
1287}
1288
1289SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1290 assert(Op.getValueType().isVector() && "Non-vector insert subvector!")((Op.getValueType().isVector() && "Non-vector insert subvector!"
) ? static_cast<void> (0) : __assert_fail ("Op.getValueType().isVector() && \"Non-vector insert subvector!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1290, __PRETTY_FUNCTION__))
;
1291
1292 SDValue Vec = Op.getOperand(0);
1293 SDValue Part = Op.getOperand(1);
1294 SDValue Idx = Op.getOperand(2);
1295 SDLoc dl(Op);
1296
1297 // Store the value to a temporary stack slot, then LOAD the returned part.
1298 EVT VecVT = Vec.getValueType();
1299 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1300 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1301 MachinePointerInfo PtrInfo =
1302 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1303
1304 // First store the whole vector.
1305 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1306
1307 // Then store the inserted part.
1308 SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1309
1310 // Store the subvector.
1311 Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
1312
1313 // Finally, load the updated vector.
1314 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1315}
1316
1317SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1318 // We can't handle this case efficiently. Allocate a sufficiently
1319 // aligned object on the stack, store each element into it, then load
1320 // the result as a vector.
1321 // Create the stack frame object.
1322 EVT VT = Node->getValueType(0);
1323 EVT EltVT = VT.getVectorElementType();
1324 SDLoc dl(Node);
1325 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1326 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1327 MachinePointerInfo PtrInfo =
1328 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1329
1330 // Emit a store of each element to the stack slot.
1331 SmallVector<SDValue, 8> Stores;
1332 unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1333 // Store (in the right endianness) the elements to memory.
1334 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1335 // Ignore undef elements.
1336 if (Node->getOperand(i).isUndef()) continue;
1337
1338 unsigned Offset = TypeByteSize*i;
1339
1340 SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1341 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1342
1343 // If the destination vector element type is narrower than the source
1344 // element type, only store the bits necessary.
1345 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1346 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1347 Node->getOperand(i), Idx,
1348 PtrInfo.getWithOffset(Offset), EltVT));
1349 } else
1350 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1351 Idx, PtrInfo.getWithOffset(Offset)));
1352 }
1353
1354 SDValue StoreChain;
1355 if (!Stores.empty()) // Not all undef elements?
1356 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1357 else
1358 StoreChain = DAG.getEntryNode();
1359
1360 // Result is a load from the stack slot.
1361 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1362}
1363
1364namespace {
1365/// Keeps track of state when getting the sign of a floating-point value as an
1366/// integer.
1367struct FloatSignAsInt {
1368 EVT FloatVT;
1369 SDValue Chain;
1370 SDValue FloatPtr;
1371 SDValue IntPtr;
1372 MachinePointerInfo IntPointerInfo;
1373 MachinePointerInfo FloatPointerInfo;
1374 SDValue IntValue;
1375 APInt SignMask;
1376 uint8_t SignBit;
1377};
1378}
1379
1380/// Bitcast a floating-point value to an integer value. Only bitcast the part
1381/// containing the sign bit if the target has no integer value capable of
1382/// holding all bits of the floating-point value.
1383void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1384 const SDLoc &DL,
1385 SDValue Value) const {
1386 EVT FloatVT = Value.getValueType();
1387 unsigned NumBits = FloatVT.getSizeInBits();
1388 State.FloatVT = FloatVT;
1389 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1390 // Convert to an integer of the same size.
1391 if (TLI.isTypeLegal(IVT)) {
1392 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1393 State.SignMask = APInt::getSignMask(NumBits);
1394 State.SignBit = NumBits - 1;
1395 return;
1396 }
1397
1398 auto &DataLayout = DAG.getDataLayout();
1399 // Store the float to memory, then load the sign part out as an integer.
1400 MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1401 // First create a temporary that is aligned for both the load and store.
1402 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1403 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1404 // Then store the float to it.
1405 State.FloatPtr = StackPtr;
1406 MachineFunction &MF = DAG.getMachineFunction();
1407 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1408 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1409 State.FloatPointerInfo);
1410
1411 SDValue IntPtr;
1412 if (DataLayout.isBigEndian()) {
1413 assert(FloatVT.isByteSized() && "Unsupported floating point type!")((FloatVT.isByteSized() && "Unsupported floating point type!"
) ? static_cast<void> (0) : __assert_fail ("FloatVT.isByteSized() && \"Unsupported floating point type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1413, __PRETTY_FUNCTION__))
;
1414 // Load out a legal integer with the same sign bit as the float.
1415 IntPtr = StackPtr;
1416 State.IntPointerInfo = State.FloatPointerInfo;
1417 } else {
1418 // Advance the pointer so that the loaded byte will contain the sign bit.
1419 unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
1420 IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
1421 DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
1422 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1423 ByteOffset);
1424 }
1425
1426 State.IntPtr = IntPtr;
1427 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1428 State.IntPointerInfo, MVT::i8);
1429 State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
1430 State.SignBit = 7;
1431}
1432
1433/// Replace the integer value produced by getSignAsIntValue() with a new value
1434/// and cast the result back to a floating-point type.
1435SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1436 const SDLoc &DL,
1437 SDValue NewIntValue) const {
1438 if (!State.Chain)
1439 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1440
1441 // Override the part containing the sign bit in the value stored on the stack.
1442 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1443 State.IntPointerInfo, MVT::i8);
1444 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1445 State.FloatPointerInfo);
1446}
1447
1448SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1449 SDLoc DL(Node);
1450 SDValue Mag = Node->getOperand(0);
1451 SDValue Sign = Node->getOperand(1);
1452
1453 // Get sign bit into an integer value.
1454 FloatSignAsInt SignAsInt;
1455 getSignAsIntValue(SignAsInt, DL, Sign);
1456
1457 EVT IntVT = SignAsInt.IntValue.getValueType();
1458 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1459 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1460 SignMask);
1461
1462 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1463 EVT FloatVT = Mag.getValueType();
1464 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1465 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1466 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1467 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1468 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1469 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1470 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1471 }
1472
1473 // Transform Mag value to integer, and clear the sign bit.
1474 FloatSignAsInt MagAsInt;
1475 getSignAsIntValue(MagAsInt, DL, Mag);
1476 EVT MagVT = MagAsInt.IntValue.getValueType();
1477 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1478 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1479 ClearSignMask);
1480
1481 // Get the signbit at the right position for MagAsInt.
1482 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1483 if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
1484 if (ShiftAmount > 0) {
1485 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, IntVT);
1486 SignBit = DAG.getNode(ISD::SRL, DL, IntVT, SignBit, ShiftCnst);
1487 } else if (ShiftAmount < 0) {
1488 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, IntVT);
1489 SignBit = DAG.getNode(ISD::SHL, DL, IntVT, SignBit, ShiftCnst);
1490 }
1491 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1492 } else if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
1493 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1494 if (ShiftAmount > 0) {
1495 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, MagVT);
1496 SignBit = DAG.getNode(ISD::SRL, DL, MagVT, SignBit, ShiftCnst);
1497 } else if (ShiftAmount < 0) {
1498 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, MagVT);
1499 SignBit = DAG.getNode(ISD::SHL, DL, MagVT, SignBit, ShiftCnst);
1500 }
1501 }
1502
1503 // Store the part with the modified sign and convert back to float.
1504 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1505 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1506}
1507
1508SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1509 SDLoc DL(Node);
1510 SDValue Value = Node->getOperand(0);
1511
1512 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1513 EVT FloatVT = Value.getValueType();
1514 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1515 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1516 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1517 }
1518
1519 // Transform value to integer, clear the sign bit and transform back.
1520 FloatSignAsInt ValueAsInt;
1521 getSignAsIntValue(ValueAsInt, DL, Value);
1522 EVT IntVT = ValueAsInt.IntValue.getValueType();
1523 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1524 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1525 ClearSignMask);
1526 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1527}
1528
1529void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1530 SmallVectorImpl<SDValue> &Results) {
1531 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1532 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"((SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? static_cast
<void> (0) : __assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1533, __PRETTY_FUNCTION__))
1533 " not tell us which reg is the stack pointer!")((SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? static_cast
<void> (0) : __assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1533, __PRETTY_FUNCTION__))
;
1534 SDLoc dl(Node);
1535 EVT VT = Node->getValueType(0);
1536 SDValue Tmp1 = SDValue(Node, 0);
1537 SDValue Tmp2 = SDValue(Node, 1);
1538 SDValue Tmp3 = Node->getOperand(2);
1539 SDValue Chain = Tmp1.getOperand(0);
1540
1541 // Chain the dynamic stack allocation so that it doesn't modify the stack
1542 // pointer when other instructions are using the stack.
1543 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1544
1545 SDValue Size = Tmp2.getOperand(1);
1546 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1547 Chain = SP.getValue(1);
1548 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1549 unsigned StackAlign =
1550 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1551 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1552 if (Align > StackAlign)
1553 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1554 DAG.getConstant(-(uint64_t)Align, dl, VT));
1555 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1556
1557 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1558 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1559
1560 Results.push_back(Tmp1);
1561 Results.push_back(Tmp2);
1562}
1563
1564/// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1565/// target.
1566///
1567/// If the SETCC has been legalized using AND / OR, then the legalized node
1568/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1569/// will be set to false.
1570///
1571/// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1572/// then the values of LHS and RHS will be swapped, CC will be set to the
1573/// new condition, and NeedInvert will be set to false.
1574///
1575/// If the SETCC has been legalized using the inverse condcode, then LHS and
1576/// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1577/// will be set to true. The caller must invert the result of the SETCC with
1578/// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1579/// of a true/false result.
1580///
1581/// \returns true if the SetCC has been legalized, false if it hasn't.
1582bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1583 SDValue &RHS, SDValue &CC,
1584 bool &NeedInvert,
1585 const SDLoc &dl) {
1586 MVT OpVT = LHS.getSimpleValueType();
1587 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1588 NeedInvert = false;
1589 switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1590 default: llvm_unreachable("Unknown condition code action!")::llvm::llvm_unreachable_internal("Unknown condition code action!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1590)
;
1591 case TargetLowering::Legal:
1592 // Nothing to do.
1593 break;
1594 case TargetLowering::Expand: {
1595 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
1596 if (TLI.isCondCodeLegal(InvCC, OpVT)) {
1597 std::swap(LHS, RHS);
1598 CC = DAG.getCondCode(InvCC);
1599 return true;
1600 }
1601 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1602 unsigned Opc = 0;
1603 switch (CCCode) {
1604 default: llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1604)
;
1605 case ISD::SETO:
1606 assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT)((TLI.getCondCodeAction(ISD::SETOEQ, OpVT) == TargetLowering::
Legal && "If SETO is expanded, SETOEQ must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETOEQ, OpVT) == TargetLowering::Legal && \"If SETO is expanded, SETOEQ must be legal!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1608, __PRETTY_FUNCTION__))
1607 == TargetLowering::Legal((TLI.getCondCodeAction(ISD::SETOEQ, OpVT) == TargetLowering::
Legal && "If SETO is expanded, SETOEQ must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETOEQ, OpVT) == TargetLowering::Legal && \"If SETO is expanded, SETOEQ must be legal!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1608, __PRETTY_FUNCTION__))
1608 && "If SETO is expanded, SETOEQ must be legal!")((TLI.getCondCodeAction(ISD::SETOEQ, OpVT) == TargetLowering::
Legal && "If SETO is expanded, SETOEQ must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETOEQ, OpVT) == TargetLowering::Legal && \"If SETO is expanded, SETOEQ must be legal!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1608, __PRETTY_FUNCTION__))
;
1609 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1610 case ISD::SETUO:
1611 assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT)((TLI.getCondCodeAction(ISD::SETUNE, OpVT) == TargetLowering::
Legal && "If SETUO is expanded, SETUNE must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETUNE, OpVT) == TargetLowering::Legal && \"If SETUO is expanded, SETUNE must be legal!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1613, __PRETTY_FUNCTION__))
1612 == TargetLowering::Legal((TLI.getCondCodeAction(ISD::SETUNE, OpVT) == TargetLowering::
Legal && "If SETUO is expanded, SETUNE must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETUNE, OpVT) == TargetLowering::Legal && \"If SETUO is expanded, SETUNE must be legal!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1613, __PRETTY_FUNCTION__))
1613 && "If SETUO is expanded, SETUNE must be legal!")((TLI.getCondCodeAction(ISD::SETUNE, OpVT) == TargetLowering::
Legal && "If SETUO is expanded, SETUNE must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETUNE, OpVT) == TargetLowering::Legal && \"If SETUO is expanded, SETUNE must be legal!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1613, __PRETTY_FUNCTION__))
;
1614 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break;
1615 case ISD::SETOEQ:
1616 case ISD::SETOGT:
1617 case ISD::SETOGE:
1618 case ISD::SETOLT:
1619 case ISD::SETOLE:
1620 case ISD::SETONE:
1621 case ISD::SETUEQ:
1622 case ISD::SETUNE:
1623 case ISD::SETUGT:
1624 case ISD::SETUGE:
1625 case ISD::SETULT:
1626 case ISD::SETULE:
1627 // If we are floating point, assign and break, otherwise fall through.
1628 if (!OpVT.isInteger()) {
1629 // We can use the 4th bit to tell if we are the unordered
1630 // or ordered version of the opcode.
1631 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
1632 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
1633 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1634 break;
1635 }
1636 // Fallthrough if we are unsigned integer.
1637 LLVM_FALLTHROUGH[[clang::fallthrough]];
1638 case ISD::SETLE:
1639 case ISD::SETGT:
1640 case ISD::SETGE:
1641 case ISD::SETLT:
1642 // We only support using the inverted operation, which is computed above
1643 // and not a different manner of supporting expanding these cases.
1644 llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1644)
;
1645 case ISD::SETNE:
1646 case ISD::SETEQ:
1647 // Try inverting the result of the inverse condition.
1648 InvCC = CCCode == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ;
1649 if (TLI.isCondCodeLegal(InvCC, OpVT)) {
1650 CC = DAG.getCondCode(InvCC);
1651 NeedInvert = true;
1652 return true;
1653 }
1654 // If inverting the condition didn't work then we have no means to expand
1655 // the condition.
1656 llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1656)
;
1657 }
1658
1659 SDValue SetCC1, SetCC2;
1660 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
1661 // If we aren't the ordered or unorder operation,
1662 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1663 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1664 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1665 } else {
1666 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1667 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1668 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1669 }
1670 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1671 RHS = SDValue();
1672 CC = SDValue();
1673 return true;
1674 }
1675 }
1676 return false;
1677}
1678
1679/// Emit a store/load combination to the stack. This stores
1680/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1681/// a load from the stack slot to DestVT, extending it if needed.
1682/// The resultant code need not be legal.
1683SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1684 EVT DestVT, const SDLoc &dl) {
1685 // Create the stack frame object.
1686 unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1687 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1688 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1689
1690 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1691 int SPFI = StackPtrFI->getIndex();
1692 MachinePointerInfo PtrInfo =
1693 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1694
1695 unsigned SrcSize = SrcOp.getValueSizeInBits();
1696 unsigned SlotSize = SlotVT.getSizeInBits();
1697 unsigned DestSize = DestVT.getSizeInBits();
1698 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1699 unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1700
1701 // Emit a store to the stack slot. Use a truncstore if the input value is
1702 // later than DestVT.
1703 SDValue Store;
1704
1705 if (SrcSize > SlotSize)
1706 Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
1707 SlotVT, SrcAlign);
1708 else {
1709 assert(SrcSize == SlotSize && "Invalid store")((SrcSize == SlotSize && "Invalid store") ? static_cast
<void> (0) : __assert_fail ("SrcSize == SlotSize && \"Invalid store\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1709, __PRETTY_FUNCTION__))
;
1710 Store =
1711 DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1712 }
1713
1714 // Result is a load from the stack slot.
1715 if (SlotSize == DestSize)
1716 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1717
1718 assert(SlotSize < DestSize && "Unknown extension!")((SlotSize < DestSize && "Unknown extension!") ? static_cast
<void> (0) : __assert_fail ("SlotSize < DestSize && \"Unknown extension!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1718, __PRETTY_FUNCTION__))
;
1719 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1720 DestAlign);
1721}
1722
1723SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1724 SDLoc dl(Node);
1725 // Create a vector sized/aligned stack slot, store the value to element #0,
1726 // then load the whole vector back out.
1727 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1728
1729 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1730 int SPFI = StackPtrFI->getIndex();
1731
1732 SDValue Ch = DAG.getTruncStore(
1733 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1734 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1735 Node->getValueType(0).getVectorElementType());
1736 return DAG.getLoad(
1737 Node->getValueType(0), dl, Ch, StackPtr,
1738 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1739}
1740
1741static bool
1742ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1743 const TargetLowering &TLI, SDValue &Res) {
1744 unsigned NumElems = Node->getNumOperands();
1745 SDLoc dl(Node);
1746 EVT VT = Node->getValueType(0);
1747
1748 // Try to group the scalars into pairs, shuffle the pairs together, then
1749 // shuffle the pairs of pairs together, etc. until the vector has
1750 // been built. This will work only if all of the necessary shuffle masks
1751 // are legal.
1752
1753 // We do this in two phases; first to check the legality of the shuffles,
1754 // and next, assuming that all shuffles are legal, to create the new nodes.
1755 for (int Phase = 0; Phase < 2; ++Phase) {
1756 SmallVector<std::pair<SDValue, SmallVector<int, 16> >, 16> IntermedVals,
1757 NewIntermedVals;
1758 for (unsigned i = 0; i < NumElems; ++i) {
1759 SDValue V = Node->getOperand(i);
1760 if (V.isUndef())
1761 continue;
1762
1763 SDValue Vec;
1764 if (Phase)
1765 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1766 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1767 }
1768
1769 while (IntermedVals.size() > 2) {
1770 NewIntermedVals.clear();
1771 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1772 // This vector and the next vector are shuffled together (simply to
1773 // append the one to the other).
1774 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1775
1776 SmallVector<int, 16> FinalIndices;
1777 FinalIndices.reserve(IntermedVals[i].second.size() +
1778 IntermedVals[i+1].second.size());
1779
1780 int k = 0;
1781 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1782 ++j, ++k) {
1783 ShuffleVec[k] = j;
1784 FinalIndices.push_back(IntermedVals[i].second[j]);
1785 }
1786 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1787 ++j, ++k) {
1788 ShuffleVec[k] = NumElems + j;
1789 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1790 }
1791
1792 SDValue Shuffle;
1793 if (Phase)
1794 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1795 IntermedVals[i+1].first,
1796 ShuffleVec);
1797 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1798 return false;
1799 NewIntermedVals.push_back(
1800 std::make_pair(Shuffle, std::move(FinalIndices)));
1801 }
1802
1803 // If we had an odd number of defined values, then append the last
1804 // element to the array of new vectors.
1805 if ((IntermedVals.size() & 1) != 0)
1806 NewIntermedVals.push_back(IntermedVals.back());
1807
1808 IntermedVals.swap(NewIntermedVals);
1809 }
1810
1811 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&((IntermedVals.size() <= 2 && IntermedVals.size() >
0 && "Invalid number of intermediate vectors") ? static_cast
<void> (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1812, __PRETTY_FUNCTION__))
1812 "Invalid number of intermediate vectors")((IntermedVals.size() <= 2 && IntermedVals.size() >
0 && "Invalid number of intermediate vectors") ? static_cast
<void> (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1812, __PRETTY_FUNCTION__))
;
1813 SDValue Vec1 = IntermedVals[0].first;
1814 SDValue Vec2;
1815 if (IntermedVals.size() > 1)
1816 Vec2 = IntermedVals[1].first;
1817 else if (Phase)
1818 Vec2 = DAG.getUNDEF(VT);
1819
1820 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1821 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1822 ShuffleVec[IntermedVals[0].second[i]] = i;
1823 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1824 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1825
1826 if (Phase)
1827 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1828 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1829 return false;
1830 }
1831
1832 return true;
1833}
1834
1835/// Expand a BUILD_VECTOR node on targets that don't
1836/// support the operation, but do support the resultant vector type.
1837SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1838 unsigned NumElems = Node->getNumOperands();
1839 SDValue Value1, Value2;
1840 SDLoc dl(Node);
1841 EVT VT = Node->getValueType(0);
1842 EVT OpVT = Node->getOperand(0).getValueType();
1843 EVT EltVT = VT.getVectorElementType();
1844
1845 // If the only non-undef value is the low element, turn this into a
1846 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1847 bool isOnlyLowElement = true;
1848 bool MoreThanTwoValues = false;
1849 bool isConstant = true;
1850 for (unsigned i = 0; i < NumElems; ++i) {
1851 SDValue V = Node->getOperand(i);
1852 if (V.isUndef())
1853 continue;
1854 if (i > 0)
1855 isOnlyLowElement = false;
1856 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1857 isConstant = false;
1858
1859 if (!Value1.getNode()) {
1860 Value1 = V;
1861 } else if (!Value2.getNode()) {
1862 if (V != Value1)
1863 Value2 = V;
1864 } else if (V != Value1 && V != Value2) {
1865 MoreThanTwoValues = true;
1866 }
1867 }
1868
1869 if (!Value1.getNode())
1870 return DAG.getUNDEF(VT);
1871
1872 if (isOnlyLowElement)
1873 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1874
1875 // If all elements are constants, create a load from the constant pool.
1876 if (isConstant) {
1877 SmallVector<Constant*, 16> CV;
1878 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1879 if (ConstantFPSDNode *V =
1880 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1881 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1882 } else if (ConstantSDNode *V =
1883 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1884 if (OpVT==EltVT)
1885 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1886 else {
1887 // If OpVT and EltVT don't match, EltVT is not legal and the
1888 // element values have been promoted/truncated earlier. Undo this;
1889 // we don't want a v16i8 to become a v16i32 for example.
1890 const ConstantInt *CI = V->getConstantIntValue();
1891 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1892 CI->getZExtValue()));
1893 }
1894 } else {
1895 assert(Node->getOperand(i).isUndef())((Node->getOperand(i).isUndef()) ? static_cast<void>
(0) : __assert_fail ("Node->getOperand(i).isUndef()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1895, __PRETTY_FUNCTION__))
;
1896 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1897 CV.push_back(UndefValue::get(OpNTy));
1898 }
1899 }
1900 Constant *CP = ConstantVector::get(CV);
1901 SDValue CPIdx =
1902 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1903 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1904 return DAG.getLoad(
1905 VT, dl, DAG.getEntryNode(), CPIdx,
1906 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1907 Alignment);
1908 }
1909
1910 SmallSet<SDValue, 16> DefinedValues;
1911 for (unsigned i = 0; i < NumElems; ++i) {
1912 if (Node->getOperand(i).isUndef())
1913 continue;
1914 DefinedValues.insert(Node->getOperand(i));
1915 }
1916
1917 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1918 if (!MoreThanTwoValues) {
1919 SmallVector<int, 8> ShuffleVec(NumElems, -1);
1920 for (unsigned i = 0; i < NumElems; ++i) {
1921 SDValue V = Node->getOperand(i);
1922 if (V.isUndef())
1923 continue;
1924 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1925 }
1926 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1927 // Get the splatted value into the low element of a vector register.
1928 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1929 SDValue Vec2;
1930 if (Value2.getNode())
1931 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1932 else
1933 Vec2 = DAG.getUNDEF(VT);
1934
1935 // Return shuffle(LowValVec, undef, <0,0,0,0>)
1936 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1937 }
1938 } else {
1939 SDValue Res;
1940 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1941 return Res;
1942 }
1943 }
1944
1945 // Otherwise, we can't handle this case efficiently.
1946 return ExpandVectorBuildThroughStack(Node);
1947}
1948
1949// Expand a node into a call to a libcall. If the result value
1950// does not fit into a register, return the lo part and set the hi part to the
1951// by-reg argument. If it does fit into a single register, return the result
1952// and leave the Hi part unset.
1953SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
1954 bool isSigned) {
1955 TargetLowering::ArgListTy Args;
1956 TargetLowering::ArgListEntry Entry;
1957 for (const SDValue &Op : Node->op_values()) {
1958 EVT ArgVT = Op.getValueType();
1959 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1960 Entry.Node = Op;
1961 Entry.Ty = ArgTy;
1962 Entry.IsSExt = isSigned;
1963 Entry.IsZExt = !isSigned;
1964 Args.push_back(Entry);
1965 }
1966 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
1967 TLI.getPointerTy(DAG.getDataLayout()));
1968
1969 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
1970
1971 // By default, the input chain to this libcall is the entry node of the
1972 // function. If the libcall is going to be emitted as a tail call then
1973 // TLI.isUsedByReturnOnly will change it to the right chain if the return
1974 // node which is being folded has a non-entry input chain.
1975 SDValue InChain = DAG.getEntryNode();
1976
1977 // isTailCall may be true since the callee does not reference caller stack
1978 // frame. Check if it's in the right position and that the return types match.
1979 SDValue TCChain = InChain;
1980 const Function *F = DAG.getMachineFunction().getFunction();
1981 bool isTailCall =
1982 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
1983 (RetTy == F->getReturnType() || F->getReturnType()->isVoidTy());
1984 if (isTailCall)
1985 InChain = TCChain;
1986
1987 TargetLowering::CallLoweringInfo CLI(DAG);
1988 CLI.setDebugLoc(SDLoc(Node))
1989 .setChain(InChain)
1990 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
1991 std::move(Args))
1992 .setTailCall(isTailCall)
1993 .setSExtResult(isSigned)
1994 .setZExtResult(!isSigned);
1995
1996 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
1997
1998 if (!CallInfo.second.getNode())
1999 // It's a tailcall, return the chain (which is the DAG root).
2000 return DAG.getRoot();
2001
2002 return CallInfo.first;
2003}
2004
2005/// Generate a libcall taking the given operands as arguments
2006/// and returning a result of type RetVT.
2007SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
2008 const SDValue *Ops, unsigned NumOps,
2009 bool isSigned, const SDLoc &dl) {
2010 TargetLowering::ArgListTy Args;
2011 Args.reserve(NumOps);
2012
2013 TargetLowering::ArgListEntry Entry;
2014 for (unsigned i = 0; i != NumOps; ++i) {
2015 Entry.Node = Ops[i];
2016 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
2017 Entry.IsSExt = isSigned;
2018 Entry.IsZExt = !isSigned;
2019 Args.push_back(Entry);
2020 }
2021 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2022 TLI.getPointerTy(DAG.getDataLayout()));
2023
2024 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2025
2026 TargetLowering::CallLoweringInfo CLI(DAG);
2027 CLI.setDebugLoc(dl)
2028 .setChain(DAG.getEntryNode())
2029 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2030 std::move(Args))
2031 .setSExtResult(isSigned)
2032 .setZExtResult(!isSigned);
2033
2034 std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
2035
2036 return CallInfo.first;
2037}
2038
2039// Expand a node into a call to a libcall. Similar to
2040// ExpandLibCall except that the first operand is the in-chain.
2041std::pair<SDValue, SDValue>
2042SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2043 SDNode *Node,
2044 bool isSigned) {
2045 SDValue InChain = Node->getOperand(0);
2046
2047 TargetLowering::ArgListTy Args;
2048 TargetLowering::ArgListEntry Entry;
2049 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
2050 EVT ArgVT = Node->getOperand(i).getValueType();
2051 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2052 Entry.Node = Node->getOperand(i);
2053 Entry.Ty = ArgTy;
2054 Entry.IsSExt = isSigned;
2055 Entry.IsZExt = !isSigned;
2056 Args.push_back(Entry);
2057 }
2058 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2059 TLI.getPointerTy(DAG.getDataLayout()));
2060
2061 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2062
2063 TargetLowering::CallLoweringInfo CLI(DAG);
2064 CLI.setDebugLoc(SDLoc(Node))
2065 .setChain(InChain)
2066 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2067 std::move(Args))
2068 .setSExtResult(isSigned)
2069 .setZExtResult(!isSigned);
2070
2071 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2072
2073 return CallInfo;
2074}
2075
2076SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2077 RTLIB::Libcall Call_F32,
2078 RTLIB::Libcall Call_F64,
2079 RTLIB::Libcall Call_F80,
2080 RTLIB::Libcall Call_F128,
2081 RTLIB::Libcall Call_PPCF128) {
2082 if (Node->isStrictFPOpcode())
2083 Node = DAG.mutateStrictFPToFP(Node);
2084
2085 RTLIB::Libcall LC;
2086 switch (Node->getSimpleValueType(0).SimpleTy) {
2087 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2087)
;
2088 case MVT::f32: LC = Call_F32; break;
2089 case MVT::f64: LC = Call_F64; break;
2090 case MVT::f80: LC = Call_F80; break;
2091 case MVT::f128: LC = Call_F128; break;
2092 case MVT::ppcf128: LC = Call_PPCF128; break;
2093 }
2094 return ExpandLibCall(LC, Node, false);
2095}
2096
2097SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2098 RTLIB::Libcall Call_I8,
2099 RTLIB::Libcall Call_I16,
2100 RTLIB::Libcall Call_I32,
2101 RTLIB::Libcall Call_I64,
2102 RTLIB::Libcall Call_I128) {
2103 RTLIB::Libcall LC;
2104 switch (Node->getSimpleValueType(0).SimpleTy) {
2105 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2105)
;
2106 case MVT::i8: LC = Call_I8; break;
2107 case MVT::i16: LC = Call_I16; break;
2108 case MVT::i32: LC = Call_I32; break;
2109 case MVT::i64: LC = Call_I64; break;
2110 case MVT::i128: LC = Call_I128; break;
2111 }
2112 return ExpandLibCall(LC, Node, isSigned);
2113}
2114
2115/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2116void
2117SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2118 SmallVectorImpl<SDValue> &Results) {
2119 unsigned Opcode = Node->getOpcode();
2120 bool isSigned = Opcode == ISD::SDIVREM;
2121
2122 RTLIB::Libcall LC;
2123 switch (Node->getSimpleValueType(0).SimpleTy) {
2124 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2124)
;
2125 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2126 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2127 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2128 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2129 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2130 }
2131
2132 // The input chain to this libcall is the entry node of the function.
2133 // Legalizing the call will automatically add the previous call to the
2134 // dependence.
2135 SDValue InChain = DAG.getEntryNode();
2136
2137 EVT RetVT = Node->getValueType(0);
2138 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2139
2140 TargetLowering::ArgListTy Args;
2141 TargetLowering::ArgListEntry Entry;
2142 for (const SDValue &Op : Node->op_values()) {
2143 EVT ArgVT = Op.getValueType();
2144 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2145 Entry.Node = Op;
2146 Entry.Ty = ArgTy;
2147 Entry.IsSExt = isSigned;
2148 Entry.IsZExt = !isSigned;
2149 Args.push_back(Entry);
2150 }
2151
2152 // Also pass the return address of the remainder.
2153 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2154 Entry.Node = FIPtr;
2155 Entry.Ty = RetTy->getPointerTo();
2156 Entry.IsSExt = isSigned;
2157 Entry.IsZExt = !isSigned;
2158 Args.push_back(Entry);
2159
2160 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2161 TLI.getPointerTy(DAG.getDataLayout()));
2162
2163 SDLoc dl(Node);
2164 TargetLowering::CallLoweringInfo CLI(DAG);
2165 CLI.setDebugLoc(dl)
2166 .setChain(InChain)
2167 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2168 std::move(Args))
2169 .setSExtResult(isSigned)
2170 .setZExtResult(!isSigned);
2171
2172 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2173
2174 // Remainder is loaded back from the stack frame.
2175 SDValue Rem =
2176 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2177 Results.push_back(CallInfo.first);
2178 Results.push_back(Rem);
2179}
2180
2181/// Return true if sincos libcall is available.
2182static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2183 RTLIB::Libcall LC;
2184 switch (Node->getSimpleValueType(0).SimpleTy) {
2185 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2185)
;
2186 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2187 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2188 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2189 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2190 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2191 }
2192 return TLI.getLibcallName(LC) != nullptr;
2193}
2194
2195/// Only issue sincos libcall if both sin and cos are needed.
2196static bool useSinCos(SDNode *Node) {
2197 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2198 ? ISD::FCOS : ISD::FSIN;
2199
2200 SDValue Op0 = Node->getOperand(0);
2201 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2202 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2203 SDNode *User = *UI;
2204 if (User == Node)
2205 continue;
2206 // The other user might have been turned into sincos already.
2207 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2208 return true;
2209 }
2210 return false;
2211}
2212
2213/// Issue libcalls to sincos to compute sin / cos pairs.
2214void
2215SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2216 SmallVectorImpl<SDValue> &Results) {
2217 RTLIB::Libcall LC;
2218 switch (Node->getSimpleValueType(0).SimpleTy) {
2219 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2219)
;
2220 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2221 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2222 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2223 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2224 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2225 }
2226
2227 // The input chain to this libcall is the entry node of the function.
2228 // Legalizing the call will automatically add the previous call to the
2229 // dependence.
2230 SDValue InChain = DAG.getEntryNode();
2231
2232 EVT RetVT = Node->getValueType(0);
2233 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2234
2235 TargetLowering::ArgListTy Args;
2236 TargetLowering::ArgListEntry Entry;
2237
2238 // Pass the argument.
2239 Entry.Node = Node->getOperand(0);
2240 Entry.Ty = RetTy;
2241 Entry.IsSExt = false;
2242 Entry.IsZExt = false;
2243 Args.push_back(Entry);
2244
2245 // Pass the return address of sin.
2246 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2247 Entry.Node = SinPtr;
2248 Entry.Ty = RetTy->getPointerTo();
2249 Entry.IsSExt = false;
2250 Entry.IsZExt = false;
2251 Args.push_back(Entry);
2252
2253 // Also pass the return address of the cos.
2254 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2255 Entry.Node = CosPtr;
2256 Entry.Ty = RetTy->getPointerTo();
2257 Entry.IsSExt = false;
2258 Entry.IsZExt = false;
2259 Args.push_back(Entry);
2260
2261 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2262 TLI.getPointerTy(DAG.getDataLayout()));
2263
2264 SDLoc dl(Node);
2265 TargetLowering::CallLoweringInfo CLI(DAG);
2266 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2267 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2268 std::move(Args));
2269
2270 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2271
2272 Results.push_back(
2273 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2274 Results.push_back(
2275 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2276}
2277
2278/// This function is responsible for legalizing a
2279/// INT_TO_FP operation of the specified operand when the target requests that
2280/// we expand it. At this point, we know that the result and operand types are
2281/// legal for the target.
2282SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2283 EVT DestVT,
2284 const SDLoc &dl) {
2285 // TODO: Should any fast-math-flags be set for the created nodes?
2286
2287 if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
2288 // simple 32-bit [signed|unsigned] integer to float/double expansion
2289
2290 // Get the stack frame index of a 8 byte buffer.
2291 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2292
2293 // word offset constant for Hi/Lo address computation
2294 SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2295 StackSlot.getValueType());
2296 // set up Hi and Lo (into buffer) address based on endian
2297 SDValue Hi = StackSlot;
2298 SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2299 StackSlot, WordOff);
2300 if (DAG.getDataLayout().isLittleEndian())
2301 std::swap(Hi, Lo);
2302
2303 // if signed map to unsigned space
2304 SDValue Op0Mapped;
2305 if (isSigned) {
2306 // constant used to invert sign bit (signed to unsigned mapping)
2307 SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2308 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2309 } else {
2310 Op0Mapped = Op0;
2311 }
2312 // store the lo of the constructed double - based on integer input
2313 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2314 MachinePointerInfo());
2315 // initial hi portion of constructed double
2316 SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2317 // store the hi of the constructed double - biased exponent
2318 SDValue Store2 =
2319 DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
2320 // load the constructed double
2321 SDValue Load =
2322 DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
2323 // FP constant to bias correct the final result
2324 SDValue Bias = DAG.getConstantFP(isSigned ?
2325 BitsToDouble(0x4330000080000000ULL) :
2326 BitsToDouble(0x4330000000000000ULL),
2327 dl, MVT::f64);
2328 // subtract the bias
2329 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2330 // final result
2331 SDValue Result;
2332 // handle final rounding
2333 if (DestVT == MVT::f64) {
2334 // do nothing
2335 Result = Sub;
2336 } else if (DestVT.bitsLT(MVT::f64)) {
2337 Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
2338 DAG.getIntPtrConstant(0, dl));
2339 } else if (DestVT.bitsGT(MVT::f64)) {
2340 Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
2341 }
2342 return Result;
2343 }
2344 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet")((!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"
) ? static_cast<void> (0) : __assert_fail ("!isSigned && \"Legalize cannot Expand SINT_TO_FP for i64 yet\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2344, __PRETTY_FUNCTION__))
;
2345 // Code below here assumes !isSigned without checking again.
2346
2347 // Implementation of unsigned i64 to f64 following the algorithm in
2348 // __floatundidf in compiler_rt. This implementation has the advantage
2349 // of performing rounding correctly, both in the default rounding mode
2350 // and in all alternate rounding modes.
2351 // TODO: Generalize this for use with other types.
2352 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) {
2353 SDValue TwoP52 =
2354 DAG.getConstant(UINT64_C(0x4330000000000000)0x4330000000000000UL, dl, MVT::i64);
2355 SDValue TwoP84PlusTwoP52 =
2356 DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)0x4530000000100000UL), dl,
2357 MVT::f64);
2358 SDValue TwoP84 =
2359 DAG.getConstant(UINT64_C(0x4530000000000000)0x4530000000000000UL, dl, MVT::i64);
2360
2361 SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32);
2362 SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
2363 DAG.getConstant(32, dl, MVT::i64));
2364 SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
2365 SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
2366 SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
2367 SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr);
2368 SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt,
2369 TwoP84PlusTwoP52);
2370 return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub);
2371 }
2372
2373 // Implementation of unsigned i64 to f32.
2374 // TODO: Generalize this for use with other types.
2375 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) {
2376 // For unsigned conversions, convert them to signed conversions using the
2377 // algorithm from the x86_64 __floatundidf in compiler_rt.
2378 if (!isSigned) {
2379 SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
2380
2381 SDValue ShiftConst = DAG.getConstant(
2382 1, dl, TLI.getShiftAmountTy(Op0.getValueType(), DAG.getDataLayout()));
2383 SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
2384 SDValue AndConst = DAG.getConstant(1, dl, MVT::i64);
2385 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
2386 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
2387
2388 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
2389 SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
2390
2391 // TODO: This really should be implemented using a branch rather than a
2392 // select. We happen to get lucky and machinesink does the right
2393 // thing most of the time. This would be a good candidate for a
2394 //pseudo-op, or, even better, for whole-function isel.
2395 SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
2396 Op0, DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
2397 return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast);
2398 }
2399
2400 // Otherwise, implement the fully general conversion.
2401
2402 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2403 DAG.getConstant(UINT64_C(0xfffffffffffff800)0xfffffffffffff800UL, dl, MVT::i64));
2404 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
2405 DAG.getConstant(UINT64_C(0x800)0x800UL, dl, MVT::i64));
2406 SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2407 DAG.getConstant(UINT64_C(0x7ff)0x7ffUL, dl, MVT::i64));
2408 SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), And2,
2409 DAG.getConstant(UINT64_C(0)0UL, dl, MVT::i64),
2410 ISD::SETNE);
2411 SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0);
2412 SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), Op0,
2413 DAG.getConstant(UINT64_C(0x0020000000000000)0x0020000000000000UL, dl,
2414 MVT::i64),
2415 ISD::SETUGE);
2416 SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0);
2417 EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType(), DAG.getDataLayout());
2418
2419 SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
2420 DAG.getConstant(32, dl, SHVT));
2421 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
2422 SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
2423 SDValue TwoP32 =
2424 DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)0x41f0000000000000UL), dl,
2425 MVT::f64);
2426 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
2427 SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
2428 SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
2429 SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
2430 return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
2431 DAG.getIntPtrConstant(0, dl));
2432 }
2433
2434 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2435
2436 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()),
2437 Op0,
2438 DAG.getConstant(0, dl, Op0.getValueType()),
2439 ISD::SETLT);
2440 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2441 Four = DAG.getIntPtrConstant(4, dl);
2442 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2443 SignSet, Four, Zero);
2444
2445 // If the sign bit of the integer is set, the large number will be treated
2446 // as a negative number. To counteract this, the dynamic code adds an
2447 // offset depending on the data type.
2448 uint64_t FF;
2449 switch (Op0.getSimpleValueType().SimpleTy) {
2450 default: llvm_unreachable("Unsupported integer type!")::llvm::llvm_unreachable_internal("Unsupported integer type!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2450)
;
2451 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2452 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2453 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2454 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2455 }
2456 if (DAG.getDataLayout().isLittleEndian())
2457 FF <<= 32;
2458 Constant *FudgeFactor = ConstantInt::get(
2459 Type::getInt64Ty(*DAG.getContext()), FF);
2460
2461 SDValue CPIdx =
2462 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2463 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2464 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2465 Alignment = std::min(Alignment, 4u);
2466 SDValue FudgeInReg;
2467 if (DestVT == MVT::f32)
2468 FudgeInReg = DAG.getLoad(
2469 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2470 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2471 Alignment);
2472 else {
2473 SDValue Load = DAG.getExtLoad(
2474 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2475 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2476 Alignment);
2477 HandleSDNode Handle(Load);
2478 LegalizeOp(Load.getNode());
2479 FudgeInReg = Handle.getValue();
2480 }
2481
2482 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2483}
2484
2485/// This function is responsible for legalizing a
2486/// *INT_TO_FP operation of the specified operand when the target requests that
2487/// we promote it. At this point, we know that the result and operand types are
2488/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2489/// operation that takes a larger input.
2490SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2491 bool isSigned,
2492 const SDLoc &dl) {
2493 // First step, figure out the appropriate *INT_TO_FP operation to use.
2494 EVT NewInTy = LegalOp.getValueType();
2495
2496 unsigned OpToUse = 0;
2497
2498 // Scan for the appropriate larger type to use.
2499 while (1) {
2500 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2501 assert(NewInTy.isInteger() && "Ran out of possibilities!")((NewInTy.isInteger() && "Ran out of possibilities!")
? static_cast<void> (0) : __assert_fail ("NewInTy.isInteger() && \"Ran out of possibilities!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2501, __PRETTY_FUNCTION__))
;
2502
2503 // If the target supports SINT_TO_FP of this type, use it.
2504 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2505 OpToUse = ISD::SINT_TO_FP;
2506 break;
2507 }
2508 if (isSigned) continue;
2509
2510 // If the target supports UINT_TO_FP of this type, use it.
2511 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2512 OpToUse = ISD::UINT_TO_FP;
2513 break;
2514 }
2515
2516 // Otherwise, try a larger type.
2517 }
2518
2519 // Okay, we found the operation and type to use. Zero extend our input to the
2520 // desired type then run the operation on it.
2521 return DAG.getNode(OpToUse, dl, DestVT,
2522 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2523 dl, NewInTy, LegalOp));
2524}
2525
2526/// This function is responsible for legalizing a
2527/// FP_TO_*INT operation of the specified operand when the target requests that
2528/// we promote it. At this point, we know that the result and operand types are
2529/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2530/// operation that returns a larger result.
2531SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2532 bool isSigned,
2533 const SDLoc &dl) {
2534 // First step, figure out the appropriate FP_TO*INT operation to use.
2535 EVT NewOutTy = DestVT;
2536
2537 unsigned OpToUse = 0;
2538
2539 // Scan for the appropriate larger type to use.
2540 while (1) {
2541 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2542 assert(NewOutTy.isInteger() && "Ran out of possibilities!")((NewOutTy.isInteger() && "Ran out of possibilities!"
) ? static_cast<void> (0) : __assert_fail ("NewOutTy.isInteger() && \"Ran out of possibilities!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2542, __PRETTY_FUNCTION__))
;
2543
2544 // A larger signed type can hold all unsigned values of the requested type,
2545 // so using FP_TO_SINT is valid
2546 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2547 OpToUse = ISD::FP_TO_SINT;
2548 break;
2549 }
2550
2551 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2552 if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2553 OpToUse = ISD::FP_TO_UINT;
2554 break;
2555 }
2556
2557 // Otherwise, try a larger type.
2558 }
2559
2560
2561 // Okay, we found the operation and type to use.
2562 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2563
2564 // Truncate the result of the extended FP_TO_*INT operation to the desired
2565 // size.
2566 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2567}
2568
2569/// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2570SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2571 EVT VT = Op.getValueType();
2572 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2573 unsigned Sz = VT.getScalarSizeInBits();
2574
2575 SDValue Tmp, Tmp2, Tmp3;
2576
2577 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2578 // and finally the i1 pairs.
2579 // TODO: We can easily support i4/i2 legal types if any target ever does.
2580 if (Sz >= 8 && isPowerOf2_32(Sz)) {
2581 // Create the masks - repeating the pattern every byte.
2582 APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0);
2583 APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0);
2584 for (unsigned J = 0; J != Sz; J += 8) {
2585 MaskHi4 = MaskHi4 | (0xF0ull << J);
2586 MaskLo4 = MaskLo4 | (0x0Full << J);
2587 MaskHi2 = MaskHi2 | (0xCCull << J);
2588 MaskLo2 = MaskLo2 | (0x33ull << J);
2589 MaskHi1 = MaskHi1 | (0xAAull << J);
2590 MaskLo1 = MaskLo1 | (0x55ull << J);
2591 }
2592
2593 // BSWAP if the type is wider than a single byte.
2594 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
2595
2596 // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2597 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
2598 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
2599 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, VT));
2600 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, VT));
2601 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2602
2603 // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2604 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
2605 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
2606 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, VT));
2607 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, VT));
2608 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2609
2610 // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2611 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
2612 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
2613 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, VT));
2614 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, VT));
2615 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2616 return Tmp;
2617 }
2618
2619 Tmp = DAG.getConstant(0, dl, VT);
2620 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
2621 if (I < J)
2622 Tmp2 =
2623 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
2624 else
2625 Tmp2 =
2626 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
2627
2628 APInt Shift(Sz, 1);
2629 Shift <<= J;
2630 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
2631 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
2632 }
2633
2634 return Tmp;
2635}
2636
2637/// Open code the operations for BSWAP of the specified operation.
2638SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2639 EVT VT = Op.getValueType();
2640 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2641 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2642 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
2643 default: llvm_unreachable("Unhandled Expand type in BSWAP!")::llvm::llvm_unreachable_internal("Unhandled Expand type in BSWAP!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2643)
;
2644 case MVT::i16:
2645 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2646 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2647 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2648 case MVT::i32:
2649 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2650 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2651 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2652 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2653 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2654 DAG.getConstant(0xFF0000, dl, VT));
2655 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2656 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2657 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2658 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2659 case MVT::i64:
2660 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2661 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2662 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2663 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2664 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2665 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2666 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2667 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2668 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2669 DAG.getConstant(255ULL<<48, dl, VT));
2670 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2671 DAG.getConstant(255ULL<<40, dl, VT));
2672 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2673 DAG.getConstant(255ULL<<32, dl, VT));
2674 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2675 DAG.getConstant(255ULL<<24, dl, VT));
2676 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2677 DAG.getConstant(255ULL<<16, dl, VT));
2678 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2679 DAG.getConstant(255ULL<<8 , dl, VT));
2680 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2681 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2682 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2683 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2684 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2685 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2686 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2687 }
2688}
2689
2690/// Expand the specified bitcount instruction into operations.
2691SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
2692 const SDLoc &dl) {
2693 switch (Opc) {
2694 default: llvm_unreachable("Cannot expand this yet!")::llvm::llvm_unreachable_internal("Cannot expand this yet!", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2694)
;
2695 case ISD::CTPOP: {
2696 EVT VT = Op.getValueType();
2697 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2698 unsigned Len = VT.getSizeInBits();
2699
2700 assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 &&((VT.isInteger() && Len <= 128 && Len % 8 ==
0 && "CTPOP not implemented for this type.") ? static_cast
<void> (0) : __assert_fail ("VT.isInteger() && Len <= 128 && Len % 8 == 0 && \"CTPOP not implemented for this type.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2701, __PRETTY_FUNCTION__))
2701 "CTPOP not implemented for this type.")((VT.isInteger() && Len <= 128 && Len % 8 ==
0 && "CTPOP not implemented for this type.") ? static_cast
<void> (0) : __assert_fail ("VT.isInteger() && Len <= 128 && Len % 8 == 0 && \"CTPOP not implemented for this type.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2701, __PRETTY_FUNCTION__))
;
2702
2703 // This is the "best" algorithm from
2704 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
2705
2706 SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)),
2707 dl, VT);
2708 SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)),
2709 dl, VT);
2710 SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)),
2711 dl, VT);
2712 SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)),
2713 dl, VT);
2714
2715 // v = v - ((v >> 1) & 0x55555555...)
2716 Op = DAG.getNode(ISD::SUB, dl, VT, Op,
2717 DAG.getNode(ISD::AND, dl, VT,
2718 DAG.getNode(ISD::SRL, dl, VT, Op,
2719 DAG.getConstant(1, dl, ShVT)),
2720 Mask55));
2721 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
2722 Op = DAG.getNode(ISD::ADD, dl, VT,
2723 DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
2724 DAG.getNode(ISD::AND, dl, VT,
2725 DAG.getNode(ISD::SRL, dl, VT, Op,
2726 DAG.getConstant(2, dl, ShVT)),
2727 Mask33));
2728 // v = (v + (v >> 4)) & 0x0F0F0F0F...
2729 Op = DAG.getNode(ISD::AND, dl, VT,
2730 DAG.getNode(ISD::ADD, dl, VT, Op,
2731 DAG.getNode(ISD::SRL, dl, VT, Op,
2732 DAG.getConstant(4, dl, ShVT))),
2733 Mask0F);
2734 // v = (v * 0x01010101...) >> (Len - 8)
2735 Op = DAG.getNode(ISD::SRL, dl, VT,
2736 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
2737 DAG.getConstant(Len - 8, dl, ShVT));
2738
2739 return Op;
2740 }
2741 case ISD::CTLZ_ZERO_UNDEF:
2742 // This trivially expands to CTLZ.
2743 return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op);
2744 case ISD::CTLZ: {
2745 EVT VT = Op.getValueType();
2746 unsigned len = VT.getSizeInBits();
2747
2748 if (TLI.isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
2749 EVT SetCCVT = getSetCCResultType(VT);
2750 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
2751 SDValue Zero = DAG.getConstant(0, dl, VT);
2752 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
2753 return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
2754 DAG.getConstant(len, dl, VT), CTLZ);
2755 }
2756
2757 // for now, we do this:
2758 // x = x | (x >> 1);
2759 // x = x | (x >> 2);
2760 // ...
2761 // x = x | (x >>16);
2762 // x = x | (x >>32); // for 64-bit input
2763 // return popcount(~x);
2764 //
2765 // Ref: "Hacker's Delight" by Henry Warren
2766 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2767 for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
2768 SDValue Tmp3 = DAG.getConstant(1ULL << i, dl, ShVT);
2769 Op = DAG.getNode(ISD::OR, dl, VT, Op,
2770 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
2771 }
2772 Op = DAG.getNOT(dl, Op, VT);
2773 return DAG.getNode(ISD::CTPOP, dl, VT, Op);
2774 }
2775 case ISD::CTTZ_ZERO_UNDEF:
2776 // This trivially expands to CTTZ.
2777 return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op);
2778 case ISD::CTTZ: {
2779 // for now, we use: { return popcount(~x & (x - 1)); }
2780 // unless the target has ctlz but not ctpop, in which case we use:
2781 // { return 32 - nlz(~x & (x-1)); }
2782 // Ref: "Hacker's Delight" by Henry Warren
2783 EVT VT = Op.getValueType();
2784 SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
2785 DAG.getNOT(dl, Op, VT),
2786 DAG.getNode(ISD::SUB, dl, VT, Op,
2787 DAG.getConstant(1, dl, VT)));
2788 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
2789 if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
2790 TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
2791 return DAG.getNode(ISD::SUB, dl, VT,
2792 DAG.getConstant(VT.getSizeInBits(), dl, VT),
2793 DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
2794 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
2795 }
2796 }
2797}
2798
2799bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2800 SmallVector<SDValue, 8> Results;
2801 SDLoc dl(Node);
2802 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2803 bool NeedInvert;
2804 switch (Node->getOpcode()) {
2805 case ISD::CTPOP:
2806 case ISD::CTLZ:
2807 case ISD::CTLZ_ZERO_UNDEF:
2808 case ISD::CTTZ:
2809 case ISD::CTTZ_ZERO_UNDEF:
2810 Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
2811 Results.push_back(Tmp1);
2812 break;
2813 case ISD::BITREVERSE:
2814 Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
2815 break;
2816 case ISD::BSWAP:
2817 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2818 break;
2819 case ISD::FRAMEADDR:
2820 case ISD::RETURNADDR:
2821 case ISD::FRAME_TO_ARGS_OFFSET:
2822 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2823 break;
2824 case ISD::EH_DWARF_CFA: {
2825 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2826 TLI.getPointerTy(DAG.getDataLayout()));
2827 SDValue Offset = DAG.getNode(ISD::ADD, dl,
2828 CfaArg.getValueType(),
2829 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2830 CfaArg.getValueType()),
2831 CfaArg);
2832 SDValue FA = DAG.getNode(
2833 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2834 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2835 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2836 FA, Offset));
2837 break;
2838 }
2839 case ISD::FLT_ROUNDS_:
2840 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2841 break;
2842 case ISD::EH_RETURN:
2843 case ISD::EH_LABEL:
2844 case ISD::PREFETCH:
2845 case ISD::VAEND:
2846 case ISD::EH_SJLJ_LONGJMP:
2847 // If the target didn't expand these, there's nothing to do, so just
2848 // preserve the chain and be done.
2849 Results.push_back(Node->getOperand(0));
2850 break;
2851 case ISD::READCYCLECOUNTER:
2852 // If the target didn't expand this, just return 'zero' and preserve the
2853 // chain.
2854 Results.append(Node->getNumValues() - 1,
2855 DAG.getConstant(0, dl, Node->getValueType(0)));
2856 Results.push_back(Node->getOperand(0));
2857 break;
2858 case ISD::EH_SJLJ_SETJMP:
2859 // If the target didn't expand this, just return 'zero' and preserve the
2860 // chain.
2861 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2862 Results.push_back(Node->getOperand(0));
2863 break;
2864 case ISD::ATOMIC_LOAD: {
2865 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2866 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2867 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2868 SDValue Swap = DAG.getAtomicCmpSwap(
2869 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2870 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2871 cast<AtomicSDNode>(Node)->getMemOperand());
2872 Results.push_back(Swap.getValue(0));
2873 Results.push_back(Swap.getValue(1));
2874 break;
2875 }
2876 case ISD::ATOMIC_STORE: {
2877 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2878 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2879 cast<AtomicSDNode>(Node)->getMemoryVT(),
2880 Node->getOperand(0),
2881 Node->getOperand(1), Node->getOperand(2),
2882 cast<AtomicSDNode>(Node)->getMemOperand());
2883 Results.push_back(Swap.getValue(1));
2884 break;
2885 }
2886 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2887 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2888 // splits out the success value as a comparison. Expanding the resulting
2889 // ATOMIC_CMP_SWAP will produce a libcall.
2890 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2891 SDValue Res = DAG.getAtomicCmpSwap(
2892 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2893 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2894 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2895
2896 SDValue ExtRes = Res;
2897 SDValue LHS = Res;
2898 SDValue RHS = Node->getOperand(1);
2899
2900 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2901 EVT OuterType = Node->getValueType(0);
2902 switch (TLI.getExtendForAtomicOps()) {
2903 case ISD::SIGN_EXTEND:
2904 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2905 DAG.getValueType(AtomicType));
2906 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2907 Node->getOperand(2), DAG.getValueType(AtomicType));
2908 ExtRes = LHS;
2909 break;
2910 case ISD::ZERO_EXTEND:
2911 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2912 DAG.getValueType(AtomicType));
2913 RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
2914 ExtRes = LHS;
2915 break;
2916 case ISD::ANY_EXTEND:
2917 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2918 RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
2919 break;
2920 default:
2921 llvm_unreachable("Invalid atomic op extension")::llvm::llvm_unreachable_internal("Invalid atomic op extension"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2921)
;
2922 }
2923
2924 SDValue Success =
2925 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2926
2927 Results.push_back(ExtRes.getValue(0));
2928 Results.push_back(Success);
2929 Results.push_back(Res.getValue(1));
2930 break;
2931 }
2932 case ISD::DYNAMIC_STACKALLOC:
2933 ExpandDYNAMIC_STACKALLOC(Node, Results);
2934 break;
2935 case ISD::MERGE_VALUES:
2936 for (unsigned i = 0; i < Node->getNumValues(); i++)
2937 Results.push_back(Node->getOperand(i));
2938 break;
2939 case ISD::UNDEF: {
2940 EVT VT = Node->getValueType(0);
2941 if (VT.isInteger())
2942 Results.push_back(DAG.getConstant(0, dl, VT));
2943 else {
2944 assert(VT.isFloatingPoint() && "Unknown value type!")((VT.isFloatingPoint() && "Unknown value type!") ? static_cast
<void> (0) : __assert_fail ("VT.isFloatingPoint() && \"Unknown value type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2944, __PRETTY_FUNCTION__))
;
2945 Results.push_back(DAG.getConstantFP(0, dl, VT));
2946 }
2947 break;
2948 }
2949 case ISD::FP_ROUND:
2950 case ISD::BITCAST:
2951 Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2952 Node->getValueType(0), dl);
2953 Results.push_back(Tmp1);
2954 break;
2955 case ISD::FP_EXTEND:
2956 Tmp1 = EmitStackConvert(Node->getOperand(0),
2957 Node->getOperand(0).getValueType(),
2958 Node->getValueType(0), dl);
2959 Results.push_back(Tmp1);
2960 break;
2961 case ISD::SIGN_EXTEND_INREG: {
2962 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2963 EVT VT = Node->getValueType(0);
2964
2965 // An in-register sign-extend of a boolean is a negation:
2966 // 'true' (1) sign-extended is -1.
2967 // 'false' (0) sign-extended is 0.
2968 // However, we must mask the high bits of the source operand because the
2969 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2970
2971 // TODO: Do this for vectors too?
2972 if (ExtraVT.getSizeInBits() == 1) {
2973 SDValue One = DAG.getConstant(1, dl, VT);
2974 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2975 SDValue Zero = DAG.getConstant(0, dl, VT);
2976 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2977 Results.push_back(Neg);
2978 break;
2979 }
2980
2981 // NOTE: we could fall back on load/store here too for targets without
2982 // SRA. However, it is doubtful that any exist.
2983 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2984 if (VT.isVector())
2985 ShiftAmountTy = VT;
2986 unsigned BitsDiff = VT.getScalarSizeInBits() -
2987 ExtraVT.getScalarSizeInBits();
2988 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2989 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2990 Node->getOperand(0), ShiftCst);
2991 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2992 Results.push_back(Tmp1);
2993 break;
2994 }
2995 case ISD::FP_ROUND_INREG: {
2996 // The only way we can lower this is to turn it into a TRUNCSTORE,
2997 // EXTLOAD pair, targeting a temporary location (a stack slot).
2998
2999 // NOTE: there is a choice here between constantly creating new stack
3000 // slots and always reusing the same one. We currently always create
3001 // new ones, as reuse may inhibit scheduling.
3002 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3003 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
3004 Node->getValueType(0), dl);
3005 Results.push_back(Tmp1);
3006 break;
3007 }
3008 case ISD::SINT_TO_FP:
3009 case ISD::UINT_TO_FP:
3010 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
3011 Node->getOperand(0), Node->getValueType(0), dl);
3012 Results.push_back(Tmp1);
3013 break;
3014 case ISD::FP_TO_SINT:
3015 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3016 Results.push_back(Tmp1);
3017 break;
3018 case ISD::FP_TO_UINT: {
3019 SDValue True, False;
3020 EVT VT = Node->getOperand(0).getValueType();
3021 EVT NVT = Node->getValueType(0);
3022 APFloat apf(DAG.EVTToAPFloatSemantics(VT),
3023 APInt::getNullValue(VT.getSizeInBits()));
3024 APInt x = APInt::getSignMask(NVT.getSizeInBits());
3025 (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
3026 Tmp1 = DAG.getConstantFP(apf, dl, VT);
3027 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT),
3028 Node->getOperand(0),
3029 Tmp1, ISD::SETLT);
3030 True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
3031 // TODO: Should any fast-math-flags be set for the FSUB?
3032 False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
3033 DAG.getNode(ISD::FSUB, dl, VT,
3034 Node->getOperand(0), Tmp1));
3035 False = DAG.getNode(ISD::XOR, dl, NVT, False,
3036 DAG.getConstant(x, dl, NVT));
3037 Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False);
3038 Results.push_back(Tmp1);
3039 break;
3040 }
3041 case ISD::VAARG:
3042 Results.push_back(DAG.expandVAArg(Node));
3043 Results.push_back(Results[0].getValue(1));
3044 break;
3045 case ISD::VACOPY:
3046 Results.push_back(DAG.expandVACopy(Node));
3047 break;
3048 case ISD::EXTRACT_VECTOR_ELT:
3049 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
3050 // This must be an access of the only element. Return it.
3051 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3052 Node->getOperand(0));
3053 else
3054 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3055 Results.push_back(Tmp1);
3056 break;
3057 case ISD::EXTRACT_SUBVECTOR:
3058 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3059 break;
3060 case ISD::INSERT_SUBVECTOR:
3061 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3062 break;
3063 case ISD::CONCAT_VECTORS: {
3064 Results.push_back(ExpandVectorBuildThroughStack(Node));
3065 break;
3066 }
3067 case ISD::SCALAR_TO_VECTOR:
3068 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3069 break;
3070 case ISD::INSERT_VECTOR_ELT:
3071 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3072 Node->getOperand(1),
3073 Node->getOperand(2), dl));
3074 break;
3075 case ISD::VECTOR_SHUFFLE: {
3076 SmallVector<int, 32> NewMask;
3077 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3078
3079 EVT VT = Node->getValueType(0);
3080 EVT EltVT = VT.getVectorElementType();
3081 SDValue Op0 = Node->getOperand(0);
3082 SDValue Op1 = Node->getOperand(1);
3083 if (!TLI.isTypeLegal(EltVT)) {
3084
3085 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3086
3087 // BUILD_VECTOR operands are allowed to be wider than the element type.
3088 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3089 // it.
3090 if (NewEltVT.bitsLT(EltVT)) {
3091
3092 // Convert shuffle node.
3093 // If original node was v4i64 and the new EltVT is i32,
3094 // cast operands to v8i32 and re-build the mask.
3095
3096 // Calculate new VT, the size of the new VT should be equal to original.
3097 EVT NewVT =
3098 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3099 VT.getSizeInBits() / NewEltVT.getSizeInBits());
3100 assert(NewVT.bitsEq(VT))((NewVT.bitsEq(VT)) ? static_cast<void> (0) : __assert_fail
("NewVT.bitsEq(VT)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3100, __PRETTY_FUNCTION__))
;
3101
3102 // cast operands to new VT
3103 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3104 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3105
3106 // Convert the shuffle mask
3107 unsigned int factor =
3108 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3109
3110 // EltVT gets smaller
3111 assert(factor > 0)((factor > 0) ? static_cast<void> (0) : __assert_fail
("factor > 0", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3111, __PRETTY_FUNCTION__))
;
3112
3113 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3114 if (Mask[i] < 0) {
3115 for (unsigned fi = 0; fi < factor; ++fi)
3116 NewMask.push_back(Mask[i]);
3117 }
3118 else {
3119 for (unsigned fi = 0; fi < factor; ++fi)
3120 NewMask.push_back(Mask[i]*factor+fi);
3121 }
3122 }
3123 Mask = NewMask;
3124 VT = NewVT;
3125 }
3126 EltVT = NewEltVT;
3127 }
3128 unsigned NumElems = VT.getVectorNumElements();
3129 SmallVector<SDValue, 16> Ops;
3130 for (unsigned i = 0; i != NumElems; ++i) {
3131 if (Mask[i] < 0) {
3132 Ops.push_back(DAG.getUNDEF(EltVT));
3133 continue;
3134 }
3135 unsigned Idx = Mask[i];
3136 if (Idx < NumElems)
3137 Ops.push_back(DAG.getNode(
3138 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3139 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3140 else
3141 Ops.push_back(DAG.getNode(
3142 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3143 DAG.getConstant(Idx - NumElems, dl,
3144 TLI.getVectorIdxTy(DAG.getDataLayout()))));
3145 }
3146
3147 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3148 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3149 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3150 Results.push_back(Tmp1);
3151 break;
3152 }
3153 case ISD::EXTRACT_ELEMENT: {
3154 EVT OpTy = Node->getOperand(0).getValueType();
3155 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3156 // 1 -> Hi
3157 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3158 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3159 TLI.getShiftAmountTy(
3160 Node->getOperand(0).getValueType(),
3161 DAG.getDataLayout())));
3162 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3163 } else {
3164 // 0 -> Lo
3165 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3166 Node->getOperand(0));
3167 }
3168 Results.push_back(Tmp1);
3169 break;
3170 }
3171 case ISD::STACKSAVE:
3172 // Expand to CopyFromReg if the target set
3173 // StackPointerRegisterToSaveRestore.
3174 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3175 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3176 Node->getValueType(0)));
3177 Results.push_back(Results[0].getValue(1));
3178 } else {
3179 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3180 Results.push_back(Node->getOperand(0));
3181 }
3182 break;
3183 case ISD::STACKRESTORE:
3184 // Expand to CopyToReg if the target set
3185 // StackPointerRegisterToSaveRestore.
3186 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3187 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3188 Node->getOperand(1)));
3189 } else {
3190 Results.push_back(Node->getOperand(0));
3191 }
3192 break;
3193 case ISD::GET_DYNAMIC_AREA_OFFSET:
3194 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3195 Results.push_back(Results[0].getValue(0));
3196 break;
3197 case ISD::FCOPYSIGN:
3198 Results.push_back(ExpandFCOPYSIGN(Node));
3199 break;
3200 case ISD::FNEG:
3201 // Expand Y = FNEG(X) -> Y = SUB -0.0, X
3202 Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3203 // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3204 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3205 Node->getOperand(0));
3206 Results.push_back(Tmp1);
3207 break;
3208 case ISD::FABS:
3209 Results.push_back(ExpandFABS(Node));
3210 break;
3211 case ISD::SMIN:
3212 case ISD::SMAX:
3213 case ISD::UMIN:
3214 case ISD::UMAX: {
3215 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3216 ISD::CondCode Pred;
3217 switch (Node->getOpcode()) {
3218 default: llvm_unreachable("How did we get here?")::llvm::llvm_unreachable_internal("How did we get here?", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3218)
;
3219 case ISD::SMAX: Pred = ISD::SETGT; break;
3220 case ISD::SMIN: Pred = ISD::SETLT; break;
3221 case ISD::UMAX: Pred = ISD::SETUGT; break;
3222 case ISD::UMIN: Pred = ISD::SETULT; break;
3223 }
3224 Tmp1 = Node->getOperand(0);
3225 Tmp2 = Node->getOperand(1);
3226 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3227 Results.push_back(Tmp1);
3228 break;
3229 }
3230
3231 case ISD::FSIN:
3232 case ISD::FCOS: {
3233 EVT VT = Node->getValueType(0);
3234 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3235 // fcos which share the same operand and both are used.
3236 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3237 isSinCosLibcallAvailable(Node, TLI))
3238 && useSinCos(Node)) {
3239 SDVTList VTs = DAG.getVTList(VT, VT);
3240 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3241 if (Node->getOpcode() == ISD::FCOS)
3242 Tmp1 = Tmp1.getValue(1);
3243 Results.push_back(Tmp1);
3244 }
3245 break;
3246 }
3247 case ISD::FMAD:
3248 llvm_unreachable("Illegal fmad should never be formed")::llvm::llvm_unreachable_internal("Illegal fmad should never be formed"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3248)
;
3249
3250 case ISD::FP16_TO_FP:
3251 if (Node->getValueType(0) != MVT::f32) {
3252 // We can extend to types bigger than f32 in two steps without changing
3253 // the result. Since "f16 -> f32" is much more commonly available, give
3254 // CodeGen the option of emitting that before resorting to a libcall.
3255 SDValue Res =
3256 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3257 Results.push_back(
3258 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3259 }
3260 break;
3261 case ISD::FP_TO_FP16:
3262 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3263 SDValue Op = Node->getOperand(0);
3264 MVT SVT = Op.getSimpleValueType();
3265 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3266 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3267 // Under fastmath, we can expand this node into a fround followed by
3268 // a float-half conversion.
3269 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3270 DAG.getIntPtrConstant(0, dl));
3271 Results.push_back(
3272 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3273 }
3274 }
3275 break;
3276 case ISD::ConstantFP: {
3277 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3278 // Check to see if this FP immediate is already legal.
3279 // If this is a legal constant, turn it into a TargetConstantFP node.
3280 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
3281 Results.push_back(ExpandConstantFP(CFP, true));
3282 break;
3283 }
3284 case ISD::Constant: {
3285 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3286 Results.push_back(ExpandConstant(CP));
3287 break;
3288 }
3289 case ISD::FSUB: {
3290 EVT VT = Node->getValueType(0);
3291 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3292 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3293 const SDNodeFlags Flags = Node->getFlags();
3294 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3295 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3296 Results.push_back(Tmp1);
3297 }
3298 break;
3299 }
3300 case ISD::SUB: {
3301 EVT VT = Node->getValueType(0);
3302 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3304, __PRETTY_FUNCTION__))
3303 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3304, __PRETTY_FUNCTION__))
3304 "Don't know how to expand this subtraction!")((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3304, __PRETTY_FUNCTION__))
;
3305 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3306 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3307 VT));
3308 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3309 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3310 break;
3311 }
3312 case ISD::UREM:
3313 case ISD::SREM: {
3314 EVT VT = Node->getValueType(0);
3315 bool isSigned = Node->getOpcode() == ISD::SREM;
3316 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3317 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3318 Tmp2 = Node->getOperand(0);
3319 Tmp3 = Node->getOperand(1);
3320 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3321 SDVTList VTs = DAG.getVTList(VT, VT);
3322 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3323 Results.push_back(Tmp1);
3324 } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3325 // X % Y -> X-X/Y*Y
3326 Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3327 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3328 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3329 Results.push_back(Tmp1);
3330 }
3331 break;
3332 }
3333 case ISD::UDIV:
3334 case ISD::SDIV: {
3335 bool isSigned = Node->getOpcode() == ISD::SDIV;
3336 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3337 EVT VT = Node->getValueType(0);
3338 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3339 SDVTList VTs = DAG.getVTList(VT, VT);
3340 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3341 Node->getOperand(1));
3342 Results.push_back(Tmp1);
3343 }
3344 break;
3345 }
3346 case ISD::MULHU:
3347 case ISD::MULHS: {
3348 unsigned ExpandOpcode =
3349 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3350 EVT VT = Node->getValueType(0);
3351 SDVTList VTs = DAG.getVTList(VT, VT);
3352
3353 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3354 Node->getOperand(1));
3355 Results.push_back(Tmp1.getValue(1));
3356 break;
3357 }
3358 case ISD::UMUL_LOHI:
3359 case ISD::SMUL_LOHI: {
3360 SDValue LHS = Node->getOperand(0);
3361 SDValue RHS = Node->getOperand(1);
3362 MVT VT = LHS.getSimpleValueType();
3363 unsigned MULHOpcode =
3364 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3365
3366 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3367 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3368 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3369 break;
3370 }
3371
3372 SmallVector<SDValue, 4> Halves;
3373 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3374 assert(TLI.isTypeLegal(HalfType))((TLI.isTypeLegal(HalfType)) ? static_cast<void> (0) : __assert_fail
("TLI.isTypeLegal(HalfType)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3374, __PRETTY_FUNCTION__))
;
3375 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
3376 HalfType, DAG,
3377 TargetLowering::MulExpansionKind::Always)) {
3378 for (unsigned i = 0; i < 2; ++i) {
3379 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3380 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3381 SDValue Shift = DAG.getConstant(
3382 HalfType.getScalarSizeInBits(), dl,
3383 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3384 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3385 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3386 }
3387 break;
3388 }
3389 break;
3390 }
3391 case ISD::MUL: {
3392 EVT VT = Node->getValueType(0);
3393 SDVTList VTs = DAG.getVTList(VT, VT);
3394 // See if multiply or divide can be lowered using two-result operations.
3395 // We just need the low half of the multiply; try both the signed
3396 // and unsigned forms. If the target supports both SMUL_LOHI and
3397 // UMUL_LOHI, form a preference by checking which forms of plain
3398 // MULH it supports.
3399 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3400 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3401 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3402 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3403 unsigned OpToUse = 0;
3404 if (HasSMUL_LOHI && !HasMULHS) {
3405 OpToUse = ISD::SMUL_LOHI;
3406 } else if (HasUMUL_LOHI && !HasMULHU) {
3407 OpToUse = ISD::UMUL_LOHI;
3408 } else if (HasSMUL_LOHI) {
3409 OpToUse = ISD::SMUL_LOHI;
3410 } else if (HasUMUL_LOHI) {
3411 OpToUse = ISD::UMUL_LOHI;
3412 }
3413 if (OpToUse) {
3414 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3415 Node->getOperand(1)));
3416 break;
3417 }
3418
3419 SDValue Lo, Hi;
3420 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3421 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3422 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3423 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3424 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3425 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3426 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3427 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3428 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3429 SDValue Shift =
3430 DAG.getConstant(HalfType.getSizeInBits(), dl,
3431 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3432 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3433 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3434 }
3435 break;
3436 }
3437 case ISD::SADDO:
3438 case ISD::SSUBO: {
3439 SDValue LHS = Node->getOperand(0);
3440 SDValue RHS = Node->getOperand(1);
3441 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3442 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3443 LHS, RHS);
3444 Results.push_back(Sum);
3445 EVT ResultType = Node->getValueType(1);
3446 EVT OType = getSetCCResultType(Node->getValueType(0));
3447
3448 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3449
3450 // LHSSign -> LHS >= 0
3451 // RHSSign -> RHS >= 0
3452 // SumSign -> Sum >= 0
3453 //
3454 // Add:
3455 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3456 // Sub:
3457 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3458 //
3459 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3460 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3461 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3462 Node->getOpcode() == ISD::SADDO ?
3463 ISD::SETEQ : ISD::SETNE);
3464
3465 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3466 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3467
3468 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3469 Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
3470 break;
3471 }
3472 case ISD::UADDO:
3473 case ISD::USUBO: {
3474 SDValue LHS = Node->getOperand(0);
3475 SDValue RHS = Node->getOperand(1);
3476 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ?
3477 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3478 LHS, RHS);
3479 Results.push_back(Sum);
3480
3481 EVT ResultType = Node->getValueType(1);
3482 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3483 ISD::CondCode CC
3484 = Node->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
3485 SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3486
3487 Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
3488 break;
3489 }
3490 case ISD::UMULO:
3491 case ISD::SMULO: {
3492 EVT VT = Node->getValueType(0);
3493 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
3494 SDValue LHS = Node->getOperand(0);
3495 SDValue RHS = Node->getOperand(1);
3496 SDValue BottomHalf;
3497 SDValue TopHalf;
3498 static const unsigned Ops[2][3] =
3499 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
3500 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
3501 bool isSigned = Node->getOpcode() == ISD::SMULO;
3502 if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
3503 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3504 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
3505 } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
3506 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
3507 RHS);
3508 TopHalf = BottomHalf.getValue(1);
3509 } else if (TLI.isTypeLegal(WideVT)) {
3510 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
3511 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
3512 Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3513 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3514 DAG.getIntPtrConstant(0, dl));
3515 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3516 DAG.getIntPtrConstant(1, dl));
3517 } else {
3518 // We can fall back to a libcall with an illegal type for the MUL if we
3519 // have a libcall big enough.
3520 // Also, we can fall back to a division in some cases, but that's a big
3521 // performance hit in the general case.
3522 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3523 if (WideVT == MVT::i16)
3524 LC = RTLIB::MUL_I16;
3525 else if (WideVT == MVT::i32)
3526 LC = RTLIB::MUL_I32;
3527 else if (WideVT == MVT::i64)
3528 LC = RTLIB::MUL_I64;
3529 else if (WideVT == MVT::i128)
3530 LC = RTLIB::MUL_I128;
3531 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!")((LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!"
) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Cannot expand this operation!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3531, __PRETTY_FUNCTION__))
;
3532
3533 SDValue HiLHS;
3534 SDValue HiRHS;
3535 if (isSigned) {
3536 // The high part is obtained by SRA'ing all but one of the bits of low
3537 // part.
3538 unsigned LoSize = VT.getSizeInBits();
3539 HiLHS =
3540 DAG.getNode(ISD::SRA, dl, VT, LHS,
3541 DAG.getConstant(LoSize - 1, dl,
3542 TLI.getPointerTy(DAG.getDataLayout())));
3543 HiRHS =
3544 DAG.getNode(ISD::SRA, dl, VT, RHS,
3545 DAG.getConstant(LoSize - 1, dl,
3546 TLI.getPointerTy(DAG.getDataLayout())));
3547 } else {
3548 HiLHS = DAG.getConstant(0, dl, VT);
3549 HiRHS = DAG.getConstant(0, dl, VT);
3550 }
3551
3552 // Here we're passing the 2 arguments explicitly as 4 arguments that are
3553 // pre-lowered to the correct types. This all depends upon WideVT not
3554 // being a legal type for the architecture and thus has to be split to
3555 // two arguments.
3556 SDValue Ret;
3557 if(DAG.getDataLayout().isLittleEndian()) {
3558 // Halves of WideVT are packed into registers in different order
3559 // depending on platform endianness. This is usually handled by
3560 // the C calling convention, but we can't defer to it in
3561 // the legalizer.
3562 SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
3563 Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3564 } else {
3565 SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
3566 Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3567 }
3568 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret,
3569 DAG.getIntPtrConstant(0, dl));
3570 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret,
3571 DAG.getIntPtrConstant(1, dl));
3572 // Ret is a node with an illegal type. Because such things are not
3573 // generally permitted during this phase of legalization, make sure the
3574 // node has no more uses. The above EXTRACT_ELEMENT nodes should have been
3575 // folded.
3576 assert(Ret->use_empty() &&((Ret->use_empty() && "Unexpected uses of illegally type from expanded lib call."
) ? static_cast<void> (0) : __assert_fail ("Ret->use_empty() && \"Unexpected uses of illegally type from expanded lib call.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3577, __PRETTY_FUNCTION__))
3577 "Unexpected uses of illegally type from expanded lib call.")((Ret->use_empty() && "Unexpected uses of illegally type from expanded lib call."
) ? static_cast<void> (0) : __assert_fail ("Ret->use_empty() && \"Unexpected uses of illegally type from expanded lib call.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3577, __PRETTY_FUNCTION__))
;
3578 }
3579
3580 if (isSigned) {
3581 Tmp1 = DAG.getConstant(
3582 VT.getSizeInBits() - 1, dl,
3583 TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
3584 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3585 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
3586 ISD::SETNE);
3587 } else {
3588 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
3589 DAG.getConstant(0, dl, VT), ISD::SETNE);
3590 }
3591
3592 // Truncate the result if SetCC returns a larger type than needed.
3593 EVT RType = Node->getValueType(1);
3594 if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
3595 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
3596
3597 assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&((RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
"Unexpected result type for S/UMULO legalization") ? static_cast
<void> (0) : __assert_fail ("RType.getSizeInBits() == TopHalf.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3598, __PRETTY_FUNCTION__))
3598 "Unexpected result type for S/UMULO legalization")((RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
"Unexpected result type for S/UMULO legalization") ? static_cast
<void> (0) : __assert_fail ("RType.getSizeInBits() == TopHalf.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3598, __PRETTY_FUNCTION__))
;
3599
3600 Results.push_back(BottomHalf);
3601 Results.push_back(TopHalf);
3602 break;
3603 }
3604 case ISD::BUILD_PAIR: {
3605 EVT PairTy = Node->getValueType(0);
3606 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3607 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3608 Tmp2 = DAG.getNode(
3609 ISD::SHL, dl, PairTy, Tmp2,
3610 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3611 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3612 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3613 break;
3614 }
3615 case ISD::SELECT:
3616 Tmp1 = Node->getOperand(0);
3617 Tmp2 = Node->getOperand(1);
3618 Tmp3 = Node->getOperand(2);
3619 if (Tmp1.getOpcode() == ISD::SETCC) {
3620 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3621 Tmp2, Tmp3,
3622 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3623 } else {
3624 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3625 DAG.getConstant(0, dl, Tmp1.getValueType()),
3626 Tmp2, Tmp3, ISD::SETNE);
3627 }
3628 Results.push_back(Tmp1);
3629 break;
3630 case ISD::BR_JT: {
3631 SDValue Chain = Node->getOperand(0);
3632 SDValue Table = Node->getOperand(1);
3633 SDValue Index = Node->getOperand(2);
3634
3635 const DataLayout &TD = DAG.getDataLayout();
3636 EVT PTy = TLI.getPointerTy(TD);
3637
3638 unsigned EntrySize =
3639 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3640
3641 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3642 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3643 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3644 Index, Table);
3645
3646 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3647 SDValue LD = DAG.getExtLoad(
3648 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3649 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3650 Addr = LD;
3651 if (TLI.isJumpTableRelative()) {
3652 // For PIC, the sequence is:
3653 // BRIND(load(Jumptable + index) + RelocBase)
3654 // RelocBase can be JumpTable, GOT or some sort of global base.
3655 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3656 TLI.getPICJumpTableRelocBase(Table, DAG));
3657 }
3658 Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr);
3659 Results.push_back(Tmp1);
3660 break;
3661 }
3662 case ISD::BRCOND:
3663 // Expand brcond's setcc into its constituent parts and create a BR_CC
3664 // Node.
3665 Tmp1 = Node->getOperand(0);
3666 Tmp2 = Node->getOperand(1);
3667 if (Tmp2.getOpcode() == ISD::SETCC) {
3668 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3669 Tmp1, Tmp2.getOperand(2),
3670 Tmp2.getOperand(0), Tmp2.getOperand(1),
3671 Node->getOperand(2));
3672 } else {
3673 // We test only the i1 bit. Skip the AND if UNDEF.
3674 Tmp3 = (Tmp2.isUndef()) ? Tmp2 :
3675 DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3676 DAG.getConstant(1, dl, Tmp2.getValueType()));
3677 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3678 DAG.getCondCode(ISD::SETNE), Tmp3,
3679 DAG.getConstant(0, dl, Tmp3.getValueType()),
3680 Node->getOperand(2));
3681 }
3682 Results.push_back(Tmp1);
3683 break;
3684 case ISD::SETCC: {
3685 Tmp1 = Node->getOperand(0);
3686 Tmp2 = Node->getOperand(1);
3687 Tmp3 = Node->getOperand(2);
3688 bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3689 Tmp3, NeedInvert, dl);
3690
3691 if (Legalized) {
3692 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3693 // condition code, create a new SETCC node.
3694 if (Tmp3.getNode())
3695 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3696 Tmp1, Tmp2, Tmp3);
3697
3698 // If we expanded the SETCC by inverting the condition code, then wrap
3699 // the existing SETCC in a NOT to restore the intended condition.
3700 if (NeedInvert)
3701 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3702
3703 Results.push_back(Tmp1);
3704 break;
3705 }
3706
3707 // Otherwise, SETCC for the given comparison type must be completely
3708 // illegal; expand it into a SELECT_CC.
3709 EVT VT = Node->getValueType(0);
3710 int TrueValue;
3711 switch (TLI.getBooleanContents(Tmp1->getValueType(0))) {
3712 case TargetLowering::ZeroOrOneBooleanContent:
3713 case TargetLowering::UndefinedBooleanContent:
3714 TrueValue = 1;
3715 break;
3716 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3717 TrueValue = -1;
3718 break;
3719 }
3720 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3721 DAG.getConstant(TrueValue, dl, VT),
3722 DAG.getConstant(0, dl, VT),
3723 Tmp3);
3724 Results.push_back(Tmp1);
3725 break;
3726 }
3727 case ISD::SELECT_CC: {
3728 Tmp1 = Node->getOperand(0); // LHS
3729 Tmp2 = Node->getOperand(1); // RHS
3730 Tmp3 = Node->getOperand(2); // True
3731 Tmp4 = Node->getOperand(3); // False
3732 EVT VT = Node->getValueType(0);
3733 SDValue CC = Node->getOperand(4);
3734 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3735
3736 if (TLI.isCondCodeLegal(CCOp, Tmp1.getSimpleValueType())) {
3737 // If the condition code is legal, then we need to expand this
3738 // node using SETCC and SELECT.
3739 EVT CmpVT = Tmp1.getValueType();
3740 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3742, __PRETTY_FUNCTION__))
3741 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3742, __PRETTY_FUNCTION__))
3742 "expanded.")((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3742, __PRETTY_FUNCTION__))
;
3743 EVT CCVT =
3744 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
3745 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
3746 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3747 break;
3748 }
3749
3750 // SELECT_CC is legal, so the condition code must not be.
3751 bool Legalized = false;
3752 // Try to legalize by inverting the condition. This is for targets that
3753 // might support an ordered version of a condition, but not the unordered
3754 // version (or vice versa).
3755 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3756 Tmp1.getValueType().isInteger());
3757 if (TLI.isCondCodeLegal(InvCC, Tmp1.getSimpleValueType())) {
3758 // Use the new condition code and swap true and false
3759 Legalized = true;
3760 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3761 } else {
3762 // If The inverse is not legal, then try to swap the arguments using
3763 // the inverse condition code.
3764 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3765 if (TLI.isCondCodeLegal(SwapInvCC, Tmp1.getSimpleValueType())) {
3766 // The swapped inverse condition is legal, so swap true and false,
3767 // lhs and rhs.
3768 Legalized = true;
3769 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3770 }
3771 }
3772
3773 if (!Legalized) {
3774 Legalized = LegalizeSetCCCondCode(
3775 getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3776 dl);
3777
3778 assert(Legalized && "Can't legalize SELECT_CC with legal condition!")((Legalized && "Can't legalize SELECT_CC with legal condition!"
) ? static_cast<void> (0) : __assert_fail ("Legalized && \"Can't legalize SELECT_CC with legal condition!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3778, __PRETTY_FUNCTION__))
;
3779
3780 // If we expanded the SETCC by inverting the condition code, then swap
3781 // the True/False operands to match.
3782 if (NeedInvert)
3783 std::swap(Tmp3, Tmp4);
3784
3785 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3786 // condition code, create a new SELECT_CC node.
3787 if (CC.getNode()) {
3788 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3789 Tmp1, Tmp2, Tmp3, Tmp4, CC);
3790 } else {
3791 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3792 CC = DAG.getCondCode(ISD::SETNE);
3793 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3794 Tmp2, Tmp3, Tmp4, CC);
3795 }
3796 }
3797 Results.push_back(Tmp1);
3798 break;
3799 }
3800 case ISD::BR_CC: {
3801 Tmp1 = Node->getOperand(0); // Chain
3802 Tmp2 = Node->getOperand(2); // LHS
3803 Tmp3 = Node->getOperand(3); // RHS
3804 Tmp4 = Node->getOperand(1); // CC
3805
3806 bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
3807 Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
3808 (void)Legalized;
3809 assert(Legalized && "Can't legalize BR_CC with legal condition!")((Legalized && "Can't legalize BR_CC with legal condition!"
) ? static_cast<void> (0) : __assert_fail ("Legalized && \"Can't legalize BR_CC with legal condition!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3809, __PRETTY_FUNCTION__))
;
3810
3811 // If we expanded the SETCC by inverting the condition code, then wrap
3812 // the existing SETCC in a NOT to restore the intended condition.
3813 if (NeedInvert)
3814 Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
3815
3816 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3817 // node.
3818 if (Tmp4.getNode()) {
3819 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3820 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3821 } else {
3822 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3823 Tmp4 = DAG.getCondCode(ISD::SETNE);
3824 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3825 Tmp2, Tmp3, Node->getOperand(4));
3826 }
3827 Results.push_back(Tmp1);
3828 break;
3829 }
3830 case ISD::BUILD_VECTOR:
3831 Results.push_back(ExpandBUILD_VECTOR(Node));
3832 break;
3833 case ISD::SRA:
3834 case ISD::SRL:
3835 case ISD::SHL: {
3836 // Scalarize vector SRA/SRL/SHL.
3837 EVT VT = Node->getValueType(0);
3838 assert(VT.isVector() && "Unable to legalize non-vector shift")((VT.isVector() && "Unable to legalize non-vector shift"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Unable to legalize non-vector shift\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3838, __PRETTY_FUNCTION__))
;
3839 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal")((TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"
) ? static_cast<void> (0) : __assert_fail ("TLI.isTypeLegal(VT.getScalarType())&& \"Element type must be legal\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3839, __PRETTY_FUNCTION__))
;
3840 unsigned NumElem = VT.getVectorNumElements();
3841
3842 SmallVector<SDValue, 8> Scalars;
3843 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3844 SDValue Ex = DAG.getNode(
3845 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3846 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3847 SDValue Sh = DAG.getNode(
3848 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3849 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3850 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3851 VT.getScalarType(), Ex, Sh));
3852 }
3853
3854 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3855 ReplaceNode(SDValue(Node, 0), Result);
3856 break;
3857 }
3858 case ISD::GLOBAL_OFFSET_TABLE:
3859 case ISD::GlobalAddress:
3860 case ISD::GlobalTLSAddress:
3861 case ISD::ExternalSymbol:
3862 case ISD::ConstantPool:
3863 case ISD::JumpTable:
3864 case ISD::INTRINSIC_W_CHAIN:
3865 case ISD::INTRINSIC_WO_CHAIN:
3866 case ISD::INTRINSIC_VOID:
3867 // FIXME: Custom lowering for these operations shouldn't return null!
3868 break;
3869 }
3870
3871 // Replace the original node with the legalized result.
3872 if (Results.empty())
3873 return false;
3874
3875 ReplaceNode(Node, Results.data());
3876 return true;
3877}
3878
3879void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3880 SmallVector<SDValue, 8> Results;
3881 SDLoc dl(Node);
3882 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3883 unsigned Opc = Node->getOpcode();
3884 switch (Opc) {
3885 case ISD::ATOMIC_FENCE: {
3886 // If the target didn't lower this, lower it to '__sync_synchronize()' call
3887 // FIXME: handle "fence singlethread" more efficiently.
3888 TargetLowering::ArgListTy Args;
3889
3890 TargetLowering::CallLoweringInfo CLI(DAG);
3891 CLI.setDebugLoc(dl)
3892 .setChain(Node->getOperand(0))
3893 .setLibCallee(
3894 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3895 DAG.getExternalSymbol("__sync_synchronize",
3896 TLI.getPointerTy(DAG.getDataLayout())),
3897 std::move(Args));
3898
3899 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3900
3901 Results.push_back(CallResult.second);
3902 break;
3903 }
3904 // By default, atomic intrinsics are marked Legal and lowered. Targets
3905 // which don't support them directly, however, may want libcalls, in which
3906 // case they mark them Expand, and we get here.
3907 case ISD::ATOMIC_SWAP:
3908 case ISD::ATOMIC_LOAD_ADD:
3909 case ISD::ATOMIC_LOAD_SUB:
3910 case ISD::ATOMIC_LOAD_AND:
3911 case ISD::ATOMIC_LOAD_OR:
3912 case ISD::ATOMIC_LOAD_XOR:
3913 case ISD::ATOMIC_LOAD_NAND:
3914 case ISD::ATOMIC_LOAD_MIN:
3915 case ISD::ATOMIC_LOAD_MAX:
3916 case ISD::ATOMIC_LOAD_UMIN:
3917 case ISD::ATOMIC_LOAD_UMAX:
3918 case ISD::ATOMIC_CMP_SWAP: {
3919 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3920 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
3921 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!")((LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"
) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected atomic op or value type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3921, __PRETTY_FUNCTION__))
;
3922
3923 std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
3924 Results.push_back(Tmp.first);
3925 Results.push_back(Tmp.second);
3926 break;
3927 }
3928 case ISD::TRAP: {
3929 // If this operation is not supported, lower it to 'abort()' call
3930 TargetLowering::ArgListTy Args;
3931 TargetLowering::CallLoweringInfo CLI(DAG);
3932 CLI.setDebugLoc(dl)
3933 .setChain(Node->getOperand(0))
3934 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3935 DAG.getExternalSymbol(
3936 "abort", TLI.getPointerTy(DAG.getDataLayout())),
3937 std::move(Args));
3938 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3939
3940 Results.push_back(CallResult.second);
3941 break;
3942 }
3943 case ISD::FMINNUM:
3944 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3945 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3946 RTLIB::FMIN_PPCF128));
3947 break;
3948 case ISD::FMAXNUM:
3949 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3950 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3951 RTLIB::FMAX_PPCF128));
3952 break;
3953 case ISD::FSQRT:
3954 case ISD::STRICT_FSQRT:
3955 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3956 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3957 RTLIB::SQRT_PPCF128));
3958 break;
3959 case ISD::FSIN:
3960 case ISD::STRICT_FSIN:
3961 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
3962 RTLIB::SIN_F80, RTLIB::SIN_F128,
3963 RTLIB::SIN_PPCF128));
3964 break;
3965 case ISD::FCOS:
3966 case ISD::STRICT_FCOS:
3967 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
3968 RTLIB::COS_F80, RTLIB::COS_F128,
3969 RTLIB::COS_PPCF128));
3970 break;
3971 case ISD::FSINCOS:
3972 // Expand into sincos libcall.
3973 ExpandSinCosLibCall(Node, Results);
3974 break;
3975 case ISD::FLOG:
3976 case ISD::STRICT_FLOG:
3977 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
3978 RTLIB::LOG_F80, RTLIB::LOG_F128,
3979 RTLIB::LOG_PPCF128));
3980 break;
3981 case ISD::FLOG2:
3982 case ISD::STRICT_FLOG2:
3983 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
3984 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
3985 RTLIB::LOG2_PPCF128));
3986 break;
3987 case ISD::FLOG10:
3988 case ISD::STRICT_FLOG10:
3989 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
3990 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
3991 RTLIB::LOG10_PPCF128));
3992 break;
3993 case ISD::FEXP:
3994 case ISD::STRICT_FEXP:
3995 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
3996 RTLIB::EXP_F80, RTLIB::EXP_F128,
3997 RTLIB::EXP_PPCF128));
3998 break;
3999 case ISD::FEXP2:
4000 case ISD::STRICT_FEXP2:
4001 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
4002 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
4003 RTLIB::EXP2_PPCF128));
4004 break;
4005 case ISD::FTRUNC:
4006 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4007 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4008 RTLIB::TRUNC_PPCF128));
4009 break;
4010 case ISD::FFLOOR:
4011 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4012 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4013 RTLIB::FLOOR_PPCF128));
4014 break;
4015 case ISD::FCEIL:
4016 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4017 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4018 RTLIB::CEIL_PPCF128));
4019 break;
4020 case ISD::FRINT:
4021 case ISD::STRICT_FRINT:
4022 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4023 RTLIB::RINT_F80, RTLIB::RINT_F128,
4024 RTLIB::RINT_PPCF128));
4025 break;
4026 case ISD::FNEARBYINT:
4027 case ISD::STRICT_FNEARBYINT:
4028 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4029 RTLIB::NEARBYINT_F64,
4030 RTLIB::NEARBYINT_F80,
4031 RTLIB::NEARBYINT_F128,
4032 RTLIB::NEARBYINT_PPCF128));
4033 break;
4034 case ISD::FROUND:
4035 Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4036 RTLIB::ROUND_F64,
4037 RTLIB::ROUND_F80,
4038 RTLIB::ROUND_F128,
4039 RTLIB::ROUND_PPCF128));
4040 break;
4041 case ISD::FPOWI:
4042 case ISD::STRICT_FPOWI:
4043 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
4044 RTLIB::POWI_F80, RTLIB::POWI_F128,
4045 RTLIB::POWI_PPCF128));
4046 break;
4047 case ISD::FPOW:
4048 case ISD::STRICT_FPOW:
4049 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
4050 RTLIB::POW_F80, RTLIB::POW_F128,
4051 RTLIB::POW_PPCF128));
4052 break;
4053 case ISD::FDIV:
4054 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4055 RTLIB::DIV_F80, RTLIB::DIV_F128,
4056 RTLIB::DIV_PPCF128));
4057 break;
4058 case ISD::FREM:
4059 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4060 RTLIB::REM_F80, RTLIB::REM_F128,
4061 RTLIB::REM_PPCF128));
4062 break;
4063 case ISD::FMA:
4064 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4065 RTLIB::FMA_F80, RTLIB::FMA_F128,
4066 RTLIB::FMA_PPCF128));
4067 break;
4068 case ISD::FADD:
4069 Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4070 RTLIB::ADD_F80, RTLIB::ADD_F128,
4071 RTLIB::ADD_PPCF128));
4072 break;
4073 case ISD::FMUL:
4074 Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4075 RTLIB::MUL_F80, RTLIB::MUL_F128,
4076 RTLIB::MUL_PPCF128));
4077 break;
4078 case ISD::FP16_TO_FP:
4079 if (Node->getValueType(0) == MVT::f32) {
4080 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4081 }
4082 break;
4083 case ISD::FP_TO_FP16: {
4084 RTLIB::Libcall LC =
4085 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4086 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16")((LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16"
) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unable to expand fp_to_fp16\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4086, __PRETTY_FUNCTION__))
;
4087 Results.push_back(ExpandLibCall(LC, Node, false));
4088 break;
4089 }
4090 case ISD::FSUB:
4091 Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4092 RTLIB::SUB_F80, RTLIB::SUB_F128,
4093 RTLIB::SUB_PPCF128));
4094 break;
4095 case ISD::SREM:
4096 Results.push_back(ExpandIntLibCall(Node, true,
4097 RTLIB::SREM_I8,
4098 RTLIB::SREM_I16, RTLIB::SREM_I32,
4099 RTLIB::SREM_I64, RTLIB::SREM_I128));
4100 break;
4101 case ISD::UREM:
4102 Results.push_back(ExpandIntLibCall(Node, false,
4103 RTLIB::UREM_I8,
4104 RTLIB::UREM_I16, RTLIB::UREM_I32,
4105 RTLIB::UREM_I64, RTLIB::UREM_I128));
4106 break;
4107 case ISD::SDIV:
4108 Results.push_back(ExpandIntLibCall(Node, true,
4109 RTLIB::SDIV_I8,
4110 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4111 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4112 break;
4113 case ISD::UDIV:
4114 Results.push_back(ExpandIntLibCall(Node, false,
4115 RTLIB::UDIV_I8,
4116 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4117 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4118 break;
4119 case ISD::SDIVREM:
4120 case ISD::UDIVREM:
4121 // Expand into divrem libcall
4122 ExpandDivRemLibCall(Node, Results);
4123 break;
4124 case ISD::MUL:
4125 Results.push_back(ExpandIntLibCall(Node, false,
4126 RTLIB::MUL_I8,
4127 RTLIB::MUL_I16, RTLIB::MUL_I32,
4128 RTLIB::MUL_I64, RTLIB::MUL_I128));
4129 break;
4130 }
4131
4132 // Replace the original node with the legalized result.
4133 if (!Results.empty())
4134 ReplaceNode(Node, Results.data());
4135}
4136
4137// Determine the vector type to use in place of an original scalar element when
4138// promoting equally sized vectors.
4139static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4140 MVT EltVT, MVT NewEltVT) {
4141 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4142 MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4143 assert(TLI.isTypeLegal(MidVT) && "unexpected")((TLI.isTypeLegal(MidVT) && "unexpected") ? static_cast
<void> (0) : __assert_fail ("TLI.isTypeLegal(MidVT) && \"unexpected\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4143, __PRETTY_FUNCTION__))
;
4144 return MidVT;
4145}
4146
4147void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4148 SmallVector<SDValue, 8> Results;
4149 MVT OVT = Node->getSimpleValueType(0);
4150 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4151 Node->getOpcode() == ISD::SINT_TO_FP ||
4152 Node->getOpcode() == ISD::SETCC ||
4153 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4154 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4155 OVT = Node->getOperand(0).getSimpleValueType();
4156 }
4157 if (Node->getOpcode() == ISD::BR_CC)
4158 OVT = Node->getOperand(2).getSimpleValueType();
4159 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4160 SDLoc dl(Node);
4161 SDValue Tmp1, Tmp2, Tmp3;
4162 switch (Node->getOpcode()) {
4163 case ISD::CTTZ:
4164 case ISD::CTTZ_ZERO_UNDEF:
4165 case ISD::CTLZ:
4166 case ISD::CTLZ_ZERO_UNDEF:
4167 case ISD::CTPOP:
4168 // Zero extend the argument.
4169 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4170 if (Node->getOpcode() == ISD::CTTZ) {
4171 // The count is the same in the promoted type except if the original
4172 // value was zero. This can be handled by setting the bit just off
4173 // the top of the original type.
4174 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4175 OVT.getSizeInBits());
4176 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4177 DAG.getConstant(TopBit, dl, NVT));
4178 }
4179 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4180 // already the correct result.
4181 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4182 if (Node->getOpcode() == ISD::CTLZ ||
4183 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4184 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4185 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4186 DAG.getConstant(NVT.getSizeInBits() -
4187 OVT.getSizeInBits(), dl, NVT));
4188 }
4189 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4190 break;
4191 case ISD::BITREVERSE:
4192 case ISD::BSWAP: {
4193 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4194 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4195 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4196 Tmp1 = DAG.getNode(
4197 ISD::SRL, dl, NVT, Tmp1,
4198 DAG.getConstant(DiffBits, dl,
4199 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4200 Results.push_back(Tmp1);
4201 break;
4202 }
4203 case ISD::FP_TO_UINT:
4204 case ISD::FP_TO_SINT:
4205 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4206 Node->getOpcode() == ISD::FP_TO_SINT, dl);
4207 Results.push_back(Tmp1);
4208 break;
4209 case ISD::UINT_TO_FP:
4210 case ISD::SINT_TO_FP:
4211 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4212 Node->getOpcode() == ISD::SINT_TO_FP, dl);
4213 Results.push_back(Tmp1);
4214 break;
4215 case ISD::VAARG: {
4216 SDValue Chain = Node->getOperand(0); // Get the chain.
4217 SDValue Ptr = Node->getOperand(1); // Get the pointer.
4218
4219 unsigned TruncOp;
4220 if (OVT.isVector()) {
4221 TruncOp = ISD::BITCAST;
4222 } else {
4223 assert(OVT.isInteger()((OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4224, __PRETTY_FUNCTION__))
4224 && "VAARG promotion is supported only for vectors or integer types")((OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4224, __PRETTY_FUNCTION__))
;
4225 TruncOp = ISD::TRUNCATE;
4226 }
4227
4228 // Perform the larger operation, then convert back
4229 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4230 Node->getConstantOperandVal(3));
4231 Chain = Tmp1.getValue(1);
4232
4233 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4234
4235 // Modified the chain result - switch anything that used the old chain to
4236 // use the new one.
4237 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4238 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4239 if (UpdatedNodes) {
4240 UpdatedNodes->insert(Tmp2.getNode());
4241 UpdatedNodes->insert(Chain.getNode());
4242 }
4243 ReplacedNode(Node);
4244 break;
4245 }
4246 case ISD::MUL:
4247 case ISD::SDIV:
4248 case ISD::SREM:
4249 case ISD::UDIV:
4250 case ISD::UREM:
4251 case ISD::AND:
4252 case ISD::OR:
4253 case ISD::XOR: {
4254 unsigned ExtOp, TruncOp;
4255 if (OVT.isVector()) {
4256 ExtOp = ISD::BITCAST;
4257 TruncOp = ISD::BITCAST;
4258 } else {
4259 assert(OVT.isInteger() && "Cannot promote logic operation")((OVT.isInteger() && "Cannot promote logic operation"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"Cannot promote logic operation\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4259, __PRETTY_FUNCTION__))
;
4260
4261 switch (Node->getOpcode()) {
4262 default:
4263 ExtOp = ISD::ANY_EXTEND;
4264 break;
4265 case ISD::SDIV:
4266 case ISD::SREM:
4267 ExtOp = ISD::SIGN_EXTEND;
4268 break;
4269 case ISD::UDIV:
4270 case ISD::UREM:
4271 ExtOp = ISD::ZERO_EXTEND;
4272 break;
4273 }
4274 TruncOp = ISD::TRUNCATE;
4275 }
4276 // Promote each of the values to the new type.
4277 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4278 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4279 // Perform the larger operation, then convert back
4280 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4281 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4282 break;
4283 }
4284 case ISD::UMUL_LOHI:
4285 case ISD::SMUL_LOHI: {
4286 // Promote to a multiply in a wider integer type.
4287 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4288 : ISD::SIGN_EXTEND;
4289 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4290 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4291 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4292
4293 auto &DL = DAG.getDataLayout();
4294 unsigned OriginalSize = OVT.getScalarSizeInBits();
4295 Tmp2 = DAG.getNode(
4296 ISD::SRL, dl, NVT, Tmp1,
4297 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4298 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4299 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4300 break;
4301 }
4302 case ISD::SELECT: {
4303 unsigned ExtOp, TruncOp;
4304 if (Node->getValueType(0).isVector() ||
4305 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4306 ExtOp = ISD::BITCAST;
4307 TruncOp = ISD::BITCAST;
4308 } else if (Node->getValueType(0).isInteger()) {
4309 ExtOp = ISD::ANY_EXTEND;
4310 TruncOp = ISD::TRUNCATE;
4311 } else {
4312 ExtOp = ISD::FP_EXTEND;
4313 TruncOp = ISD::FP_ROUND;
4314 }
4315 Tmp1 = Node->getOperand(0);
4316 // Promote each of the values to the new type.
4317 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4318 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4319 // Perform the larger operation, then round down.
4320 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4321 if (TruncOp != ISD::FP_ROUND)
4322 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4323 else
4324 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4325 DAG.getIntPtrConstant(0, dl));
4326 Results.push_back(Tmp1);
4327 break;
4328 }
4329 case ISD::VECTOR_SHUFFLE: {
4330 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4331
4332 // Cast the two input vectors.
4333 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4334 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4335
4336 // Convert the shuffle mask to the right # elements.
4337 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4338 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4339 Results.push_back(Tmp1);
4340 break;
4341 }
4342 case ISD::SETCC: {
4343 unsigned ExtOp = ISD::FP_EXTEND;
4344 if (NVT.isInteger()) {
4345 ISD::CondCode CCCode =
4346 cast<CondCodeSDNode>(Node->getOperand(2))->get();
4347 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4348 }
4349 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4350 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4351 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
4352 Tmp1, Tmp2, Node->getOperand(2)));
4353 break;
4354 }
4355 case ISD::BR_CC: {
4356 unsigned ExtOp = ISD::FP_EXTEND;
4357 if (NVT.isInteger()) {
4358 ISD::CondCode CCCode =
4359 cast<CondCodeSDNode>(Node->getOperand(1))->get();
4360 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4361 }
4362 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4363 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4364 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4365 Node->getOperand(0), Node->getOperand(1),
4366 Tmp1, Tmp2, Node->getOperand(4)));
4367 break;
4368 }
4369 case ISD::FADD:
4370 case ISD::FSUB:
4371 case ISD::FMUL:
4372 case ISD::FDIV:
4373 case ISD::FREM:
4374 case ISD::FMINNUM:
4375 case ISD::FMAXNUM:
4376 case ISD::FPOW: {
4377 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4378 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4379 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4380 Node->getFlags());
4381 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4382 Tmp3, DAG.getIntPtrConstant(0, dl)));
4383 break;
4384 }
4385 case ISD::FMA: {
4386 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4387 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4388 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4389 Results.push_back(
4390 DAG.getNode(ISD::FP_ROUND, dl, OVT,
4391 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4392 DAG.getIntPtrConstant(0, dl)));
4393 break;
4394 }
4395 case ISD::FCOPYSIGN:
4396 case ISD::FPOWI: {
4397 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4398 Tmp2 = Node->getOperand(1);
4399 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4400
4401 // fcopysign doesn't change anything but the sign bit, so
4402 // (fp_round (fcopysign (fpext a), b))
4403 // is as precise as
4404 // (fp_round (fpext a))
4405 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4406 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4407 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4408 Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4409 break;
4410 }
4411 case ISD::FFLOOR:
4412 case ISD::FCEIL:
4413 case ISD::FRINT:
4414 case ISD::FNEARBYINT:
4415 case ISD::FROUND:
4416 case ISD::FTRUNC:
4417 case ISD::FNEG:
4418 case ISD::FSQRT:
4419 case ISD::FSIN:
4420 case ISD::FCOS:
4421 case ISD::FLOG:
4422 case ISD::FLOG2:
4423 case ISD::FLOG10:
4424 case ISD::FABS:
4425 case ISD::FEXP:
4426 case ISD::FEXP2: {
4427 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4428 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4429 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4430 Tmp2, DAG.getIntPtrConstant(0, dl)));
4431 break;
4432 }
4433 case ISD::BUILD_VECTOR: {
4434 MVT EltVT = OVT.getVectorElementType();
4435 MVT NewEltVT = NVT.getVectorElementType();
4436
4437 // Handle bitcasts to a different vector type with the same total bit size
4438 //
4439 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4440 // =>
4441 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4442
4443 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for build_vector") ? static_cast
<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4444, __PRETTY_FUNCTION__))
4444 "Invalid promote type for build_vector")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for build_vector") ? static_cast
<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4444, __PRETTY_FUNCTION__))
;
4445 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4445, __PRETTY_FUNCTION__))
;
4446
4447 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4448
4449 SmallVector<SDValue, 8> NewOps;
4450 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4451 SDValue Op = Node->getOperand(I);
4452 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4453 }
4454
4455 SDLoc SL(Node);
4456 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4457 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4458 Results.push_back(CvtVec);
4459 break;
4460 }
4461 case ISD::EXTRACT_VECTOR_ELT: {
4462 MVT EltVT = OVT.getVectorElementType();
4463 MVT NewEltVT = NVT.getVectorElementType();
4464
4465 // Handle bitcasts to a different vector type with the same total bit size.
4466 //
4467 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4468 // =>
4469 // v4i32:castx = bitcast x:v2i64
4470 //
4471 // i64 = bitcast
4472 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4473 // (i32 (extract_vector_elt castx, (2 * y + 1)))
4474 //
4475
4476 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for extract_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4477, __PRETTY_FUNCTION__))
4477 "Invalid promote type for extract_vector_elt")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for extract_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4477, __PRETTY_FUNCTION__))
;
4478 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4478, __PRETTY_FUNCTION__))
;
4479
4480 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4481 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4482
4483 SDValue Idx = Node->getOperand(1);
4484 EVT IdxVT = Idx.getValueType();
4485 SDLoc SL(Node);
4486 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4487 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4488
4489 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4490
4491 SmallVector<SDValue, 8> NewOps;
4492 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4493 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4494 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4495
4496 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4497 CastVec, TmpIdx);
4498 NewOps.push_back(Elt);
4499 }
4500
4501 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4502 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4503 break;
4504 }
4505 case ISD::INSERT_VECTOR_ELT: {
4506 MVT EltVT = OVT.getVectorElementType();
4507 MVT NewEltVT = NVT.getVectorElementType();
4508
4509 // Handle bitcasts to a different vector type with the same total bit size
4510 //
4511 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4512 // =>
4513 // v4i32:castx = bitcast x:v2i64
4514 // v2i32:casty = bitcast y:i64
4515 //
4516 // v2i64 = bitcast
4517 // (v4i32 insert_vector_elt
4518 // (v4i32 insert_vector_elt v4i32:castx,
4519 // (extract_vector_elt casty, 0), 2 * z),
4520 // (extract_vector_elt casty, 1), (2 * z + 1))
4521
4522 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for insert_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4523, __PRETTY_FUNCTION__))
4523 "Invalid promote type for insert_vector_elt")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for insert_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4523, __PRETTY_FUNCTION__))
;
4524 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4524, __PRETTY_FUNCTION__))
;
4525
4526 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4527 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4528
4529 SDValue Val = Node->getOperand(1);
4530 SDValue Idx = Node->getOperand(2);
4531 EVT IdxVT = Idx.getValueType();
4532 SDLoc SL(Node);
4533
4534 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4535 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4536
4537 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4538 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4539
4540 SDValue NewVec = CastVec;
4541 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4542 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4543 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4544
4545 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4546 CastVal, IdxOffset);
4547
4548 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4549 NewVec, Elt, InEltIdx);
4550 }
4551
4552 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4553 break;
4554 }
4555 case ISD::SCALAR_TO_VECTOR: {
4556 MVT EltVT = OVT.getVectorElementType();
4557 MVT NewEltVT = NVT.getVectorElementType();
4558
4559 // Handle bitcasts to different vector type with the same total bit size.
4560 //
4561 // e.g. v2i64 = scalar_to_vector x:i64
4562 // =>
4563 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4564 //
4565
4566 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4567 SDValue Val = Node->getOperand(0);
4568 SDLoc SL(Node);
4569
4570 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4571 SDValue Undef = DAG.getUNDEF(MidVT);
4572
4573 SmallVector<SDValue, 8> NewElts;
4574 NewElts.push_back(CastVal);
4575 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4576 NewElts.push_back(Undef);
4577
4578 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4579 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4580 Results.push_back(CvtVec);
4581 break;
4582 }
4583 }
4584
4585 // Replace the original node with the legalized result.
4586 if (!Results.empty())
4587 ReplaceNode(Node, Results.data());
4588}
4589
4590/// This is the entry point for the file.
4591void SelectionDAG::Legalize() {
4592 AssignTopologicalOrder();
4593
4594 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4595 // Use a delete listener to remove nodes which were deleted during
4596 // legalization from LegalizeNodes. This is needed to handle the situation
4597 // where a new node is allocated by the object pool to the same address of a
4598 // previously deleted node.
4599 DAGNodeDeletedListener DeleteListener(
4600 *this,
4601 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4602
4603 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4604
4605 // Visit all the nodes. We start in topological order, so that we see
4606 // nodes with their original operands intact. Legalization can produce
4607 // new nodes which may themselves need to be legalized. Iterate until all
4608 // nodes have been legalized.
4609 for (;;) {
4610 bool AnyLegalized = false;
4611 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4612 --NI;
4613
4614 SDNode *N = &*NI;
4615 if (N->use_empty() && N != getRoot().getNode()) {
4616 ++NI;
4617 DeleteNode(N);
4618 continue;
4619 }
4620
4621 if (LegalizedNodes.insert(N).second) {
4622 AnyLegalized = true;
4623 Legalizer.LegalizeOp(N);
4624
4625 if (N->use_empty() && N != getRoot().getNode()) {
4626 ++NI;
4627 DeleteNode(N);
4628 }
4629 }
4630 }
4631 if (!AnyLegalized)
4632 break;
4633
4634 }
4635
4636 // Remove dead nodes now.
4637 RemoveDeadNodes();
4638}
4639
4640bool SelectionDAG::LegalizeOp(SDNode *N,
4641 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4642 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4643 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4644
4645 // Directly insert the node in question, and legalize it. This will recurse
4646 // as needed through operands.
4647 LegalizedNodes.insert(N);
4648 Legalizer.LegalizeOp(N);
4649
4650 return LegalizedNodes.count(N);
4651}