Bug Summary

File:lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
Warning:line 751, column 29
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'

Annotated Source Code

1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the SelectionDAG::Legalize method.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ADT/APFloat.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/CodeGen/ISDOpcodes.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineMemOperand.h"
25#include "llvm/CodeGen/MachineValueType.h"
26#include "llvm/CodeGen/RuntimeLibcalls.h"
27#include "llvm/CodeGen/SelectionDAG.h"
28#include "llvm/CodeGen/SelectionDAGNodes.h"
29#include "llvm/CodeGen/TargetFrameLowering.h"
30#include "llvm/CodeGen/TargetLowering.h"
31#include "llvm/CodeGen/TargetSubtargetInfo.h"
32#include "llvm/CodeGen/ValueTypes.h"
33#include "llvm/IR/CallingConv.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/DerivedTypes.h"
37#include "llvm/IR/Function.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/IR/Type.h"
40#include "llvm/Support/Casting.h"
41#include "llvm/Support/Compiler.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/ErrorHandling.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/raw_ostream.h"
46#include "llvm/Target/TargetMachine.h"
47#include "llvm/Target/TargetOptions.h"
48#include <algorithm>
49#include <cassert>
50#include <cstdint>
51#include <tuple>
52#include <utility>
53
54using namespace llvm;
55
56#define DEBUG_TYPE"legalizedag" "legalizedag"
57
58namespace {
59
60/// Keeps track of state when getting the sign of a floating-point value as an
61/// integer.
62struct FloatSignAsInt {
63 EVT FloatVT;
64 SDValue Chain;
65 SDValue FloatPtr;
66 SDValue IntPtr;
67 MachinePointerInfo IntPointerInfo;
68 MachinePointerInfo FloatPointerInfo;
69 SDValue IntValue;
70 APInt SignMask;
71 uint8_t SignBit;
72};
73
74//===----------------------------------------------------------------------===//
75/// This takes an arbitrary SelectionDAG as input and
76/// hacks on it until the target machine can handle it. This involves
77/// eliminating value sizes the machine cannot handle (promoting small sizes to
78/// large sizes or splitting up large values into small values) as well as
79/// eliminating operations the machine cannot handle.
80///
81/// This code also does a small amount of optimization and recognition of idioms
82/// as part of its processing. For example, if a target does not support a
83/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
84/// will attempt merge setcc and brc instructions into brcc's.
85class SelectionDAGLegalize {
86 const TargetMachine &TM;
87 const TargetLowering &TLI;
88 SelectionDAG &DAG;
89
90 /// \brief The set of nodes which have already been legalized. We hold a
91 /// reference to it in order to update as necessary on node deletion.
92 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93
94 /// \brief A set of all the nodes updated during legalization.
95 SmallSetVector<SDNode *, 16> *UpdatedNodes;
96
97 EVT getSetCCResultType(EVT VT) const {
98 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
99 }
100
101 // Libcall insertion helpers.
102
103public:
104 SelectionDAGLegalize(SelectionDAG &DAG,
105 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
106 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
107 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
108 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109
110 /// \brief Legalizes the given operation.
111 void LegalizeOp(SDNode *Node);
112
113private:
114 SDValue OptimizeFloatStore(StoreSDNode *ST);
115
116 void LegalizeLoadOps(SDNode *Node);
117 void LegalizeStoreOps(SDNode *Node);
118
119 /// Some targets cannot handle a variable
120 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
121 /// is necessary to spill the vector being inserted into to memory, perform
122 /// the insert there, and then read the result back.
123 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
124 const SDLoc &dl);
125 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
126 const SDLoc &dl);
127
128 /// Return a vector shuffle operation which
129 /// performs the same shuffe in terms of order or result bytes, but on a type
130 /// whose vector element type is narrower than the original shuffle type.
131 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
132 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133 SDValue N1, SDValue N2,
134 ArrayRef<int> Mask) const;
135
136 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
137 bool &NeedInvert, const SDLoc &dl);
138
139 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
140 SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
141 unsigned NumOps, bool isSigned, const SDLoc &dl);
142
143 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
144 SDNode *Node, bool isSigned);
145 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
146 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
147 RTLIB::Libcall Call_F128,
148 RTLIB::Libcall Call_PPCF128);
149 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
150 RTLIB::Libcall Call_I8,
151 RTLIB::Libcall Call_I16,
152 RTLIB::Libcall Call_I32,
153 RTLIB::Libcall Call_I64,
154 RTLIB::Libcall Call_I128);
155 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157
158 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
159 const SDLoc &dl);
160 SDValue ExpandBUILD_VECTOR(SDNode *Node);
161 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
162 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
163 SmallVectorImpl<SDValue> &Results);
164 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
165 SDValue Value) const;
166 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
167 SDValue NewIntValue) const;
168 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
169 SDValue ExpandFABS(SDNode *Node) const;
170 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT,
171 const SDLoc &dl);
172 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
173 const SDLoc &dl);
174 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
175 const SDLoc &dl);
176
177 SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
178 SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
179 SDValue ExpandBitCount(unsigned Opc, SDValue Op, const SDLoc &dl);
180
181 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
182 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
183 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
184
185 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
186 SDValue ExpandConstant(ConstantSDNode *CP);
187
188 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
189 bool ExpandNode(SDNode *Node);
190 void ConvertNodeToLibcall(SDNode *Node);
191 void PromoteNode(SDNode *Node);
192
193public:
194 // Node replacement helpers
195
196 void ReplacedNode(SDNode *N) {
197 LegalizedNodes.erase(N);
198 if (UpdatedNodes)
199 UpdatedNodes->insert(N);
200 }
201
202 void ReplaceNode(SDNode *Old, SDNode *New) {
203 DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
204 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
205
206 assert(Old->getNumValues() == New->getNumValues() &&(static_cast <bool> (Old->getNumValues() == New->
getNumValues() && "Replacing one node with another that produces a different number "
"of values!") ? void (0) : __assert_fail ("Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 208, __extension__ __PRETTY_FUNCTION__))
207 "Replacing one node with another that produces a different number "(static_cast <bool> (Old->getNumValues() == New->
getNumValues() && "Replacing one node with another that produces a different number "
"of values!") ? void (0) : __assert_fail ("Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 208, __extension__ __PRETTY_FUNCTION__))
208 "of values!")(static_cast <bool> (Old->getNumValues() == New->
getNumValues() && "Replacing one node with another that produces a different number "
"of values!") ? void (0) : __assert_fail ("Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 208, __extension__ __PRETTY_FUNCTION__))
;
209 DAG.ReplaceAllUsesWith(Old, New);
210 if (UpdatedNodes)
211 UpdatedNodes->insert(New);
212 ReplacedNode(Old);
213 }
214
215 void ReplaceNode(SDValue Old, SDValue New) {
216 DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
217 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
218
219 DAG.ReplaceAllUsesWith(Old, New);
220 if (UpdatedNodes)
221 UpdatedNodes->insert(New.getNode());
222 ReplacedNode(Old.getNode());
223 }
224
225 void ReplaceNode(SDNode *Old, const SDValue *New) {
226 DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); } } while (false)
;
227
228 DAG.ReplaceAllUsesWith(Old, New);
229 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
230 DEBUG(dbgs() << (i == 0 ? " with: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
231 : " and: ");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
232 New[i]->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
;
233 if (UpdatedNodes)
234 UpdatedNodes->insert(New[i].getNode());
235 }
236 ReplacedNode(Old);
237 }
238};
239
240} // end anonymous namespace
241
242/// Return a vector shuffle operation which
243/// performs the same shuffe in terms of order or result bytes, but on a type
244/// whose vector element type is narrower than the original shuffle type.
245/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
246SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
247 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
248 ArrayRef<int> Mask) const {
249 unsigned NumMaskElts = VT.getVectorNumElements();
250 unsigned NumDestElts = NVT.getVectorNumElements();
251 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
252
253 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!")(static_cast <bool> (NumEltsGrowth && "Cannot promote to vector type with fewer elts!"
) ? void (0) : __assert_fail ("NumEltsGrowth && \"Cannot promote to vector type with fewer elts!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 253, __extension__ __PRETTY_FUNCTION__))
;
254
255 if (NumEltsGrowth == 1)
256 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
257
258 SmallVector<int, 8> NewMask;
259 for (unsigned i = 0; i != NumMaskElts; ++i) {
260 int Idx = Mask[i];
261 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
262 if (Idx < 0)
263 NewMask.push_back(-1);
264 else
265 NewMask.push_back(Idx * NumEltsGrowth + j);
266 }
267 }
268 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?")(static_cast <bool> (NewMask.size() == NumDestElts &&
"Non-integer NumEltsGrowth?") ? void (0) : __assert_fail ("NewMask.size() == NumDestElts && \"Non-integer NumEltsGrowth?\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 268, __extension__ __PRETTY_FUNCTION__))
;
269 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?")(static_cast <bool> (TLI.isShuffleMaskLegal(NewMask, NVT
) && "Shuffle not legal?") ? void (0) : __assert_fail
("TLI.isShuffleMaskLegal(NewMask, NVT) && \"Shuffle not legal?\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 269, __extension__ __PRETTY_FUNCTION__))
;
270 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
271}
272
273/// Expands the ConstantFP node to an integer constant or
274/// a load from the constant pool.
275SDValue
276SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
277 bool Extend = false;
278 SDLoc dl(CFP);
279
280 // If a FP immediate is precise when represented as a float and if the
281 // target can do an extending load from float to double, we put it into
282 // the constant pool as a float, even if it's is statically typed as a
283 // double. This shrinks FP constants and canonicalizes them for targets where
284 // an FP extending load is the same cost as a normal load (such as on the x87
285 // fp stack or PPC FP unit).
286 EVT VT = CFP->getValueType(0);
287 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
288 if (!UseCP) {
289 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion")(static_cast <bool> ((VT == MVT::f64 || VT == MVT::f32)
&& "Invalid type expansion") ? void (0) : __assert_fail
("(VT == MVT::f64 || VT == MVT::f32) && \"Invalid type expansion\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 289, __extension__ __PRETTY_FUNCTION__))
;
290 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
291 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
292 }
293
294 APFloat APF = CFP->getValueAPF();
295 EVT OrigVT = VT;
296 EVT SVT = VT;
297
298 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
299 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
300 if (!APF.isSignaling()) {
301 while (SVT != MVT::f32 && SVT != MVT::f16) {
302 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
303 if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
304 // Only do this if the target has a native EXTLOAD instruction from
305 // smaller type.
306 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
307 TLI.ShouldShrinkFPConstant(OrigVT)) {
308 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
309 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
310 VT = SVT;
311 Extend = true;
312 }
313 }
314 }
315
316 SDValue CPIdx =
317 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
318 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
319 if (Extend) {
320 SDValue Result = DAG.getExtLoad(
321 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
322 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
323 Alignment);
324 return Result;
325 }
326 SDValue Result = DAG.getLoad(
327 OrigVT, dl, DAG.getEntryNode(), CPIdx,
328 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
329 return Result;
330}
331
332/// Expands the Constant node to a load from the constant pool.
333SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
334 SDLoc dl(CP);
335 EVT VT = CP->getValueType(0);
336 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
337 TLI.getPointerTy(DAG.getDataLayout()));
338 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
339 SDValue Result = DAG.getLoad(
340 VT, dl, DAG.getEntryNode(), CPIdx,
341 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
342 return Result;
343}
344
345/// Some target cannot handle a variable insertion index for the
346/// INSERT_VECTOR_ELT instruction. In this case, it
347/// is necessary to spill the vector being inserted into to memory, perform
348/// the insert there, and then read the result back.
349SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
350 SDValue Val,
351 SDValue Idx,
352 const SDLoc &dl) {
353 SDValue Tmp1 = Vec;
354 SDValue Tmp2 = Val;
355 SDValue Tmp3 = Idx;
356
357 // If the target doesn't support this, we have to spill the input vector
358 // to a temporary stack slot, update the element, then reload it. This is
359 // badness. We could also load the value into a vector register (either
360 // with a "move to register" or "extload into register" instruction, then
361 // permute it into place, if the idx is a constant and if the idx is
362 // supported by the target.
363 EVT VT = Tmp1.getValueType();
364 EVT EltVT = VT.getVectorElementType();
365 SDValue StackPtr = DAG.CreateStackTemporary(VT);
366
367 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
368
369 // Store the vector.
370 SDValue Ch = DAG.getStore(
371 DAG.getEntryNode(), dl, Tmp1, StackPtr,
372 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
373
374 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
375
376 // Store the scalar value.
377 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
378 // Load the updated vector.
379 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
380 DAG.getMachineFunction(), SPFI));
381}
382
383SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
384 SDValue Idx,
385 const SDLoc &dl) {
386 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
387 // SCALAR_TO_VECTOR requires that the type of the value being inserted
388 // match the element type of the vector being created, except for
389 // integers in which case the inserted value can be over width.
390 EVT EltVT = Vec.getValueType().getVectorElementType();
391 if (Val.getValueType() == EltVT ||
392 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
393 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
394 Vec.getValueType(), Val);
395
396 unsigned NumElts = Vec.getValueType().getVectorNumElements();
397 // We generate a shuffle of InVec and ScVec, so the shuffle mask
398 // should be 0,1,2,3,4,5... with the appropriate element replaced with
399 // elt 0 of the RHS.
400 SmallVector<int, 8> ShufOps;
401 for (unsigned i = 0; i != NumElts; ++i)
402 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
403
404 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
405 }
406 }
407 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
408}
409
410SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
411 DEBUG(dbgs() << "Optimizing float store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Optimizing float store operations\n"
; } } while (false)
;
412 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
413 // FIXME: We shouldn't do this for TargetConstantFP's.
414 // FIXME: move this to the DAG Combiner! Note that we can't regress due
415 // to phase ordering between legalized code and the dag combiner. This
416 // probably means that we need to integrate dag combiner and legalizer
417 // together.
418 // We generally can't do this one for long doubles.
419 SDValue Chain = ST->getChain();
420 SDValue Ptr = ST->getBasePtr();
421 unsigned Alignment = ST->getAlignment();
422 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
423 AAMDNodes AAInfo = ST->getAAInfo();
424 SDLoc dl(ST);
425 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
426 if (CFP->getValueType(0) == MVT::f32 &&
427 TLI.isTypeLegal(MVT::i32)) {
428 SDValue Con = DAG.getConstant(CFP->getValueAPF().
429 bitcastToAPInt().zextOrTrunc(32),
430 SDLoc(CFP), MVT::i32);
431 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
432 MMOFlags, AAInfo);
433 }
434
435 if (CFP->getValueType(0) == MVT::f64) {
436 // If this target supports 64-bit registers, do a single 64-bit store.
437 if (TLI.isTypeLegal(MVT::i64)) {
438 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
439 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
440 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
441 Alignment, MMOFlags, AAInfo);
442 }
443
444 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
445 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
446 // stores. If the target supports neither 32- nor 64-bits, this
447 // xform is certainly not worth it.
448 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
449 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
450 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
451 if (DAG.getDataLayout().isBigEndian())
452 std::swap(Lo, Hi);
453
454 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
455 MMOFlags, AAInfo);
456 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
457 DAG.getConstant(4, dl, Ptr.getValueType()));
458 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
459 ST->getPointerInfo().getWithOffset(4),
460 MinAlign(Alignment, 4U), MMOFlags, AAInfo);
461
462 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
463 }
464 }
465 }
466 return SDValue(nullptr, 0);
467}
468
469void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
470 StoreSDNode *ST = cast<StoreSDNode>(Node);
471 SDValue Chain = ST->getChain();
472 SDValue Ptr = ST->getBasePtr();
473 SDLoc dl(Node);
474
475 unsigned Alignment = ST->getAlignment();
476 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
477 AAMDNodes AAInfo = ST->getAAInfo();
478
479 if (!ST->isTruncatingStore()) {
480 DEBUG(dbgs() << "Legalizing store operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing store operation\n"
; } } while (false)
;
481 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
482 ReplaceNode(ST, OptStore);
483 return;
484 }
485
486 SDValue Value = ST->getValue();
487 MVT VT = Value.getSimpleValueType();
488 switch (TLI.getOperationAction(ISD::STORE, VT)) {
489 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 489)
;
490 case TargetLowering::Legal: {
491 // If this is an unaligned store and the target doesn't support it,
492 // expand it.
493 EVT MemVT = ST->getMemoryVT();
494 unsigned AS = ST->getAddressSpace();
495 unsigned Align = ST->getAlignment();
496 const DataLayout &DL = DAG.getDataLayout();
497 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
498 DEBUG(dbgs() << "Expanding unsupported unaligned store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Expanding unsupported unaligned store\n"
; } } while (false)
;
499 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
500 ReplaceNode(SDValue(ST, 0), Result);
501 } else
502 DEBUG(dbgs() << "Legal store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legal store\n"; } } while
(false)
;
503 break;
504 }
505 case TargetLowering::Custom: {
506 DEBUG(dbgs() << "Trying custom lowering\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying custom lowering\n"
; } } while (false)
;
507 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
508 if (Res && Res != SDValue(Node, 0))
509 ReplaceNode(SDValue(Node, 0), Res);
510 return;
511 }
512 case TargetLowering::Promote: {
513 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
514 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&(static_cast <bool> (NVT.getSizeInBits() == VT.getSizeInBits
() && "Can only promote stores to same size type") ? void
(0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 515, __extension__ __PRETTY_FUNCTION__))
515 "Can only promote stores to same size type")(static_cast <bool> (NVT.getSizeInBits() == VT.getSizeInBits
() && "Can only promote stores to same size type") ? void
(0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 515, __extension__ __PRETTY_FUNCTION__))
;
516 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
517 SDValue Result =
518 DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
519 Alignment, MMOFlags, AAInfo);
520 ReplaceNode(SDValue(Node, 0), Result);
521 break;
522 }
523 }
524 return;
525 }
526
527 DEBUG(dbgs() << "Legalizing truncating store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing truncating store operations\n"
; } } while (false)
;
528 SDValue Value = ST->getValue();
529 EVT StVT = ST->getMemoryVT();
530 unsigned StWidth = StVT.getSizeInBits();
531 auto &DL = DAG.getDataLayout();
532
533 if (StWidth != StVT.getStoreSizeInBits()) {
534 // Promote to a byte-sized store with upper bits zero if not
535 // storing an integral number of bytes. For example, promote
536 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
537 EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
538 StVT.getStoreSizeInBits());
539 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
540 SDValue Result =
541 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
542 Alignment, MMOFlags, AAInfo);
543 ReplaceNode(SDValue(Node, 0), Result);
544 } else if (StWidth & (StWidth - 1)) {
545 // If not storing a power-of-2 number of bits, expand as two stores.
546 assert(!StVT.isVector() && "Unsupported truncstore!")(static_cast <bool> (!StVT.isVector() && "Unsupported truncstore!"
) ? void (0) : __assert_fail ("!StVT.isVector() && \"Unsupported truncstore!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 546, __extension__ __PRETTY_FUNCTION__))
;
547 unsigned RoundWidth = 1 << Log2_32(StWidth);
548 assert(RoundWidth < StWidth)(static_cast <bool> (RoundWidth < StWidth) ? void (0
) : __assert_fail ("RoundWidth < StWidth", "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 548, __extension__ __PRETTY_FUNCTION__))
;
549 unsigned ExtraWidth = StWidth - RoundWidth;
550 assert(ExtraWidth < RoundWidth)(static_cast <bool> (ExtraWidth < RoundWidth) ? void
(0) : __assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 550, __extension__ __PRETTY_FUNCTION__))
;
551 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&(static_cast <bool> (!(RoundWidth % 8) && !(ExtraWidth
% 8) && "Store size not an integral number of bytes!"
) ? void (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 552, __extension__ __PRETTY_FUNCTION__))
552 "Store size not an integral number of bytes!")(static_cast <bool> (!(RoundWidth % 8) && !(ExtraWidth
% 8) && "Store size not an integral number of bytes!"
) ? void (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 552, __extension__ __PRETTY_FUNCTION__))
;
553 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
554 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
555 SDValue Lo, Hi;
556 unsigned IncrementSize;
557
558 if (DL.isLittleEndian()) {
559 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
560 // Store the bottom RoundWidth bits.
561 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
562 RoundVT, Alignment, MMOFlags, AAInfo);
563
564 // Store the remaining ExtraWidth bits.
565 IncrementSize = RoundWidth / 8;
566 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
567 DAG.getConstant(IncrementSize, dl,
568 Ptr.getValueType()));
569 Hi = DAG.getNode(
570 ISD::SRL, dl, Value.getValueType(), Value,
571 DAG.getConstant(RoundWidth, dl,
572 TLI.getShiftAmountTy(Value.getValueType(), DL)));
573 Hi = DAG.getTruncStore(
574 Chain, dl, Hi, Ptr,
575 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
576 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
577 } else {
578 // Big endian - avoid unaligned stores.
579 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
580 // Store the top RoundWidth bits.
581 Hi = DAG.getNode(
582 ISD::SRL, dl, Value.getValueType(), Value,
583 DAG.getConstant(ExtraWidth, dl,
584 TLI.getShiftAmountTy(Value.getValueType(), DL)));
585 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
586 RoundVT, Alignment, MMOFlags, AAInfo);
587
588 // Store the remaining ExtraWidth bits.
589 IncrementSize = RoundWidth / 8;
590 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
591 DAG.getConstant(IncrementSize, dl,
592 Ptr.getValueType()));
593 Lo = DAG.getTruncStore(
594 Chain, dl, Value, Ptr,
595 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
596 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
597 }
598
599 // The order of the stores doesn't matter.
600 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
601 ReplaceNode(SDValue(Node, 0), Result);
602 } else {
603 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
604 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 604)
;
605 case TargetLowering::Legal: {
606 EVT MemVT = ST->getMemoryVT();
607 unsigned AS = ST->getAddressSpace();
608 unsigned Align = ST->getAlignment();
609 // If this is an unaligned store and the target doesn't support it,
610 // expand it.
611 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
612 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
613 ReplaceNode(SDValue(ST, 0), Result);
614 }
615 break;
616 }
617 case TargetLowering::Custom: {
618 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
619 if (Res && Res != SDValue(Node, 0))
620 ReplaceNode(SDValue(Node, 0), Res);
621 return;
622 }
623 case TargetLowering::Expand:
624 assert(!StVT.isVector() &&(static_cast <bool> (!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? void (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 625, __extension__ __PRETTY_FUNCTION__))
625 "Vector Stores are handled in LegalizeVectorOps")(static_cast <bool> (!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? void (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 625, __extension__ __PRETTY_FUNCTION__))
;
626
627 // TRUNCSTORE:i16 i32 -> STORE i16
628 assert(TLI.isTypeLegal(StVT) &&(static_cast <bool> (TLI.isTypeLegal(StVT) && "Do not know how to expand this store!"
) ? void (0) : __assert_fail ("TLI.isTypeLegal(StVT) && \"Do not know how to expand this store!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 629, __extension__ __PRETTY_FUNCTION__))
629 "Do not know how to expand this store!")(static_cast <bool> (TLI.isTypeLegal(StVT) && "Do not know how to expand this store!"
) ? void (0) : __assert_fail ("TLI.isTypeLegal(StVT) && \"Do not know how to expand this store!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 629, __extension__ __PRETTY_FUNCTION__))
;
630 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
631 SDValue Result =
632 DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
633 Alignment, MMOFlags, AAInfo);
634 ReplaceNode(SDValue(Node, 0), Result);
635 break;
636 }
637 }
638}
639
640void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
641 LoadSDNode *LD = cast<LoadSDNode>(Node);
642 SDValue Chain = LD->getChain(); // The chain.
643 SDValue Ptr = LD->getBasePtr(); // The base pointer.
644 SDValue Value; // The value returned by the load op.
645 SDLoc dl(Node);
646
647 ISD::LoadExtType ExtType = LD->getExtensionType();
648 if (ExtType == ISD::NON_EXTLOAD) {
1
Assuming 'ExtType' is not equal to NON_EXTLOAD
2
Taking false branch
649 DEBUG(dbgs() << "Legalizing non-extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing non-extending load operation\n"
; } } while (false)
;
650 MVT VT = Node->getSimpleValueType(0);
651 SDValue RVal = SDValue(Node, 0);
652 SDValue RChain = SDValue(Node, 1);
653
654 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
655 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 655)
;
656 case TargetLowering::Legal: {
657 EVT MemVT = LD->getMemoryVT();
658 unsigned AS = LD->getAddressSpace();
659 unsigned Align = LD->getAlignment();
660 const DataLayout &DL = DAG.getDataLayout();
661 // If this is an unaligned load and the target doesn't support it,
662 // expand it.
663 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
664 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
665 }
666 break;
667 }
668 case TargetLowering::Custom:
669 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
670 RVal = Res;
671 RChain = Res.getValue(1);
672 }
673 break;
674
675 case TargetLowering::Promote: {
676 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
677 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&(static_cast <bool> (NVT.getSizeInBits() == VT.getSizeInBits
() && "Can only promote loads to same size type") ? void
(0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 678, __extension__ __PRETTY_FUNCTION__))
678 "Can only promote loads to same size type")(static_cast <bool> (NVT.getSizeInBits() == VT.getSizeInBits
() && "Can only promote loads to same size type") ? void
(0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 678, __extension__ __PRETTY_FUNCTION__))
;
679
680 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
681 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
682 RChain = Res.getValue(1);
683 break;
684 }
685 }
686 if (RChain.getNode() != Node) {
687 assert(RVal.getNode() != Node && "Load must be completely replaced")(static_cast <bool> (RVal.getNode() != Node && "Load must be completely replaced"
) ? void (0) : __assert_fail ("RVal.getNode() != Node && \"Load must be completely replaced\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 687, __extension__ __PRETTY_FUNCTION__))
;
688 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
689 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
690 if (UpdatedNodes) {
691 UpdatedNodes->insert(RVal.getNode());
692 UpdatedNodes->insert(RChain.getNode());
693 }
694 ReplacedNode(Node);
695 }
696 return;
697 }
698
699 DEBUG(dbgs() << "Legalizing extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing extending load operation\n"
; } } while (false)
;
700 EVT SrcVT = LD->getMemoryVT();
701 unsigned SrcWidth = SrcVT.getSizeInBits();
702 unsigned Alignment = LD->getAlignment();
703 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
704 AAMDNodes AAInfo = LD->getAAInfo();
705
706 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
3
Taking false branch
707 // Some targets pretend to have an i1 loading operation, and actually
708 // load an i8. This trick is correct for ZEXTLOAD because the top 7
709 // bits are guaranteed to be zero; it helps the optimizers understand
710 // that these bits are zero. It is also useful for EXTLOAD, since it
711 // tells the optimizers that those bits are undefined. It would be
712 // nice to have an effective generic way of getting these benefits...
713 // Until such a way is found, don't insist on promoting i1 here.
714 (SrcVT != MVT::i1 ||
715 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
716 TargetLowering::Promote)) {
717 // Promote to a byte-sized load if not loading an integral number of
718 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
719 unsigned NewWidth = SrcVT.getStoreSizeInBits();
720 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
721 SDValue Ch;
722
723 // The extra bits are guaranteed to be zero, since we stored them that
724 // way. A zext load from NVT thus automatically gives zext from SrcVT.
725
726 ISD::LoadExtType NewExtType =
727 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
728
729 SDValue Result =
730 DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
731 LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
732
733 Ch = Result.getValue(1); // The chain.
734
735 if (ExtType == ISD::SEXTLOAD)
736 // Having the top bits zero doesn't help when sign extending.
737 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
738 Result.getValueType(),
739 Result, DAG.getValueType(SrcVT));
740 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
741 // All the top bits are guaranteed to be zero - inform the optimizers.
742 Result = DAG.getNode(ISD::AssertZext, dl,
743 Result.getValueType(), Result,
744 DAG.getValueType(SrcVT));
745
746 Value = Result;
747 Chain = Ch;
748 } else if (SrcWidth & (SrcWidth - 1)) {
4
Taking true branch
749 // If not loading a power-of-2 number of bits, expand as two loads.
750 assert(!SrcVT.isVector() && "Unsupported extload!")(static_cast <bool> (!SrcVT.isVector() && "Unsupported extload!"
) ? void (0) : __assert_fail ("!SrcVT.isVector() && \"Unsupported extload!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 750, __extension__ __PRETTY_FUNCTION__))
;
751 unsigned RoundWidth = 1 << Log2_32(SrcWidth);
5
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'
752 assert(RoundWidth < SrcWidth)(static_cast <bool> (RoundWidth < SrcWidth) ? void (
0) : __assert_fail ("RoundWidth < SrcWidth", "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 752, __extension__ __PRETTY_FUNCTION__))
;
753 unsigned ExtraWidth = SrcWidth - RoundWidth;
754 assert(ExtraWidth < RoundWidth)(static_cast <bool> (ExtraWidth < RoundWidth) ? void
(0) : __assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 754, __extension__ __PRETTY_FUNCTION__))
;
755 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&(static_cast <bool> (!(RoundWidth % 8) && !(ExtraWidth
% 8) && "Load size not an integral number of bytes!"
) ? void (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 756, __extension__ __PRETTY_FUNCTION__))
756 "Load size not an integral number of bytes!")(static_cast <bool> (!(RoundWidth % 8) && !(ExtraWidth
% 8) && "Load size not an integral number of bytes!"
) ? void (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 756, __extension__ __PRETTY_FUNCTION__))
;
757 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
758 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
759 SDValue Lo, Hi, Ch;
760 unsigned IncrementSize;
761 auto &DL = DAG.getDataLayout();
762
763 if (DL.isLittleEndian()) {
764 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
765 // Load the bottom RoundWidth bits.
766 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
767 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
768 AAInfo);
769
770 // Load the remaining ExtraWidth bits.
771 IncrementSize = RoundWidth / 8;
772 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
773 DAG.getConstant(IncrementSize, dl,
774 Ptr.getValueType()));
775 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
776 LD->getPointerInfo().getWithOffset(IncrementSize),
777 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
778 AAInfo);
779
780 // Build a factor node to remember that this load is independent of
781 // the other one.
782 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
783 Hi.getValue(1));
784
785 // Move the top bits to the right place.
786 Hi = DAG.getNode(
787 ISD::SHL, dl, Hi.getValueType(), Hi,
788 DAG.getConstant(RoundWidth, dl,
789 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
790
791 // Join the hi and lo parts.
792 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
793 } else {
794 // Big endian - avoid unaligned loads.
795 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
796 // Load the top RoundWidth bits.
797 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
798 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
799 AAInfo);
800
801 // Load the remaining ExtraWidth bits.
802 IncrementSize = RoundWidth / 8;
803 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
804 DAG.getConstant(IncrementSize, dl,
805 Ptr.getValueType()));
806 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
807 LD->getPointerInfo().getWithOffset(IncrementSize),
808 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
809 AAInfo);
810
811 // Build a factor node to remember that this load is independent of
812 // the other one.
813 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
814 Hi.getValue(1));
815
816 // Move the top bits to the right place.
817 Hi = DAG.getNode(
818 ISD::SHL, dl, Hi.getValueType(), Hi,
819 DAG.getConstant(ExtraWidth, dl,
820 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
821
822 // Join the hi and lo parts.
823 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
824 }
825
826 Chain = Ch;
827 } else {
828 bool isCustom = false;
829 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
830 SrcVT.getSimpleVT())) {
831 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 831)
;
832 case TargetLowering::Custom:
833 isCustom = true;
834 LLVM_FALLTHROUGH[[clang::fallthrough]];
835 case TargetLowering::Legal:
836 Value = SDValue(Node, 0);
837 Chain = SDValue(Node, 1);
838
839 if (isCustom) {
840 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
841 Value = Res;
842 Chain = Res.getValue(1);
843 }
844 } else {
845 // If this is an unaligned load and the target doesn't support it,
846 // expand it.
847 EVT MemVT = LD->getMemoryVT();
848 unsigned AS = LD->getAddressSpace();
849 unsigned Align = LD->getAlignment();
850 const DataLayout &DL = DAG.getDataLayout();
851 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
852 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
853 }
854 }
855 break;
856
857 case TargetLowering::Expand: {
858 EVT DestVT = Node->getValueType(0);
859 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
860 // If the source type is not legal, see if there is a legal extload to
861 // an intermediate type that we can then extend further.
862 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
863 if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
864 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
865 // If we are loading a legal type, this is a non-extload followed by a
866 // full extend.
867 ISD::LoadExtType MidExtType =
868 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
869
870 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
871 SrcVT, LD->getMemOperand());
872 unsigned ExtendOp =
873 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
874 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
875 Chain = Load.getValue(1);
876 break;
877 }
878
879 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
880 // normal undefined upper bits behavior to allow using an in-reg extend
881 // with the illegal FP type, so load as an integer and do the
882 // from-integer conversion.
883 if (SrcVT.getScalarType() == MVT::f16) {
884 EVT ISrcVT = SrcVT.changeTypeToInteger();
885 EVT IDestVT = DestVT.changeTypeToInteger();
886 EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
887
888 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
889 Chain, Ptr, ISrcVT,
890 LD->getMemOperand());
891 Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
892 Chain = Result.getValue(1);
893 break;
894 }
895 }
896
897 assert(!SrcVT.isVector() &&(static_cast <bool> (!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? void (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 898, __extension__ __PRETTY_FUNCTION__))
898 "Vector Loads are handled in LegalizeVectorOps")(static_cast <bool> (!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? void (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 898, __extension__ __PRETTY_FUNCTION__))
;
899
900 // FIXME: This does not work for vectors on most targets. Sign-
901 // and zero-extend operations are currently folded into extending
902 // loads, whether they are legal or not, and then we end up here
903 // without any support for legalizing them.
904 assert(ExtType != ISD::EXTLOAD &&(static_cast <bool> (ExtType != ISD::EXTLOAD &&
"EXTLOAD should always be supported!") ? void (0) : __assert_fail
("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 905, __extension__ __PRETTY_FUNCTION__))
905 "EXTLOAD should always be supported!")(static_cast <bool> (ExtType != ISD::EXTLOAD &&
"EXTLOAD should always be supported!") ? void (0) : __assert_fail
("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 905, __extension__ __PRETTY_FUNCTION__))
;
906 // Turn the unsupported load into an EXTLOAD followed by an
907 // explicit zero/sign extend inreg.
908 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
909 Node->getValueType(0),
910 Chain, Ptr, SrcVT,
911 LD->getMemOperand());
912 SDValue ValRes;
913 if (ExtType == ISD::SEXTLOAD)
914 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
915 Result.getValueType(),
916 Result, DAG.getValueType(SrcVT));
917 else
918 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
919 Value = ValRes;
920 Chain = Result.getValue(1);
921 break;
922 }
923 }
924 }
925
926 // Since loads produce two values, make sure to remember that we legalized
927 // both of them.
928 if (Chain.getNode() != Node) {
929 assert(Value.getNode() != Node && "Load must be completely replaced")(static_cast <bool> (Value.getNode() != Node &&
"Load must be completely replaced") ? void (0) : __assert_fail
("Value.getNode() != Node && \"Load must be completely replaced\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 929, __extension__ __PRETTY_FUNCTION__))
;
930 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
931 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
932 if (UpdatedNodes) {
933 UpdatedNodes->insert(Value.getNode());
934 UpdatedNodes->insert(Chain.getNode());
935 }
936 ReplacedNode(Node);
937 }
938}
939
940static TargetLowering::LegalizeAction
941getStrictFPOpcodeAction(const TargetLowering &TLI, unsigned Opcode, EVT VT) {
942 unsigned EqOpc;
943 switch (Opcode) {
944 default: llvm_unreachable("Unexpected FP pseudo-opcode")::llvm::llvm_unreachable_internal("Unexpected FP pseudo-opcode"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 944)
;
945 case ISD::STRICT_FSQRT: EqOpc = ISD::FSQRT; break;
946 case ISD::STRICT_FPOW: EqOpc = ISD::FPOW; break;
947 case ISD::STRICT_FPOWI: EqOpc = ISD::FPOWI; break;
948 case ISD::STRICT_FMA: EqOpc = ISD::FMA; break;
949 case ISD::STRICT_FSIN: EqOpc = ISD::FSIN; break;
950 case ISD::STRICT_FCOS: EqOpc = ISD::FCOS; break;
951 case ISD::STRICT_FEXP: EqOpc = ISD::FEXP; break;
952 case ISD::STRICT_FEXP2: EqOpc = ISD::FEXP2; break;
953 case ISD::STRICT_FLOG: EqOpc = ISD::FLOG; break;
954 case ISD::STRICT_FLOG10: EqOpc = ISD::FLOG10; break;
955 case ISD::STRICT_FLOG2: EqOpc = ISD::FLOG2; break;
956 case ISD::STRICT_FRINT: EqOpc = ISD::FRINT; break;
957 case ISD::STRICT_FNEARBYINT: EqOpc = ISD::FNEARBYINT; break;
958 }
959
960 auto Action = TLI.getOperationAction(EqOpc, VT);
961
962 // We don't currently handle Custom or Promote for strict FP pseudo-ops.
963 // For now, we just expand for those cases.
964 if (Action != TargetLowering::Legal)
965 Action = TargetLowering::Expand;
966
967 return Action;
968}
969
970/// Return a legal replacement for the given operation, with all legal operands.
971void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
972 DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "\nLegalizing: "; Node->
dump(&DAG); } } while (false)
;
973
974 // Allow illegal target nodes and illegal registers.
975 if (Node->getOpcode() == ISD::TargetConstant ||
976 Node->getOpcode() == ISD::Register)
977 return;
978
979#ifndef NDEBUG
980 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
981 assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Node->getValueType(i)) == TargetLowering::TypeLegal ||
TLI.isTypeLegal(Node->getValueType(i))) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 984, __extension__ __PRETTY_FUNCTION__))
982 TargetLowering::TypeLegal ||(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Node->getValueType(i)) == TargetLowering::TypeLegal ||
TLI.isTypeLegal(Node->getValueType(i))) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 984, __extension__ __PRETTY_FUNCTION__))
983 TLI.isTypeLegal(Node->getValueType(i))) &&(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Node->getValueType(i)) == TargetLowering::TypeLegal ||
TLI.isTypeLegal(Node->getValueType(i))) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 984, __extension__ __PRETTY_FUNCTION__))
984 "Unexpected illegal type!")(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Node->getValueType(i)) == TargetLowering::TypeLegal ||
TLI.isTypeLegal(Node->getValueType(i))) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 984, __extension__ __PRETTY_FUNCTION__))
;
985
986 for (const SDValue &Op : Node->op_values())
987 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 992, __extension__ __PRETTY_FUNCTION__))
988 TargetLowering::TypeLegal ||(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 992, __extension__ __PRETTY_FUNCTION__))
989 TLI.isTypeLegal(Op.getValueType()) ||(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 992, __extension__ __PRETTY_FUNCTION__))
990 Op.getOpcode() == ISD::TargetConstant ||(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 992, __extension__ __PRETTY_FUNCTION__))
991 Op.getOpcode() == ISD::Register) &&(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 992, __extension__ __PRETTY_FUNCTION__))
992 "Unexpected illegal type!")(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 992, __extension__ __PRETTY_FUNCTION__))
;
993#endif
994
995 // Figure out the correct action; the way to query this varies by opcode
996 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
997 bool SimpleFinishLegalizing = true;
998 switch (Node->getOpcode()) {
999 case ISD::INTRINSIC_W_CHAIN:
1000 case ISD::INTRINSIC_WO_CHAIN:
1001 case ISD::INTRINSIC_VOID:
1002 case ISD::STACKSAVE:
1003 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1004 break;
1005 case ISD::GET_DYNAMIC_AREA_OFFSET:
1006 Action = TLI.getOperationAction(Node->getOpcode(),
1007 Node->getValueType(0));
1008 break;
1009 case ISD::VAARG:
1010 Action = TLI.getOperationAction(Node->getOpcode(),
1011 Node->getValueType(0));
1012 if (Action != TargetLowering::Promote)
1013 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1014 break;
1015 case ISD::FP_TO_FP16:
1016 case ISD::SINT_TO_FP:
1017 case ISD::UINT_TO_FP:
1018 case ISD::EXTRACT_VECTOR_ELT:
1019 Action = TLI.getOperationAction(Node->getOpcode(),
1020 Node->getOperand(0).getValueType());
1021 break;
1022 case ISD::FP_ROUND_INREG:
1023 case ISD::SIGN_EXTEND_INREG: {
1024 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1025 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1026 break;
1027 }
1028 case ISD::ATOMIC_STORE:
1029 Action = TLI.getOperationAction(Node->getOpcode(),
1030 Node->getOperand(2).getValueType());
1031 break;
1032 case ISD::SELECT_CC:
1033 case ISD::SETCC:
1034 case ISD::BR_CC: {
1035 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1036 Node->getOpcode() == ISD::SETCC ? 2 :
1037 Node->getOpcode() == ISD::SETCCE ? 3 : 1;
1038 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
1039 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1040 ISD::CondCode CCCode =
1041 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1042 Action = TLI.getCondCodeAction(CCCode, OpVT);
1043 if (Action == TargetLowering::Legal) {
1044 if (Node->getOpcode() == ISD::SELECT_CC)
1045 Action = TLI.getOperationAction(Node->getOpcode(),
1046 Node->getValueType(0));
1047 else
1048 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1049 }
1050 break;
1051 }
1052 case ISD::LOAD:
1053 case ISD::STORE:
1054 // FIXME: Model these properly. LOAD and STORE are complicated, and
1055 // STORE expects the unlegalized operand in some cases.
1056 SimpleFinishLegalizing = false;
1057 break;
1058 case ISD::CALLSEQ_START:
1059 case ISD::CALLSEQ_END:
1060 // FIXME: This shouldn't be necessary. These nodes have special properties
1061 // dealing with the recursive nature of legalization. Removing this
1062 // special case should be done as part of making LegalizeDAG non-recursive.
1063 SimpleFinishLegalizing = false;
1064 break;
1065 case ISD::EXTRACT_ELEMENT:
1066 case ISD::FLT_ROUNDS_:
1067 case ISD::MERGE_VALUES:
1068 case ISD::EH_RETURN:
1069 case ISD::FRAME_TO_ARGS_OFFSET:
1070 case ISD::EH_DWARF_CFA:
1071 case ISD::EH_SJLJ_SETJMP:
1072 case ISD::EH_SJLJ_LONGJMP:
1073 case ISD::EH_SJLJ_SETUP_DISPATCH:
1074 // These operations lie about being legal: when they claim to be legal,
1075 // they should actually be expanded.
1076 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1077 if (Action == TargetLowering::Legal)
1078 Action = TargetLowering::Expand;
1079 break;
1080 case ISD::INIT_TRAMPOLINE:
1081 case ISD::ADJUST_TRAMPOLINE:
1082 case ISD::FRAMEADDR:
1083 case ISD::RETURNADDR:
1084 case ISD::ADDROFRETURNADDR:
1085 // These operations lie about being legal: when they claim to be legal,
1086 // they should actually be custom-lowered.
1087 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1088 if (Action == TargetLowering::Legal)
1089 Action = TargetLowering::Custom;
1090 break;
1091 case ISD::READCYCLECOUNTER:
1092 // READCYCLECOUNTER returns an i64, even if type legalization might have
1093 // expanded that to several smaller types.
1094 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1095 break;
1096 case ISD::READ_REGISTER:
1097 case ISD::WRITE_REGISTER:
1098 // Named register is legal in the DAG, but blocked by register name
1099 // selection if not implemented by target (to chose the correct register)
1100 // They'll be converted to Copy(To/From)Reg.
1101 Action = TargetLowering::Legal;
1102 break;
1103 case ISD::DEBUGTRAP:
1104 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1105 if (Action == TargetLowering::Expand) {
1106 // replace ISD::DEBUGTRAP with ISD::TRAP
1107 SDValue NewVal;
1108 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1109 Node->getOperand(0));
1110 ReplaceNode(Node, NewVal.getNode());
1111 LegalizeOp(NewVal.getNode());
1112 return;
1113 }
1114 break;
1115 case ISD::STRICT_FSQRT:
1116 case ISD::STRICT_FMA:
1117 case ISD::STRICT_FPOW:
1118 case ISD::STRICT_FPOWI:
1119 case ISD::STRICT_FSIN:
1120 case ISD::STRICT_FCOS:
1121 case ISD::STRICT_FEXP:
1122 case ISD::STRICT_FEXP2:
1123 case ISD::STRICT_FLOG:
1124 case ISD::STRICT_FLOG10:
1125 case ISD::STRICT_FLOG2:
1126 case ISD::STRICT_FRINT:
1127 case ISD::STRICT_FNEARBYINT:
1128 // These pseudo-ops get legalized as if they were their non-strict
1129 // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
1130 // is also legal, but if ISD::FSQRT requires expansion then so does
1131 // ISD::STRICT_FSQRT.
1132 Action = getStrictFPOpcodeAction(TLI, Node->getOpcode(),
1133 Node->getValueType(0));
1134 break;
1135 default:
1136 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1137 Action = TargetLowering::Legal;
1138 } else {
1139 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1140 }
1141 break;
1142 }
1143
1144 if (SimpleFinishLegalizing) {
1145 SDNode *NewNode = Node;
1146 switch (Node->getOpcode()) {
1147 default: break;
1148 case ISD::SHL:
1149 case ISD::SRL:
1150 case ISD::SRA:
1151 case ISD::ROTL:
1152 case ISD::ROTR: {
1153 // Legalizing shifts/rotates requires adjusting the shift amount
1154 // to the appropriate width.
1155 SDValue Op0 = Node->getOperand(0);
1156 SDValue Op1 = Node->getOperand(1);
1157 if (!Op1.getValueType().isVector()) {
1158 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1159 // The getShiftAmountOperand() may create a new operand node or
1160 // return the existing one. If new operand is created we need
1161 // to update the parent node.
1162 // Do not try to legalize SAO here! It will be automatically legalized
1163 // in the next round.
1164 if (SAO != Op1)
1165 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1166 }
1167 }
1168 break;
1169 case ISD::SRL_PARTS:
1170 case ISD::SRA_PARTS:
1171 case ISD::SHL_PARTS: {
1172 // Legalizing shifts/rotates requires adjusting the shift amount
1173 // to the appropriate width.
1174 SDValue Op0 = Node->getOperand(0);
1175 SDValue Op1 = Node->getOperand(1);
1176 SDValue Op2 = Node->getOperand(2);
1177 if (!Op2.getValueType().isVector()) {
1178 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1179 // The getShiftAmountOperand() may create a new operand node or
1180 // return the existing one. If new operand is created we need
1181 // to update the parent node.
1182 if (SAO != Op2)
1183 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1184 }
1185 break;
1186 }
1187 }
1188
1189 if (NewNode != Node) {
1190 ReplaceNode(Node, NewNode);
1191 Node = NewNode;
1192 }
1193 switch (Action) {
1194 case TargetLowering::Legal:
1195 DEBUG(dbgs() << "Legal node: nothing to do\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legal node: nothing to do\n"
; } } while (false)
;
1196 return;
1197 case TargetLowering::Custom:
1198 DEBUG(dbgs() << "Trying custom legalization\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying custom legalization\n"
; } } while (false)
;
1199 // FIXME: The handling for custom lowering with multiple results is
1200 // a complete mess.
1201 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1202 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1203 return;
1204
1205 if (Node->getNumValues() == 1) {
1206 DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully custom legalized node\n"
; } } while (false)
;
1207 // We can just directly replace this node with the lowered value.
1208 ReplaceNode(SDValue(Node, 0), Res);
1209 return;
1210 }
1211
1212 SmallVector<SDValue, 8> ResultVals;
1213 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1214 ResultVals.push_back(Res.getValue(i));
1215 DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully custom legalized node\n"
; } } while (false)
;
1216 ReplaceNode(Node, ResultVals.data());
1217 return;
1218 }
1219 DEBUG(dbgs() << "Could not custom legalize node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not custom legalize node\n"
; } } while (false)
;
1220 LLVM_FALLTHROUGH[[clang::fallthrough]];
1221 case TargetLowering::Expand:
1222 if (ExpandNode(Node))
1223 return;
1224 LLVM_FALLTHROUGH[[clang::fallthrough]];
1225 case TargetLowering::LibCall:
1226 ConvertNodeToLibcall(Node);
1227 return;
1228 case TargetLowering::Promote:
1229 PromoteNode(Node);
1230 return;
1231 }
1232 }
1233
1234 switch (Node->getOpcode()) {
1235 default:
1236#ifndef NDEBUG
1237 dbgs() << "NODE: ";
1238 Node->dump( &DAG);
1239 dbgs() << "\n";
1240#endif
1241 llvm_unreachable("Do not know how to legalize this operator!")::llvm::llvm_unreachable_internal("Do not know how to legalize this operator!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1241)
;
1242
1243 case ISD::CALLSEQ_START:
1244 case ISD::CALLSEQ_END:
1245 break;
1246 case ISD::LOAD:
1247 return LegalizeLoadOps(Node);
1248 case ISD::STORE:
1249 return LegalizeStoreOps(Node);
1250 }
1251}
1252
1253SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1254 SDValue Vec = Op.getOperand(0);
1255 SDValue Idx = Op.getOperand(1);
1256 SDLoc dl(Op);
1257
1258 // Before we generate a new store to a temporary stack slot, see if there is
1259 // already one that we can use. There often is because when we scalarize
1260 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1261 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1262 // the vector. If all are expanded here, we don't want one store per vector
1263 // element.
1264
1265 // Caches for hasPredecessorHelper
1266 SmallPtrSet<const SDNode *, 32> Visited;
1267 SmallVector<const SDNode *, 16> Worklist;
1268 Worklist.push_back(Idx.getNode());
1269 SDValue StackPtr, Ch;
1270 for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1271 UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1272 SDNode *User = *UI;
1273 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1274 if (ST->isIndexed() || ST->isTruncatingStore() ||
1275 ST->getValue() != Vec)
1276 continue;
1277
1278 // Make sure that nothing else could have stored into the destination of
1279 // this store.
1280 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1281 continue;
1282
1283 // If the index is dependent on the store we will introduce a cycle when
1284 // creating the load (the load uses the index, and by replacing the chain
1285 // we will make the index dependent on the load). Also, the store might be
1286 // dependent on the extractelement and introduce a cycle when creating
1287 // the load.
1288 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1289 ST->hasPredecessor(Op.getNode()))
1290 continue;
1291
1292 StackPtr = ST->getBasePtr();
1293 Ch = SDValue(ST, 0);
1294 break;
1295 }
1296 }
1297
1298 EVT VecVT = Vec.getValueType();
1299
1300 if (!Ch.getNode()) {
1301 // Store the value to a temporary stack slot, then LOAD the returned part.
1302 StackPtr = DAG.CreateStackTemporary(VecVT);
1303 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1304 MachinePointerInfo());
1305 }
1306
1307 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1308
1309 SDValue NewLoad;
1310
1311 if (Op.getValueType().isVector())
1312 NewLoad =
1313 DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1314 else
1315 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1316 MachinePointerInfo(),
1317 VecVT.getVectorElementType());
1318
1319 // Replace the chain going out of the store, by the one out of the load.
1320 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1321
1322 // We introduced a cycle though, so update the loads operands, making sure
1323 // to use the original store's chain as an incoming chain.
1324 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1325 NewLoad->op_end());
1326 NewLoadOperands[0] = Ch;
1327 NewLoad =
1328 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1329 return NewLoad;
1330}
1331
1332SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1333 assert(Op.getValueType().isVector() && "Non-vector insert subvector!")(static_cast <bool> (Op.getValueType().isVector() &&
"Non-vector insert subvector!") ? void (0) : __assert_fail (
"Op.getValueType().isVector() && \"Non-vector insert subvector!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1333, __extension__ __PRETTY_FUNCTION__))
;
1334
1335 SDValue Vec = Op.getOperand(0);
1336 SDValue Part = Op.getOperand(1);
1337 SDValue Idx = Op.getOperand(2);
1338 SDLoc dl(Op);
1339
1340 // Store the value to a temporary stack slot, then LOAD the returned part.
1341 EVT VecVT = Vec.getValueType();
1342 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1343 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1344 MachinePointerInfo PtrInfo =
1345 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1346
1347 // First store the whole vector.
1348 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1349
1350 // Then store the inserted part.
1351 SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1352
1353 // Store the subvector.
1354 Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
1355
1356 // Finally, load the updated vector.
1357 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1358}
1359
1360SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1361 // We can't handle this case efficiently. Allocate a sufficiently
1362 // aligned object on the stack, store each element into it, then load
1363 // the result as a vector.
1364 // Create the stack frame object.
1365 EVT VT = Node->getValueType(0);
1366 EVT EltVT = VT.getVectorElementType();
1367 SDLoc dl(Node);
1368 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1369 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1370 MachinePointerInfo PtrInfo =
1371 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1372
1373 // Emit a store of each element to the stack slot.
1374 SmallVector<SDValue, 8> Stores;
1375 unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1376 // Store (in the right endianness) the elements to memory.
1377 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1378 // Ignore undef elements.
1379 if (Node->getOperand(i).isUndef()) continue;
1380
1381 unsigned Offset = TypeByteSize*i;
1382
1383 SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1384 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1385
1386 // If the destination vector element type is narrower than the source
1387 // element type, only store the bits necessary.
1388 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1389 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1390 Node->getOperand(i), Idx,
1391 PtrInfo.getWithOffset(Offset), EltVT));
1392 } else
1393 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1394 Idx, PtrInfo.getWithOffset(Offset)));
1395 }
1396
1397 SDValue StoreChain;
1398 if (!Stores.empty()) // Not all undef elements?
1399 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1400 else
1401 StoreChain = DAG.getEntryNode();
1402
1403 // Result is a load from the stack slot.
1404 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1405}
1406
1407/// Bitcast a floating-point value to an integer value. Only bitcast the part
1408/// containing the sign bit if the target has no integer value capable of
1409/// holding all bits of the floating-point value.
1410void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1411 const SDLoc &DL,
1412 SDValue Value) const {
1413 EVT FloatVT = Value.getValueType();
1414 unsigned NumBits = FloatVT.getSizeInBits();
1415 State.FloatVT = FloatVT;
1416 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1417 // Convert to an integer of the same size.
1418 if (TLI.isTypeLegal(IVT)) {
1419 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1420 State.SignMask = APInt::getSignMask(NumBits);
1421 State.SignBit = NumBits - 1;
1422 return;
1423 }
1424
1425 auto &DataLayout = DAG.getDataLayout();
1426 // Store the float to memory, then load the sign part out as an integer.
1427 MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1428 // First create a temporary that is aligned for both the load and store.
1429 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1430 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1431 // Then store the float to it.
1432 State.FloatPtr = StackPtr;
1433 MachineFunction &MF = DAG.getMachineFunction();
1434 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1435 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1436 State.FloatPointerInfo);
1437
1438 SDValue IntPtr;
1439 if (DataLayout.isBigEndian()) {
1440 assert(FloatVT.isByteSized() && "Unsupported floating point type!")(static_cast <bool> (FloatVT.isByteSized() && "Unsupported floating point type!"
) ? void (0) : __assert_fail ("FloatVT.isByteSized() && \"Unsupported floating point type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1440, __extension__ __PRETTY_FUNCTION__))
;
1441 // Load out a legal integer with the same sign bit as the float.
1442 IntPtr = StackPtr;
1443 State.IntPointerInfo = State.FloatPointerInfo;
1444 } else {
1445 // Advance the pointer so that the loaded byte will contain the sign bit.
1446 unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
1447 IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
1448 DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
1449 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1450 ByteOffset);
1451 }
1452
1453 State.IntPtr = IntPtr;
1454 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1455 State.IntPointerInfo, MVT::i8);
1456 State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
1457 State.SignBit = 7;
1458}
1459
1460/// Replace the integer value produced by getSignAsIntValue() with a new value
1461/// and cast the result back to a floating-point type.
1462SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1463 const SDLoc &DL,
1464 SDValue NewIntValue) const {
1465 if (!State.Chain)
1466 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1467
1468 // Override the part containing the sign bit in the value stored on the stack.
1469 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1470 State.IntPointerInfo, MVT::i8);
1471 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1472 State.FloatPointerInfo);
1473}
1474
1475SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1476 SDLoc DL(Node);
1477 SDValue Mag = Node->getOperand(0);
1478 SDValue Sign = Node->getOperand(1);
1479
1480 // Get sign bit into an integer value.
1481 FloatSignAsInt SignAsInt;
1482 getSignAsIntValue(SignAsInt, DL, Sign);
1483
1484 EVT IntVT = SignAsInt.IntValue.getValueType();
1485 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1486 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1487 SignMask);
1488
1489 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1490 EVT FloatVT = Mag.getValueType();
1491 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1492 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1493 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1494 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1495 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1496 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1497 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1498 }
1499
1500 // Transform Mag value to integer, and clear the sign bit.
1501 FloatSignAsInt MagAsInt;
1502 getSignAsIntValue(MagAsInt, DL, Mag);
1503 EVT MagVT = MagAsInt.IntValue.getValueType();
1504 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1505 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1506 ClearSignMask);
1507
1508 // Get the signbit at the right position for MagAsInt.
1509 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1510 if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
1511 if (ShiftAmount > 0) {
1512 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, IntVT);
1513 SignBit = DAG.getNode(ISD::SRL, DL, IntVT, SignBit, ShiftCnst);
1514 } else if (ShiftAmount < 0) {
1515 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, IntVT);
1516 SignBit = DAG.getNode(ISD::SHL, DL, IntVT, SignBit, ShiftCnst);
1517 }
1518 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1519 } else if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
1520 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1521 if (ShiftAmount > 0) {
1522 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, MagVT);
1523 SignBit = DAG.getNode(ISD::SRL, DL, MagVT, SignBit, ShiftCnst);
1524 } else if (ShiftAmount < 0) {
1525 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, MagVT);
1526 SignBit = DAG.getNode(ISD::SHL, DL, MagVT, SignBit, ShiftCnst);
1527 }
1528 }
1529
1530 // Store the part with the modified sign and convert back to float.
1531 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1532 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1533}
1534
1535SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1536 SDLoc DL(Node);
1537 SDValue Value = Node->getOperand(0);
1538
1539 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1540 EVT FloatVT = Value.getValueType();
1541 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1542 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1543 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1544 }
1545
1546 // Transform value to integer, clear the sign bit and transform back.
1547 FloatSignAsInt ValueAsInt;
1548 getSignAsIntValue(ValueAsInt, DL, Value);
1549 EVT IntVT = ValueAsInt.IntValue.getValueType();
1550 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1551 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1552 ClearSignMask);
1553 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1554}
1555
1556void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1557 SmallVectorImpl<SDValue> &Results) {
1558 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1559 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"(static_cast <bool> (SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? void (0) :
__assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1560, __extension__ __PRETTY_FUNCTION__))
1560 " not tell us which reg is the stack pointer!")(static_cast <bool> (SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? void (0) :
__assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1560, __extension__ __PRETTY_FUNCTION__))
;
1561 SDLoc dl(Node);
1562 EVT VT = Node->getValueType(0);
1563 SDValue Tmp1 = SDValue(Node, 0);
1564 SDValue Tmp2 = SDValue(Node, 1);
1565 SDValue Tmp3 = Node->getOperand(2);
1566 SDValue Chain = Tmp1.getOperand(0);
1567
1568 // Chain the dynamic stack allocation so that it doesn't modify the stack
1569 // pointer when other instructions are using the stack.
1570 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1571
1572 SDValue Size = Tmp2.getOperand(1);
1573 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1574 Chain = SP.getValue(1);
1575 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1576 unsigned StackAlign =
1577 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1578 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1579 if (Align > StackAlign)
1580 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1581 DAG.getConstant(-(uint64_t)Align, dl, VT));
1582 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1583
1584 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1585 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1586
1587 Results.push_back(Tmp1);
1588 Results.push_back(Tmp2);
1589}
1590
1591/// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1592/// target.
1593///
1594/// If the SETCC has been legalized using AND / OR, then the legalized node
1595/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1596/// will be set to false.
1597///
1598/// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1599/// then the values of LHS and RHS will be swapped, CC will be set to the
1600/// new condition, and NeedInvert will be set to false.
1601///
1602/// If the SETCC has been legalized using the inverse condcode, then LHS and
1603/// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1604/// will be set to true. The caller must invert the result of the SETCC with
1605/// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1606/// of a true/false result.
1607///
1608/// \returns true if the SetCC has been legalized, false if it hasn't.
1609bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1610 SDValue &RHS, SDValue &CC,
1611 bool &NeedInvert,
1612 const SDLoc &dl) {
1613 MVT OpVT = LHS.getSimpleValueType();
1614 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1615 NeedInvert = false;
1616 switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1617 default: llvm_unreachable("Unknown condition code action!")::llvm::llvm_unreachable_internal("Unknown condition code action!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1617)
;
1618 case TargetLowering::Legal:
1619 // Nothing to do.
1620 break;
1621 case TargetLowering::Expand: {
1622 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
1623 if (TLI.isCondCodeLegal(InvCC, OpVT)) {
1624 std::swap(LHS, RHS);
1625 CC = DAG.getCondCode(InvCC);
1626 return true;
1627 }
1628 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1629 unsigned Opc = 0;
1630 switch (CCCode) {
1631 default: llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1631)
;
1632 case ISD::SETO:
1633 assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT)(static_cast <bool> (TLI.getCondCodeAction(ISD::SETOEQ,
OpVT) == TargetLowering::Legal && "If SETO is expanded, SETOEQ must be legal!"
) ? void (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETOEQ, OpVT) == TargetLowering::Legal && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1635, __extension__ __PRETTY_FUNCTION__))
1634 == TargetLowering::Legal(static_cast <bool> (TLI.getCondCodeAction(ISD::SETOEQ,
OpVT) == TargetLowering::Legal && "If SETO is expanded, SETOEQ must be legal!"
) ? void (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETOEQ, OpVT) == TargetLowering::Legal && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1635, __extension__ __PRETTY_FUNCTION__))
1635 && "If SETO is expanded, SETOEQ must be legal!")(static_cast <bool> (TLI.getCondCodeAction(ISD::SETOEQ,
OpVT) == TargetLowering::Legal && "If SETO is expanded, SETOEQ must be legal!"
) ? void (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETOEQ, OpVT) == TargetLowering::Legal && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1635, __extension__ __PRETTY_FUNCTION__))
;
1636 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1637 case ISD::SETUO:
1638 assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT)(static_cast <bool> (TLI.getCondCodeAction(ISD::SETUNE,
OpVT) == TargetLowering::Legal && "If SETUO is expanded, SETUNE must be legal!"
) ? void (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETUNE, OpVT) == TargetLowering::Legal && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1640, __extension__ __PRETTY_FUNCTION__))
1639 == TargetLowering::Legal(static_cast <bool> (TLI.getCondCodeAction(ISD::SETUNE,
OpVT) == TargetLowering::Legal && "If SETUO is expanded, SETUNE must be legal!"
) ? void (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETUNE, OpVT) == TargetLowering::Legal && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1640, __extension__ __PRETTY_FUNCTION__))
1640 && "If SETUO is expanded, SETUNE must be legal!")(static_cast <bool> (TLI.getCondCodeAction(ISD::SETUNE,
OpVT) == TargetLowering::Legal && "If SETUO is expanded, SETUNE must be legal!"
) ? void (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETUNE, OpVT) == TargetLowering::Legal && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1640, __extension__ __PRETTY_FUNCTION__))
;
1641 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break;
1642 case ISD::SETOEQ:
1643 case ISD::SETOGT:
1644 case ISD::SETOGE:
1645 case ISD::SETOLT:
1646 case ISD::SETOLE:
1647 case ISD::SETONE:
1648 case ISD::SETUEQ:
1649 case ISD::SETUNE:
1650 case ISD::SETUGT:
1651 case ISD::SETUGE:
1652 case ISD::SETULT:
1653 case ISD::SETULE:
1654 // If we are floating point, assign and break, otherwise fall through.
1655 if (!OpVT.isInteger()) {
1656 // We can use the 4th bit to tell if we are the unordered
1657 // or ordered version of the opcode.
1658 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
1659 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
1660 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1661 break;
1662 }
1663 // Fallthrough if we are unsigned integer.
1664 LLVM_FALLTHROUGH[[clang::fallthrough]];
1665 case ISD::SETLE:
1666 case ISD::SETGT:
1667 case ISD::SETGE:
1668 case ISD::SETLT:
1669 // We only support using the inverted operation, which is computed above
1670 // and not a different manner of supporting expanding these cases.
1671 llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1671)
;
1672 case ISD::SETNE:
1673 case ISD::SETEQ:
1674 // Try inverting the result of the inverse condition.
1675 InvCC = CCCode == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ;
1676 if (TLI.isCondCodeLegal(InvCC, OpVT)) {
1677 CC = DAG.getCondCode(InvCC);
1678 NeedInvert = true;
1679 return true;
1680 }
1681 // If inverting the condition didn't work then we have no means to expand
1682 // the condition.
1683 llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1683)
;
1684 }
1685
1686 SDValue SetCC1, SetCC2;
1687 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
1688 // If we aren't the ordered or unorder operation,
1689 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1690 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1691 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1692 } else {
1693 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1694 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1695 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1696 }
1697 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1698 RHS = SDValue();
1699 CC = SDValue();
1700 return true;
1701 }
1702 }
1703 return false;
1704}
1705
1706/// Emit a store/load combination to the stack. This stores
1707/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1708/// a load from the stack slot to DestVT, extending it if needed.
1709/// The resultant code need not be legal.
1710SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1711 EVT DestVT, const SDLoc &dl) {
1712 // Create the stack frame object.
1713 unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1714 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1715 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1716
1717 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1718 int SPFI = StackPtrFI->getIndex();
1719 MachinePointerInfo PtrInfo =
1720 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1721
1722 unsigned SrcSize = SrcOp.getValueSizeInBits();
1723 unsigned SlotSize = SlotVT.getSizeInBits();
1724 unsigned DestSize = DestVT.getSizeInBits();
1725 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1726 unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1727
1728 // Emit a store to the stack slot. Use a truncstore if the input value is
1729 // later than DestVT.
1730 SDValue Store;
1731
1732 if (SrcSize > SlotSize)
1733 Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
1734 SlotVT, SrcAlign);
1735 else {
1736 assert(SrcSize == SlotSize && "Invalid store")(static_cast <bool> (SrcSize == SlotSize && "Invalid store"
) ? void (0) : __assert_fail ("SrcSize == SlotSize && \"Invalid store\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1736, __extension__ __PRETTY_FUNCTION__))
;
1737 Store =
1738 DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1739 }
1740
1741 // Result is a load from the stack slot.
1742 if (SlotSize == DestSize)
1743 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1744
1745 assert(SlotSize < DestSize && "Unknown extension!")(static_cast <bool> (SlotSize < DestSize && "Unknown extension!"
) ? void (0) : __assert_fail ("SlotSize < DestSize && \"Unknown extension!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1745, __extension__ __PRETTY_FUNCTION__))
;
1746 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1747 DestAlign);
1748}
1749
1750SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1751 SDLoc dl(Node);
1752 // Create a vector sized/aligned stack slot, store the value to element #0,
1753 // then load the whole vector back out.
1754 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1755
1756 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1757 int SPFI = StackPtrFI->getIndex();
1758
1759 SDValue Ch = DAG.getTruncStore(
1760 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1761 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1762 Node->getValueType(0).getVectorElementType());
1763 return DAG.getLoad(
1764 Node->getValueType(0), dl, Ch, StackPtr,
1765 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1766}
1767
1768static bool
1769ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1770 const TargetLowering &TLI, SDValue &Res) {
1771 unsigned NumElems = Node->getNumOperands();
1772 SDLoc dl(Node);
1773 EVT VT = Node->getValueType(0);
1774
1775 // Try to group the scalars into pairs, shuffle the pairs together, then
1776 // shuffle the pairs of pairs together, etc. until the vector has
1777 // been built. This will work only if all of the necessary shuffle masks
1778 // are legal.
1779
1780 // We do this in two phases; first to check the legality of the shuffles,
1781 // and next, assuming that all shuffles are legal, to create the new nodes.
1782 for (int Phase = 0; Phase < 2; ++Phase) {
1783 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1784 NewIntermedVals;
1785 for (unsigned i = 0; i < NumElems; ++i) {
1786 SDValue V = Node->getOperand(i);
1787 if (V.isUndef())
1788 continue;
1789
1790 SDValue Vec;
1791 if (Phase)
1792 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1793 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1794 }
1795
1796 while (IntermedVals.size() > 2) {
1797 NewIntermedVals.clear();
1798 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1799 // This vector and the next vector are shuffled together (simply to
1800 // append the one to the other).
1801 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1802
1803 SmallVector<int, 16> FinalIndices;
1804 FinalIndices.reserve(IntermedVals[i].second.size() +
1805 IntermedVals[i+1].second.size());
1806
1807 int k = 0;
1808 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1809 ++j, ++k) {
1810 ShuffleVec[k] = j;
1811 FinalIndices.push_back(IntermedVals[i].second[j]);
1812 }
1813 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1814 ++j, ++k) {
1815 ShuffleVec[k] = NumElems + j;
1816 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1817 }
1818
1819 SDValue Shuffle;
1820 if (Phase)
1821 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1822 IntermedVals[i+1].first,
1823 ShuffleVec);
1824 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1825 return false;
1826 NewIntermedVals.push_back(
1827 std::make_pair(Shuffle, std::move(FinalIndices)));
1828 }
1829
1830 // If we had an odd number of defined values, then append the last
1831 // element to the array of new vectors.
1832 if ((IntermedVals.size() & 1) != 0)
1833 NewIntermedVals.push_back(IntermedVals.back());
1834
1835 IntermedVals.swap(NewIntermedVals);
1836 }
1837
1838 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&(static_cast <bool> (IntermedVals.size() <= 2 &&
IntermedVals.size() > 0 && "Invalid number of intermediate vectors"
) ? void (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1839, __extension__ __PRETTY_FUNCTION__))
1839 "Invalid number of intermediate vectors")(static_cast <bool> (IntermedVals.size() <= 2 &&
IntermedVals.size() > 0 && "Invalid number of intermediate vectors"
) ? void (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1839, __extension__ __PRETTY_FUNCTION__))
;
1840 SDValue Vec1 = IntermedVals[0].first;
1841 SDValue Vec2;
1842 if (IntermedVals.size() > 1)
1843 Vec2 = IntermedVals[1].first;
1844 else if (Phase)
1845 Vec2 = DAG.getUNDEF(VT);
1846
1847 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1848 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1849 ShuffleVec[IntermedVals[0].second[i]] = i;
1850 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1851 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1852
1853 if (Phase)
1854 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1855 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1856 return false;
1857 }
1858
1859 return true;
1860}
1861
1862/// Expand a BUILD_VECTOR node on targets that don't
1863/// support the operation, but do support the resultant vector type.
1864SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1865 unsigned NumElems = Node->getNumOperands();
1866 SDValue Value1, Value2;
1867 SDLoc dl(Node);
1868 EVT VT = Node->getValueType(0);
1869 EVT OpVT = Node->getOperand(0).getValueType();
1870 EVT EltVT = VT.getVectorElementType();
1871
1872 // If the only non-undef value is the low element, turn this into a
1873 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1874 bool isOnlyLowElement = true;
1875 bool MoreThanTwoValues = false;
1876 bool isConstant = true;
1877 for (unsigned i = 0; i < NumElems; ++i) {
1878 SDValue V = Node->getOperand(i);
1879 if (V.isUndef())
1880 continue;
1881 if (i > 0)
1882 isOnlyLowElement = false;
1883 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1884 isConstant = false;
1885
1886 if (!Value1.getNode()) {
1887 Value1 = V;
1888 } else if (!Value2.getNode()) {
1889 if (V != Value1)
1890 Value2 = V;
1891 } else if (V != Value1 && V != Value2) {
1892 MoreThanTwoValues = true;
1893 }
1894 }
1895
1896 if (!Value1.getNode())
1897 return DAG.getUNDEF(VT);
1898
1899 if (isOnlyLowElement)
1900 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1901
1902 // If all elements are constants, create a load from the constant pool.
1903 if (isConstant) {
1904 SmallVector<Constant*, 16> CV;
1905 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1906 if (ConstantFPSDNode *V =
1907 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1908 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1909 } else if (ConstantSDNode *V =
1910 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1911 if (OpVT==EltVT)
1912 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1913 else {
1914 // If OpVT and EltVT don't match, EltVT is not legal and the
1915 // element values have been promoted/truncated earlier. Undo this;
1916 // we don't want a v16i8 to become a v16i32 for example.
1917 const ConstantInt *CI = V->getConstantIntValue();
1918 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1919 CI->getZExtValue()));
1920 }
1921 } else {
1922 assert(Node->getOperand(i).isUndef())(static_cast <bool> (Node->getOperand(i).isUndef()) ?
void (0) : __assert_fail ("Node->getOperand(i).isUndef()"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1922, __extension__ __PRETTY_FUNCTION__))
;
1923 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1924 CV.push_back(UndefValue::get(OpNTy));
1925 }
1926 }
1927 Constant *CP = ConstantVector::get(CV);
1928 SDValue CPIdx =
1929 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1930 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1931 return DAG.getLoad(
1932 VT, dl, DAG.getEntryNode(), CPIdx,
1933 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1934 Alignment);
1935 }
1936
1937 SmallSet<SDValue, 16> DefinedValues;
1938 for (unsigned i = 0; i < NumElems; ++i) {
1939 if (Node->getOperand(i).isUndef())
1940 continue;
1941 DefinedValues.insert(Node->getOperand(i));
1942 }
1943
1944 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1945 if (!MoreThanTwoValues) {
1946 SmallVector<int, 8> ShuffleVec(NumElems, -1);
1947 for (unsigned i = 0; i < NumElems; ++i) {
1948 SDValue V = Node->getOperand(i);
1949 if (V.isUndef())
1950 continue;
1951 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1952 }
1953 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1954 // Get the splatted value into the low element of a vector register.
1955 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1956 SDValue Vec2;
1957 if (Value2.getNode())
1958 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1959 else
1960 Vec2 = DAG.getUNDEF(VT);
1961
1962 // Return shuffle(LowValVec, undef, <0,0,0,0>)
1963 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1964 }
1965 } else {
1966 SDValue Res;
1967 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1968 return Res;
1969 }
1970 }
1971
1972 // Otherwise, we can't handle this case efficiently.
1973 return ExpandVectorBuildThroughStack(Node);
1974}
1975
1976// Expand a node into a call to a libcall. If the result value
1977// does not fit into a register, return the lo part and set the hi part to the
1978// by-reg argument. If it does fit into a single register, return the result
1979// and leave the Hi part unset.
1980SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
1981 bool isSigned) {
1982 TargetLowering::ArgListTy Args;
1983 TargetLowering::ArgListEntry Entry;
1984 for (const SDValue &Op : Node->op_values()) {
1985 EVT ArgVT = Op.getValueType();
1986 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1987 Entry.Node = Op;
1988 Entry.Ty = ArgTy;
1989 Entry.IsSExt = isSigned;
1990 Entry.IsZExt = !isSigned;
1991 Args.push_back(Entry);
1992 }
1993 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
1994 TLI.getPointerTy(DAG.getDataLayout()));
1995
1996 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
1997
1998 // By default, the input chain to this libcall is the entry node of the
1999 // function. If the libcall is going to be emitted as a tail call then
2000 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2001 // node which is being folded has a non-entry input chain.
2002 SDValue InChain = DAG.getEntryNode();
2003
2004 // isTailCall may be true since the callee does not reference caller stack
2005 // frame. Check if it's in the right position and that the return types match.
2006 SDValue TCChain = InChain;
2007 const Function *F = DAG.getMachineFunction().getFunction();
2008 bool isTailCall =
2009 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2010 (RetTy == F->getReturnType() || F->getReturnType()->isVoidTy());
2011 if (isTailCall)
2012 InChain = TCChain;
2013
2014 TargetLowering::CallLoweringInfo CLI(DAG);
2015 CLI.setDebugLoc(SDLoc(Node))
2016 .setChain(InChain)
2017 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2018 std::move(Args))
2019 .setTailCall(isTailCall)
2020 .setSExtResult(isSigned)
2021 .setZExtResult(!isSigned)
2022 .setIsPostTypeLegalization(true);
2023
2024 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2025
2026 if (!CallInfo.second.getNode()) {
2027 DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Created tailcall: "; DAG.
getRoot().dump(); } } while (false)
;
2028 // It's a tailcall, return the chain (which is the DAG root).
2029 return DAG.getRoot();
2030 }
2031
2032 DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Created libcall: "; CallInfo
.first.dump(); } } while (false)
;
2033 return CallInfo.first;
2034}
2035
2036/// Generate a libcall taking the given operands as arguments
2037/// and returning a result of type RetVT.
2038SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
2039 const SDValue *Ops, unsigned NumOps,
2040 bool isSigned, const SDLoc &dl) {
2041 TargetLowering::ArgListTy Args;
2042 Args.reserve(NumOps);
2043
2044 TargetLowering::ArgListEntry Entry;
2045 for (unsigned i = 0; i != NumOps; ++i) {
2046 Entry.Node = Ops[i];
2047 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
2048 Entry.IsSExt = isSigned;
2049 Entry.IsZExt = !isSigned;
2050 Args.push_back(Entry);
2051 }
2052 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2053 TLI.getPointerTy(DAG.getDataLayout()));
2054
2055 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2056
2057 TargetLowering::CallLoweringInfo CLI(DAG);
2058 CLI.setDebugLoc(dl)
2059 .setChain(DAG.getEntryNode())
2060 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2061 std::move(Args))
2062 .setSExtResult(isSigned)
2063 .setZExtResult(!isSigned)
2064 .setIsPostTypeLegalization(true);
2065
2066 std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
2067
2068 return CallInfo.first;
2069}
2070
2071// Expand a node into a call to a libcall. Similar to
2072// ExpandLibCall except that the first operand is the in-chain.
2073std::pair<SDValue, SDValue>
2074SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2075 SDNode *Node,
2076 bool isSigned) {
2077 SDValue InChain = Node->getOperand(0);
2078
2079 TargetLowering::ArgListTy Args;
2080 TargetLowering::ArgListEntry Entry;
2081 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
2082 EVT ArgVT = Node->getOperand(i).getValueType();
2083 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2084 Entry.Node = Node->getOperand(i);
2085 Entry.Ty = ArgTy;
2086 Entry.IsSExt = isSigned;
2087 Entry.IsZExt = !isSigned;
2088 Args.push_back(Entry);
2089 }
2090 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2091 TLI.getPointerTy(DAG.getDataLayout()));
2092
2093 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2094
2095 TargetLowering::CallLoweringInfo CLI(DAG);
2096 CLI.setDebugLoc(SDLoc(Node))
2097 .setChain(InChain)
2098 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2099 std::move(Args))
2100 .setSExtResult(isSigned)
2101 .setZExtResult(!isSigned);
2102
2103 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2104
2105 return CallInfo;
2106}
2107
2108SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2109 RTLIB::Libcall Call_F32,
2110 RTLIB::Libcall Call_F64,
2111 RTLIB::Libcall Call_F80,
2112 RTLIB::Libcall Call_F128,
2113 RTLIB::Libcall Call_PPCF128) {
2114 if (Node->isStrictFPOpcode())
2115 Node = DAG.mutateStrictFPToFP(Node);
2116
2117 RTLIB::Libcall LC;
2118 switch (Node->getSimpleValueType(0).SimpleTy) {
2119 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2119)
;
2120 case MVT::f32: LC = Call_F32; break;
2121 case MVT::f64: LC = Call_F64; break;
2122 case MVT::f80: LC = Call_F80; break;
2123 case MVT::f128: LC = Call_F128; break;
2124 case MVT::ppcf128: LC = Call_PPCF128; break;
2125 }
2126 return ExpandLibCall(LC, Node, false);
2127}
2128
2129SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2130 RTLIB::Libcall Call_I8,
2131 RTLIB::Libcall Call_I16,
2132 RTLIB::Libcall Call_I32,
2133 RTLIB::Libcall Call_I64,
2134 RTLIB::Libcall Call_I128) {
2135 RTLIB::Libcall LC;
2136 switch (Node->getSimpleValueType(0).SimpleTy) {
2137 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2137)
;
2138 case MVT::i8: LC = Call_I8; break;
2139 case MVT::i16: LC = Call_I16; break;
2140 case MVT::i32: LC = Call_I32; break;
2141 case MVT::i64: LC = Call_I64; break;
2142 case MVT::i128: LC = Call_I128; break;
2143 }
2144 return ExpandLibCall(LC, Node, isSigned);
2145}
2146
2147/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2148void
2149SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2150 SmallVectorImpl<SDValue> &Results) {
2151 unsigned Opcode = Node->getOpcode();
2152 bool isSigned = Opcode == ISD::SDIVREM;
2153
2154 RTLIB::Libcall LC;
2155 switch (Node->getSimpleValueType(0).SimpleTy) {
2156 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2156)
;
2157 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2158 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2159 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2160 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2161 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2162 }
2163
2164 // The input chain to this libcall is the entry node of the function.
2165 // Legalizing the call will automatically add the previous call to the
2166 // dependence.
2167 SDValue InChain = DAG.getEntryNode();
2168
2169 EVT RetVT = Node->getValueType(0);
2170 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2171
2172 TargetLowering::ArgListTy Args;
2173 TargetLowering::ArgListEntry Entry;
2174 for (const SDValue &Op : Node->op_values()) {
2175 EVT ArgVT = Op.getValueType();
2176 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2177 Entry.Node = Op;
2178 Entry.Ty = ArgTy;
2179 Entry.IsSExt = isSigned;
2180 Entry.IsZExt = !isSigned;
2181 Args.push_back(Entry);
2182 }
2183
2184 // Also pass the return address of the remainder.
2185 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2186 Entry.Node = FIPtr;
2187 Entry.Ty = RetTy->getPointerTo();
2188 Entry.IsSExt = isSigned;
2189 Entry.IsZExt = !isSigned;
2190 Args.push_back(Entry);
2191
2192 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2193 TLI.getPointerTy(DAG.getDataLayout()));
2194
2195 SDLoc dl(Node);
2196 TargetLowering::CallLoweringInfo CLI(DAG);
2197 CLI.setDebugLoc(dl)
2198 .setChain(InChain)
2199 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2200 std::move(Args))
2201 .setSExtResult(isSigned)
2202 .setZExtResult(!isSigned);
2203
2204 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2205
2206 // Remainder is loaded back from the stack frame.
2207 SDValue Rem =
2208 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2209 Results.push_back(CallInfo.first);
2210 Results.push_back(Rem);
2211}
2212
2213/// Return true if sincos libcall is available.
2214static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2215 RTLIB::Libcall LC;
2216 switch (Node->getSimpleValueType(0).SimpleTy) {
2217 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2217)
;
2218 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2219 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2220 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2221 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2222 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2223 }
2224 return TLI.getLibcallName(LC) != nullptr;
2225}
2226
2227/// Only issue sincos libcall if both sin and cos are needed.
2228static bool useSinCos(SDNode *Node) {
2229 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2230 ? ISD::FCOS : ISD::FSIN;
2231
2232 SDValue Op0 = Node->getOperand(0);
2233 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2234 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2235 SDNode *User = *UI;
2236 if (User == Node)
2237 continue;
2238 // The other user might have been turned into sincos already.
2239 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2240 return true;
2241 }
2242 return false;
2243}
2244
2245/// Issue libcalls to sincos to compute sin / cos pairs.
2246void
2247SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2248 SmallVectorImpl<SDValue> &Results) {
2249 RTLIB::Libcall LC;
2250 switch (Node->getSimpleValueType(0).SimpleTy) {
2251 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2251)
;
2252 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2253 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2254 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2255 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2256 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2257 }
2258
2259 // The input chain to this libcall is the entry node of the function.
2260 // Legalizing the call will automatically add the previous call to the
2261 // dependence.
2262 SDValue InChain = DAG.getEntryNode();
2263
2264 EVT RetVT = Node->getValueType(0);
2265 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2266
2267 TargetLowering::ArgListTy Args;
2268 TargetLowering::ArgListEntry Entry;
2269
2270 // Pass the argument.
2271 Entry.Node = Node->getOperand(0);
2272 Entry.Ty = RetTy;
2273 Entry.IsSExt = false;
2274 Entry.IsZExt = false;
2275 Args.push_back(Entry);
2276
2277 // Pass the return address of sin.
2278 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2279 Entry.Node = SinPtr;
2280 Entry.Ty = RetTy->getPointerTo();
2281 Entry.IsSExt = false;
2282 Entry.IsZExt = false;
2283 Args.push_back(Entry);
2284
2285 // Also pass the return address of the cos.
2286 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2287 Entry.Node = CosPtr;
2288 Entry.Ty = RetTy->getPointerTo();
2289 Entry.IsSExt = false;
2290 Entry.IsZExt = false;
2291 Args.push_back(Entry);
2292
2293 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2294 TLI.getPointerTy(DAG.getDataLayout()));
2295
2296 SDLoc dl(Node);
2297 TargetLowering::CallLoweringInfo CLI(DAG);
2298 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2299 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2300 std::move(Args));
2301
2302 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2303
2304 Results.push_back(
2305 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2306 Results.push_back(
2307 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2308}
2309
2310/// This function is responsible for legalizing a
2311/// INT_TO_FP operation of the specified operand when the target requests that
2312/// we expand it. At this point, we know that the result and operand types are
2313/// legal for the target.
2314SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2315 EVT DestVT,
2316 const SDLoc &dl) {
2317 // TODO: Should any fast-math-flags be set for the created nodes?
2318 DEBUG(dbgs() << "Legalizing INT_TO_FP\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing INT_TO_FP\n"; }
} while (false)
;
2319 if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
2320 DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double "
"expansion\n"; } } while (false)
2321 "expansion\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double "
"expansion\n"; } } while (false)
;
2322
2323 // Get the stack frame index of a 8 byte buffer.
2324 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2325
2326 // word offset constant for Hi/Lo address computation
2327 SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2328 StackSlot.getValueType());
2329 // set up Hi and Lo (into buffer) address based on endian
2330 SDValue Hi = StackSlot;
2331 SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2332 StackSlot, WordOff);
2333 if (DAG.getDataLayout().isLittleEndian())
2334 std::swap(Hi, Lo);
2335
2336 // if signed map to unsigned space
2337 SDValue Op0Mapped;
2338 if (isSigned) {
2339 // constant used to invert sign bit (signed to unsigned mapping)
2340 SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2341 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2342 } else {
2343 Op0Mapped = Op0;
2344 }
2345 // store the lo of the constructed double - based on integer input
2346 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2347 MachinePointerInfo());
2348 // initial hi portion of constructed double
2349 SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2350 // store the hi of the constructed double - biased exponent
2351 SDValue Store2 =
2352 DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
2353 // load the constructed double
2354 SDValue Load =
2355 DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
2356 // FP constant to bias correct the final result
2357 SDValue Bias = DAG.getConstantFP(isSigned ?
2358 BitsToDouble(0x4330000080000000ULL) :
2359 BitsToDouble(0x4330000000000000ULL),
2360 dl, MVT::f64);
2361 // subtract the bias
2362 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2363 // final result
2364 SDValue Result;
2365 // handle final rounding
2366 if (DestVT == MVT::f64) {
2367 // do nothing
2368 Result = Sub;
2369 } else if (DestVT.bitsLT(MVT::f64)) {
2370 Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
2371 DAG.getIntPtrConstant(0, dl));
2372 } else if (DestVT.bitsGT(MVT::f64)) {
2373 Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
2374 }
2375 return Result;
2376 }
2377 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet")(static_cast <bool> (!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"
) ? void (0) : __assert_fail ("!isSigned && \"Legalize cannot Expand SINT_TO_FP for i64 yet\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2377, __extension__ __PRETTY_FUNCTION__))
;
2378 // Code below here assumes !isSigned without checking again.
2379
2380 // Implementation of unsigned i64 to f64 following the algorithm in
2381 // __floatundidf in compiler_rt. This implementation has the advantage
2382 // of performing rounding correctly, both in the default rounding mode
2383 // and in all alternate rounding modes.
2384 // TODO: Generalize this for use with other types.
2385 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) {
2386 DEBUG(dbgs() << "Converting unsigned i64 to f64\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Converting unsigned i64 to f64\n"
; } } while (false)
;
2387 SDValue TwoP52 =
2388 DAG.getConstant(UINT64_C(0x4330000000000000)0x4330000000000000UL, dl, MVT::i64);
2389 SDValue TwoP84PlusTwoP52 =
2390 DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)0x4530000000100000UL), dl,
2391 MVT::f64);
2392 SDValue TwoP84 =
2393 DAG.getConstant(UINT64_C(0x4530000000000000)0x4530000000000000UL, dl, MVT::i64);
2394
2395 SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32);
2396 SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
2397 DAG.getConstant(32, dl, MVT::i64));
2398 SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
2399 SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
2400 SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
2401 SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr);
2402 SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt,
2403 TwoP84PlusTwoP52);
2404 return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub);
2405 }
2406
2407 // TODO: Generalize this for use with other types.
2408 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) {
2409 DEBUG(dbgs() << "Converting unsigned i64 to f32\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Converting unsigned i64 to f32\n"
; } } while (false)
;
2410 // For unsigned conversions, convert them to signed conversions using the
2411 // algorithm from the x86_64 __floatundidf in compiler_rt.
2412 if (!isSigned) {
2413 SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
2414
2415 SDValue ShiftConst = DAG.getConstant(
2416 1, dl, TLI.getShiftAmountTy(Op0.getValueType(), DAG.getDataLayout()));
2417 SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
2418 SDValue AndConst = DAG.getConstant(1, dl, MVT::i64);
2419 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
2420 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
2421
2422 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
2423 SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
2424
2425 // TODO: This really should be implemented using a branch rather than a
2426 // select. We happen to get lucky and machinesink does the right
2427 // thing most of the time. This would be a good candidate for a
2428 //pseudo-op, or, even better, for whole-function isel.
2429 SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
2430 Op0, DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
2431 return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast);
2432 }
2433
2434 // Otherwise, implement the fully general conversion.
2435
2436 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2437 DAG.getConstant(UINT64_C(0xfffffffffffff800)0xfffffffffffff800UL, dl, MVT::i64));
2438 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
2439 DAG.getConstant(UINT64_C(0x800)0x800UL, dl, MVT::i64));
2440 SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2441 DAG.getConstant(UINT64_C(0x7ff)0x7ffUL, dl, MVT::i64));
2442 SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), And2,
2443 DAG.getConstant(UINT64_C(0)0UL, dl, MVT::i64),
2444 ISD::SETNE);
2445 SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0);
2446 SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), Op0,
2447 DAG.getConstant(UINT64_C(0x0020000000000000)0x0020000000000000UL, dl,
2448 MVT::i64),
2449 ISD::SETUGE);
2450 SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0);
2451 EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType(), DAG.getDataLayout());
2452
2453 SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
2454 DAG.getConstant(32, dl, SHVT));
2455 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
2456 SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
2457 SDValue TwoP32 =
2458 DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)0x41f0000000000000UL), dl,
2459 MVT::f64);
2460 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
2461 SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
2462 SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
2463 SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
2464 return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
2465 DAG.getIntPtrConstant(0, dl));
2466 }
2467
2468 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2469
2470 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()),
2471 Op0,
2472 DAG.getConstant(0, dl, Op0.getValueType()),
2473 ISD::SETLT);
2474 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2475 Four = DAG.getIntPtrConstant(4, dl);
2476 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2477 SignSet, Four, Zero);
2478
2479 // If the sign bit of the integer is set, the large number will be treated
2480 // as a negative number. To counteract this, the dynamic code adds an
2481 // offset depending on the data type.
2482 uint64_t FF;
2483 switch (Op0.getSimpleValueType().SimpleTy) {
2484 default: llvm_unreachable("Unsupported integer type!")::llvm::llvm_unreachable_internal("Unsupported integer type!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2484)
;
2485 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2486 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2487 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2488 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2489 }
2490 if (DAG.getDataLayout().isLittleEndian())
2491 FF <<= 32;
2492 Constant *FudgeFactor = ConstantInt::get(
2493 Type::getInt64Ty(*DAG.getContext()), FF);
2494
2495 SDValue CPIdx =
2496 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2497 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2498 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2499 Alignment = std::min(Alignment, 4u);
2500 SDValue FudgeInReg;
2501 if (DestVT == MVT::f32)
2502 FudgeInReg = DAG.getLoad(
2503 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2504 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2505 Alignment);
2506 else {
2507 SDValue Load = DAG.getExtLoad(
2508 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2509 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2510 Alignment);
2511 HandleSDNode Handle(Load);
2512 LegalizeOp(Load.getNode());
2513 FudgeInReg = Handle.getValue();
2514 }
2515
2516 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2517}
2518
2519/// This function is responsible for legalizing a
2520/// *INT_TO_FP operation of the specified operand when the target requests that
2521/// we promote it. At this point, we know that the result and operand types are
2522/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2523/// operation that takes a larger input.
2524SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2525 bool isSigned,
2526 const SDLoc &dl) {
2527 // First step, figure out the appropriate *INT_TO_FP operation to use.
2528 EVT NewInTy = LegalOp.getValueType();
2529
2530 unsigned OpToUse = 0;
2531
2532 // Scan for the appropriate larger type to use.
2533 while (true) {
2534 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2535 assert(NewInTy.isInteger() && "Ran out of possibilities!")(static_cast <bool> (NewInTy.isInteger() && "Ran out of possibilities!"
) ? void (0) : __assert_fail ("NewInTy.isInteger() && \"Ran out of possibilities!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2535, __extension__ __PRETTY_FUNCTION__))
;
2536
2537 // If the target supports SINT_TO_FP of this type, use it.
2538 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2539 OpToUse = ISD::SINT_TO_FP;
2540 break;
2541 }
2542 if (isSigned) continue;
2543
2544 // If the target supports UINT_TO_FP of this type, use it.
2545 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2546 OpToUse = ISD::UINT_TO_FP;
2547 break;
2548 }
2549
2550 // Otherwise, try a larger type.
2551 }
2552
2553 // Okay, we found the operation and type to use. Zero extend our input to the
2554 // desired type then run the operation on it.
2555 return DAG.getNode(OpToUse, dl, DestVT,
2556 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2557 dl, NewInTy, LegalOp));
2558}
2559
2560/// This function is responsible for legalizing a
2561/// FP_TO_*INT operation of the specified operand when the target requests that
2562/// we promote it. At this point, we know that the result and operand types are
2563/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2564/// operation that returns a larger result.
2565SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2566 bool isSigned,
2567 const SDLoc &dl) {
2568 // First step, figure out the appropriate FP_TO*INT operation to use.
2569 EVT NewOutTy = DestVT;
2570
2571 unsigned OpToUse = 0;
2572
2573 // Scan for the appropriate larger type to use.
2574 while (true) {
2575 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2576 assert(NewOutTy.isInteger() && "Ran out of possibilities!")(static_cast <bool> (NewOutTy.isInteger() && "Ran out of possibilities!"
) ? void (0) : __assert_fail ("NewOutTy.isInteger() && \"Ran out of possibilities!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2576, __extension__ __PRETTY_FUNCTION__))
;
2577
2578 // A larger signed type can hold all unsigned values of the requested type,
2579 // so using FP_TO_SINT is valid
2580 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2581 OpToUse = ISD::FP_TO_SINT;
2582 break;
2583 }
2584
2585 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2586 if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2587 OpToUse = ISD::FP_TO_UINT;
2588 break;
2589 }
2590
2591 // Otherwise, try a larger type.
2592 }
2593
2594 // Okay, we found the operation and type to use.
2595 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2596
2597 // Truncate the result of the extended FP_TO_*INT operation to the desired
2598 // size.
2599 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2600}
2601
2602/// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2603SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2604 EVT VT = Op.getValueType();
2605 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2606 unsigned Sz = VT.getScalarSizeInBits();
2607
2608 SDValue Tmp, Tmp2, Tmp3;
2609
2610 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2611 // and finally the i1 pairs.
2612 // TODO: We can easily support i4/i2 legal types if any target ever does.
2613 if (Sz >= 8 && isPowerOf2_32(Sz)) {
2614 // Create the masks - repeating the pattern every byte.
2615 APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0);
2616 APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0);
2617 for (unsigned J = 0; J != Sz; J += 8) {
2618 MaskHi4 = MaskHi4 | (0xF0ull << J);
2619 MaskLo4 = MaskLo4 | (0x0Full << J);
2620 MaskHi2 = MaskHi2 | (0xCCull << J);
2621 MaskLo2 = MaskLo2 | (0x33ull << J);
2622 MaskHi1 = MaskHi1 | (0xAAull << J);
2623 MaskLo1 = MaskLo1 | (0x55ull << J);
2624 }
2625
2626 // BSWAP if the type is wider than a single byte.
2627 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
2628
2629 // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2630 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
2631 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
2632 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, VT));
2633 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, VT));
2634 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2635
2636 // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2637 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
2638 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
2639 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, VT));
2640 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, VT));
2641 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2642
2643 // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2644 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
2645 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
2646 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, VT));
2647 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, VT));
2648 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2649 return Tmp;
2650 }
2651
2652 Tmp = DAG.getConstant(0, dl, VT);
2653 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
2654 if (I < J)
2655 Tmp2 =
2656 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
2657 else
2658 Tmp2 =
2659 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
2660
2661 APInt Shift(Sz, 1);
2662 Shift <<= J;
2663 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
2664 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
2665 }
2666
2667 return Tmp;
2668}
2669
2670/// Open code the operations for BSWAP of the specified operation.
2671SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2672 EVT VT = Op.getValueType();
2673 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2674 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2675 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
2676 default: llvm_unreachable("Unhandled Expand type in BSWAP!")::llvm::llvm_unreachable_internal("Unhandled Expand type in BSWAP!"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2676)
;
2677 case MVT::i16:
2678 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2679 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2680 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2681 case MVT::i32:
2682 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2683 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2684 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2685 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2686 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2687 DAG.getConstant(0xFF0000, dl, VT));
2688 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2689 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2690 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2691 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2692 case MVT::i64:
2693 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2694 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2695 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2696 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2697 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2698 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2699 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2700 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2701 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2702 DAG.getConstant(255ULL<<48, dl, VT));
2703 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2704 DAG.getConstant(255ULL<<40, dl, VT));
2705 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2706 DAG.getConstant(255ULL<<32, dl, VT));
2707 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2708 DAG.getConstant(255ULL<<24, dl, VT));
2709 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2710 DAG.getConstant(255ULL<<16, dl, VT));
2711 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2712 DAG.getConstant(255ULL<<8 , dl, VT));
2713 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2714 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2715 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2716 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2717 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2718 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2719 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2720 }
2721}
2722
2723/// Expand the specified bitcount instruction into operations.
2724SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
2725 const SDLoc &dl) {
2726 switch (Opc) {
2727 default: llvm_unreachable("Cannot expand this yet!")::llvm::llvm_unreachable_internal("Cannot expand this yet!", "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2727)
;
2728 case ISD::CTPOP: {
2729 EVT VT = Op.getValueType();
2730 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2731 unsigned Len = VT.getSizeInBits();
2732
2733 assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 &&(static_cast <bool> (VT.isInteger() && Len <=
128 && Len % 8 == 0 && "CTPOP not implemented for this type."
) ? void (0) : __assert_fail ("VT.isInteger() && Len <= 128 && Len % 8 == 0 && \"CTPOP not implemented for this type.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2734, __extension__ __PRETTY_FUNCTION__))
2734 "CTPOP not implemented for this type.")(static_cast <bool> (VT.isInteger() && Len <=
128 && Len % 8 == 0 && "CTPOP not implemented for this type."
) ? void (0) : __assert_fail ("VT.isInteger() && Len <= 128 && Len % 8 == 0 && \"CTPOP not implemented for this type.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2734, __extension__ __PRETTY_FUNCTION__))
;
2735
2736 // This is the "best" algorithm from
2737 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
2738
2739 SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)),
2740 dl, VT);
2741 SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)),
2742 dl, VT);
2743 SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)),
2744 dl, VT);
2745 SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)),
2746 dl, VT);
2747
2748 // v = v - ((v >> 1) & 0x55555555...)
2749 Op = DAG.getNode(ISD::SUB, dl, VT, Op,
2750 DAG.getNode(ISD::AND, dl, VT,
2751 DAG.getNode(ISD::SRL, dl, VT, Op,
2752 DAG.getConstant(1, dl, ShVT)),
2753 Mask55));
2754 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
2755 Op = DAG.getNode(ISD::ADD, dl, VT,
2756 DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
2757 DAG.getNode(ISD::AND, dl, VT,
2758 DAG.getNode(ISD::SRL, dl, VT, Op,
2759 DAG.getConstant(2, dl, ShVT)),
2760 Mask33));
2761 // v = (v + (v >> 4)) & 0x0F0F0F0F...
2762 Op = DAG.getNode(ISD::AND, dl, VT,
2763 DAG.getNode(ISD::ADD, dl, VT, Op,
2764 DAG.getNode(ISD::SRL, dl, VT, Op,
2765 DAG.getConstant(4, dl, ShVT))),
2766 Mask0F);
2767 // v = (v * 0x01010101...) >> (Len - 8)
2768 Op = DAG.getNode(ISD::SRL, dl, VT,
2769 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
2770 DAG.getConstant(Len - 8, dl, ShVT));
2771
2772 return Op;
2773 }
2774 case ISD::CTLZ_ZERO_UNDEF:
2775 // This trivially expands to CTLZ.
2776 return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op);
2777 case ISD::CTLZ: {
2778 EVT VT = Op.getValueType();
2779 unsigned Len = VT.getSizeInBits();
2780
2781 if (TLI.isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
2782 EVT SetCCVT = getSetCCResultType(VT);
2783 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
2784 SDValue Zero = DAG.getConstant(0, dl, VT);
2785 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
2786 return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
2787 DAG.getConstant(Len, dl, VT), CTLZ);
2788 }
2789
2790 // for now, we do this:
2791 // x = x | (x >> 1);
2792 // x = x | (x >> 2);
2793 // ...
2794 // x = x | (x >>16);
2795 // x = x | (x >>32); // for 64-bit input
2796 // return popcount(~x);
2797 //
2798 // Ref: "Hacker's Delight" by Henry Warren
2799 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2800 for (unsigned i = 0; (1U << i) <= (Len / 2); ++i) {
2801 SDValue Tmp3 = DAG.getConstant(1ULL << i, dl, ShVT);
2802 Op = DAG.getNode(ISD::OR, dl, VT, Op,
2803 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
2804 }
2805 Op = DAG.getNOT(dl, Op, VT);
2806 return DAG.getNode(ISD::CTPOP, dl, VT, Op);
2807 }
2808 case ISD::CTTZ_ZERO_UNDEF:
2809 // This trivially expands to CTTZ.
2810 return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op);
2811 case ISD::CTTZ: {
2812 EVT VT = Op.getValueType();
2813 unsigned Len = VT.getSizeInBits();
2814
2815 if (TLI.isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
2816 EVT SetCCVT = getSetCCResultType(VT);
2817 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
2818 SDValue Zero = DAG.getConstant(0, dl, VT);
2819 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
2820 return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
2821 DAG.getConstant(Len, dl, VT), CTTZ);
2822 }
2823
2824 // for now, we use: { return popcount(~x & (x - 1)); }
2825 // unless the target has ctlz but not ctpop, in which case we use:
2826 // { return 32 - nlz(~x & (x-1)); }
2827 // Ref: "Hacker's Delight" by Henry Warren
2828 SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
2829 DAG.getNOT(dl, Op, VT),
2830 DAG.getNode(ISD::SUB, dl, VT, Op,
2831 DAG.getConstant(1, dl, VT)));
2832 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
2833 if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
2834 TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
2835 return DAG.getNode(ISD::SUB, dl, VT,
2836 DAG.getConstant(VT.getSizeInBits(), dl, VT),
2837 DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
2838 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
2839 }
2840 }
2841}
2842
2843bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2844 DEBUG(dbgs() << "Trying to expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to expand node\n";
} } while (false)
;
2845 SmallVector<SDValue, 8> Results;
2846 SDLoc dl(Node);
2847 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2848 bool NeedInvert;
2849 switch (Node->getOpcode()) {
2850 case ISD::CTPOP:
2851 case ISD::CTLZ:
2852 case ISD::CTLZ_ZERO_UNDEF:
2853 case ISD::CTTZ:
2854 case ISD::CTTZ_ZERO_UNDEF:
2855 Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
2856 Results.push_back(Tmp1);
2857 break;
2858 case ISD::BITREVERSE:
2859 Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
2860 break;
2861 case ISD::BSWAP:
2862 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2863 break;
2864 case ISD::FRAMEADDR:
2865 case ISD::RETURNADDR:
2866 case ISD::FRAME_TO_ARGS_OFFSET:
2867 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2868 break;
2869 case ISD::EH_DWARF_CFA: {
2870 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2871 TLI.getPointerTy(DAG.getDataLayout()));
2872 SDValue Offset = DAG.getNode(ISD::ADD, dl,
2873 CfaArg.getValueType(),
2874 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2875 CfaArg.getValueType()),
2876 CfaArg);
2877 SDValue FA = DAG.getNode(
2878 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2879 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2880 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2881 FA, Offset));
2882 break;
2883 }
2884 case ISD::FLT_ROUNDS_:
2885 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2886 break;
2887 case ISD::EH_RETURN:
2888 case ISD::EH_LABEL:
2889 case ISD::PREFETCH:
2890 case ISD::VAEND:
2891 case ISD::EH_SJLJ_LONGJMP:
2892 // If the target didn't expand these, there's nothing to do, so just
2893 // preserve the chain and be done.
2894 Results.push_back(Node->getOperand(0));
2895 break;
2896 case ISD::READCYCLECOUNTER:
2897 // If the target didn't expand this, just return 'zero' and preserve the
2898 // chain.
2899 Results.append(Node->getNumValues() - 1,
2900 DAG.getConstant(0, dl, Node->getValueType(0)));
2901 Results.push_back(Node->getOperand(0));
2902 break;
2903 case ISD::EH_SJLJ_SETJMP:
2904 // If the target didn't expand this, just return 'zero' and preserve the
2905 // chain.
2906 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2907 Results.push_back(Node->getOperand(0));
2908 break;
2909 case ISD::ATOMIC_LOAD: {
2910 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2911 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2912 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2913 SDValue Swap = DAG.getAtomicCmpSwap(
2914 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2915 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2916 cast<AtomicSDNode>(Node)->getMemOperand());
2917 Results.push_back(Swap.getValue(0));
2918 Results.push_back(Swap.getValue(1));
2919 break;
2920 }
2921 case ISD::ATOMIC_STORE: {
2922 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2923 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2924 cast<AtomicSDNode>(Node)->getMemoryVT(),
2925 Node->getOperand(0),
2926 Node->getOperand(1), Node->getOperand(2),
2927 cast<AtomicSDNode>(Node)->getMemOperand());
2928 Results.push_back(Swap.getValue(1));
2929 break;
2930 }
2931 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2932 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2933 // splits out the success value as a comparison. Expanding the resulting
2934 // ATOMIC_CMP_SWAP will produce a libcall.
2935 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2936 SDValue Res = DAG.getAtomicCmpSwap(
2937 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2938 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2939 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2940
2941 SDValue ExtRes = Res;
2942 SDValue LHS = Res;
2943 SDValue RHS = Node->getOperand(1);
2944
2945 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2946 EVT OuterType = Node->getValueType(0);
2947 switch (TLI.getExtendForAtomicOps()) {
2948 case ISD::SIGN_EXTEND:
2949 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2950 DAG.getValueType(AtomicType));
2951 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2952 Node->getOperand(2), DAG.getValueType(AtomicType));
2953 ExtRes = LHS;
2954 break;
2955 case ISD::ZERO_EXTEND:
2956 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2957 DAG.getValueType(AtomicType));
2958 RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
2959 ExtRes = LHS;
2960 break;
2961 case ISD::ANY_EXTEND:
2962 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2963 RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
2964 break;
2965 default:
2966 llvm_unreachable("Invalid atomic op extension")::llvm::llvm_unreachable_internal("Invalid atomic op extension"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2966)
;
2967 }
2968
2969 SDValue Success =
2970 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2971
2972 Results.push_back(ExtRes.getValue(0));
2973 Results.push_back(Success);
2974 Results.push_back(Res.getValue(1));
2975 break;
2976 }
2977 case ISD::DYNAMIC_STACKALLOC:
2978 ExpandDYNAMIC_STACKALLOC(Node, Results);
2979 break;
2980 case ISD::MERGE_VALUES:
2981 for (unsigned i = 0; i < Node->getNumValues(); i++)
2982 Results.push_back(Node->getOperand(i));
2983 break;
2984 case ISD::UNDEF: {
2985 EVT VT = Node->getValueType(0);
2986 if (VT.isInteger())
2987 Results.push_back(DAG.getConstant(0, dl, VT));
2988 else {
2989 assert(VT.isFloatingPoint() && "Unknown value type!")(static_cast <bool> (VT.isFloatingPoint() && "Unknown value type!"
) ? void (0) : __assert_fail ("VT.isFloatingPoint() && \"Unknown value type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))
;
2990 Results.push_back(DAG.getConstantFP(0, dl, VT));
2991 }
2992 break;
2993 }
2994 case ISD::FP_ROUND:
2995 case ISD::BITCAST:
2996 Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2997 Node->getValueType(0), dl);
2998 Results.push_back(Tmp1);
2999 break;
3000 case ISD::FP_EXTEND:
3001 Tmp1 = EmitStackConvert(Node->getOperand(0),
3002 Node->getOperand(0).getValueType(),
3003 Node->getValueType(0), dl);
3004 Results.push_back(Tmp1);
3005 break;
3006 case ISD::SIGN_EXTEND_INREG: {
3007 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3008 EVT VT = Node->getValueType(0);
3009
3010 // An in-register sign-extend of a boolean is a negation:
3011 // 'true' (1) sign-extended is -1.
3012 // 'false' (0) sign-extended is 0.
3013 // However, we must mask the high bits of the source operand because the
3014 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3015
3016 // TODO: Do this for vectors too?
3017 if (ExtraVT.getSizeInBits() == 1) {
3018 SDValue One = DAG.getConstant(1, dl, VT);
3019 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
3020 SDValue Zero = DAG.getConstant(0, dl, VT);
3021 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
3022 Results.push_back(Neg);
3023 break;
3024 }
3025
3026 // NOTE: we could fall back on load/store here too for targets without
3027 // SRA. However, it is doubtful that any exist.
3028 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3029 unsigned BitsDiff = VT.getScalarSizeInBits() -
3030 ExtraVT.getScalarSizeInBits();
3031 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3032 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3033 Node->getOperand(0), ShiftCst);
3034 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3035 Results.push_back(Tmp1);
3036 break;
3037 }
3038 case ISD::FP_ROUND_INREG: {
3039 // The only way we can lower this is to turn it into a TRUNCSTORE,
3040 // EXTLOAD pair, targeting a temporary location (a stack slot).
3041
3042 // NOTE: there is a choice here between constantly creating new stack
3043 // slots and always reusing the same one. We currently always create
3044 // new ones, as reuse may inhibit scheduling.
3045 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3046 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
3047 Node->getValueType(0), dl);
3048 Results.push_back(Tmp1);
3049 break;
3050 }
3051 case ISD::SINT_TO_FP:
3052 case ISD::UINT_TO_FP:
3053 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
3054 Node->getOperand(0), Node->getValueType(0), dl);
3055 Results.push_back(Tmp1);
3056 break;
3057 case ISD::FP_TO_SINT:
3058 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3059 Results.push_back(Tmp1);
3060 break;
3061 case ISD::FP_TO_UINT: {
3062 SDValue True, False;
3063 EVT VT = Node->getOperand(0).getValueType();
3064 EVT NVT = Node->getValueType(0);
3065 APFloat apf(DAG.EVTToAPFloatSemantics(VT),
3066 APInt::getNullValue(VT.getSizeInBits()));
3067 APInt x = APInt::getSignMask(NVT.getSizeInBits());
3068 (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
3069 Tmp1 = DAG.getConstantFP(apf, dl, VT);
3070 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT),
3071 Node->getOperand(0),
3072 Tmp1, ISD::SETLT);
3073 True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
3074 // TODO: Should any fast-math-flags be set for the FSUB?
3075 False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
3076 DAG.getNode(ISD::FSUB, dl, VT,
3077 Node->getOperand(0), Tmp1));
3078 False = DAG.getNode(ISD::XOR, dl, NVT, False,
3079 DAG.getConstant(x, dl, NVT));
3080 Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False);
3081 Results.push_back(Tmp1);
3082 break;
3083 }
3084 case ISD::VAARG:
3085 Results.push_back(DAG.expandVAArg(Node));
3086 Results.push_back(Results[0].getValue(1));
3087 break;
3088 case ISD::VACOPY:
3089 Results.push_back(DAG.expandVACopy(Node));
3090 break;
3091 case ISD::EXTRACT_VECTOR_ELT:
3092 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
3093 // This must be an access of the only element. Return it.
3094 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3095 Node->getOperand(0));
3096 else
3097 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3098 Results.push_back(Tmp1);
3099 break;
3100 case ISD::EXTRACT_SUBVECTOR:
3101 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3102 break;
3103 case ISD::INSERT_SUBVECTOR:
3104 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3105 break;
3106 case ISD::CONCAT_VECTORS:
3107 Results.push_back(ExpandVectorBuildThroughStack(Node));
3108 break;
3109 case ISD::SCALAR_TO_VECTOR:
3110 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3111 break;
3112 case ISD::INSERT_VECTOR_ELT:
3113 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3114 Node->getOperand(1),
3115 Node->getOperand(2), dl));
3116 break;
3117 case ISD::VECTOR_SHUFFLE: {
3118 SmallVector<int, 32> NewMask;
3119 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3120
3121 EVT VT = Node->getValueType(0);
3122 EVT EltVT = VT.getVectorElementType();
3123 SDValue Op0 = Node->getOperand(0);
3124 SDValue Op1 = Node->getOperand(1);
3125 if (!TLI.isTypeLegal(EltVT)) {
3126 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3127
3128 // BUILD_VECTOR operands are allowed to be wider than the element type.
3129 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3130 // it.
3131 if (NewEltVT.bitsLT(EltVT)) {
3132 // Convert shuffle node.
3133 // If original node was v4i64 and the new EltVT is i32,
3134 // cast operands to v8i32 and re-build the mask.
3135
3136 // Calculate new VT, the size of the new VT should be equal to original.
3137 EVT NewVT =
3138 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3139 VT.getSizeInBits() / NewEltVT.getSizeInBits());
3140 assert(NewVT.bitsEq(VT))(static_cast <bool> (NewVT.bitsEq(VT)) ? void (0) : __assert_fail
("NewVT.bitsEq(VT)", "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3140, __extension__ __PRETTY_FUNCTION__))
;
3141
3142 // cast operands to new VT
3143 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3144 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3145
3146 // Convert the shuffle mask
3147 unsigned int factor =
3148 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3149
3150 // EltVT gets smaller
3151 assert(factor > 0)(static_cast <bool> (factor > 0) ? void (0) : __assert_fail
("factor > 0", "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3151, __extension__ __PRETTY_FUNCTION__))
;
3152
3153 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3154 if (Mask[i] < 0) {
3155 for (unsigned fi = 0; fi < factor; ++fi)
3156 NewMask.push_back(Mask[i]);
3157 }
3158 else {
3159 for (unsigned fi = 0; fi < factor; ++fi)
3160 NewMask.push_back(Mask[i]*factor+fi);
3161 }
3162 }
3163 Mask = NewMask;
3164 VT = NewVT;
3165 }
3166 EltVT = NewEltVT;
3167 }
3168 unsigned NumElems = VT.getVectorNumElements();
3169 SmallVector<SDValue, 16> Ops;
3170 for (unsigned i = 0; i != NumElems; ++i) {
3171 if (Mask[i] < 0) {
3172 Ops.push_back(DAG.getUNDEF(EltVT));
3173 continue;
3174 }
3175 unsigned Idx = Mask[i];
3176 if (Idx < NumElems)
3177 Ops.push_back(DAG.getNode(
3178 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3179 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3180 else
3181 Ops.push_back(DAG.getNode(
3182 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3183 DAG.getConstant(Idx - NumElems, dl,
3184 TLI.getVectorIdxTy(DAG.getDataLayout()))));
3185 }
3186
3187 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3188 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3189 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3190 Results.push_back(Tmp1);
3191 break;
3192 }
3193 case ISD::EXTRACT_ELEMENT: {
3194 EVT OpTy = Node->getOperand(0).getValueType();
3195 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3196 // 1 -> Hi
3197 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3198 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3199 TLI.getShiftAmountTy(
3200 Node->getOperand(0).getValueType(),
3201 DAG.getDataLayout())));
3202 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3203 } else {
3204 // 0 -> Lo
3205 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3206 Node->getOperand(0));
3207 }
3208 Results.push_back(Tmp1);
3209 break;
3210 }
3211 case ISD::STACKSAVE:
3212 // Expand to CopyFromReg if the target set
3213 // StackPointerRegisterToSaveRestore.
3214 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3215 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3216 Node->getValueType(0)));
3217 Results.push_back(Results[0].getValue(1));
3218 } else {
3219 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3220 Results.push_back(Node->getOperand(0));
3221 }
3222 break;
3223 case ISD::STACKRESTORE:
3224 // Expand to CopyToReg if the target set
3225 // StackPointerRegisterToSaveRestore.
3226 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3227 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3228 Node->getOperand(1)));
3229 } else {
3230 Results.push_back(Node->getOperand(0));
3231 }
3232 break;
3233 case ISD::GET_DYNAMIC_AREA_OFFSET:
3234 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3235 Results.push_back(Results[0].getValue(0));
3236 break;
3237 case ISD::FCOPYSIGN:
3238 Results.push_back(ExpandFCOPYSIGN(Node));
3239 break;
3240 case ISD::FNEG:
3241 // Expand Y = FNEG(X) -> Y = SUB -0.0, X
3242 Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3243 // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3244 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3245 Node->getOperand(0));
3246 Results.push_back(Tmp1);
3247 break;
3248 case ISD::FABS:
3249 Results.push_back(ExpandFABS(Node));
3250 break;
3251 case ISD::SMIN:
3252 case ISD::SMAX:
3253 case ISD::UMIN:
3254 case ISD::UMAX: {
3255 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3256 ISD::CondCode Pred;
3257 switch (Node->getOpcode()) {
3258 default: llvm_unreachable("How did we get here?")::llvm::llvm_unreachable_internal("How did we get here?", "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3258)
;
3259 case ISD::SMAX: Pred = ISD::SETGT; break;
3260 case ISD::SMIN: Pred = ISD::SETLT; break;
3261 case ISD::UMAX: Pred = ISD::SETUGT; break;
3262 case ISD::UMIN: Pred = ISD::SETULT; break;
3263 }
3264 Tmp1 = Node->getOperand(0);
3265 Tmp2 = Node->getOperand(1);
3266 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3267 Results.push_back(Tmp1);
3268 break;
3269 }
3270
3271 case ISD::FSIN:
3272 case ISD::FCOS: {
3273 EVT VT = Node->getValueType(0);
3274 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3275 // fcos which share the same operand and both are used.
3276 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3277 isSinCosLibcallAvailable(Node, TLI))
3278 && useSinCos(Node)) {
3279 SDVTList VTs = DAG.getVTList(VT, VT);
3280 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3281 if (Node->getOpcode() == ISD::FCOS)
3282 Tmp1 = Tmp1.getValue(1);
3283 Results.push_back(Tmp1);
3284 }
3285 break;
3286 }
3287 case ISD::FMAD:
3288 llvm_unreachable("Illegal fmad should never be formed")::llvm::llvm_unreachable_internal("Illegal fmad should never be formed"
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3288)
;
3289
3290 case ISD::FP16_TO_FP:
3291 if (Node->getValueType(0) != MVT::f32) {
3292 // We can extend to types bigger than f32 in two steps without changing
3293 // the result. Since "f16 -> f32" is much more commonly available, give
3294 // CodeGen the option of emitting that before resorting to a libcall.
3295 SDValue Res =
3296 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3297 Results.push_back(
3298 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3299 }
3300 break;
3301 case ISD::FP_TO_FP16:
3302 DEBUG(dbgs() << "Legalizing FP_TO_FP16\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing FP_TO_FP16\n";
} } while (false)
;
3303 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3304 SDValue Op = Node->getOperand(0);
3305 MVT SVT = Op.getSimpleValueType();
3306 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3307 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3308 // Under fastmath, we can expand this node into a fround followed by
3309 // a float-half conversion.
3310 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3311 DAG.getIntPtrConstant(0, dl));
3312 Results.push_back(
3313 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3314 }
3315 }
3316 break;
3317 case ISD::ConstantFP: {
3318 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3319 // Check to see if this FP immediate is already legal.
3320 // If this is a legal constant, turn it into a TargetConstantFP node.
3321 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
3322 Results.push_back(ExpandConstantFP(CFP, true));
3323 break;
3324 }
3325 case ISD::Constant: {
3326 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3327 Results.push_back(ExpandConstant(CP));
3328 break;
3329 }
3330 case ISD::FSUB: {
3331 EVT VT = Node->getValueType(0);
3332 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3333 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3334 const SDNodeFlags Flags = Node->getFlags();
3335 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3336 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3337 Results.push_back(Tmp1);
3338 }
3339 break;
3340 }
3341 case ISD::SUB: {
3342 EVT VT = Node->getValueType(0);
3343 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&(static_cast <bool> (TLI.isOperationLegalOrCustom(ISD::
ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT
) && "Don't know how to expand this subtraction!") ? void
(0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3345, __extension__ __PRETTY_FUNCTION__))
3344 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&(static_cast <bool> (TLI.isOperationLegalOrCustom(ISD::
ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT
) && "Don't know how to expand this subtraction!") ? void
(0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3345, __extension__ __PRETTY_FUNCTION__))
3345 "Don't know how to expand this subtraction!")(static_cast <bool> (TLI.isOperationLegalOrCustom(ISD::
ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT
) && "Don't know how to expand this subtraction!") ? void
(0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3345, __extension__ __PRETTY_FUNCTION__))
;
3346 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3347 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3348 VT));
3349 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3350 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3351 break;
3352 }
3353 case ISD::UREM:
3354 case ISD::SREM: {
3355 EVT VT = Node->getValueType(0);
3356 bool isSigned = Node->getOpcode() == ISD::SREM;
3357 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3358 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3359 Tmp2 = Node->getOperand(0);
3360 Tmp3 = Node->getOperand(1);
3361 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3362 SDVTList VTs = DAG.getVTList(VT, VT);
3363 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3364 Results.push_back(Tmp1);
3365 } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3366 // X % Y -> X-X/Y*Y
3367 Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3368 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3369 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3370 Results.push_back(Tmp1);
3371 }
3372 break;
3373 }
3374 case ISD::UDIV:
3375 case ISD::SDIV: {
3376 bool isSigned = Node->getOpcode() == ISD::SDIV;
3377 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3378 EVT VT = Node->getValueType(0);
3379 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3380 SDVTList VTs = DAG.getVTList(VT, VT);
3381 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3382 Node->getOperand(1));
3383 Results.push_back(Tmp1);
3384 }
3385 break;
3386 }
3387 case ISD::MULHU:
3388 case ISD::MULHS: {
3389 unsigned ExpandOpcode =
3390 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3391 EVT VT = Node->getValueType(0);
3392 SDVTList VTs = DAG.getVTList(VT, VT);
3393
3394 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3395 Node->getOperand(1));
3396 Results.push_back(Tmp1.getValue(1));
3397 break;
3398 }
3399 case ISD::UMUL_LOHI:
3400 case ISD::SMUL_LOHI: {
3401 SDValue LHS = Node->getOperand(0);
3402 SDValue RHS = Node->getOperand(1);
3403 MVT VT = LHS.getSimpleValueType();
3404 unsigned MULHOpcode =
3405 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3406
3407 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3408 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3409 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3410 break;
3411 }
3412
3413 SmallVector<SDValue, 4> Halves;
3414 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3415 assert(TLI.isTypeLegal(HalfType))(static_cast <bool> (TLI.isTypeLegal(HalfType)) ? void (
0) : __assert_fail ("TLI.isTypeLegal(HalfType)", "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3415, __extension__ __PRETTY_FUNCTION__))
;
3416 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
3417 HalfType, DAG,
3418 TargetLowering::MulExpansionKind::Always)) {
3419 for (unsigned i = 0; i < 2; ++i) {
3420 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3421 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3422 SDValue Shift = DAG.getConstant(
3423 HalfType.getScalarSizeInBits(), dl,
3424 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3425 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3426 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3427 }
3428 break;
3429 }
3430 break;
3431 }
3432 case ISD::MUL: {
3433 EVT VT = Node->getValueType(0);
3434 SDVTList VTs = DAG.getVTList(VT, VT);
3435 // See if multiply or divide can be lowered using two-result operations.
3436 // We just need the low half of the multiply; try both the signed
3437 // and unsigned forms. If the target supports both SMUL_LOHI and
3438 // UMUL_LOHI, form a preference by checking which forms of plain
3439 // MULH it supports.
3440 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3441 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3442 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3443 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3444 unsigned OpToUse = 0;
3445 if (HasSMUL_LOHI && !HasMULHS) {
3446 OpToUse = ISD::SMUL_LOHI;
3447 } else if (HasUMUL_LOHI && !HasMULHU) {
3448 OpToUse = ISD::UMUL_LOHI;
3449 } else if (HasSMUL_LOHI) {
3450 OpToUse = ISD::SMUL_LOHI;
3451 } else if (HasUMUL_LOHI) {
3452 OpToUse = ISD::UMUL_LOHI;
3453 }
3454 if (OpToUse) {
3455 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3456 Node->getOperand(1)));
3457 break;
3458 }
3459
3460 SDValue Lo, Hi;
3461 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3462 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3463 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3464 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3465 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3466 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3467 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3468 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3469 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3470 SDValue Shift =
3471 DAG.getConstant(HalfType.getSizeInBits(), dl,
3472 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3473 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3474 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3475 }
3476 break;
3477 }
3478 case ISD::SADDO:
3479 case ISD::SSUBO: {
3480 SDValue LHS = Node->getOperand(0);
3481 SDValue RHS = Node->getOperand(1);
3482 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3483 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3484 LHS, RHS);
3485 Results.push_back(Sum);
3486 EVT ResultType = Node->getValueType(1);
3487 EVT OType = getSetCCResultType(Node->getValueType(0));
3488
3489 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3490
3491 // LHSSign -> LHS >= 0
3492 // RHSSign -> RHS >= 0
3493 // SumSign -> Sum >= 0
3494 //
3495 // Add:
3496 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3497 // Sub:
3498 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3499 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3500 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3501 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3502 Node->getOpcode() == ISD::SADDO ?
3503 ISD::SETEQ : ISD::SETNE);
3504
3505 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3506 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3507
3508 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3509 Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
3510 break;
3511 }
3512 case ISD::UADDO:
3513 case ISD::USUBO: {
3514 SDValue LHS = Node->getOperand(0);
3515 SDValue RHS = Node->getOperand(1);
3516 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ?
3517 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3518 LHS, RHS);
3519 Results.push_back(Sum);
3520
3521 EVT ResultType = Node->getValueType(1);
3522 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3523 ISD::CondCode CC
3524 = Node->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
3525 SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3526
3527 Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
3528 break;
3529 }
3530 case ISD::UMULO:
3531 case ISD::SMULO: {
3532 EVT VT = Node->getValueType(0);
3533 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
3534 SDValue LHS = Node->getOperand(0);
3535 SDValue RHS = Node->getOperand(1);
3536 SDValue BottomHalf;
3537 SDValue TopHalf;
3538 static const unsigned Ops[2][3] =
3539 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
3540 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
3541 bool isSigned = Node->getOpcode() == ISD::SMULO;
3542 if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
3543 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3544 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
3545 } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
3546 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
3547 RHS);
3548 TopHalf = BottomHalf.getValue(1);
3549 } else if (TLI.isTypeLegal(WideVT)) {
3550 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
3551 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
3552 Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3553 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3554 DAG.getIntPtrConstant(0, dl));
3555 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3556 DAG.getIntPtrConstant(1, dl));
3557 } else {
3558 // We can fall back to a libcall with an illegal type for the MUL if we
3559 // have a libcall big enough.
3560 // Also, we can fall back to a division in some cases, but that's a big
3561 // performance hit in the general case.
3562 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3563 if (WideVT == MVT::i16)
3564 LC = RTLIB::MUL_I16;
3565 else if (WideVT == MVT::i32)
3566 LC = RTLIB::MUL_I32;
3567 else if (WideVT == MVT::i64)
3568 LC = RTLIB::MUL_I64;
3569 else if (WideVT == MVT::i128)
3570 LC = RTLIB::MUL_I128;
3571 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Cannot expand this operation!") ? void (0) : __assert_fail (
"LC != RTLIB::UNKNOWN_LIBCALL && \"Cannot expand this operation!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3571, __extension__ __PRETTY_FUNCTION__))
;
3572
3573 SDValue HiLHS;
3574 SDValue HiRHS;
3575 if (isSigned) {
3576 // The high part is obtained by SRA'ing all but one of the bits of low
3577 // part.
3578 unsigned LoSize = VT.getSizeInBits();
3579 HiLHS =
3580 DAG.getNode(ISD::SRA, dl, VT, LHS,
3581 DAG.getConstant(LoSize - 1, dl,
3582 TLI.getPointerTy(DAG.getDataLayout())));
3583 HiRHS =
3584 DAG.getNode(ISD::SRA, dl, VT, RHS,
3585 DAG.getConstant(LoSize - 1, dl,
3586 TLI.getPointerTy(DAG.getDataLayout())));
3587 } else {
3588 HiLHS = DAG.getConstant(0, dl, VT);
3589 HiRHS = DAG.getConstant(0, dl, VT);
3590 }
3591
3592 // Here we're passing the 2 arguments explicitly as 4 arguments that are
3593 // pre-lowered to the correct types. This all depends upon WideVT not
3594 // being a legal type for the architecture and thus has to be split to
3595 // two arguments.
3596 SDValue Ret;
3597 if(DAG.getDataLayout().isLittleEndian()) {
3598 // Halves of WideVT are packed into registers in different order
3599 // depending on platform endianness. This is usually handled by
3600 // the C calling convention, but we can't defer to it in
3601 // the legalizer.
3602 SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
3603 Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3604 } else {
3605 SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
3606 Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3607 }
3608 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&(static_cast <bool> (Ret.getOpcode() == ISD::MERGE_VALUES
&& "Ret value is a collection of constituent nodes holding result."
) ? void (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3609, __extension__ __PRETTY_FUNCTION__))
3609 "Ret value is a collection of constituent nodes holding result.")(static_cast <bool> (Ret.getOpcode() == ISD::MERGE_VALUES
&& "Ret value is a collection of constituent nodes holding result."
) ? void (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3609, __extension__ __PRETTY_FUNCTION__))
;
3610 BottomHalf = Ret.getOperand(0);
3611 TopHalf = Ret.getOperand(1);
3612 }
3613
3614 if (isSigned) {
3615 Tmp1 = DAG.getConstant(
3616 VT.getSizeInBits() - 1, dl,
3617 TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
3618 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3619 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
3620 ISD::SETNE);
3621 } else {
3622 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
3623 DAG.getConstant(0, dl, VT), ISD::SETNE);
3624 }
3625
3626 // Truncate the result if SetCC returns a larger type than needed.
3627 EVT RType = Node->getValueType(1);
3628 if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
3629 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
3630
3631 assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&(static_cast <bool> (RType.getSizeInBits() == TopHalf.getValueSizeInBits
() && "Unexpected result type for S/UMULO legalization"
) ? void (0) : __assert_fail ("RType.getSizeInBits() == TopHalf.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3632, __extension__ __PRETTY_FUNCTION__))
3632 "Unexpected result type for S/UMULO legalization")(static_cast <bool> (RType.getSizeInBits() == TopHalf.getValueSizeInBits
() && "Unexpected result type for S/UMULO legalization"
) ? void (0) : __assert_fail ("RType.getSizeInBits() == TopHalf.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3632, __extension__ __PRETTY_FUNCTION__))
;
3633
3634 Results.push_back(BottomHalf);
3635 Results.push_back(TopHalf);
3636 break;
3637 }
3638 case ISD::BUILD_PAIR: {
3639 EVT PairTy = Node->getValueType(0);
3640 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3641 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3642 Tmp2 = DAG.getNode(
3643 ISD::SHL, dl, PairTy, Tmp2,
3644 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3645 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3646 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3647 break;
3648 }
3649 case ISD::SELECT:
3650 Tmp1 = Node->getOperand(0);
3651 Tmp2 = Node->getOperand(1);
3652 Tmp3 = Node->getOperand(2);
3653 if (Tmp1.getOpcode() == ISD::SETCC) {
3654 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3655 Tmp2, Tmp3,
3656 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3657 } else {
3658 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3659 DAG.getConstant(0, dl, Tmp1.getValueType()),
3660 Tmp2, Tmp3, ISD::SETNE);
3661 }
3662 Results.push_back(Tmp1);
3663 break;
3664 case ISD::BR_JT: {
3665 SDValue Chain = Node->getOperand(0);
3666 SDValue Table = Node->getOperand(1);
3667 SDValue Index = Node->getOperand(2);
3668
3669 const DataLayout &TD = DAG.getDataLayout();
3670 EVT PTy = TLI.getPointerTy(TD);
3671
3672 unsigned EntrySize =
3673 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3674
3675 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3676 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3677 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3678 Index, Table);
3679
3680 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3681 SDValue LD = DAG.getExtLoad(
3682 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3683 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3684 Addr = LD;
3685 if (TLI.isJumpTableRelative()) {
3686 // For PIC, the sequence is:
3687 // BRIND(load(Jumptable + index) + RelocBase)
3688 // RelocBase can be JumpTable, GOT or some sort of global base.
3689 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3690 TLI.getPICJumpTableRelocBase(Table, DAG));
3691 }
3692 Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr);
3693 Results.push_back(Tmp1);
3694 break;
3695 }
3696 case ISD::BRCOND:
3697 // Expand brcond's setcc into its constituent parts and create a BR_CC
3698 // Node.
3699 Tmp1 = Node->getOperand(0);
3700 Tmp2 = Node->getOperand(1);
3701 if (Tmp2.getOpcode() == ISD::SETCC) {
3702 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3703 Tmp1, Tmp2.getOperand(2),
3704 Tmp2.getOperand(0), Tmp2.getOperand(1),
3705 Node->getOperand(2));
3706 } else {
3707 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3708 if (Tmp2.isUndef() ||
3709 (Tmp2.getOpcode() == ISD::AND &&
3710 isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3711 dyn_cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3712 Tmp3 = Tmp2;
3713 else
3714 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3715 DAG.getConstant(1, dl, Tmp2.getValueType()));
3716 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3717 DAG.getCondCode(ISD::SETNE), Tmp3,
3718 DAG.getConstant(0, dl, Tmp3.getValueType()),
3719 Node->getOperand(2));
3720 }
3721 Results.push_back(Tmp1);
3722 break;
3723 case ISD::SETCC: {
3724 Tmp1 = Node->getOperand(0);
3725 Tmp2 = Node->getOperand(1);
3726 Tmp3 = Node->getOperand(2);
3727 bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3728 Tmp3, NeedInvert, dl);
3729
3730 if (Legalized) {
3731 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3732 // condition code, create a new SETCC node.
3733 if (Tmp3.getNode())
3734 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3735 Tmp1, Tmp2, Tmp3);
3736
3737 // If we expanded the SETCC by inverting the condition code, then wrap
3738 // the existing SETCC in a NOT to restore the intended condition.
3739 if (NeedInvert)
3740 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3741
3742 Results.push_back(Tmp1);
3743 break;
3744 }
3745
3746 // Otherwise, SETCC for the given comparison type must be completely
3747 // illegal; expand it into a SELECT_CC.
3748 EVT VT = Node->getValueType(0);
3749 int TrueValue;
3750 switch (TLI.getBooleanContents(Tmp1->getValueType(0))) {
3751 case TargetLowering::ZeroOrOneBooleanContent:
3752 case TargetLowering::UndefinedBooleanContent:
3753 TrueValue = 1;
3754 break;
3755 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3756 TrueValue = -1;
3757 break;
3758 }
3759 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3760 DAG.getConstant(TrueValue, dl, VT),
3761 DAG.getConstant(0, dl, VT),
3762 Tmp3);
3763 Results.push_back(Tmp1);
3764 break;
3765 }
3766 case ISD::SELECT_CC: {
3767 Tmp1 = Node->getOperand(0); // LHS
3768 Tmp2 = Node->getOperand(1); // RHS
3769 Tmp3 = Node->getOperand(2); // True
3770 Tmp4 = Node->getOperand(3); // False
3771 EVT VT = Node->getValueType(0);
3772 SDValue CC = Node->getOperand(4);
3773 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3774
3775 if (TLI.isCondCodeLegal(CCOp, Tmp1.getSimpleValueType())) {
3776 // If the condition code is legal, then we need to expand this
3777 // node using SETCC and SELECT.
3778 EVT CmpVT = Tmp1.getValueType();
3779 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&(static_cast <bool> (!TLI.isOperationExpand(ISD::SELECT
, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? void (0) : __assert_fail ("!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3781, __extension__ __PRETTY_FUNCTION__))
3780 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "(static_cast <bool> (!TLI.isOperationExpand(ISD::SELECT
, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? void (0) : __assert_fail ("!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3781, __extension__ __PRETTY_FUNCTION__))
3781 "expanded.")(static_cast <bool> (!TLI.isOperationExpand(ISD::SELECT
, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? void (0) : __assert_fail ("!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3781, __extension__ __PRETTY_FUNCTION__))
;
3782 EVT CCVT =
3783 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
3784 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
3785 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3786 break;
3787 }
3788
3789 // SELECT_CC is legal, so the condition code must not be.
3790 bool Legalized = false;
3791 // Try to legalize by inverting the condition. This is for targets that
3792 // might support an ordered version of a condition, but not the unordered
3793 // version (or vice versa).
3794 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3795 Tmp1.getValueType().isInteger());
3796 if (TLI.isCondCodeLegal(InvCC, Tmp1.getSimpleValueType())) {
3797 // Use the new condition code and swap true and false
3798 Legalized = true;
3799 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3800 } else {
3801 // If The inverse is not legal, then try to swap the arguments using
3802 // the inverse condition code.
3803 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3804 if (TLI.isCondCodeLegal(SwapInvCC, Tmp1.getSimpleValueType())) {
3805 // The swapped inverse condition is legal, so swap true and false,
3806 // lhs and rhs.
3807 Legalized = true;
3808 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3809 }
3810 }
3811
3812 if (!Legalized) {
3813 Legalized = LegalizeSetCCCondCode(
3814 getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3815 dl);
3816
3817 assert(Legalized && "Can't legalize SELECT_CC with legal condition!")(static_cast <bool> (Legalized && "Can't legalize SELECT_CC with legal condition!"
) ? void (0) : __assert_fail ("Legalized && \"Can't legalize SELECT_CC with legal condition!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3817, __extension__ __PRETTY_FUNCTION__))
;
3818
3819 // If we expanded the SETCC by inverting the condition code, then swap
3820 // the True/False operands to match.
3821 if (NeedInvert)
3822 std::swap(Tmp3, Tmp4);
3823
3824 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3825 // condition code, create a new SELECT_CC node.
3826 if (CC.getNode()) {
3827 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3828 Tmp1, Tmp2, Tmp3, Tmp4, CC);
3829 } else {
3830 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3831 CC = DAG.getCondCode(ISD::SETNE);
3832 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3833 Tmp2, Tmp3, Tmp4, CC);
3834 }
3835 }
3836 Results.push_back(Tmp1);
3837 break;
3838 }
3839 case ISD::BR_CC: {
3840 Tmp1 = Node->getOperand(0); // Chain
3841 Tmp2 = Node->getOperand(2); // LHS
3842 Tmp3 = Node->getOperand(3); // RHS
3843 Tmp4 = Node->getOperand(1); // CC
3844
3845 bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
3846 Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
3847 (void)Legalized;
3848 assert(Legalized && "Can't legalize BR_CC with legal condition!")(static_cast <bool> (Legalized && "Can't legalize BR_CC with legal condition!"
) ? void (0) : __assert_fail ("Legalized && \"Can't legalize BR_CC with legal condition!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3848, __extension__ __PRETTY_FUNCTION__))
;
3849
3850 // If we expanded the SETCC by inverting the condition code, then wrap
3851 // the existing SETCC in a NOT to restore the intended condition.
3852 if (NeedInvert)
3853 Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
3854
3855 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3856 // node.
3857 if (Tmp4.getNode()) {
3858 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3859 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3860 } else {
3861 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3862 Tmp4 = DAG.getCondCode(ISD::SETNE);
3863 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3864 Tmp2, Tmp3, Node->getOperand(4));
3865 }
3866 Results.push_back(Tmp1);
3867 break;
3868 }
3869 case ISD::BUILD_VECTOR:
3870 Results.push_back(ExpandBUILD_VECTOR(Node));
3871 break;
3872 case ISD::SRA:
3873 case ISD::SRL:
3874 case ISD::SHL: {
3875 // Scalarize vector SRA/SRL/SHL.
3876 EVT VT = Node->getValueType(0);
3877 assert(VT.isVector() && "Unable to legalize non-vector shift")(static_cast <bool> (VT.isVector() && "Unable to legalize non-vector shift"
) ? void (0) : __assert_fail ("VT.isVector() && \"Unable to legalize non-vector shift\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3877, __extension__ __PRETTY_FUNCTION__))
;
3878 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal")(static_cast <bool> (TLI.isTypeLegal(VT.getScalarType()
)&& "Element type must be legal") ? void (0) : __assert_fail
("TLI.isTypeLegal(VT.getScalarType())&& \"Element type must be legal\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3878, __extension__ __PRETTY_FUNCTION__))
;
3879 unsigned NumElem = VT.getVectorNumElements();
3880
3881 SmallVector<SDValue, 8> Scalars;
3882 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3883 SDValue Ex = DAG.getNode(
3884 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3885 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3886 SDValue Sh = DAG.getNode(
3887 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3888 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3889 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3890 VT.getScalarType(), Ex, Sh));
3891 }
3892
3893 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3894 ReplaceNode(SDValue(Node, 0), Result);
3895 break;
3896 }
3897 case ISD::GLOBAL_OFFSET_TABLE:
3898 case ISD::GlobalAddress:
3899 case ISD::GlobalTLSAddress:
3900 case ISD::ExternalSymbol:
3901 case ISD::ConstantPool:
3902 case ISD::JumpTable:
3903 case ISD::INTRINSIC_W_CHAIN:
3904 case ISD::INTRINSIC_WO_CHAIN:
3905 case ISD::INTRINSIC_VOID:
3906 // FIXME: Custom lowering for these operations shouldn't return null!
3907 break;
3908 }
3909
3910 // Replace the original node with the legalized result.
3911 if (Results.empty()) {
3912 DEBUG(dbgs() << "Cannot expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Cannot expand node\n"; } }
while (false)
;
3913 return false;
3914 }
3915
3916 DEBUG(dbgs() << "Succesfully expanded node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Succesfully expanded node\n"
; } } while (false)
;
3917 ReplaceNode(Node, Results.data());
3918 return true;
3919}
3920
3921void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3922 DEBUG(dbgs() << "Trying to convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to convert node to libcall\n"
; } } while (false)
;
3923 SmallVector<SDValue, 8> Results;
3924 SDLoc dl(Node);
3925 unsigned Opc = Node->getOpcode();
3926 switch (Opc) {
3927 case ISD::ATOMIC_FENCE: {
3928 // If the target didn't lower this, lower it to '__sync_synchronize()' call
3929 // FIXME: handle "fence singlethread" more efficiently.
3930 TargetLowering::ArgListTy Args;
3931
3932 TargetLowering::CallLoweringInfo CLI(DAG);
3933 CLI.setDebugLoc(dl)
3934 .setChain(Node->getOperand(0))
3935 .setLibCallee(
3936 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3937 DAG.getExternalSymbol("__sync_synchronize",
3938 TLI.getPointerTy(DAG.getDataLayout())),
3939 std::move(Args));
3940
3941 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3942
3943 Results.push_back(CallResult.second);
3944 break;
3945 }
3946 // By default, atomic intrinsics are marked Legal and lowered. Targets
3947 // which don't support them directly, however, may want libcalls, in which
3948 // case they mark them Expand, and we get here.
3949 case ISD::ATOMIC_SWAP:
3950 case ISD::ATOMIC_LOAD_ADD:
3951 case ISD::ATOMIC_LOAD_SUB:
3952 case ISD::ATOMIC_LOAD_AND:
3953 case ISD::ATOMIC_LOAD_OR:
3954 case ISD::ATOMIC_LOAD_XOR:
3955 case ISD::ATOMIC_LOAD_NAND:
3956 case ISD::ATOMIC_LOAD_MIN:
3957 case ISD::ATOMIC_LOAD_MAX:
3958 case ISD::ATOMIC_LOAD_UMIN:
3959 case ISD::ATOMIC_LOAD_UMAX:
3960 case ISD::ATOMIC_CMP_SWAP: {
3961 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3962 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
3963 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected atomic op or value type!") ? void (0) : __assert_fail
("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected atomic op or value type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3963, __extension__ __PRETTY_FUNCTION__))
;
3964
3965 std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
3966 Results.push_back(Tmp.first);
3967 Results.push_back(Tmp.second);
3968 break;
3969 }
3970 case ISD::TRAP: {
3971 // If this operation is not supported, lower it to 'abort()' call
3972 TargetLowering::ArgListTy Args;
3973 TargetLowering::CallLoweringInfo CLI(DAG);
3974 CLI.setDebugLoc(dl)
3975 .setChain(Node->getOperand(0))
3976 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3977 DAG.getExternalSymbol(
3978 "abort", TLI.getPointerTy(DAG.getDataLayout())),
3979 std::move(Args));
3980 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3981
3982 Results.push_back(CallResult.second);
3983 break;
3984 }
3985 case ISD::FMINNUM:
3986 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3987 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3988 RTLIB::FMIN_PPCF128));
3989 break;
3990 case ISD::FMAXNUM:
3991 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3992 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3993 RTLIB::FMAX_PPCF128));
3994 break;
3995 case ISD::FSQRT:
3996 case ISD::STRICT_FSQRT:
3997 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3998 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3999 RTLIB::SQRT_PPCF128));
4000 break;
4001 case ISD::FSIN:
4002 case ISD::STRICT_FSIN:
4003 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4004 RTLIB::SIN_F80, RTLIB::SIN_F128,
4005 RTLIB::SIN_PPCF128));
4006 break;
4007 case ISD::FCOS:
4008 case ISD::STRICT_FCOS:
4009 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4010 RTLIB::COS_F80, RTLIB::COS_F128,
4011 RTLIB::COS_PPCF128));
4012 break;
4013 case ISD::FSINCOS:
4014 // Expand into sincos libcall.
4015 ExpandSinCosLibCall(Node, Results);
4016 break;
4017 case ISD::FLOG:
4018 case ISD::STRICT_FLOG:
4019 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
4020 RTLIB::LOG_F80, RTLIB::LOG_F128,
4021 RTLIB::LOG_PPCF128));
4022 break;
4023 case ISD::FLOG2:
4024 case ISD::STRICT_FLOG2:
4025 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
4026 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
4027 RTLIB::LOG2_PPCF128));
4028 break;
4029 case ISD::FLOG10:
4030 case ISD::STRICT_FLOG10:
4031 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
4032 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
4033 RTLIB::LOG10_PPCF128));
4034 break;
4035 case ISD::FEXP:
4036 case ISD::STRICT_FEXP:
4037 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
4038 RTLIB::EXP_F80, RTLIB::EXP_F128,
4039 RTLIB::EXP_PPCF128));
4040 break;
4041 case ISD::FEXP2:
4042 case ISD::STRICT_FEXP2:
4043 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
4044 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
4045 RTLIB::EXP2_PPCF128));
4046 break;
4047 case ISD::FTRUNC:
4048 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4049 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4050 RTLIB::TRUNC_PPCF128));
4051 break;
4052 case ISD::FFLOOR:
4053 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4054 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4055 RTLIB::FLOOR_PPCF128));
4056 break;
4057 case ISD::FCEIL:
4058 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4059 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4060 RTLIB::CEIL_PPCF128));
4061 break;
4062 case ISD::FRINT:
4063 case ISD::STRICT_FRINT:
4064 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4065 RTLIB::RINT_F80, RTLIB::RINT_F128,
4066 RTLIB::RINT_PPCF128));
4067 break;
4068 case ISD::FNEARBYINT:
4069 case ISD::STRICT_FNEARBYINT:
4070 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4071 RTLIB::NEARBYINT_F64,
4072 RTLIB::NEARBYINT_F80,
4073 RTLIB::NEARBYINT_F128,
4074 RTLIB::NEARBYINT_PPCF128));
4075 break;
4076 case ISD::FROUND:
4077 Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4078 RTLIB::ROUND_F64,
4079 RTLIB::ROUND_F80,
4080 RTLIB::ROUND_F128,
4081 RTLIB::ROUND_PPCF128));
4082 break;
4083 case ISD::FPOWI:
4084 case ISD::STRICT_FPOWI:
4085 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
4086 RTLIB::POWI_F80, RTLIB::POWI_F128,
4087 RTLIB::POWI_PPCF128));
4088 break;
4089 case ISD::FPOW:
4090 case ISD::STRICT_FPOW:
4091 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
4092 RTLIB::POW_F80, RTLIB::POW_F128,
4093 RTLIB::POW_PPCF128));
4094 break;
4095 case ISD::FDIV:
4096 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4097 RTLIB::DIV_F80, RTLIB::DIV_F128,
4098 RTLIB::DIV_PPCF128));
4099 break;
4100 case ISD::FREM:
4101 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4102 RTLIB::REM_F80, RTLIB::REM_F128,
4103 RTLIB::REM_PPCF128));
4104 break;
4105 case ISD::FMA:
4106 case ISD::STRICT_FMA:
4107 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4108 RTLIB::FMA_F80, RTLIB::FMA_F128,
4109 RTLIB::FMA_PPCF128));
4110 break;
4111 case ISD::FADD:
4112 Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4113 RTLIB::ADD_F80, RTLIB::ADD_F128,
4114 RTLIB::ADD_PPCF128));
4115 break;
4116 case ISD::FMUL:
4117 Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4118 RTLIB::MUL_F80, RTLIB::MUL_F128,
4119 RTLIB::MUL_PPCF128));
4120 break;
4121 case ISD::FP16_TO_FP:
4122 if (Node->getValueType(0) == MVT::f32) {
4123 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4124 }
4125 break;
4126 case ISD::FP_TO_FP16: {
4127 RTLIB::Libcall LC =
4128 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4129 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_fp16") ? void (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unable to expand fp_to_fp16\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4129, __extension__ __PRETTY_FUNCTION__))
;
4130 Results.push_back(ExpandLibCall(LC, Node, false));
4131 break;
4132 }
4133 case ISD::FSUB:
4134 Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4135 RTLIB::SUB_F80, RTLIB::SUB_F128,
4136 RTLIB::SUB_PPCF128));
4137 break;
4138 case ISD::SREM:
4139 Results.push_back(ExpandIntLibCall(Node, true,
4140 RTLIB::SREM_I8,
4141 RTLIB::SREM_I16, RTLIB::SREM_I32,
4142 RTLIB::SREM_I64, RTLIB::SREM_I128));
4143 break;
4144 case ISD::UREM:
4145 Results.push_back(ExpandIntLibCall(Node, false,
4146 RTLIB::UREM_I8,
4147 RTLIB::UREM_I16, RTLIB::UREM_I32,
4148 RTLIB::UREM_I64, RTLIB::UREM_I128));
4149 break;
4150 case ISD::SDIV:
4151 Results.push_back(ExpandIntLibCall(Node, true,
4152 RTLIB::SDIV_I8,
4153 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4154 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4155 break;
4156 case ISD::UDIV:
4157 Results.push_back(ExpandIntLibCall(Node, false,
4158 RTLIB::UDIV_I8,
4159 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4160 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4161 break;
4162 case ISD::SDIVREM:
4163 case ISD::UDIVREM:
4164 // Expand into divrem libcall
4165 ExpandDivRemLibCall(Node, Results);
4166 break;
4167 case ISD::MUL:
4168 Results.push_back(ExpandIntLibCall(Node, false,
4169 RTLIB::MUL_I8,
4170 RTLIB::MUL_I16, RTLIB::MUL_I32,
4171 RTLIB::MUL_I64, RTLIB::MUL_I128));
4172 break;
4173 }
4174
4175 // Replace the original node with the legalized result.
4176 if (!Results.empty()) {
4177 DEBUG(dbgs() << "Successfully converted node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully converted node to libcall\n"
; } } while (false)
;
4178 ReplaceNode(Node, Results.data());
4179 } else
4180 DEBUG(dbgs() << "Could not convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not convert node to libcall\n"
; } } while (false)
;
4181}
4182
4183// Determine the vector type to use in place of an original scalar element when
4184// promoting equally sized vectors.
4185static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4186 MVT EltVT, MVT NewEltVT) {
4187 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4188 MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4189 assert(TLI.isTypeLegal(MidVT) && "unexpected")(static_cast <bool> (TLI.isTypeLegal(MidVT) && "unexpected"
) ? void (0) : __assert_fail ("TLI.isTypeLegal(MidVT) && \"unexpected\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4189, __extension__ __PRETTY_FUNCTION__))
;
4190 return MidVT;
4191}
4192
4193void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4194 DEBUG(dbgs() << "Trying to promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to promote node\n"
; } } while (false)
;
4195 SmallVector<SDValue, 8> Results;
4196 MVT OVT = Node->getSimpleValueType(0);
4197 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4198 Node->getOpcode() == ISD::SINT_TO_FP ||
4199 Node->getOpcode() == ISD::SETCC ||
4200 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4201 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4202 OVT = Node->getOperand(0).getSimpleValueType();
4203 }
4204 if (Node->getOpcode() == ISD::BR_CC)
4205 OVT = Node->getOperand(2).getSimpleValueType();
4206 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4207 SDLoc dl(Node);
4208 SDValue Tmp1, Tmp2, Tmp3;
4209 switch (Node->getOpcode()) {
4210 case ISD::CTTZ:
4211 case ISD::CTTZ_ZERO_UNDEF:
4212 case ISD::CTLZ:
4213 case ISD::CTLZ_ZERO_UNDEF:
4214 case ISD::CTPOP:
4215 // Zero extend the argument.
4216 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4217 if (Node->getOpcode() == ISD::CTTZ) {
4218 // The count is the same in the promoted type except if the original
4219 // value was zero. This can be handled by setting the bit just off
4220 // the top of the original type.
4221 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4222 OVT.getSizeInBits());
4223 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4224 DAG.getConstant(TopBit, dl, NVT));
4225 }
4226 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4227 // already the correct result.
4228 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4229 if (Node->getOpcode() == ISD::CTLZ ||
4230 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4231 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4232 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4233 DAG.getConstant(NVT.getSizeInBits() -
4234 OVT.getSizeInBits(), dl, NVT));
4235 }
4236 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4237 break;
4238 case ISD::BITREVERSE:
4239 case ISD::BSWAP: {
4240 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4241 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4242 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4243 Tmp1 = DAG.getNode(
4244 ISD::SRL, dl, NVT, Tmp1,
4245 DAG.getConstant(DiffBits, dl,
4246 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4247 Results.push_back(Tmp1);
4248 break;
4249 }
4250 case ISD::FP_TO_UINT:
4251 case ISD::FP_TO_SINT:
4252 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4253 Node->getOpcode() == ISD::FP_TO_SINT, dl);
4254 Results.push_back(Tmp1);
4255 break;
4256 case ISD::UINT_TO_FP:
4257 case ISD::SINT_TO_FP:
4258 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4259 Node->getOpcode() == ISD::SINT_TO_FP, dl);
4260 Results.push_back(Tmp1);
4261 break;
4262 case ISD::VAARG: {
4263 SDValue Chain = Node->getOperand(0); // Get the chain.
4264 SDValue Ptr = Node->getOperand(1); // Get the pointer.
4265
4266 unsigned TruncOp;
4267 if (OVT.isVector()) {
4268 TruncOp = ISD::BITCAST;
4269 } else {
4270 assert(OVT.isInteger()(static_cast <bool> (OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? void (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4271, __extension__ __PRETTY_FUNCTION__))
4271 && "VAARG promotion is supported only for vectors or integer types")(static_cast <bool> (OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? void (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4271, __extension__ __PRETTY_FUNCTION__))
;
4272 TruncOp = ISD::TRUNCATE;
4273 }
4274
4275 // Perform the larger operation, then convert back
4276 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4277 Node->getConstantOperandVal(3));
4278 Chain = Tmp1.getValue(1);
4279
4280 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4281
4282 // Modified the chain result - switch anything that used the old chain to
4283 // use the new one.
4284 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4285 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4286 if (UpdatedNodes) {
4287 UpdatedNodes->insert(Tmp2.getNode());
4288 UpdatedNodes->insert(Chain.getNode());
4289 }
4290 ReplacedNode(Node);
4291 break;
4292 }
4293 case ISD::MUL:
4294 case ISD::SDIV:
4295 case ISD::SREM:
4296 case ISD::UDIV:
4297 case ISD::UREM:
4298 case ISD::AND:
4299 case ISD::OR:
4300 case ISD::XOR: {
4301 unsigned ExtOp, TruncOp;
4302 if (OVT.isVector()) {
4303 ExtOp = ISD::BITCAST;
4304 TruncOp = ISD::BITCAST;
4305 } else {
4306 assert(OVT.isInteger() && "Cannot promote logic operation")(static_cast <bool> (OVT.isInteger() && "Cannot promote logic operation"
) ? void (0) : __assert_fail ("OVT.isInteger() && \"Cannot promote logic operation\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4306, __extension__ __PRETTY_FUNCTION__))
;
4307
4308 switch (Node->getOpcode()) {
4309 default:
4310 ExtOp = ISD::ANY_EXTEND;
4311 break;
4312 case ISD::SDIV:
4313 case ISD::SREM:
4314 ExtOp = ISD::SIGN_EXTEND;
4315 break;
4316 case ISD::UDIV:
4317 case ISD::UREM:
4318 ExtOp = ISD::ZERO_EXTEND;
4319 break;
4320 }
4321 TruncOp = ISD::TRUNCATE;
4322 }
4323 // Promote each of the values to the new type.
4324 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4325 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4326 // Perform the larger operation, then convert back
4327 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4328 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4329 break;
4330 }
4331 case ISD::UMUL_LOHI:
4332 case ISD::SMUL_LOHI: {
4333 // Promote to a multiply in a wider integer type.
4334 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4335 : ISD::SIGN_EXTEND;
4336 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4337 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4338 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4339
4340 auto &DL = DAG.getDataLayout();
4341 unsigned OriginalSize = OVT.getScalarSizeInBits();
4342 Tmp2 = DAG.getNode(
4343 ISD::SRL, dl, NVT, Tmp1,
4344 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4345 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4346 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4347 break;
4348 }
4349 case ISD::SELECT: {
4350 unsigned ExtOp, TruncOp;
4351 if (Node->getValueType(0).isVector() ||
4352 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4353 ExtOp = ISD::BITCAST;
4354 TruncOp = ISD::BITCAST;
4355 } else if (Node->getValueType(0).isInteger()) {
4356 ExtOp = ISD::ANY_EXTEND;
4357 TruncOp = ISD::TRUNCATE;
4358 } else {
4359 ExtOp = ISD::FP_EXTEND;
4360 TruncOp = ISD::FP_ROUND;
4361 }
4362 Tmp1 = Node->getOperand(0);
4363 // Promote each of the values to the new type.
4364 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4365 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4366 // Perform the larger operation, then round down.
4367 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4368 if (TruncOp != ISD::FP_ROUND)
4369 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4370 else
4371 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4372 DAG.getIntPtrConstant(0, dl));
4373 Results.push_back(Tmp1);
4374 break;
4375 }
4376 case ISD::VECTOR_SHUFFLE: {
4377 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4378
4379 // Cast the two input vectors.
4380 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4381 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4382
4383 // Convert the shuffle mask to the right # elements.
4384 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4385 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4386 Results.push_back(Tmp1);
4387 break;
4388 }
4389 case ISD::SETCC: {
4390 unsigned ExtOp = ISD::FP_EXTEND;
4391 if (NVT.isInteger()) {
4392 ISD::CondCode CCCode =
4393 cast<CondCodeSDNode>(Node->getOperand(2))->get();
4394 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4395 }
4396 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4397 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4398 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
4399 Tmp1, Tmp2, Node->getOperand(2)));
4400 break;
4401 }
4402 case ISD::BR_CC: {
4403 unsigned ExtOp = ISD::FP_EXTEND;
4404 if (NVT.isInteger()) {
4405 ISD::CondCode CCCode =
4406 cast<CondCodeSDNode>(Node->getOperand(1))->get();
4407 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4408 }
4409 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4410 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4411 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4412 Node->getOperand(0), Node->getOperand(1),
4413 Tmp1, Tmp2, Node->getOperand(4)));
4414 break;
4415 }
4416 case ISD::FADD:
4417 case ISD::FSUB:
4418 case ISD::FMUL:
4419 case ISD::FDIV:
4420 case ISD::FREM:
4421 case ISD::FMINNUM:
4422 case ISD::FMAXNUM:
4423 case ISD::FPOW:
4424 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4425 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4426 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4427 Node->getFlags());
4428 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4429 Tmp3, DAG.getIntPtrConstant(0, dl)));
4430 break;
4431 case ISD::FMA:
4432 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4433 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4434 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4435 Results.push_back(
4436 DAG.getNode(ISD::FP_ROUND, dl, OVT,
4437 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4438 DAG.getIntPtrConstant(0, dl)));
4439 break;
4440 case ISD::FCOPYSIGN:
4441 case ISD::FPOWI: {
4442 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4443 Tmp2 = Node->getOperand(1);
4444 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4445
4446 // fcopysign doesn't change anything but the sign bit, so
4447 // (fp_round (fcopysign (fpext a), b))
4448 // is as precise as
4449 // (fp_round (fpext a))
4450 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4451 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4452 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4453 Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4454 break;
4455 }
4456 case ISD::FFLOOR:
4457 case ISD::FCEIL:
4458 case ISD::FRINT:
4459 case ISD::FNEARBYINT:
4460 case ISD::FROUND:
4461 case ISD::FTRUNC:
4462 case ISD::FNEG:
4463 case ISD::FSQRT:
4464 case ISD::FSIN:
4465 case ISD::FCOS:
4466 case ISD::FLOG:
4467 case ISD::FLOG2:
4468 case ISD::FLOG10:
4469 case ISD::FABS:
4470 case ISD::FEXP:
4471 case ISD::FEXP2:
4472 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4473 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4474 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4475 Tmp2, DAG.getIntPtrConstant(0, dl)));
4476 break;
4477 case ISD::BUILD_VECTOR: {
4478 MVT EltVT = OVT.getVectorElementType();
4479 MVT NewEltVT = NVT.getVectorElementType();
4480
4481 // Handle bitcasts to a different vector type with the same total bit size
4482 //
4483 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4484 // =>
4485 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4486
4487 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for build_vector"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4488, __extension__ __PRETTY_FUNCTION__))
4488 "Invalid promote type for build_vector")(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for build_vector"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4488, __extension__ __PRETTY_FUNCTION__))
;
4489 assert(NewEltVT.bitsLT(EltVT) && "not handled")(static_cast <bool> (NewEltVT.bitsLT(EltVT) && "not handled"
) ? void (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4489, __extension__ __PRETTY_FUNCTION__))
;
4490
4491 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4492
4493 SmallVector<SDValue, 8> NewOps;
4494 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4495 SDValue Op = Node->getOperand(I);
4496 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4497 }
4498
4499 SDLoc SL(Node);
4500 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4501 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4502 Results.push_back(CvtVec);
4503 break;
4504 }
4505 case ISD::EXTRACT_VECTOR_ELT: {
4506 MVT EltVT = OVT.getVectorElementType();
4507 MVT NewEltVT = NVT.getVectorElementType();
4508
4509 // Handle bitcasts to a different vector type with the same total bit size.
4510 //
4511 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4512 // =>
4513 // v4i32:castx = bitcast x:v2i64
4514 //
4515 // i64 = bitcast
4516 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4517 // (i32 (extract_vector_elt castx, (2 * y + 1)))
4518 //
4519
4520 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for extract_vector_elt"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4521, __extension__ __PRETTY_FUNCTION__))
4521 "Invalid promote type for extract_vector_elt")(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for extract_vector_elt"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4521, __extension__ __PRETTY_FUNCTION__))
;
4522 assert(NewEltVT.bitsLT(EltVT) && "not handled")(static_cast <bool> (NewEltVT.bitsLT(EltVT) && "not handled"
) ? void (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4522, __extension__ __PRETTY_FUNCTION__))
;
4523
4524 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4525 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4526
4527 SDValue Idx = Node->getOperand(1);
4528 EVT IdxVT = Idx.getValueType();
4529 SDLoc SL(Node);
4530 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4531 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4532
4533 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4534
4535 SmallVector<SDValue, 8> NewOps;
4536 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4537 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4538 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4539
4540 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4541 CastVec, TmpIdx);
4542 NewOps.push_back(Elt);
4543 }
4544
4545 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4546 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4547 break;
4548 }
4549 case ISD::INSERT_VECTOR_ELT: {
4550 MVT EltVT = OVT.getVectorElementType();
4551 MVT NewEltVT = NVT.getVectorElementType();
4552
4553 // Handle bitcasts to a different vector type with the same total bit size
4554 //
4555 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4556 // =>
4557 // v4i32:castx = bitcast x:v2i64
4558 // v2i32:casty = bitcast y:i64
4559 //
4560 // v2i64 = bitcast
4561 // (v4i32 insert_vector_elt
4562 // (v4i32 insert_vector_elt v4i32:castx,
4563 // (extract_vector_elt casty, 0), 2 * z),
4564 // (extract_vector_elt casty, 1), (2 * z + 1))
4565
4566 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for insert_vector_elt"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4567, __extension__ __PRETTY_FUNCTION__))
4567 "Invalid promote type for insert_vector_elt")(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for insert_vector_elt"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4567, __extension__ __PRETTY_FUNCTION__))
;
4568 assert(NewEltVT.bitsLT(EltVT) && "not handled")(static_cast <bool> (NewEltVT.bitsLT(EltVT) && "not handled"
) ? void (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-6.0~svn318601/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4568, __extension__ __PRETTY_FUNCTION__))
;
4569
4570 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4571 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4572
4573 SDValue Val = Node->getOperand(1);
4574 SDValue Idx = Node->getOperand(2);
4575 EVT IdxVT = Idx.getValueType();
4576 SDLoc SL(Node);
4577
4578 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4579 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4580
4581 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4582 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4583
4584 SDValue NewVec = CastVec;
4585 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4586 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4587 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4588
4589 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4590 CastVal, IdxOffset);
4591
4592 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4593 NewVec, Elt, InEltIdx);
4594 }
4595
4596 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4597 break;
4598 }
4599 case ISD::SCALAR_TO_VECTOR: {
4600 MVT EltVT = OVT.getVectorElementType();
4601 MVT NewEltVT = NVT.getVectorElementType();
4602
4603 // Handle bitcasts to different vector type with the same total bit size.
4604 //
4605 // e.g. v2i64 = scalar_to_vector x:i64
4606 // =>
4607 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4608 //
4609
4610 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4611 SDValue Val = Node->getOperand(0);
4612 SDLoc SL(Node);
4613
4614 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4615 SDValue Undef = DAG.getUNDEF(MidVT);
4616
4617 SmallVector<SDValue, 8> NewElts;
4618 NewElts.push_back(CastVal);
4619 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4620 NewElts.push_back(Undef);
4621
4622 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4623 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4624 Results.push_back(CvtVec);
4625 break;
4626 }
4627 }
4628
4629 // Replace the original node with the legalized result.
4630 if (!Results.empty()) {
4631 DEBUG(dbgs() << "Successfully promoted node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully promoted node\n"
; } } while (false)
;
4632 ReplaceNode(Node, Results.data());
4633 } else
4634 DEBUG(dbgs() << "Could not promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not promote node\n"
; } } while (false)
;
4635}
4636
4637/// This is the entry point for the file.
4638void SelectionDAG::Legalize() {
4639 AssignTopologicalOrder();
4640
4641 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4642 // Use a delete listener to remove nodes which were deleted during
4643 // legalization from LegalizeNodes. This is needed to handle the situation
4644 // where a new node is allocated by the object pool to the same address of a
4645 // previously deleted node.
4646 DAGNodeDeletedListener DeleteListener(
4647 *this,
4648 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4649
4650 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4651
4652 // Visit all the nodes. We start in topological order, so that we see
4653 // nodes with their original operands intact. Legalization can produce
4654 // new nodes which may themselves need to be legalized. Iterate until all
4655 // nodes have been legalized.
4656 while (true) {
4657 bool AnyLegalized = false;
4658 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4659 --NI;
4660
4661 SDNode *N = &*NI;
4662 if (N->use_empty() && N != getRoot().getNode()) {
4663 ++NI;
4664 DeleteNode(N);
4665 continue;
4666 }
4667
4668 if (LegalizedNodes.insert(N).second) {
4669 AnyLegalized = true;
4670 Legalizer.LegalizeOp(N);
4671
4672 if (N->use_empty() && N != getRoot().getNode()) {
4673 ++NI;
4674 DeleteNode(N);
4675 }
4676 }
4677 }
4678 if (!AnyLegalized)
4679 break;
4680
4681 }
4682
4683 // Remove dead nodes now.
4684 RemoveDeadNodes();
4685}
4686
4687bool SelectionDAG::LegalizeOp(SDNode *N,
4688 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4689 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4690 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4691
4692 // Directly insert the node in question, and legalize it. This will recurse
4693 // as needed through operands.
4694 LegalizedNodes.insert(N);
4695 Legalizer.LegalizeOp(N);
4696
4697 return LegalizedNodes.count(N);
4698}