Bug Summary

File:lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
Warning:line 547, column 29
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'

Annotated Source Code

1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the SelectionDAG::Legalize method.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ADT/APFloat.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/CodeGen/ISDOpcodes.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineMemOperand.h"
25#include "llvm/CodeGen/MachineValueType.h"
26#include "llvm/CodeGen/RuntimeLibcalls.h"
27#include "llvm/CodeGen/SelectionDAG.h"
28#include "llvm/CodeGen/SelectionDAGNodes.h"
29#include "llvm/CodeGen/TargetFrameLowering.h"
30#include "llvm/CodeGen/TargetLowering.h"
31#include "llvm/CodeGen/TargetSubtargetInfo.h"
32#include "llvm/CodeGen/ValueTypes.h"
33#include "llvm/IR/CallingConv.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/DerivedTypes.h"
37#include "llvm/IR/Function.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/IR/Type.h"
40#include "llvm/Support/Casting.h"
41#include "llvm/Support/Compiler.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/ErrorHandling.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/raw_ostream.h"
46#include "llvm/Target/TargetMachine.h"
47#include "llvm/Target/TargetOptions.h"
48#include <algorithm>
49#include <cassert>
50#include <cstdint>
51#include <tuple>
52#include <utility>
53
54using namespace llvm;
55
56#define DEBUG_TYPE"legalizedag" "legalizedag"
57
58namespace {
59
60/// Keeps track of state when getting the sign of a floating-point value as an
61/// integer.
62struct FloatSignAsInt {
63 EVT FloatVT;
64 SDValue Chain;
65 SDValue FloatPtr;
66 SDValue IntPtr;
67 MachinePointerInfo IntPointerInfo;
68 MachinePointerInfo FloatPointerInfo;
69 SDValue IntValue;
70 APInt SignMask;
71 uint8_t SignBit;
72};
73
74//===----------------------------------------------------------------------===//
75/// This takes an arbitrary SelectionDAG as input and
76/// hacks on it until the target machine can handle it. This involves
77/// eliminating value sizes the machine cannot handle (promoting small sizes to
78/// large sizes or splitting up large values into small values) as well as
79/// eliminating operations the machine cannot handle.
80///
81/// This code also does a small amount of optimization and recognition of idioms
82/// as part of its processing. For example, if a target does not support a
83/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
84/// will attempt merge setcc and brc instructions into brcc's.
85class SelectionDAGLegalize {
86 const TargetMachine &TM;
87 const TargetLowering &TLI;
88 SelectionDAG &DAG;
89
90 /// \brief The set of nodes which have already been legalized. We hold a
91 /// reference to it in order to update as necessary on node deletion.
92 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93
94 /// \brief A set of all the nodes updated during legalization.
95 SmallSetVector<SDNode *, 16> *UpdatedNodes;
96
97 EVT getSetCCResultType(EVT VT) const {
98 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
99 }
100
101 // Libcall insertion helpers.
102
103public:
104 SelectionDAGLegalize(SelectionDAG &DAG,
105 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
106 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
107 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
108 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109
110 /// \brief Legalizes the given operation.
111 void LegalizeOp(SDNode *Node);
112
113private:
114 SDValue OptimizeFloatStore(StoreSDNode *ST);
115
116 void LegalizeLoadOps(SDNode *Node);
117 void LegalizeStoreOps(SDNode *Node);
118
119 /// Some targets cannot handle a variable
120 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
121 /// is necessary to spill the vector being inserted into to memory, perform
122 /// the insert there, and then read the result back.
123 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
124 const SDLoc &dl);
125 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
126 const SDLoc &dl);
127
128 /// Return a vector shuffle operation which
129 /// performs the same shuffe in terms of order or result bytes, but on a type
130 /// whose vector element type is narrower than the original shuffle type.
131 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
132 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133 SDValue N1, SDValue N2,
134 ArrayRef<int> Mask) const;
135
136 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
137 bool &NeedInvert, const SDLoc &dl);
138
139 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
140 SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
141 unsigned NumOps, bool isSigned, const SDLoc &dl);
142
143 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
144 SDNode *Node, bool isSigned);
145 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
146 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
147 RTLIB::Libcall Call_F128,
148 RTLIB::Libcall Call_PPCF128);
149 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
150 RTLIB::Libcall Call_I8,
151 RTLIB::Libcall Call_I16,
152 RTLIB::Libcall Call_I32,
153 RTLIB::Libcall Call_I64,
154 RTLIB::Libcall Call_I128);
155 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157
158 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
159 const SDLoc &dl);
160 SDValue ExpandBUILD_VECTOR(SDNode *Node);
161 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
162 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
163 SmallVectorImpl<SDValue> &Results);
164 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
165 SDValue Value) const;
166 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
167 SDValue NewIntValue) const;
168 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
169 SDValue ExpandFABS(SDNode *Node) const;
170 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT,
171 const SDLoc &dl);
172 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
173 const SDLoc &dl);
174 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
175 const SDLoc &dl);
176
177 SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
178 SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
179 SDValue ExpandBitCount(unsigned Opc, SDValue Op, const SDLoc &dl);
180
181 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
182 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
183 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
184
185 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
186 SDValue ExpandConstant(ConstantSDNode *CP);
187
188 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
189 bool ExpandNode(SDNode *Node);
190 void ConvertNodeToLibcall(SDNode *Node);
191 void PromoteNode(SDNode *Node);
192
193public:
194 // Node replacement helpers
195
196 void ReplacedNode(SDNode *N) {
197 LegalizedNodes.erase(N);
198 if (UpdatedNodes)
199 UpdatedNodes->insert(N);
200 }
201
202 void ReplaceNode(SDNode *Old, SDNode *New) {
203 DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
204 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
205
206 assert(Old->getNumValues() == New->getNumValues() &&(static_cast <bool> (Old->getNumValues() == New->
getNumValues() && "Replacing one node with another that produces a different number "
"of values!") ? void (0) : __assert_fail ("Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 208, __extension__ __PRETTY_FUNCTION__))
207 "Replacing one node with another that produces a different number "(static_cast <bool> (Old->getNumValues() == New->
getNumValues() && "Replacing one node with another that produces a different number "
"of values!") ? void (0) : __assert_fail ("Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 208, __extension__ __PRETTY_FUNCTION__))
208 "of values!")(static_cast <bool> (Old->getNumValues() == New->
getNumValues() && "Replacing one node with another that produces a different number "
"of values!") ? void (0) : __assert_fail ("Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 208, __extension__ __PRETTY_FUNCTION__))
;
209 DAG.ReplaceAllUsesWith(Old, New);
210 if (UpdatedNodes)
211 UpdatedNodes->insert(New);
212 ReplacedNode(Old);
213 }
214
215 void ReplaceNode(SDValue Old, SDValue New) {
216 DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
217 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
218
219 DAG.ReplaceAllUsesWith(Old, New);
220 if (UpdatedNodes)
221 UpdatedNodes->insert(New.getNode());
222 ReplacedNode(Old.getNode());
223 }
224
225 void ReplaceNode(SDNode *Old, const SDValue *New) {
226 DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); } } while (false)
;
227
228 DAG.ReplaceAllUsesWith(Old, New);
229 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
230 DEBUG(dbgs() << (i == 0 ? " with: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
231 : " and: ");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
232 New[i]->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
;
233 if (UpdatedNodes)
234 UpdatedNodes->insert(New[i].getNode());
235 }
236 ReplacedNode(Old);
237 }
238};
239
240} // end anonymous namespace
241
242/// Return a vector shuffle operation which
243/// performs the same shuffe in terms of order or result bytes, but on a type
244/// whose vector element type is narrower than the original shuffle type.
245/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
246SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
247 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
248 ArrayRef<int> Mask) const {
249 unsigned NumMaskElts = VT.getVectorNumElements();
250 unsigned NumDestElts = NVT.getVectorNumElements();
251 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
252
253 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!")(static_cast <bool> (NumEltsGrowth && "Cannot promote to vector type with fewer elts!"
) ? void (0) : __assert_fail ("NumEltsGrowth && \"Cannot promote to vector type with fewer elts!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 253, __extension__ __PRETTY_FUNCTION__))
;
254
255 if (NumEltsGrowth == 1)
256 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
257
258 SmallVector<int, 8> NewMask;
259 for (unsigned i = 0; i != NumMaskElts; ++i) {
260 int Idx = Mask[i];
261 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
262 if (Idx < 0)
263 NewMask.push_back(-1);
264 else
265 NewMask.push_back(Idx * NumEltsGrowth + j);
266 }
267 }
268 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?")(static_cast <bool> (NewMask.size() == NumDestElts &&
"Non-integer NumEltsGrowth?") ? void (0) : __assert_fail ("NewMask.size() == NumDestElts && \"Non-integer NumEltsGrowth?\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 268, __extension__ __PRETTY_FUNCTION__))
;
269 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?")(static_cast <bool> (TLI.isShuffleMaskLegal(NewMask, NVT
) && "Shuffle not legal?") ? void (0) : __assert_fail
("TLI.isShuffleMaskLegal(NewMask, NVT) && \"Shuffle not legal?\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 269, __extension__ __PRETTY_FUNCTION__))
;
270 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
271}
272
273/// Expands the ConstantFP node to an integer constant or
274/// a load from the constant pool.
275SDValue
276SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
277 bool Extend = false;
278 SDLoc dl(CFP);
279
280 // If a FP immediate is precise when represented as a float and if the
281 // target can do an extending load from float to double, we put it into
282 // the constant pool as a float, even if it's is statically typed as a
283 // double. This shrinks FP constants and canonicalizes them for targets where
284 // an FP extending load is the same cost as a normal load (such as on the x87
285 // fp stack or PPC FP unit).
286 EVT VT = CFP->getValueType(0);
287 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
288 if (!UseCP) {
289 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion")(static_cast <bool> ((VT == MVT::f64 || VT == MVT::f32)
&& "Invalid type expansion") ? void (0) : __assert_fail
("(VT == MVT::f64 || VT == MVT::f32) && \"Invalid type expansion\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 289, __extension__ __PRETTY_FUNCTION__))
;
290 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
291 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
292 }
293
294 APFloat APF = CFP->getValueAPF();
295 EVT OrigVT = VT;
296 EVT SVT = VT;
297
298 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
299 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
300 if (!APF.isSignaling()) {
301 while (SVT != MVT::f32 && SVT != MVT::f16) {
302 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
303 if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
304 // Only do this if the target has a native EXTLOAD instruction from
305 // smaller type.
306 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
307 TLI.ShouldShrinkFPConstant(OrigVT)) {
308 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
309 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
310 VT = SVT;
311 Extend = true;
312 }
313 }
314 }
315
316 SDValue CPIdx =
317 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
318 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
319 if (Extend) {
320 SDValue Result = DAG.getExtLoad(
321 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
322 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
323 Alignment);
324 return Result;
325 }
326 SDValue Result = DAG.getLoad(
327 OrigVT, dl, DAG.getEntryNode(), CPIdx,
328 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
329 return Result;
330}
331
332/// Expands the Constant node to a load from the constant pool.
333SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
334 SDLoc dl(CP);
335 EVT VT = CP->getValueType(0);
336 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
337 TLI.getPointerTy(DAG.getDataLayout()));
338 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
339 SDValue Result = DAG.getLoad(
340 VT, dl, DAG.getEntryNode(), CPIdx,
341 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
342 return Result;
343}
344
345/// Some target cannot handle a variable insertion index for the
346/// INSERT_VECTOR_ELT instruction. In this case, it
347/// is necessary to spill the vector being inserted into to memory, perform
348/// the insert there, and then read the result back.
349SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
350 SDValue Val,
351 SDValue Idx,
352 const SDLoc &dl) {
353 SDValue Tmp1 = Vec;
354 SDValue Tmp2 = Val;
355 SDValue Tmp3 = Idx;
356
357 // If the target doesn't support this, we have to spill the input vector
358 // to a temporary stack slot, update the element, then reload it. This is
359 // badness. We could also load the value into a vector register (either
360 // with a "move to register" or "extload into register" instruction, then
361 // permute it into place, if the idx is a constant and if the idx is
362 // supported by the target.
363 EVT VT = Tmp1.getValueType();
364 EVT EltVT = VT.getVectorElementType();
365 SDValue StackPtr = DAG.CreateStackTemporary(VT);
366
367 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
368
369 // Store the vector.
370 SDValue Ch = DAG.getStore(
371 DAG.getEntryNode(), dl, Tmp1, StackPtr,
372 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
373
374 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
375
376 // Store the scalar value.
377 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
378 // Load the updated vector.
379 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
380 DAG.getMachineFunction(), SPFI));
381}
382
383SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
384 SDValue Idx,
385 const SDLoc &dl) {
386 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
387 // SCALAR_TO_VECTOR requires that the type of the value being inserted
388 // match the element type of the vector being created, except for
389 // integers in which case the inserted value can be over width.
390 EVT EltVT = Vec.getValueType().getVectorElementType();
391 if (Val.getValueType() == EltVT ||
392 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
393 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
394 Vec.getValueType(), Val);
395
396 unsigned NumElts = Vec.getValueType().getVectorNumElements();
397 // We generate a shuffle of InVec and ScVec, so the shuffle mask
398 // should be 0,1,2,3,4,5... with the appropriate element replaced with
399 // elt 0 of the RHS.
400 SmallVector<int, 8> ShufOps;
401 for (unsigned i = 0; i != NumElts; ++i)
402 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
403
404 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
405 }
406 }
407 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
408}
409
410SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
411 DEBUG(dbgs() << "Optimizing float store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Optimizing float store operations\n"
; } } while (false)
;
412 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
413 // FIXME: We shouldn't do this for TargetConstantFP's.
414 // FIXME: move this to the DAG Combiner! Note that we can't regress due
415 // to phase ordering between legalized code and the dag combiner. This
416 // probably means that we need to integrate dag combiner and legalizer
417 // together.
418 // We generally can't do this one for long doubles.
419 SDValue Chain = ST->getChain();
420 SDValue Ptr = ST->getBasePtr();
421 unsigned Alignment = ST->getAlignment();
422 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
423 AAMDNodes AAInfo = ST->getAAInfo();
424 SDLoc dl(ST);
425 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
426 if (CFP->getValueType(0) == MVT::f32 &&
427 TLI.isTypeLegal(MVT::i32)) {
428 SDValue Con = DAG.getConstant(CFP->getValueAPF().
429 bitcastToAPInt().zextOrTrunc(32),
430 SDLoc(CFP), MVT::i32);
431 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
432 MMOFlags, AAInfo);
433 }
434
435 if (CFP->getValueType(0) == MVT::f64) {
436 // If this target supports 64-bit registers, do a single 64-bit store.
437 if (TLI.isTypeLegal(MVT::i64)) {
438 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
439 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
440 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
441 Alignment, MMOFlags, AAInfo);
442 }
443
444 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
445 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
446 // stores. If the target supports neither 32- nor 64-bits, this
447 // xform is certainly not worth it.
448 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
449 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
450 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
451 if (DAG.getDataLayout().isBigEndian())
452 std::swap(Lo, Hi);
453
454 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
455 MMOFlags, AAInfo);
456 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
457 DAG.getConstant(4, dl, Ptr.getValueType()));
458 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
459 ST->getPointerInfo().getWithOffset(4),
460 MinAlign(Alignment, 4U), MMOFlags, AAInfo);
461
462 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
463 }
464 }
465 }
466 return SDValue(nullptr, 0);
467}
468
469void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
470 StoreSDNode *ST = cast<StoreSDNode>(Node);
471 SDValue Chain = ST->getChain();
472 SDValue Ptr = ST->getBasePtr();
473 SDLoc dl(Node);
474
475 unsigned Alignment = ST->getAlignment();
476 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
477 AAMDNodes AAInfo = ST->getAAInfo();
478
479 if (!ST->isTruncatingStore()) {
1
Taking false branch
480 DEBUG(dbgs() << "Legalizing store operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing store operation\n"
; } } while (false)
;
481 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
482 ReplaceNode(ST, OptStore);
483 return;
484 }
485
486 SDValue Value = ST->getValue();
487 MVT VT = Value.getSimpleValueType();
488 switch (TLI.getOperationAction(ISD::STORE, VT)) {
489 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 489)
;
490 case TargetLowering::Legal: {
491 // If this is an unaligned store and the target doesn't support it,
492 // expand it.
493 EVT MemVT = ST->getMemoryVT();
494 unsigned AS = ST->getAddressSpace();
495 unsigned Align = ST->getAlignment();
496 const DataLayout &DL = DAG.getDataLayout();
497 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
498 DEBUG(dbgs() << "Expanding unsupported unaligned store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Expanding unsupported unaligned store\n"
; } } while (false)
;
499 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
500 ReplaceNode(SDValue(ST, 0), Result);
501 } else
502 DEBUG(dbgs() << "Legal store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legal store\n"; } } while
(false)
;
503 break;
504 }
505 case TargetLowering::Custom: {
506 DEBUG(dbgs() << "Trying custom lowering\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying custom lowering\n"
; } } while (false)
;
507 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
508 if (Res && Res != SDValue(Node, 0))
509 ReplaceNode(SDValue(Node, 0), Res);
510 return;
511 }
512 case TargetLowering::Promote: {
513 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
514 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&(static_cast <bool> (NVT.getSizeInBits() == VT.getSizeInBits
() && "Can only promote stores to same size type") ? void
(0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 515, __extension__ __PRETTY_FUNCTION__))
515 "Can only promote stores to same size type")(static_cast <bool> (NVT.getSizeInBits() == VT.getSizeInBits
() && "Can only promote stores to same size type") ? void
(0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 515, __extension__ __PRETTY_FUNCTION__))
;
516 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
517 SDValue Result =
518 DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
519 Alignment, MMOFlags, AAInfo);
520 ReplaceNode(SDValue(Node, 0), Result);
521 break;
522 }
523 }
524 return;
525 }
526
527 DEBUG(dbgs() << "Legalizing truncating store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing truncating store operations\n"
; } } while (false)
;
528 SDValue Value = ST->getValue();
529 EVT StVT = ST->getMemoryVT();
530 unsigned StWidth = StVT.getSizeInBits();
531 auto &DL = DAG.getDataLayout();
532
533 if (StWidth != StVT.getStoreSizeInBits()) {
2
Taking false branch
534 // Promote to a byte-sized store with upper bits zero if not
535 // storing an integral number of bytes. For example, promote
536 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
537 EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
538 StVT.getStoreSizeInBits());
539 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
540 SDValue Result =
541 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
542 Alignment, MMOFlags, AAInfo);
543 ReplaceNode(SDValue(Node, 0), Result);
544 } else if (StWidth & (StWidth - 1)) {
3
Taking true branch
545 // If not storing a power-of-2 number of bits, expand as two stores.
546 assert(!StVT.isVector() && "Unsupported truncstore!")(static_cast <bool> (!StVT.isVector() && "Unsupported truncstore!"
) ? void (0) : __assert_fail ("!StVT.isVector() && \"Unsupported truncstore!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 546, __extension__ __PRETTY_FUNCTION__))
;
547 unsigned RoundWidth = 1 << Log2_32(StWidth);
4
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'
548 assert(RoundWidth < StWidth)(static_cast <bool> (RoundWidth < StWidth) ? void (0
) : __assert_fail ("RoundWidth < StWidth", "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 548, __extension__ __PRETTY_FUNCTION__))
;
549 unsigned ExtraWidth = StWidth - RoundWidth;
550 assert(ExtraWidth < RoundWidth)(static_cast <bool> (ExtraWidth < RoundWidth) ? void
(0) : __assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 550, __extension__ __PRETTY_FUNCTION__))
;
551 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&(static_cast <bool> (!(RoundWidth % 8) && !(ExtraWidth
% 8) && "Store size not an integral number of bytes!"
) ? void (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 552, __extension__ __PRETTY_FUNCTION__))
552 "Store size not an integral number of bytes!")(static_cast <bool> (!(RoundWidth % 8) && !(ExtraWidth
% 8) && "Store size not an integral number of bytes!"
) ? void (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 552, __extension__ __PRETTY_FUNCTION__))
;
553 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
554 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
555 SDValue Lo, Hi;
556 unsigned IncrementSize;
557
558 if (DL.isLittleEndian()) {
559 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
560 // Store the bottom RoundWidth bits.
561 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
562 RoundVT, Alignment, MMOFlags, AAInfo);
563
564 // Store the remaining ExtraWidth bits.
565 IncrementSize = RoundWidth / 8;
566 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
567 DAG.getConstant(IncrementSize, dl,
568 Ptr.getValueType()));
569 Hi = DAG.getNode(
570 ISD::SRL, dl, Value.getValueType(), Value,
571 DAG.getConstant(RoundWidth, dl,
572 TLI.getShiftAmountTy(Value.getValueType(), DL)));
573 Hi = DAG.getTruncStore(
574 Chain, dl, Hi, Ptr,
575 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
576 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
577 } else {
578 // Big endian - avoid unaligned stores.
579 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
580 // Store the top RoundWidth bits.
581 Hi = DAG.getNode(
582 ISD::SRL, dl, Value.getValueType(), Value,
583 DAG.getConstant(ExtraWidth, dl,
584 TLI.getShiftAmountTy(Value.getValueType(), DL)));
585 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
586 RoundVT, Alignment, MMOFlags, AAInfo);
587
588 // Store the remaining ExtraWidth bits.
589 IncrementSize = RoundWidth / 8;
590 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
591 DAG.getConstant(IncrementSize, dl,
592 Ptr.getValueType()));
593 Lo = DAG.getTruncStore(
594 Chain, dl, Value, Ptr,
595 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
596 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
597 }
598
599 // The order of the stores doesn't matter.
600 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
601 ReplaceNode(SDValue(Node, 0), Result);
602 } else {
603 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
604 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 604)
;
605 case TargetLowering::Legal: {
606 EVT MemVT = ST->getMemoryVT();
607 unsigned AS = ST->getAddressSpace();
608 unsigned Align = ST->getAlignment();
609 // If this is an unaligned store and the target doesn't support it,
610 // expand it.
611 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
612 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
613 ReplaceNode(SDValue(ST, 0), Result);
614 }
615 break;
616 }
617 case TargetLowering::Custom: {
618 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
619 if (Res && Res != SDValue(Node, 0))
620 ReplaceNode(SDValue(Node, 0), Res);
621 return;
622 }
623 case TargetLowering::Expand:
624 assert(!StVT.isVector() &&(static_cast <bool> (!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? void (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 625, __extension__ __PRETTY_FUNCTION__))
625 "Vector Stores are handled in LegalizeVectorOps")(static_cast <bool> (!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? void (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 625, __extension__ __PRETTY_FUNCTION__))
;
626
627 SDValue Result;
628
629 // TRUNCSTORE:i16 i32 -> STORE i16
630 if (TLI.isTypeLegal(StVT)) {
631 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
632 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
633 Alignment, MMOFlags, AAInfo);
634 } else {
635 // The in-memory type isn't legal. Truncate to the type it would promote
636 // to, and then do a truncstore.
637 Value = DAG.getNode(ISD::TRUNCATE, dl,
638 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
639 Value);
640 Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
641 StVT, Alignment, MMOFlags, AAInfo);
642 }
643
644 ReplaceNode(SDValue(Node, 0), Result);
645 break;
646 }
647 }
648}
649
650void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
651 LoadSDNode *LD = cast<LoadSDNode>(Node);
652 SDValue Chain = LD->getChain(); // The chain.
653 SDValue Ptr = LD->getBasePtr(); // The base pointer.
654 SDValue Value; // The value returned by the load op.
655 SDLoc dl(Node);
656
657 ISD::LoadExtType ExtType = LD->getExtensionType();
658 if (ExtType == ISD::NON_EXTLOAD) {
659 DEBUG(dbgs() << "Legalizing non-extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing non-extending load operation\n"
; } } while (false)
;
660 MVT VT = Node->getSimpleValueType(0);
661 SDValue RVal = SDValue(Node, 0);
662 SDValue RChain = SDValue(Node, 1);
663
664 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
665 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 665)
;
666 case TargetLowering::Legal: {
667 EVT MemVT = LD->getMemoryVT();
668 unsigned AS = LD->getAddressSpace();
669 unsigned Align = LD->getAlignment();
670 const DataLayout &DL = DAG.getDataLayout();
671 // If this is an unaligned load and the target doesn't support it,
672 // expand it.
673 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
674 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
675 }
676 break;
677 }
678 case TargetLowering::Custom:
679 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
680 RVal = Res;
681 RChain = Res.getValue(1);
682 }
683 break;
684
685 case TargetLowering::Promote: {
686 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
687 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&(static_cast <bool> (NVT.getSizeInBits() == VT.getSizeInBits
() && "Can only promote loads to same size type") ? void
(0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 688, __extension__ __PRETTY_FUNCTION__))
688 "Can only promote loads to same size type")(static_cast <bool> (NVT.getSizeInBits() == VT.getSizeInBits
() && "Can only promote loads to same size type") ? void
(0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 688, __extension__ __PRETTY_FUNCTION__))
;
689
690 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
691 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
692 RChain = Res.getValue(1);
693 break;
694 }
695 }
696 if (RChain.getNode() != Node) {
697 assert(RVal.getNode() != Node && "Load must be completely replaced")(static_cast <bool> (RVal.getNode() != Node && "Load must be completely replaced"
) ? void (0) : __assert_fail ("RVal.getNode() != Node && \"Load must be completely replaced\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 697, __extension__ __PRETTY_FUNCTION__))
;
698 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
699 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
700 if (UpdatedNodes) {
701 UpdatedNodes->insert(RVal.getNode());
702 UpdatedNodes->insert(RChain.getNode());
703 }
704 ReplacedNode(Node);
705 }
706 return;
707 }
708
709 DEBUG(dbgs() << "Legalizing extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing extending load operation\n"
; } } while (false)
;
710 EVT SrcVT = LD->getMemoryVT();
711 unsigned SrcWidth = SrcVT.getSizeInBits();
712 unsigned Alignment = LD->getAlignment();
713 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
714 AAMDNodes AAInfo = LD->getAAInfo();
715
716 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
717 // Some targets pretend to have an i1 loading operation, and actually
718 // load an i8. This trick is correct for ZEXTLOAD because the top 7
719 // bits are guaranteed to be zero; it helps the optimizers understand
720 // that these bits are zero. It is also useful for EXTLOAD, since it
721 // tells the optimizers that those bits are undefined. It would be
722 // nice to have an effective generic way of getting these benefits...
723 // Until such a way is found, don't insist on promoting i1 here.
724 (SrcVT != MVT::i1 ||
725 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
726 TargetLowering::Promote)) {
727 // Promote to a byte-sized load if not loading an integral number of
728 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
729 unsigned NewWidth = SrcVT.getStoreSizeInBits();
730 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
731 SDValue Ch;
732
733 // The extra bits are guaranteed to be zero, since we stored them that
734 // way. A zext load from NVT thus automatically gives zext from SrcVT.
735
736 ISD::LoadExtType NewExtType =
737 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
738
739 SDValue Result =
740 DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
741 LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
742
743 Ch = Result.getValue(1); // The chain.
744
745 if (ExtType == ISD::SEXTLOAD)
746 // Having the top bits zero doesn't help when sign extending.
747 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
748 Result.getValueType(),
749 Result, DAG.getValueType(SrcVT));
750 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
751 // All the top bits are guaranteed to be zero - inform the optimizers.
752 Result = DAG.getNode(ISD::AssertZext, dl,
753 Result.getValueType(), Result,
754 DAG.getValueType(SrcVT));
755
756 Value = Result;
757 Chain = Ch;
758 } else if (SrcWidth & (SrcWidth - 1)) {
759 // If not loading a power-of-2 number of bits, expand as two loads.
760 assert(!SrcVT.isVector() && "Unsupported extload!")(static_cast <bool> (!SrcVT.isVector() && "Unsupported extload!"
) ? void (0) : __assert_fail ("!SrcVT.isVector() && \"Unsupported extload!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 760, __extension__ __PRETTY_FUNCTION__))
;
761 unsigned RoundWidth = 1 << Log2_32(SrcWidth);
762 assert(RoundWidth < SrcWidth)(static_cast <bool> (RoundWidth < SrcWidth) ? void (
0) : __assert_fail ("RoundWidth < SrcWidth", "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 762, __extension__ __PRETTY_FUNCTION__))
;
763 unsigned ExtraWidth = SrcWidth - RoundWidth;
764 assert(ExtraWidth < RoundWidth)(static_cast <bool> (ExtraWidth < RoundWidth) ? void
(0) : __assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 764, __extension__ __PRETTY_FUNCTION__))
;
765 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&(static_cast <bool> (!(RoundWidth % 8) && !(ExtraWidth
% 8) && "Load size not an integral number of bytes!"
) ? void (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 766, __extension__ __PRETTY_FUNCTION__))
766 "Load size not an integral number of bytes!")(static_cast <bool> (!(RoundWidth % 8) && !(ExtraWidth
% 8) && "Load size not an integral number of bytes!"
) ? void (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 766, __extension__ __PRETTY_FUNCTION__))
;
767 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
768 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
769 SDValue Lo, Hi, Ch;
770 unsigned IncrementSize;
771 auto &DL = DAG.getDataLayout();
772
773 if (DL.isLittleEndian()) {
774 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
775 // Load the bottom RoundWidth bits.
776 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
777 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
778 AAInfo);
779
780 // Load the remaining ExtraWidth bits.
781 IncrementSize = RoundWidth / 8;
782 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
783 DAG.getConstant(IncrementSize, dl,
784 Ptr.getValueType()));
785 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
786 LD->getPointerInfo().getWithOffset(IncrementSize),
787 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
788 AAInfo);
789
790 // Build a factor node to remember that this load is independent of
791 // the other one.
792 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
793 Hi.getValue(1));
794
795 // Move the top bits to the right place.
796 Hi = DAG.getNode(
797 ISD::SHL, dl, Hi.getValueType(), Hi,
798 DAG.getConstant(RoundWidth, dl,
799 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
800
801 // Join the hi and lo parts.
802 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
803 } else {
804 // Big endian - avoid unaligned loads.
805 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
806 // Load the top RoundWidth bits.
807 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
808 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
809 AAInfo);
810
811 // Load the remaining ExtraWidth bits.
812 IncrementSize = RoundWidth / 8;
813 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
814 DAG.getConstant(IncrementSize, dl,
815 Ptr.getValueType()));
816 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
817 LD->getPointerInfo().getWithOffset(IncrementSize),
818 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
819 AAInfo);
820
821 // Build a factor node to remember that this load is independent of
822 // the other one.
823 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
824 Hi.getValue(1));
825
826 // Move the top bits to the right place.
827 Hi = DAG.getNode(
828 ISD::SHL, dl, Hi.getValueType(), Hi,
829 DAG.getConstant(ExtraWidth, dl,
830 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
831
832 // Join the hi and lo parts.
833 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
834 }
835
836 Chain = Ch;
837 } else {
838 bool isCustom = false;
839 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
840 SrcVT.getSimpleVT())) {
841 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 841)
;
842 case TargetLowering::Custom:
843 isCustom = true;
844 LLVM_FALLTHROUGH[[clang::fallthrough]];
845 case TargetLowering::Legal:
846 Value = SDValue(Node, 0);
847 Chain = SDValue(Node, 1);
848
849 if (isCustom) {
850 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
851 Value = Res;
852 Chain = Res.getValue(1);
853 }
854 } else {
855 // If this is an unaligned load and the target doesn't support it,
856 // expand it.
857 EVT MemVT = LD->getMemoryVT();
858 unsigned AS = LD->getAddressSpace();
859 unsigned Align = LD->getAlignment();
860 const DataLayout &DL = DAG.getDataLayout();
861 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
862 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
863 }
864 }
865 break;
866
867 case TargetLowering::Expand: {
868 EVT DestVT = Node->getValueType(0);
869 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
870 // If the source type is not legal, see if there is a legal extload to
871 // an intermediate type that we can then extend further.
872 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
873 if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
874 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
875 // If we are loading a legal type, this is a non-extload followed by a
876 // full extend.
877 ISD::LoadExtType MidExtType =
878 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
879
880 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
881 SrcVT, LD->getMemOperand());
882 unsigned ExtendOp =
883 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
884 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
885 Chain = Load.getValue(1);
886 break;
887 }
888
889 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
890 // normal undefined upper bits behavior to allow using an in-reg extend
891 // with the illegal FP type, so load as an integer and do the
892 // from-integer conversion.
893 if (SrcVT.getScalarType() == MVT::f16) {
894 EVT ISrcVT = SrcVT.changeTypeToInteger();
895 EVT IDestVT = DestVT.changeTypeToInteger();
896 EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
897
898 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
899 Chain, Ptr, ISrcVT,
900 LD->getMemOperand());
901 Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
902 Chain = Result.getValue(1);
903 break;
904 }
905 }
906
907 assert(!SrcVT.isVector() &&(static_cast <bool> (!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? void (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 908, __extension__ __PRETTY_FUNCTION__))
908 "Vector Loads are handled in LegalizeVectorOps")(static_cast <bool> (!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? void (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 908, __extension__ __PRETTY_FUNCTION__))
;
909
910 // FIXME: This does not work for vectors on most targets. Sign-
911 // and zero-extend operations are currently folded into extending
912 // loads, whether they are legal or not, and then we end up here
913 // without any support for legalizing them.
914 assert(ExtType != ISD::EXTLOAD &&(static_cast <bool> (ExtType != ISD::EXTLOAD &&
"EXTLOAD should always be supported!") ? void (0) : __assert_fail
("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 915, __extension__ __PRETTY_FUNCTION__))
915 "EXTLOAD should always be supported!")(static_cast <bool> (ExtType != ISD::EXTLOAD &&
"EXTLOAD should always be supported!") ? void (0) : __assert_fail
("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 915, __extension__ __PRETTY_FUNCTION__))
;
916 // Turn the unsupported load into an EXTLOAD followed by an
917 // explicit zero/sign extend inreg.
918 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
919 Node->getValueType(0),
920 Chain, Ptr, SrcVT,
921 LD->getMemOperand());
922 SDValue ValRes;
923 if (ExtType == ISD::SEXTLOAD)
924 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
925 Result.getValueType(),
926 Result, DAG.getValueType(SrcVT));
927 else
928 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
929 Value = ValRes;
930 Chain = Result.getValue(1);
931 break;
932 }
933 }
934 }
935
936 // Since loads produce two values, make sure to remember that we legalized
937 // both of them.
938 if (Chain.getNode() != Node) {
939 assert(Value.getNode() != Node && "Load must be completely replaced")(static_cast <bool> (Value.getNode() != Node &&
"Load must be completely replaced") ? void (0) : __assert_fail
("Value.getNode() != Node && \"Load must be completely replaced\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 939, __extension__ __PRETTY_FUNCTION__))
;
940 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
941 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
942 if (UpdatedNodes) {
943 UpdatedNodes->insert(Value.getNode());
944 UpdatedNodes->insert(Chain.getNode());
945 }
946 ReplacedNode(Node);
947 }
948}
949
950static TargetLowering::LegalizeAction
951getStrictFPOpcodeAction(const TargetLowering &TLI, unsigned Opcode, EVT VT) {
952 unsigned EqOpc;
953 switch (Opcode) {
954 default: llvm_unreachable("Unexpected FP pseudo-opcode")::llvm::llvm_unreachable_internal("Unexpected FP pseudo-opcode"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 954)
;
955 case ISD::STRICT_FSQRT: EqOpc = ISD::FSQRT; break;
956 case ISD::STRICT_FPOW: EqOpc = ISD::FPOW; break;
957 case ISD::STRICT_FPOWI: EqOpc = ISD::FPOWI; break;
958 case ISD::STRICT_FMA: EqOpc = ISD::FMA; break;
959 case ISD::STRICT_FSIN: EqOpc = ISD::FSIN; break;
960 case ISD::STRICT_FCOS: EqOpc = ISD::FCOS; break;
961 case ISD::STRICT_FEXP: EqOpc = ISD::FEXP; break;
962 case ISD::STRICT_FEXP2: EqOpc = ISD::FEXP2; break;
963 case ISD::STRICT_FLOG: EqOpc = ISD::FLOG; break;
964 case ISD::STRICT_FLOG10: EqOpc = ISD::FLOG10; break;
965 case ISD::STRICT_FLOG2: EqOpc = ISD::FLOG2; break;
966 case ISD::STRICT_FRINT: EqOpc = ISD::FRINT; break;
967 case ISD::STRICT_FNEARBYINT: EqOpc = ISD::FNEARBYINT; break;
968 }
969
970 auto Action = TLI.getOperationAction(EqOpc, VT);
971
972 // We don't currently handle Custom or Promote for strict FP pseudo-ops.
973 // For now, we just expand for those cases.
974 if (Action != TargetLowering::Legal)
975 Action = TargetLowering::Expand;
976
977 return Action;
978}
979
980/// Return a legal replacement for the given operation, with all legal operands.
981void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
982 DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "\nLegalizing: "; Node->
dump(&DAG); } } while (false)
;
983
984 // Allow illegal target nodes and illegal registers.
985 if (Node->getOpcode() == ISD::TargetConstant ||
986 Node->getOpcode() == ISD::Register)
987 return;
988
989#ifndef NDEBUG
990 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
991 assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Node->getValueType(i)) == TargetLowering::TypeLegal ||
TLI.isTypeLegal(Node->getValueType(i))) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 994, __extension__ __PRETTY_FUNCTION__))
992 TargetLowering::TypeLegal ||(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Node->getValueType(i)) == TargetLowering::TypeLegal ||
TLI.isTypeLegal(Node->getValueType(i))) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 994, __extension__ __PRETTY_FUNCTION__))
993 TLI.isTypeLegal(Node->getValueType(i))) &&(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Node->getValueType(i)) == TargetLowering::TypeLegal ||
TLI.isTypeLegal(Node->getValueType(i))) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 994, __extension__ __PRETTY_FUNCTION__))
994 "Unexpected illegal type!")(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Node->getValueType(i)) == TargetLowering::TypeLegal ||
TLI.isTypeLegal(Node->getValueType(i))) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 994, __extension__ __PRETTY_FUNCTION__))
;
995
996 for (const SDValue &Op : Node->op_values())
997 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1002, __extension__ __PRETTY_FUNCTION__))
998 TargetLowering::TypeLegal ||(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1002, __extension__ __PRETTY_FUNCTION__))
999 TLI.isTypeLegal(Op.getValueType()) ||(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1002, __extension__ __PRETTY_FUNCTION__))
1000 Op.getOpcode() == ISD::TargetConstant ||(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1002, __extension__ __PRETTY_FUNCTION__))
1001 Op.getOpcode() == ISD::Register) &&(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1002, __extension__ __PRETTY_FUNCTION__))
1002 "Unexpected illegal type!")(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1002, __extension__ __PRETTY_FUNCTION__))
;
1003#endif
1004
1005 // Figure out the correct action; the way to query this varies by opcode
1006 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
1007 bool SimpleFinishLegalizing = true;
1008 switch (Node->getOpcode()) {
1009 case ISD::INTRINSIC_W_CHAIN:
1010 case ISD::INTRINSIC_WO_CHAIN:
1011 case ISD::INTRINSIC_VOID:
1012 case ISD::STACKSAVE:
1013 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1014 break;
1015 case ISD::GET_DYNAMIC_AREA_OFFSET:
1016 Action = TLI.getOperationAction(Node->getOpcode(),
1017 Node->getValueType(0));
1018 break;
1019 case ISD::VAARG:
1020 Action = TLI.getOperationAction(Node->getOpcode(),
1021 Node->getValueType(0));
1022 if (Action != TargetLowering::Promote)
1023 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1024 break;
1025 case ISD::FP_TO_FP16:
1026 case ISD::SINT_TO_FP:
1027 case ISD::UINT_TO_FP:
1028 case ISD::EXTRACT_VECTOR_ELT:
1029 Action = TLI.getOperationAction(Node->getOpcode(),
1030 Node->getOperand(0).getValueType());
1031 break;
1032 case ISD::FP_ROUND_INREG:
1033 case ISD::SIGN_EXTEND_INREG: {
1034 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1035 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1036 break;
1037 }
1038 case ISD::ATOMIC_STORE:
1039 Action = TLI.getOperationAction(Node->getOpcode(),
1040 Node->getOperand(2).getValueType());
1041 break;
1042 case ISD::SELECT_CC:
1043 case ISD::SETCC:
1044 case ISD::BR_CC: {
1045 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1046 Node->getOpcode() == ISD::SETCC ? 2 :
1047 Node->getOpcode() == ISD::SETCCE ? 3 : 1;
1048 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
1049 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1050 ISD::CondCode CCCode =
1051 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1052 Action = TLI.getCondCodeAction(CCCode, OpVT);
1053 if (Action == TargetLowering::Legal) {
1054 if (Node->getOpcode() == ISD::SELECT_CC)
1055 Action = TLI.getOperationAction(Node->getOpcode(),
1056 Node->getValueType(0));
1057 else
1058 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1059 }
1060 break;
1061 }
1062 case ISD::LOAD:
1063 case ISD::STORE:
1064 // FIXME: Model these properly. LOAD and STORE are complicated, and
1065 // STORE expects the unlegalized operand in some cases.
1066 SimpleFinishLegalizing = false;
1067 break;
1068 case ISD::CALLSEQ_START:
1069 case ISD::CALLSEQ_END:
1070 // FIXME: This shouldn't be necessary. These nodes have special properties
1071 // dealing with the recursive nature of legalization. Removing this
1072 // special case should be done as part of making LegalizeDAG non-recursive.
1073 SimpleFinishLegalizing = false;
1074 break;
1075 case ISD::EXTRACT_ELEMENT:
1076 case ISD::FLT_ROUNDS_:
1077 case ISD::MERGE_VALUES:
1078 case ISD::EH_RETURN:
1079 case ISD::FRAME_TO_ARGS_OFFSET:
1080 case ISD::EH_DWARF_CFA:
1081 case ISD::EH_SJLJ_SETJMP:
1082 case ISD::EH_SJLJ_LONGJMP:
1083 case ISD::EH_SJLJ_SETUP_DISPATCH:
1084 // These operations lie about being legal: when they claim to be legal,
1085 // they should actually be expanded.
1086 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1087 if (Action == TargetLowering::Legal)
1088 Action = TargetLowering::Expand;
1089 break;
1090 case ISD::INIT_TRAMPOLINE:
1091 case ISD::ADJUST_TRAMPOLINE:
1092 case ISD::FRAMEADDR:
1093 case ISD::RETURNADDR:
1094 case ISD::ADDROFRETURNADDR:
1095 // These operations lie about being legal: when they claim to be legal,
1096 // they should actually be custom-lowered.
1097 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1098 if (Action == TargetLowering::Legal)
1099 Action = TargetLowering::Custom;
1100 break;
1101 case ISD::READCYCLECOUNTER:
1102 // READCYCLECOUNTER returns an i64, even if type legalization might have
1103 // expanded that to several smaller types.
1104 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1105 break;
1106 case ISD::READ_REGISTER:
1107 case ISD::WRITE_REGISTER:
1108 // Named register is legal in the DAG, but blocked by register name
1109 // selection if not implemented by target (to chose the correct register)
1110 // They'll be converted to Copy(To/From)Reg.
1111 Action = TargetLowering::Legal;
1112 break;
1113 case ISD::DEBUGTRAP:
1114 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1115 if (Action == TargetLowering::Expand) {
1116 // replace ISD::DEBUGTRAP with ISD::TRAP
1117 SDValue NewVal;
1118 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1119 Node->getOperand(0));
1120 ReplaceNode(Node, NewVal.getNode());
1121 LegalizeOp(NewVal.getNode());
1122 return;
1123 }
1124 break;
1125 case ISD::STRICT_FSQRT:
1126 case ISD::STRICT_FMA:
1127 case ISD::STRICT_FPOW:
1128 case ISD::STRICT_FPOWI:
1129 case ISD::STRICT_FSIN:
1130 case ISD::STRICT_FCOS:
1131 case ISD::STRICT_FEXP:
1132 case ISD::STRICT_FEXP2:
1133 case ISD::STRICT_FLOG:
1134 case ISD::STRICT_FLOG10:
1135 case ISD::STRICT_FLOG2:
1136 case ISD::STRICT_FRINT:
1137 case ISD::STRICT_FNEARBYINT:
1138 // These pseudo-ops get legalized as if they were their non-strict
1139 // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
1140 // is also legal, but if ISD::FSQRT requires expansion then so does
1141 // ISD::STRICT_FSQRT.
1142 Action = getStrictFPOpcodeAction(TLI, Node->getOpcode(),
1143 Node->getValueType(0));
1144 break;
1145 default:
1146 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1147 Action = TargetLowering::Legal;
1148 } else {
1149 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1150 }
1151 break;
1152 }
1153
1154 if (SimpleFinishLegalizing) {
1155 SDNode *NewNode = Node;
1156 switch (Node->getOpcode()) {
1157 default: break;
1158 case ISD::SHL:
1159 case ISD::SRL:
1160 case ISD::SRA:
1161 case ISD::ROTL:
1162 case ISD::ROTR: {
1163 // Legalizing shifts/rotates requires adjusting the shift amount
1164 // to the appropriate width.
1165 SDValue Op0 = Node->getOperand(0);
1166 SDValue Op1 = Node->getOperand(1);
1167 if (!Op1.getValueType().isVector()) {
1168 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1169 // The getShiftAmountOperand() may create a new operand node or
1170 // return the existing one. If new operand is created we need
1171 // to update the parent node.
1172 // Do not try to legalize SAO here! It will be automatically legalized
1173 // in the next round.
1174 if (SAO != Op1)
1175 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1176 }
1177 }
1178 break;
1179 case ISD::SRL_PARTS:
1180 case ISD::SRA_PARTS:
1181 case ISD::SHL_PARTS: {
1182 // Legalizing shifts/rotates requires adjusting the shift amount
1183 // to the appropriate width.
1184 SDValue Op0 = Node->getOperand(0);
1185 SDValue Op1 = Node->getOperand(1);
1186 SDValue Op2 = Node->getOperand(2);
1187 if (!Op2.getValueType().isVector()) {
1188 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1189 // The getShiftAmountOperand() may create a new operand node or
1190 // return the existing one. If new operand is created we need
1191 // to update the parent node.
1192 if (SAO != Op2)
1193 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1194 }
1195 break;
1196 }
1197 }
1198
1199 if (NewNode != Node) {
1200 ReplaceNode(Node, NewNode);
1201 Node = NewNode;
1202 }
1203 switch (Action) {
1204 case TargetLowering::Legal:
1205 DEBUG(dbgs() << "Legal node: nothing to do\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legal node: nothing to do\n"
; } } while (false)
;
1206 return;
1207 case TargetLowering::Custom:
1208 DEBUG(dbgs() << "Trying custom legalization\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying custom legalization\n"
; } } while (false)
;
1209 // FIXME: The handling for custom lowering with multiple results is
1210 // a complete mess.
1211 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1212 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1213 return;
1214
1215 if (Node->getNumValues() == 1) {
1216 DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully custom legalized node\n"
; } } while (false)
;
1217 // We can just directly replace this node with the lowered value.
1218 ReplaceNode(SDValue(Node, 0), Res);
1219 return;
1220 }
1221
1222 SmallVector<SDValue, 8> ResultVals;
1223 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1224 ResultVals.push_back(Res.getValue(i));
1225 DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully custom legalized node\n"
; } } while (false)
;
1226 ReplaceNode(Node, ResultVals.data());
1227 return;
1228 }
1229 DEBUG(dbgs() << "Could not custom legalize node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not custom legalize node\n"
; } } while (false)
;
1230 LLVM_FALLTHROUGH[[clang::fallthrough]];
1231 case TargetLowering::Expand:
1232 if (ExpandNode(Node))
1233 return;
1234 LLVM_FALLTHROUGH[[clang::fallthrough]];
1235 case TargetLowering::LibCall:
1236 ConvertNodeToLibcall(Node);
1237 return;
1238 case TargetLowering::Promote:
1239 PromoteNode(Node);
1240 return;
1241 }
1242 }
1243
1244 switch (Node->getOpcode()) {
1245 default:
1246#ifndef NDEBUG
1247 dbgs() << "NODE: ";
1248 Node->dump( &DAG);
1249 dbgs() << "\n";
1250#endif
1251 llvm_unreachable("Do not know how to legalize this operator!")::llvm::llvm_unreachable_internal("Do not know how to legalize this operator!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1251)
;
1252
1253 case ISD::CALLSEQ_START:
1254 case ISD::CALLSEQ_END:
1255 break;
1256 case ISD::LOAD:
1257 return LegalizeLoadOps(Node);
1258 case ISD::STORE:
1259 return LegalizeStoreOps(Node);
1260 }
1261}
1262
1263SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1264 SDValue Vec = Op.getOperand(0);
1265 SDValue Idx = Op.getOperand(1);
1266 SDLoc dl(Op);
1267
1268 // Before we generate a new store to a temporary stack slot, see if there is
1269 // already one that we can use. There often is because when we scalarize
1270 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1271 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1272 // the vector. If all are expanded here, we don't want one store per vector
1273 // element.
1274
1275 // Caches for hasPredecessorHelper
1276 SmallPtrSet<const SDNode *, 32> Visited;
1277 SmallVector<const SDNode *, 16> Worklist;
1278 Worklist.push_back(Idx.getNode());
1279 SDValue StackPtr, Ch;
1280 for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1281 UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1282 SDNode *User = *UI;
1283 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1284 if (ST->isIndexed() || ST->isTruncatingStore() ||
1285 ST->getValue() != Vec)
1286 continue;
1287
1288 // Make sure that nothing else could have stored into the destination of
1289 // this store.
1290 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1291 continue;
1292
1293 // If the index is dependent on the store we will introduce a cycle when
1294 // creating the load (the load uses the index, and by replacing the chain
1295 // we will make the index dependent on the load). Also, the store might be
1296 // dependent on the extractelement and introduce a cycle when creating
1297 // the load.
1298 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1299 ST->hasPredecessor(Op.getNode()))
1300 continue;
1301
1302 StackPtr = ST->getBasePtr();
1303 Ch = SDValue(ST, 0);
1304 break;
1305 }
1306 }
1307
1308 EVT VecVT = Vec.getValueType();
1309
1310 if (!Ch.getNode()) {
1311 // Store the value to a temporary stack slot, then LOAD the returned part.
1312 StackPtr = DAG.CreateStackTemporary(VecVT);
1313 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1314 MachinePointerInfo());
1315 }
1316
1317 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1318
1319 SDValue NewLoad;
1320
1321 if (Op.getValueType().isVector())
1322 NewLoad =
1323 DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1324 else
1325 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1326 MachinePointerInfo(),
1327 VecVT.getVectorElementType());
1328
1329 // Replace the chain going out of the store, by the one out of the load.
1330 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1331
1332 // We introduced a cycle though, so update the loads operands, making sure
1333 // to use the original store's chain as an incoming chain.
1334 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1335 NewLoad->op_end());
1336 NewLoadOperands[0] = Ch;
1337 NewLoad =
1338 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1339 return NewLoad;
1340}
1341
1342SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1343 assert(Op.getValueType().isVector() && "Non-vector insert subvector!")(static_cast <bool> (Op.getValueType().isVector() &&
"Non-vector insert subvector!") ? void (0) : __assert_fail (
"Op.getValueType().isVector() && \"Non-vector insert subvector!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1343, __extension__ __PRETTY_FUNCTION__))
;
1344
1345 SDValue Vec = Op.getOperand(0);
1346 SDValue Part = Op.getOperand(1);
1347 SDValue Idx = Op.getOperand(2);
1348 SDLoc dl(Op);
1349
1350 // Store the value to a temporary stack slot, then LOAD the returned part.
1351 EVT VecVT = Vec.getValueType();
1352 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1353 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1354 MachinePointerInfo PtrInfo =
1355 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1356
1357 // First store the whole vector.
1358 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1359
1360 // Then store the inserted part.
1361 SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1362
1363 // Store the subvector.
1364 Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
1365
1366 // Finally, load the updated vector.
1367 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1368}
1369
1370SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1371 // We can't handle this case efficiently. Allocate a sufficiently
1372 // aligned object on the stack, store each element into it, then load
1373 // the result as a vector.
1374 // Create the stack frame object.
1375 EVT VT = Node->getValueType(0);
1376 EVT EltVT = VT.getVectorElementType();
1377 SDLoc dl(Node);
1378 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1379 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1380 MachinePointerInfo PtrInfo =
1381 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1382
1383 // Emit a store of each element to the stack slot.
1384 SmallVector<SDValue, 8> Stores;
1385 unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1386 // Store (in the right endianness) the elements to memory.
1387 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1388 // Ignore undef elements.
1389 if (Node->getOperand(i).isUndef()) continue;
1390
1391 unsigned Offset = TypeByteSize*i;
1392
1393 SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1394 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1395
1396 // If the destination vector element type is narrower than the source
1397 // element type, only store the bits necessary.
1398 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1399 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1400 Node->getOperand(i), Idx,
1401 PtrInfo.getWithOffset(Offset), EltVT));
1402 } else
1403 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1404 Idx, PtrInfo.getWithOffset(Offset)));
1405 }
1406
1407 SDValue StoreChain;
1408 if (!Stores.empty()) // Not all undef elements?
1409 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1410 else
1411 StoreChain = DAG.getEntryNode();
1412
1413 // Result is a load from the stack slot.
1414 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1415}
1416
1417/// Bitcast a floating-point value to an integer value. Only bitcast the part
1418/// containing the sign bit if the target has no integer value capable of
1419/// holding all bits of the floating-point value.
1420void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1421 const SDLoc &DL,
1422 SDValue Value) const {
1423 EVT FloatVT = Value.getValueType();
1424 unsigned NumBits = FloatVT.getSizeInBits();
1425 State.FloatVT = FloatVT;
1426 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1427 // Convert to an integer of the same size.
1428 if (TLI.isTypeLegal(IVT)) {
1429 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1430 State.SignMask = APInt::getSignMask(NumBits);
1431 State.SignBit = NumBits - 1;
1432 return;
1433 }
1434
1435 auto &DataLayout = DAG.getDataLayout();
1436 // Store the float to memory, then load the sign part out as an integer.
1437 MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1438 // First create a temporary that is aligned for both the load and store.
1439 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1440 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1441 // Then store the float to it.
1442 State.FloatPtr = StackPtr;
1443 MachineFunction &MF = DAG.getMachineFunction();
1444 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1445 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1446 State.FloatPointerInfo);
1447
1448 SDValue IntPtr;
1449 if (DataLayout.isBigEndian()) {
1450 assert(FloatVT.isByteSized() && "Unsupported floating point type!")(static_cast <bool> (FloatVT.isByteSized() && "Unsupported floating point type!"
) ? void (0) : __assert_fail ("FloatVT.isByteSized() && \"Unsupported floating point type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1450, __extension__ __PRETTY_FUNCTION__))
;
1451 // Load out a legal integer with the same sign bit as the float.
1452 IntPtr = StackPtr;
1453 State.IntPointerInfo = State.FloatPointerInfo;
1454 } else {
1455 // Advance the pointer so that the loaded byte will contain the sign bit.
1456 unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
1457 IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
1458 DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
1459 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1460 ByteOffset);
1461 }
1462
1463 State.IntPtr = IntPtr;
1464 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1465 State.IntPointerInfo, MVT::i8);
1466 State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
1467 State.SignBit = 7;
1468}
1469
1470/// Replace the integer value produced by getSignAsIntValue() with a new value
1471/// and cast the result back to a floating-point type.
1472SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1473 const SDLoc &DL,
1474 SDValue NewIntValue) const {
1475 if (!State.Chain)
1476 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1477
1478 // Override the part containing the sign bit in the value stored on the stack.
1479 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1480 State.IntPointerInfo, MVT::i8);
1481 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1482 State.FloatPointerInfo);
1483}
1484
1485SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1486 SDLoc DL(Node);
1487 SDValue Mag = Node->getOperand(0);
1488 SDValue Sign = Node->getOperand(1);
1489
1490 // Get sign bit into an integer value.
1491 FloatSignAsInt SignAsInt;
1492 getSignAsIntValue(SignAsInt, DL, Sign);
1493
1494 EVT IntVT = SignAsInt.IntValue.getValueType();
1495 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1496 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1497 SignMask);
1498
1499 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1500 EVT FloatVT = Mag.getValueType();
1501 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1502 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1503 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1504 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1505 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1506 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1507 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1508 }
1509
1510 // Transform Mag value to integer, and clear the sign bit.
1511 FloatSignAsInt MagAsInt;
1512 getSignAsIntValue(MagAsInt, DL, Mag);
1513 EVT MagVT = MagAsInt.IntValue.getValueType();
1514 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1515 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1516 ClearSignMask);
1517
1518 // Get the signbit at the right position for MagAsInt.
1519 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1520 if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
1521 if (ShiftAmount > 0) {
1522 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, IntVT);
1523 SignBit = DAG.getNode(ISD::SRL, DL, IntVT, SignBit, ShiftCnst);
1524 } else if (ShiftAmount < 0) {
1525 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, IntVT);
1526 SignBit = DAG.getNode(ISD::SHL, DL, IntVT, SignBit, ShiftCnst);
1527 }
1528 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1529 } else if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
1530 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1531 if (ShiftAmount > 0) {
1532 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, MagVT);
1533 SignBit = DAG.getNode(ISD::SRL, DL, MagVT, SignBit, ShiftCnst);
1534 } else if (ShiftAmount < 0) {
1535 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, MagVT);
1536 SignBit = DAG.getNode(ISD::SHL, DL, MagVT, SignBit, ShiftCnst);
1537 }
1538 }
1539
1540 // Store the part with the modified sign and convert back to float.
1541 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1542 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1543}
1544
1545SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1546 SDLoc DL(Node);
1547 SDValue Value = Node->getOperand(0);
1548
1549 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1550 EVT FloatVT = Value.getValueType();
1551 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1552 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1553 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1554 }
1555
1556 // Transform value to integer, clear the sign bit and transform back.
1557 FloatSignAsInt ValueAsInt;
1558 getSignAsIntValue(ValueAsInt, DL, Value);
1559 EVT IntVT = ValueAsInt.IntValue.getValueType();
1560 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1561 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1562 ClearSignMask);
1563 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1564}
1565
1566void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1567 SmallVectorImpl<SDValue> &Results) {
1568 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1569 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"(static_cast <bool> (SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? void (0) :
__assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1570, __extension__ __PRETTY_FUNCTION__))
1570 " not tell us which reg is the stack pointer!")(static_cast <bool> (SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? void (0) :
__assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1570, __extension__ __PRETTY_FUNCTION__))
;
1571 SDLoc dl(Node);
1572 EVT VT = Node->getValueType(0);
1573 SDValue Tmp1 = SDValue(Node, 0);
1574 SDValue Tmp2 = SDValue(Node, 1);
1575 SDValue Tmp3 = Node->getOperand(2);
1576 SDValue Chain = Tmp1.getOperand(0);
1577
1578 // Chain the dynamic stack allocation so that it doesn't modify the stack
1579 // pointer when other instructions are using the stack.
1580 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1581
1582 SDValue Size = Tmp2.getOperand(1);
1583 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1584 Chain = SP.getValue(1);
1585 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1586 unsigned StackAlign =
1587 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1588 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1589 if (Align > StackAlign)
1590 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1591 DAG.getConstant(-(uint64_t)Align, dl, VT));
1592 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1593
1594 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1595 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1596
1597 Results.push_back(Tmp1);
1598 Results.push_back(Tmp2);
1599}
1600
1601/// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1602/// target.
1603///
1604/// If the SETCC has been legalized using AND / OR, then the legalized node
1605/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1606/// will be set to false.
1607///
1608/// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1609/// then the values of LHS and RHS will be swapped, CC will be set to the
1610/// new condition, and NeedInvert will be set to false.
1611///
1612/// If the SETCC has been legalized using the inverse condcode, then LHS and
1613/// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1614/// will be set to true. The caller must invert the result of the SETCC with
1615/// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1616/// of a true/false result.
1617///
1618/// \returns true if the SetCC has been legalized, false if it hasn't.
1619bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1620 SDValue &RHS, SDValue &CC,
1621 bool &NeedInvert,
1622 const SDLoc &dl) {
1623 MVT OpVT = LHS.getSimpleValueType();
1624 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1625 NeedInvert = false;
1626 switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1627 default: llvm_unreachable("Unknown condition code action!")::llvm::llvm_unreachable_internal("Unknown condition code action!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1627)
;
1628 case TargetLowering::Legal:
1629 // Nothing to do.
1630 break;
1631 case TargetLowering::Expand: {
1632 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
1633 if (TLI.isCondCodeLegal(InvCC, OpVT)) {
1634 std::swap(LHS, RHS);
1635 CC = DAG.getCondCode(InvCC);
1636 return true;
1637 }
1638 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1639 unsigned Opc = 0;
1640 switch (CCCode) {
1641 default: llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1641)
;
1642 case ISD::SETO:
1643 assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT)(static_cast <bool> (TLI.getCondCodeAction(ISD::SETOEQ,
OpVT) == TargetLowering::Legal && "If SETO is expanded, SETOEQ must be legal!"
) ? void (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETOEQ, OpVT) == TargetLowering::Legal && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1645, __extension__ __PRETTY_FUNCTION__))
1644 == TargetLowering::Legal(static_cast <bool> (TLI.getCondCodeAction(ISD::SETOEQ,
OpVT) == TargetLowering::Legal && "If SETO is expanded, SETOEQ must be legal!"
) ? void (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETOEQ, OpVT) == TargetLowering::Legal && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1645, __extension__ __PRETTY_FUNCTION__))
1645 && "If SETO is expanded, SETOEQ must be legal!")(static_cast <bool> (TLI.getCondCodeAction(ISD::SETOEQ,
OpVT) == TargetLowering::Legal && "If SETO is expanded, SETOEQ must be legal!"
) ? void (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETOEQ, OpVT) == TargetLowering::Legal && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1645, __extension__ __PRETTY_FUNCTION__))
;
1646 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1647 case ISD::SETUO:
1648 assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT)(static_cast <bool> (TLI.getCondCodeAction(ISD::SETUNE,
OpVT) == TargetLowering::Legal && "If SETUO is expanded, SETUNE must be legal!"
) ? void (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETUNE, OpVT) == TargetLowering::Legal && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1650, __extension__ __PRETTY_FUNCTION__))
1649 == TargetLowering::Legal(static_cast <bool> (TLI.getCondCodeAction(ISD::SETUNE,
OpVT) == TargetLowering::Legal && "If SETUO is expanded, SETUNE must be legal!"
) ? void (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETUNE, OpVT) == TargetLowering::Legal && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1650, __extension__ __PRETTY_FUNCTION__))
1650 && "If SETUO is expanded, SETUNE must be legal!")(static_cast <bool> (TLI.getCondCodeAction(ISD::SETUNE,
OpVT) == TargetLowering::Legal && "If SETUO is expanded, SETUNE must be legal!"
) ? void (0) : __assert_fail ("TLI.getCondCodeAction(ISD::SETUNE, OpVT) == TargetLowering::Legal && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1650, __extension__ __PRETTY_FUNCTION__))
;
1651 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break;
1652 case ISD::SETOEQ:
1653 case ISD::SETOGT:
1654 case ISD::SETOGE:
1655 case ISD::SETOLT:
1656 case ISD::SETOLE:
1657 case ISD::SETONE:
1658 case ISD::SETUEQ:
1659 case ISD::SETUNE:
1660 case ISD::SETUGT:
1661 case ISD::SETUGE:
1662 case ISD::SETULT:
1663 case ISD::SETULE:
1664 // If we are floating point, assign and break, otherwise fall through.
1665 if (!OpVT.isInteger()) {
1666 // We can use the 4th bit to tell if we are the unordered
1667 // or ordered version of the opcode.
1668 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
1669 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
1670 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1671 break;
1672 }
1673 // Fallthrough if we are unsigned integer.
1674 LLVM_FALLTHROUGH[[clang::fallthrough]];
1675 case ISD::SETLE:
1676 case ISD::SETGT:
1677 case ISD::SETGE:
1678 case ISD::SETLT:
1679 // We only support using the inverted operation, which is computed above
1680 // and not a different manner of supporting expanding these cases.
1681 llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1681)
;
1682 case ISD::SETNE:
1683 case ISD::SETEQ:
1684 // Try inverting the result of the inverse condition.
1685 InvCC = CCCode == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ;
1686 if (TLI.isCondCodeLegal(InvCC, OpVT)) {
1687 CC = DAG.getCondCode(InvCC);
1688 NeedInvert = true;
1689 return true;
1690 }
1691 // If inverting the condition didn't work then we have no means to expand
1692 // the condition.
1693 llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1693)
;
1694 }
1695
1696 SDValue SetCC1, SetCC2;
1697 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
1698 // If we aren't the ordered or unorder operation,
1699 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1700 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1701 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1702 } else {
1703 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1704 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1705 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1706 }
1707 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1708 RHS = SDValue();
1709 CC = SDValue();
1710 return true;
1711 }
1712 }
1713 return false;
1714}
1715
1716/// Emit a store/load combination to the stack. This stores
1717/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1718/// a load from the stack slot to DestVT, extending it if needed.
1719/// The resultant code need not be legal.
1720SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1721 EVT DestVT, const SDLoc &dl) {
1722 // Create the stack frame object.
1723 unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1724 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1725 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1726
1727 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1728 int SPFI = StackPtrFI->getIndex();
1729 MachinePointerInfo PtrInfo =
1730 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1731
1732 unsigned SrcSize = SrcOp.getValueSizeInBits();
1733 unsigned SlotSize = SlotVT.getSizeInBits();
1734 unsigned DestSize = DestVT.getSizeInBits();
1735 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1736 unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1737
1738 // Emit a store to the stack slot. Use a truncstore if the input value is
1739 // later than DestVT.
1740 SDValue Store;
1741
1742 if (SrcSize > SlotSize)
1743 Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
1744 SlotVT, SrcAlign);
1745 else {
1746 assert(SrcSize == SlotSize && "Invalid store")(static_cast <bool> (SrcSize == SlotSize && "Invalid store"
) ? void (0) : __assert_fail ("SrcSize == SlotSize && \"Invalid store\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1746, __extension__ __PRETTY_FUNCTION__))
;
1747 Store =
1748 DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1749 }
1750
1751 // Result is a load from the stack slot.
1752 if (SlotSize == DestSize)
1753 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1754
1755 assert(SlotSize < DestSize && "Unknown extension!")(static_cast <bool> (SlotSize < DestSize && "Unknown extension!"
) ? void (0) : __assert_fail ("SlotSize < DestSize && \"Unknown extension!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1755, __extension__ __PRETTY_FUNCTION__))
;
1756 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1757 DestAlign);
1758}
1759
1760SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1761 SDLoc dl(Node);
1762 // Create a vector sized/aligned stack slot, store the value to element #0,
1763 // then load the whole vector back out.
1764 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1765
1766 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1767 int SPFI = StackPtrFI->getIndex();
1768
1769 SDValue Ch = DAG.getTruncStore(
1770 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1771 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1772 Node->getValueType(0).getVectorElementType());
1773 return DAG.getLoad(
1774 Node->getValueType(0), dl, Ch, StackPtr,
1775 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1776}
1777
1778static bool
1779ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1780 const TargetLowering &TLI, SDValue &Res) {
1781 unsigned NumElems = Node->getNumOperands();
1782 SDLoc dl(Node);
1783 EVT VT = Node->getValueType(0);
1784
1785 // Try to group the scalars into pairs, shuffle the pairs together, then
1786 // shuffle the pairs of pairs together, etc. until the vector has
1787 // been built. This will work only if all of the necessary shuffle masks
1788 // are legal.
1789
1790 // We do this in two phases; first to check the legality of the shuffles,
1791 // and next, assuming that all shuffles are legal, to create the new nodes.
1792 for (int Phase = 0; Phase < 2; ++Phase) {
1793 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1794 NewIntermedVals;
1795 for (unsigned i = 0; i < NumElems; ++i) {
1796 SDValue V = Node->getOperand(i);
1797 if (V.isUndef())
1798 continue;
1799
1800 SDValue Vec;
1801 if (Phase)
1802 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1803 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1804 }
1805
1806 while (IntermedVals.size() > 2) {
1807 NewIntermedVals.clear();
1808 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1809 // This vector and the next vector are shuffled together (simply to
1810 // append the one to the other).
1811 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1812
1813 SmallVector<int, 16> FinalIndices;
1814 FinalIndices.reserve(IntermedVals[i].second.size() +
1815 IntermedVals[i+1].second.size());
1816
1817 int k = 0;
1818 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1819 ++j, ++k) {
1820 ShuffleVec[k] = j;
1821 FinalIndices.push_back(IntermedVals[i].second[j]);
1822 }
1823 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1824 ++j, ++k) {
1825 ShuffleVec[k] = NumElems + j;
1826 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1827 }
1828
1829 SDValue Shuffle;
1830 if (Phase)
1831 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1832 IntermedVals[i+1].first,
1833 ShuffleVec);
1834 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1835 return false;
1836 NewIntermedVals.push_back(
1837 std::make_pair(Shuffle, std::move(FinalIndices)));
1838 }
1839
1840 // If we had an odd number of defined values, then append the last
1841 // element to the array of new vectors.
1842 if ((IntermedVals.size() & 1) != 0)
1843 NewIntermedVals.push_back(IntermedVals.back());
1844
1845 IntermedVals.swap(NewIntermedVals);
1846 }
1847
1848 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&(static_cast <bool> (IntermedVals.size() <= 2 &&
IntermedVals.size() > 0 && "Invalid number of intermediate vectors"
) ? void (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1849, __extension__ __PRETTY_FUNCTION__))
1849 "Invalid number of intermediate vectors")(static_cast <bool> (IntermedVals.size() <= 2 &&
IntermedVals.size() > 0 && "Invalid number of intermediate vectors"
) ? void (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1849, __extension__ __PRETTY_FUNCTION__))
;
1850 SDValue Vec1 = IntermedVals[0].first;
1851 SDValue Vec2;
1852 if (IntermedVals.size() > 1)
1853 Vec2 = IntermedVals[1].first;
1854 else if (Phase)
1855 Vec2 = DAG.getUNDEF(VT);
1856
1857 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1858 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1859 ShuffleVec[IntermedVals[0].second[i]] = i;
1860 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1861 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1862
1863 if (Phase)
1864 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1865 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1866 return false;
1867 }
1868
1869 return true;
1870}
1871
1872/// Expand a BUILD_VECTOR node on targets that don't
1873/// support the operation, but do support the resultant vector type.
1874SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1875 unsigned NumElems = Node->getNumOperands();
1876 SDValue Value1, Value2;
1877 SDLoc dl(Node);
1878 EVT VT = Node->getValueType(0);
1879 EVT OpVT = Node->getOperand(0).getValueType();
1880 EVT EltVT = VT.getVectorElementType();
1881
1882 // If the only non-undef value is the low element, turn this into a
1883 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1884 bool isOnlyLowElement = true;
1885 bool MoreThanTwoValues = false;
1886 bool isConstant = true;
1887 for (unsigned i = 0; i < NumElems; ++i) {
1888 SDValue V = Node->getOperand(i);
1889 if (V.isUndef())
1890 continue;
1891 if (i > 0)
1892 isOnlyLowElement = false;
1893 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1894 isConstant = false;
1895
1896 if (!Value1.getNode()) {
1897 Value1 = V;
1898 } else if (!Value2.getNode()) {
1899 if (V != Value1)
1900 Value2 = V;
1901 } else if (V != Value1 && V != Value2) {
1902 MoreThanTwoValues = true;
1903 }
1904 }
1905
1906 if (!Value1.getNode())
1907 return DAG.getUNDEF(VT);
1908
1909 if (isOnlyLowElement)
1910 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1911
1912 // If all elements are constants, create a load from the constant pool.
1913 if (isConstant) {
1914 SmallVector<Constant*, 16> CV;
1915 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1916 if (ConstantFPSDNode *V =
1917 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1918 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1919 } else if (ConstantSDNode *V =
1920 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1921 if (OpVT==EltVT)
1922 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1923 else {
1924 // If OpVT and EltVT don't match, EltVT is not legal and the
1925 // element values have been promoted/truncated earlier. Undo this;
1926 // we don't want a v16i8 to become a v16i32 for example.
1927 const ConstantInt *CI = V->getConstantIntValue();
1928 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1929 CI->getZExtValue()));
1930 }
1931 } else {
1932 assert(Node->getOperand(i).isUndef())(static_cast <bool> (Node->getOperand(i).isUndef()) ?
void (0) : __assert_fail ("Node->getOperand(i).isUndef()"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1932, __extension__ __PRETTY_FUNCTION__))
;
1933 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1934 CV.push_back(UndefValue::get(OpNTy));
1935 }
1936 }
1937 Constant *CP = ConstantVector::get(CV);
1938 SDValue CPIdx =
1939 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1940 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1941 return DAG.getLoad(
1942 VT, dl, DAG.getEntryNode(), CPIdx,
1943 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1944 Alignment);
1945 }
1946
1947 SmallSet<SDValue, 16> DefinedValues;
1948 for (unsigned i = 0; i < NumElems; ++i) {
1949 if (Node->getOperand(i).isUndef())
1950 continue;
1951 DefinedValues.insert(Node->getOperand(i));
1952 }
1953
1954 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1955 if (!MoreThanTwoValues) {
1956 SmallVector<int, 8> ShuffleVec(NumElems, -1);
1957 for (unsigned i = 0; i < NumElems; ++i) {
1958 SDValue V = Node->getOperand(i);
1959 if (V.isUndef())
1960 continue;
1961 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1962 }
1963 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1964 // Get the splatted value into the low element of a vector register.
1965 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1966 SDValue Vec2;
1967 if (Value2.getNode())
1968 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1969 else
1970 Vec2 = DAG.getUNDEF(VT);
1971
1972 // Return shuffle(LowValVec, undef, <0,0,0,0>)
1973 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1974 }
1975 } else {
1976 SDValue Res;
1977 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1978 return Res;
1979 }
1980 }
1981
1982 // Otherwise, we can't handle this case efficiently.
1983 return ExpandVectorBuildThroughStack(Node);
1984}
1985
1986// Expand a node into a call to a libcall. If the result value
1987// does not fit into a register, return the lo part and set the hi part to the
1988// by-reg argument. If it does fit into a single register, return the result
1989// and leave the Hi part unset.
1990SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
1991 bool isSigned) {
1992 TargetLowering::ArgListTy Args;
1993 TargetLowering::ArgListEntry Entry;
1994 for (const SDValue &Op : Node->op_values()) {
1995 EVT ArgVT = Op.getValueType();
1996 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1997 Entry.Node = Op;
1998 Entry.Ty = ArgTy;
1999 Entry.IsSExt = isSigned;
2000 Entry.IsZExt = !isSigned;
2001 Args.push_back(Entry);
2002 }
2003 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2004 TLI.getPointerTy(DAG.getDataLayout()));
2005
2006 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2007
2008 // By default, the input chain to this libcall is the entry node of the
2009 // function. If the libcall is going to be emitted as a tail call then
2010 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2011 // node which is being folded has a non-entry input chain.
2012 SDValue InChain = DAG.getEntryNode();
2013
2014 // isTailCall may be true since the callee does not reference caller stack
2015 // frame. Check if it's in the right position and that the return types match.
2016 SDValue TCChain = InChain;
2017 const Function *F = DAG.getMachineFunction().getFunction();
2018 bool isTailCall =
2019 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2020 (RetTy == F->getReturnType() || F->getReturnType()->isVoidTy());
2021 if (isTailCall)
2022 InChain = TCChain;
2023
2024 TargetLowering::CallLoweringInfo CLI(DAG);
2025 CLI.setDebugLoc(SDLoc(Node))
2026 .setChain(InChain)
2027 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2028 std::move(Args))
2029 .setTailCall(isTailCall)
2030 .setSExtResult(isSigned)
2031 .setZExtResult(!isSigned)
2032 .setIsPostTypeLegalization(true);
2033
2034 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2035
2036 if (!CallInfo.second.getNode()) {
2037 DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Created tailcall: "; DAG.
getRoot().dump(); } } while (false)
;
2038 // It's a tailcall, return the chain (which is the DAG root).
2039 return DAG.getRoot();
2040 }
2041
2042 DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Created libcall: "; CallInfo
.first.dump(); } } while (false)
;
2043 return CallInfo.first;
2044}
2045
2046/// Generate a libcall taking the given operands as arguments
2047/// and returning a result of type RetVT.
2048SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
2049 const SDValue *Ops, unsigned NumOps,
2050 bool isSigned, const SDLoc &dl) {
2051 TargetLowering::ArgListTy Args;
2052 Args.reserve(NumOps);
2053
2054 TargetLowering::ArgListEntry Entry;
2055 for (unsigned i = 0; i != NumOps; ++i) {
2056 Entry.Node = Ops[i];
2057 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
2058 Entry.IsSExt = isSigned;
2059 Entry.IsZExt = !isSigned;
2060 Args.push_back(Entry);
2061 }
2062 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2063 TLI.getPointerTy(DAG.getDataLayout()));
2064
2065 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2066
2067 TargetLowering::CallLoweringInfo CLI(DAG);
2068 CLI.setDebugLoc(dl)
2069 .setChain(DAG.getEntryNode())
2070 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2071 std::move(Args))
2072 .setSExtResult(isSigned)
2073 .setZExtResult(!isSigned)
2074 .setIsPostTypeLegalization(true);
2075
2076 std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
2077
2078 return CallInfo.first;
2079}
2080
2081// Expand a node into a call to a libcall. Similar to
2082// ExpandLibCall except that the first operand is the in-chain.
2083std::pair<SDValue, SDValue>
2084SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2085 SDNode *Node,
2086 bool isSigned) {
2087 SDValue InChain = Node->getOperand(0);
2088
2089 TargetLowering::ArgListTy Args;
2090 TargetLowering::ArgListEntry Entry;
2091 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
2092 EVT ArgVT = Node->getOperand(i).getValueType();
2093 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2094 Entry.Node = Node->getOperand(i);
2095 Entry.Ty = ArgTy;
2096 Entry.IsSExt = isSigned;
2097 Entry.IsZExt = !isSigned;
2098 Args.push_back(Entry);
2099 }
2100 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2101 TLI.getPointerTy(DAG.getDataLayout()));
2102
2103 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2104
2105 TargetLowering::CallLoweringInfo CLI(DAG);
2106 CLI.setDebugLoc(SDLoc(Node))
2107 .setChain(InChain)
2108 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2109 std::move(Args))
2110 .setSExtResult(isSigned)
2111 .setZExtResult(!isSigned);
2112
2113 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2114
2115 return CallInfo;
2116}
2117
2118SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2119 RTLIB::Libcall Call_F32,
2120 RTLIB::Libcall Call_F64,
2121 RTLIB::Libcall Call_F80,
2122 RTLIB::Libcall Call_F128,
2123 RTLIB::Libcall Call_PPCF128) {
2124 if (Node->isStrictFPOpcode())
2125 Node = DAG.mutateStrictFPToFP(Node);
2126
2127 RTLIB::Libcall LC;
2128 switch (Node->getSimpleValueType(0).SimpleTy) {
2129 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2129)
;
2130 case MVT::f32: LC = Call_F32; break;
2131 case MVT::f64: LC = Call_F64; break;
2132 case MVT::f80: LC = Call_F80; break;
2133 case MVT::f128: LC = Call_F128; break;
2134 case MVT::ppcf128: LC = Call_PPCF128; break;
2135 }
2136 return ExpandLibCall(LC, Node, false);
2137}
2138
2139SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2140 RTLIB::Libcall Call_I8,
2141 RTLIB::Libcall Call_I16,
2142 RTLIB::Libcall Call_I32,
2143 RTLIB::Libcall Call_I64,
2144 RTLIB::Libcall Call_I128) {
2145 RTLIB::Libcall LC;
2146 switch (Node->getSimpleValueType(0).SimpleTy) {
2147 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2147)
;
2148 case MVT::i8: LC = Call_I8; break;
2149 case MVT::i16: LC = Call_I16; break;
2150 case MVT::i32: LC = Call_I32; break;
2151 case MVT::i64: LC = Call_I64; break;
2152 case MVT::i128: LC = Call_I128; break;
2153 }
2154 return ExpandLibCall(LC, Node, isSigned);
2155}
2156
2157/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2158void
2159SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2160 SmallVectorImpl<SDValue> &Results) {
2161 unsigned Opcode = Node->getOpcode();
2162 bool isSigned = Opcode == ISD::SDIVREM;
2163
2164 RTLIB::Libcall LC;
2165 switch (Node->getSimpleValueType(0).SimpleTy) {
2166 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2166)
;
2167 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2168 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2169 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2170 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2171 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2172 }
2173
2174 // The input chain to this libcall is the entry node of the function.
2175 // Legalizing the call will automatically add the previous call to the
2176 // dependence.
2177 SDValue InChain = DAG.getEntryNode();
2178
2179 EVT RetVT = Node->getValueType(0);
2180 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2181
2182 TargetLowering::ArgListTy Args;
2183 TargetLowering::ArgListEntry Entry;
2184 for (const SDValue &Op : Node->op_values()) {
2185 EVT ArgVT = Op.getValueType();
2186 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2187 Entry.Node = Op;
2188 Entry.Ty = ArgTy;
2189 Entry.IsSExt = isSigned;
2190 Entry.IsZExt = !isSigned;
2191 Args.push_back(Entry);
2192 }
2193
2194 // Also pass the return address of the remainder.
2195 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2196 Entry.Node = FIPtr;
2197 Entry.Ty = RetTy->getPointerTo();
2198 Entry.IsSExt = isSigned;
2199 Entry.IsZExt = !isSigned;
2200 Args.push_back(Entry);
2201
2202 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2203 TLI.getPointerTy(DAG.getDataLayout()));
2204
2205 SDLoc dl(Node);
2206 TargetLowering::CallLoweringInfo CLI(DAG);
2207 CLI.setDebugLoc(dl)
2208 .setChain(InChain)
2209 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2210 std::move(Args))
2211 .setSExtResult(isSigned)
2212 .setZExtResult(!isSigned);
2213
2214 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2215
2216 // Remainder is loaded back from the stack frame.
2217 SDValue Rem =
2218 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2219 Results.push_back(CallInfo.first);
2220 Results.push_back(Rem);
2221}
2222
2223/// Return true if sincos libcall is available.
2224static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2225 RTLIB::Libcall LC;
2226 switch (Node->getSimpleValueType(0).SimpleTy) {
2227 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2227)
;
2228 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2229 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2230 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2231 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2232 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2233 }
2234 return TLI.getLibcallName(LC) != nullptr;
2235}
2236
2237/// Only issue sincos libcall if both sin and cos are needed.
2238static bool useSinCos(SDNode *Node) {
2239 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2240 ? ISD::FCOS : ISD::FSIN;
2241
2242 SDValue Op0 = Node->getOperand(0);
2243 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2244 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2245 SDNode *User = *UI;
2246 if (User == Node)
2247 continue;
2248 // The other user might have been turned into sincos already.
2249 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2250 return true;
2251 }
2252 return false;
2253}
2254
2255/// Issue libcalls to sincos to compute sin / cos pairs.
2256void
2257SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2258 SmallVectorImpl<SDValue> &Results) {
2259 RTLIB::Libcall LC;
2260 switch (Node->getSimpleValueType(0).SimpleTy) {
2261 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2261)
;
2262 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2263 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2264 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2265 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2266 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2267 }
2268
2269 // The input chain to this libcall is the entry node of the function.
2270 // Legalizing the call will automatically add the previous call to the
2271 // dependence.
2272 SDValue InChain = DAG.getEntryNode();
2273
2274 EVT RetVT = Node->getValueType(0);
2275 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2276
2277 TargetLowering::ArgListTy Args;
2278 TargetLowering::ArgListEntry Entry;
2279
2280 // Pass the argument.
2281 Entry.Node = Node->getOperand(0);
2282 Entry.Ty = RetTy;
2283 Entry.IsSExt = false;
2284 Entry.IsZExt = false;
2285 Args.push_back(Entry);
2286
2287 // Pass the return address of sin.
2288 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2289 Entry.Node = SinPtr;
2290 Entry.Ty = RetTy->getPointerTo();
2291 Entry.IsSExt = false;
2292 Entry.IsZExt = false;
2293 Args.push_back(Entry);
2294
2295 // Also pass the return address of the cos.
2296 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2297 Entry.Node = CosPtr;
2298 Entry.Ty = RetTy->getPointerTo();
2299 Entry.IsSExt = false;
2300 Entry.IsZExt = false;
2301 Args.push_back(Entry);
2302
2303 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2304 TLI.getPointerTy(DAG.getDataLayout()));
2305
2306 SDLoc dl(Node);
2307 TargetLowering::CallLoweringInfo CLI(DAG);
2308 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2309 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2310 std::move(Args));
2311
2312 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2313
2314 Results.push_back(
2315 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2316 Results.push_back(
2317 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2318}
2319
2320/// This function is responsible for legalizing a
2321/// INT_TO_FP operation of the specified operand when the target requests that
2322/// we expand it. At this point, we know that the result and operand types are
2323/// legal for the target.
2324SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2325 EVT DestVT,
2326 const SDLoc &dl) {
2327 // TODO: Should any fast-math-flags be set for the created nodes?
2328 DEBUG(dbgs() << "Legalizing INT_TO_FP\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing INT_TO_FP\n"; }
} while (false)
;
2329 if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
2330 DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double "
"expansion\n"; } } while (false)
2331 "expansion\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double "
"expansion\n"; } } while (false)
;
2332
2333 // Get the stack frame index of a 8 byte buffer.
2334 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2335
2336 // word offset constant for Hi/Lo address computation
2337 SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2338 StackSlot.getValueType());
2339 // set up Hi and Lo (into buffer) address based on endian
2340 SDValue Hi = StackSlot;
2341 SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2342 StackSlot, WordOff);
2343 if (DAG.getDataLayout().isLittleEndian())
2344 std::swap(Hi, Lo);
2345
2346 // if signed map to unsigned space
2347 SDValue Op0Mapped;
2348 if (isSigned) {
2349 // constant used to invert sign bit (signed to unsigned mapping)
2350 SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2351 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2352 } else {
2353 Op0Mapped = Op0;
2354 }
2355 // store the lo of the constructed double - based on integer input
2356 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2357 MachinePointerInfo());
2358 // initial hi portion of constructed double
2359 SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2360 // store the hi of the constructed double - biased exponent
2361 SDValue Store2 =
2362 DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
2363 // load the constructed double
2364 SDValue Load =
2365 DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
2366 // FP constant to bias correct the final result
2367 SDValue Bias = DAG.getConstantFP(isSigned ?
2368 BitsToDouble(0x4330000080000000ULL) :
2369 BitsToDouble(0x4330000000000000ULL),
2370 dl, MVT::f64);
2371 // subtract the bias
2372 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2373 // final result
2374 SDValue Result;
2375 // handle final rounding
2376 if (DestVT == MVT::f64) {
2377 // do nothing
2378 Result = Sub;
2379 } else if (DestVT.bitsLT(MVT::f64)) {
2380 Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
2381 DAG.getIntPtrConstant(0, dl));
2382 } else if (DestVT.bitsGT(MVT::f64)) {
2383 Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
2384 }
2385 return Result;
2386 }
2387 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet")(static_cast <bool> (!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"
) ? void (0) : __assert_fail ("!isSigned && \"Legalize cannot Expand SINT_TO_FP for i64 yet\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2387, __extension__ __PRETTY_FUNCTION__))
;
2388 // Code below here assumes !isSigned without checking again.
2389
2390 // Implementation of unsigned i64 to f64 following the algorithm in
2391 // __floatundidf in compiler_rt. This implementation has the advantage
2392 // of performing rounding correctly, both in the default rounding mode
2393 // and in all alternate rounding modes.
2394 // TODO: Generalize this for use with other types.
2395 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) {
2396 DEBUG(dbgs() << "Converting unsigned i64 to f64\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Converting unsigned i64 to f64\n"
; } } while (false)
;
2397 SDValue TwoP52 =
2398 DAG.getConstant(UINT64_C(0x4330000000000000)0x4330000000000000UL, dl, MVT::i64);
2399 SDValue TwoP84PlusTwoP52 =
2400 DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)0x4530000000100000UL), dl,
2401 MVT::f64);
2402 SDValue TwoP84 =
2403 DAG.getConstant(UINT64_C(0x4530000000000000)0x4530000000000000UL, dl, MVT::i64);
2404
2405 SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32);
2406 SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
2407 DAG.getConstant(32, dl, MVT::i64));
2408 SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
2409 SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
2410 SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
2411 SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr);
2412 SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt,
2413 TwoP84PlusTwoP52);
2414 return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub);
2415 }
2416
2417 // TODO: Generalize this for use with other types.
2418 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) {
2419 DEBUG(dbgs() << "Converting unsigned i64 to f32\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Converting unsigned i64 to f32\n"
; } } while (false)
;
2420 // For unsigned conversions, convert them to signed conversions using the
2421 // algorithm from the x86_64 __floatundidf in compiler_rt.
2422 if (!isSigned) {
2423 SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
2424
2425 SDValue ShiftConst = DAG.getConstant(
2426 1, dl, TLI.getShiftAmountTy(Op0.getValueType(), DAG.getDataLayout()));
2427 SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
2428 SDValue AndConst = DAG.getConstant(1, dl, MVT::i64);
2429 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
2430 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
2431
2432 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
2433 SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
2434
2435 // TODO: This really should be implemented using a branch rather than a
2436 // select. We happen to get lucky and machinesink does the right
2437 // thing most of the time. This would be a good candidate for a
2438 //pseudo-op, or, even better, for whole-function isel.
2439 SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
2440 Op0, DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
2441 return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast);
2442 }
2443
2444 // Otherwise, implement the fully general conversion.
2445
2446 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2447 DAG.getConstant(UINT64_C(0xfffffffffffff800)0xfffffffffffff800UL, dl, MVT::i64));
2448 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
2449 DAG.getConstant(UINT64_C(0x800)0x800UL, dl, MVT::i64));
2450 SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2451 DAG.getConstant(UINT64_C(0x7ff)0x7ffUL, dl, MVT::i64));
2452 SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), And2,
2453 DAG.getConstant(UINT64_C(0)0UL, dl, MVT::i64),
2454 ISD::SETNE);
2455 SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0);
2456 SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), Op0,
2457 DAG.getConstant(UINT64_C(0x0020000000000000)0x0020000000000000UL, dl,
2458 MVT::i64),
2459 ISD::SETUGE);
2460 SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0);
2461 EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType(), DAG.getDataLayout());
2462
2463 SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
2464 DAG.getConstant(32, dl, SHVT));
2465 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
2466 SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
2467 SDValue TwoP32 =
2468 DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)0x41f0000000000000UL), dl,
2469 MVT::f64);
2470 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
2471 SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
2472 SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
2473 SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
2474 return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
2475 DAG.getIntPtrConstant(0, dl));
2476 }
2477
2478 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2479
2480 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()),
2481 Op0,
2482 DAG.getConstant(0, dl, Op0.getValueType()),
2483 ISD::SETLT);
2484 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2485 Four = DAG.getIntPtrConstant(4, dl);
2486 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2487 SignSet, Four, Zero);
2488
2489 // If the sign bit of the integer is set, the large number will be treated
2490 // as a negative number. To counteract this, the dynamic code adds an
2491 // offset depending on the data type.
2492 uint64_t FF;
2493 switch (Op0.getSimpleValueType().SimpleTy) {
2494 default: llvm_unreachable("Unsupported integer type!")::llvm::llvm_unreachable_internal("Unsupported integer type!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2494)
;
2495 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2496 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2497 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2498 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2499 }
2500 if (DAG.getDataLayout().isLittleEndian())
2501 FF <<= 32;
2502 Constant *FudgeFactor = ConstantInt::get(
2503 Type::getInt64Ty(*DAG.getContext()), FF);
2504
2505 SDValue CPIdx =
2506 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2507 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2508 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2509 Alignment = std::min(Alignment, 4u);
2510 SDValue FudgeInReg;
2511 if (DestVT == MVT::f32)
2512 FudgeInReg = DAG.getLoad(
2513 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2514 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2515 Alignment);
2516 else {
2517 SDValue Load = DAG.getExtLoad(
2518 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2519 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2520 Alignment);
2521 HandleSDNode Handle(Load);
2522 LegalizeOp(Load.getNode());
2523 FudgeInReg = Handle.getValue();
2524 }
2525
2526 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2527}
2528
2529/// This function is responsible for legalizing a
2530/// *INT_TO_FP operation of the specified operand when the target requests that
2531/// we promote it. At this point, we know that the result and operand types are
2532/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2533/// operation that takes a larger input.
2534SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2535 bool isSigned,
2536 const SDLoc &dl) {
2537 // First step, figure out the appropriate *INT_TO_FP operation to use.
2538 EVT NewInTy = LegalOp.getValueType();
2539
2540 unsigned OpToUse = 0;
2541
2542 // Scan for the appropriate larger type to use.
2543 while (true) {
2544 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2545 assert(NewInTy.isInteger() && "Ran out of possibilities!")(static_cast <bool> (NewInTy.isInteger() && "Ran out of possibilities!"
) ? void (0) : __assert_fail ("NewInTy.isInteger() && \"Ran out of possibilities!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2545, __extension__ __PRETTY_FUNCTION__))
;
2546
2547 // If the target supports SINT_TO_FP of this type, use it.
2548 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2549 OpToUse = ISD::SINT_TO_FP;
2550 break;
2551 }
2552 if (isSigned) continue;
2553
2554 // If the target supports UINT_TO_FP of this type, use it.
2555 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2556 OpToUse = ISD::UINT_TO_FP;
2557 break;
2558 }
2559
2560 // Otherwise, try a larger type.
2561 }
2562
2563 // Okay, we found the operation and type to use. Zero extend our input to the
2564 // desired type then run the operation on it.
2565 return DAG.getNode(OpToUse, dl, DestVT,
2566 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2567 dl, NewInTy, LegalOp));
2568}
2569
2570/// This function is responsible for legalizing a
2571/// FP_TO_*INT operation of the specified operand when the target requests that
2572/// we promote it. At this point, we know that the result and operand types are
2573/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2574/// operation that returns a larger result.
2575SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2576 bool isSigned,
2577 const SDLoc &dl) {
2578 // First step, figure out the appropriate FP_TO*INT operation to use.
2579 EVT NewOutTy = DestVT;
2580
2581 unsigned OpToUse = 0;
2582
2583 // Scan for the appropriate larger type to use.
2584 while (true) {
2585 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2586 assert(NewOutTy.isInteger() && "Ran out of possibilities!")(static_cast <bool> (NewOutTy.isInteger() && "Ran out of possibilities!"
) ? void (0) : __assert_fail ("NewOutTy.isInteger() && \"Ran out of possibilities!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2586, __extension__ __PRETTY_FUNCTION__))
;
2587
2588 // A larger signed type can hold all unsigned values of the requested type,
2589 // so using FP_TO_SINT is valid
2590 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2591 OpToUse = ISD::FP_TO_SINT;
2592 break;
2593 }
2594
2595 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2596 if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2597 OpToUse = ISD::FP_TO_UINT;
2598 break;
2599 }
2600
2601 // Otherwise, try a larger type.
2602 }
2603
2604 // Okay, we found the operation and type to use.
2605 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2606
2607 // Truncate the result of the extended FP_TO_*INT operation to the desired
2608 // size.
2609 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2610}
2611
2612/// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2613SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2614 EVT VT = Op.getValueType();
2615 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2616 unsigned Sz = VT.getScalarSizeInBits();
2617
2618 SDValue Tmp, Tmp2, Tmp3;
2619
2620 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2621 // and finally the i1 pairs.
2622 // TODO: We can easily support i4/i2 legal types if any target ever does.
2623 if (Sz >= 8 && isPowerOf2_32(Sz)) {
2624 // Create the masks - repeating the pattern every byte.
2625 APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0);
2626 APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0);
2627 for (unsigned J = 0; J != Sz; J += 8) {
2628 MaskHi4 = MaskHi4 | (0xF0ull << J);
2629 MaskLo4 = MaskLo4 | (0x0Full << J);
2630 MaskHi2 = MaskHi2 | (0xCCull << J);
2631 MaskLo2 = MaskLo2 | (0x33ull << J);
2632 MaskHi1 = MaskHi1 | (0xAAull << J);
2633 MaskLo1 = MaskLo1 | (0x55ull << J);
2634 }
2635
2636 // BSWAP if the type is wider than a single byte.
2637 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
2638
2639 // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2640 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
2641 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
2642 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, VT));
2643 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, VT));
2644 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2645
2646 // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2647 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
2648 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
2649 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, VT));
2650 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, VT));
2651 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2652
2653 // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2654 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
2655 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
2656 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, VT));
2657 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, VT));
2658 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2659 return Tmp;
2660 }
2661
2662 Tmp = DAG.getConstant(0, dl, VT);
2663 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
2664 if (I < J)
2665 Tmp2 =
2666 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
2667 else
2668 Tmp2 =
2669 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
2670
2671 APInt Shift(Sz, 1);
2672 Shift <<= J;
2673 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
2674 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
2675 }
2676
2677 return Tmp;
2678}
2679
2680/// Open code the operations for BSWAP of the specified operation.
2681SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2682 EVT VT = Op.getValueType();
2683 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2684 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2685 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
2686 default: llvm_unreachable("Unhandled Expand type in BSWAP!")::llvm::llvm_unreachable_internal("Unhandled Expand type in BSWAP!"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2686)
;
2687 case MVT::i16:
2688 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2689 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2690 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2691 case MVT::i32:
2692 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2693 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2694 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2695 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2696 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2697 DAG.getConstant(0xFF0000, dl, VT));
2698 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2699 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2700 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2701 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2702 case MVT::i64:
2703 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2704 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2705 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2706 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2707 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2708 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2709 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2710 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2711 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2712 DAG.getConstant(255ULL<<48, dl, VT));
2713 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2714 DAG.getConstant(255ULL<<40, dl, VT));
2715 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2716 DAG.getConstant(255ULL<<32, dl, VT));
2717 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2718 DAG.getConstant(255ULL<<24, dl, VT));
2719 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2720 DAG.getConstant(255ULL<<16, dl, VT));
2721 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2722 DAG.getConstant(255ULL<<8 , dl, VT));
2723 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2724 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2725 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2726 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2727 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2728 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2729 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2730 }
2731}
2732
2733/// Expand the specified bitcount instruction into operations.
2734SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
2735 const SDLoc &dl) {
2736 switch (Opc) {
2737 default: llvm_unreachable("Cannot expand this yet!")::llvm::llvm_unreachable_internal("Cannot expand this yet!", "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2737)
;
2738 case ISD::CTPOP: {
2739 EVT VT = Op.getValueType();
2740 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2741 unsigned Len = VT.getSizeInBits();
2742
2743 assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 &&(static_cast <bool> (VT.isInteger() && Len <=
128 && Len % 8 == 0 && "CTPOP not implemented for this type."
) ? void (0) : __assert_fail ("VT.isInteger() && Len <= 128 && Len % 8 == 0 && \"CTPOP not implemented for this type.\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2744, __extension__ __PRETTY_FUNCTION__))
2744 "CTPOP not implemented for this type.")(static_cast <bool> (VT.isInteger() && Len <=
128 && Len % 8 == 0 && "CTPOP not implemented for this type."
) ? void (0) : __assert_fail ("VT.isInteger() && Len <= 128 && Len % 8 == 0 && \"CTPOP not implemented for this type.\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2744, __extension__ __PRETTY_FUNCTION__))
;
2745
2746 // This is the "best" algorithm from
2747 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
2748
2749 SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)),
2750 dl, VT);
2751 SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)),
2752 dl, VT);
2753 SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)),
2754 dl, VT);
2755 SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)),
2756 dl, VT);
2757
2758 // v = v - ((v >> 1) & 0x55555555...)
2759 Op = DAG.getNode(ISD::SUB, dl, VT, Op,
2760 DAG.getNode(ISD::AND, dl, VT,
2761 DAG.getNode(ISD::SRL, dl, VT, Op,
2762 DAG.getConstant(1, dl, ShVT)),
2763 Mask55));
2764 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
2765 Op = DAG.getNode(ISD::ADD, dl, VT,
2766 DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
2767 DAG.getNode(ISD::AND, dl, VT,
2768 DAG.getNode(ISD::SRL, dl, VT, Op,
2769 DAG.getConstant(2, dl, ShVT)),
2770 Mask33));
2771 // v = (v + (v >> 4)) & 0x0F0F0F0F...
2772 Op = DAG.getNode(ISD::AND, dl, VT,
2773 DAG.getNode(ISD::ADD, dl, VT, Op,
2774 DAG.getNode(ISD::SRL, dl, VT, Op,
2775 DAG.getConstant(4, dl, ShVT))),
2776 Mask0F);
2777 // v = (v * 0x01010101...) >> (Len - 8)
2778 Op = DAG.getNode(ISD::SRL, dl, VT,
2779 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
2780 DAG.getConstant(Len - 8, dl, ShVT));
2781
2782 return Op;
2783 }
2784 case ISD::CTLZ_ZERO_UNDEF:
2785 // This trivially expands to CTLZ.
2786 return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op);
2787 case ISD::CTLZ: {
2788 EVT VT = Op.getValueType();
2789 unsigned Len = VT.getSizeInBits();
2790
2791 if (TLI.isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
2792 EVT SetCCVT = getSetCCResultType(VT);
2793 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
2794 SDValue Zero = DAG.getConstant(0, dl, VT);
2795 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
2796 return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
2797 DAG.getConstant(Len, dl, VT), CTLZ);
2798 }
2799
2800 // for now, we do this:
2801 // x = x | (x >> 1);
2802 // x = x | (x >> 2);
2803 // ...
2804 // x = x | (x >>16);
2805 // x = x | (x >>32); // for 64-bit input
2806 // return popcount(~x);
2807 //
2808 // Ref: "Hacker's Delight" by Henry Warren
2809 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2810 for (unsigned i = 0; (1U << i) <= (Len / 2); ++i) {
2811 SDValue Tmp3 = DAG.getConstant(1ULL << i, dl, ShVT);
2812 Op = DAG.getNode(ISD::OR, dl, VT, Op,
2813 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
2814 }
2815 Op = DAG.getNOT(dl, Op, VT);
2816 return DAG.getNode(ISD::CTPOP, dl, VT, Op);
2817 }
2818 case ISD::CTTZ_ZERO_UNDEF:
2819 // This trivially expands to CTTZ.
2820 return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op);
2821 case ISD::CTTZ: {
2822 EVT VT = Op.getValueType();
2823 unsigned Len = VT.getSizeInBits();
2824
2825 if (TLI.isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
2826 EVT SetCCVT = getSetCCResultType(VT);
2827 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
2828 SDValue Zero = DAG.getConstant(0, dl, VT);
2829 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
2830 return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
2831 DAG.getConstant(Len, dl, VT), CTTZ);
2832 }
2833
2834 // for now, we use: { return popcount(~x & (x - 1)); }
2835 // unless the target has ctlz but not ctpop, in which case we use:
2836 // { return 32 - nlz(~x & (x-1)); }
2837 // Ref: "Hacker's Delight" by Henry Warren
2838 SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
2839 DAG.getNOT(dl, Op, VT),
2840 DAG.getNode(ISD::SUB, dl, VT, Op,
2841 DAG.getConstant(1, dl, VT)));
2842 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
2843 if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
2844 TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
2845 return DAG.getNode(ISD::SUB, dl, VT,
2846 DAG.getConstant(VT.getSizeInBits(), dl, VT),
2847 DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
2848 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
2849 }
2850 }
2851}
2852
2853bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2854 DEBUG(dbgs() << "Trying to expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to expand node\n";
} } while (false)
;
2855 SmallVector<SDValue, 8> Results;
2856 SDLoc dl(Node);
2857 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2858 bool NeedInvert;
2859 switch (Node->getOpcode()) {
2860 case ISD::CTPOP:
2861 case ISD::CTLZ:
2862 case ISD::CTLZ_ZERO_UNDEF:
2863 case ISD::CTTZ:
2864 case ISD::CTTZ_ZERO_UNDEF:
2865 Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
2866 Results.push_back(Tmp1);
2867 break;
2868 case ISD::BITREVERSE:
2869 Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
2870 break;
2871 case ISD::BSWAP:
2872 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2873 break;
2874 case ISD::FRAMEADDR:
2875 case ISD::RETURNADDR:
2876 case ISD::FRAME_TO_ARGS_OFFSET:
2877 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2878 break;
2879 case ISD::EH_DWARF_CFA: {
2880 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2881 TLI.getPointerTy(DAG.getDataLayout()));
2882 SDValue Offset = DAG.getNode(ISD::ADD, dl,
2883 CfaArg.getValueType(),
2884 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2885 CfaArg.getValueType()),
2886 CfaArg);
2887 SDValue FA = DAG.getNode(
2888 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2889 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2890 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2891 FA, Offset));
2892 break;
2893 }
2894 case ISD::FLT_ROUNDS_:
2895 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2896 break;
2897 case ISD::EH_RETURN:
2898 case ISD::EH_LABEL:
2899 case ISD::PREFETCH:
2900 case ISD::VAEND:
2901 case ISD::EH_SJLJ_LONGJMP:
2902 // If the target didn't expand these, there's nothing to do, so just
2903 // preserve the chain and be done.
2904 Results.push_back(Node->getOperand(0));
2905 break;
2906 case ISD::READCYCLECOUNTER:
2907 // If the target didn't expand this, just return 'zero' and preserve the
2908 // chain.
2909 Results.append(Node->getNumValues() - 1,
2910 DAG.getConstant(0, dl, Node->getValueType(0)));
2911 Results.push_back(Node->getOperand(0));
2912 break;
2913 case ISD::EH_SJLJ_SETJMP:
2914 // If the target didn't expand this, just return 'zero' and preserve the
2915 // chain.
2916 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2917 Results.push_back(Node->getOperand(0));
2918 break;
2919 case ISD::ATOMIC_LOAD: {
2920 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2921 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2922 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2923 SDValue Swap = DAG.getAtomicCmpSwap(
2924 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2925 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2926 cast<AtomicSDNode>(Node)->getMemOperand());
2927 Results.push_back(Swap.getValue(0));
2928 Results.push_back(Swap.getValue(1));
2929 break;
2930 }
2931 case ISD::ATOMIC_STORE: {
2932 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2933 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2934 cast<AtomicSDNode>(Node)->getMemoryVT(),
2935 Node->getOperand(0),
2936 Node->getOperand(1), Node->getOperand(2),
2937 cast<AtomicSDNode>(Node)->getMemOperand());
2938 Results.push_back(Swap.getValue(1));
2939 break;
2940 }
2941 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2942 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2943 // splits out the success value as a comparison. Expanding the resulting
2944 // ATOMIC_CMP_SWAP will produce a libcall.
2945 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2946 SDValue Res = DAG.getAtomicCmpSwap(
2947 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2948 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2949 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2950
2951 SDValue ExtRes = Res;
2952 SDValue LHS = Res;
2953 SDValue RHS = Node->getOperand(1);
2954
2955 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2956 EVT OuterType = Node->getValueType(0);
2957 switch (TLI.getExtendForAtomicOps()) {
2958 case ISD::SIGN_EXTEND:
2959 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2960 DAG.getValueType(AtomicType));
2961 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2962 Node->getOperand(2), DAG.getValueType(AtomicType));
2963 ExtRes = LHS;
2964 break;
2965 case ISD::ZERO_EXTEND:
2966 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2967 DAG.getValueType(AtomicType));
2968 RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
2969 ExtRes = LHS;
2970 break;
2971 case ISD::ANY_EXTEND:
2972 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2973 RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
2974 break;
2975 default:
2976 llvm_unreachable("Invalid atomic op extension")::llvm::llvm_unreachable_internal("Invalid atomic op extension"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2976)
;
2977 }
2978
2979 SDValue Success =
2980 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2981
2982 Results.push_back(ExtRes.getValue(0));
2983 Results.push_back(Success);
2984 Results.push_back(Res.getValue(1));
2985 break;
2986 }
2987 case ISD::DYNAMIC_STACKALLOC:
2988 ExpandDYNAMIC_STACKALLOC(Node, Results);
2989 break;
2990 case ISD::MERGE_VALUES:
2991 for (unsigned i = 0; i < Node->getNumValues(); i++)
2992 Results.push_back(Node->getOperand(i));
2993 break;
2994 case ISD::UNDEF: {
2995 EVT VT = Node->getValueType(0);
2996 if (VT.isInteger())
2997 Results.push_back(DAG.getConstant(0, dl, VT));
2998 else {
2999 assert(VT.isFloatingPoint() && "Unknown value type!")(static_cast <bool> (VT.isFloatingPoint() && "Unknown value type!"
) ? void (0) : __assert_fail ("VT.isFloatingPoint() && \"Unknown value type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2999, __extension__ __PRETTY_FUNCTION__))
;
3000 Results.push_back(DAG.getConstantFP(0, dl, VT));
3001 }
3002 break;
3003 }
3004 case ISD::FP_ROUND:
3005 case ISD::BITCAST:
3006 Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
3007 Node->getValueType(0), dl);
3008 Results.push_back(Tmp1);
3009 break;
3010 case ISD::FP_EXTEND:
3011 Tmp1 = EmitStackConvert(Node->getOperand(0),
3012 Node->getOperand(0).getValueType(),
3013 Node->getValueType(0), dl);
3014 Results.push_back(Tmp1);
3015 break;
3016 case ISD::SIGN_EXTEND_INREG: {
3017 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3018 EVT VT = Node->getValueType(0);
3019
3020 // An in-register sign-extend of a boolean is a negation:
3021 // 'true' (1) sign-extended is -1.
3022 // 'false' (0) sign-extended is 0.
3023 // However, we must mask the high bits of the source operand because the
3024 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3025
3026 // TODO: Do this for vectors too?
3027 if (ExtraVT.getSizeInBits() == 1) {
3028 SDValue One = DAG.getConstant(1, dl, VT);
3029 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
3030 SDValue Zero = DAG.getConstant(0, dl, VT);
3031 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
3032 Results.push_back(Neg);
3033 break;
3034 }
3035
3036 // NOTE: we could fall back on load/store here too for targets without
3037 // SRA. However, it is doubtful that any exist.
3038 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3039 unsigned BitsDiff = VT.getScalarSizeInBits() -
3040 ExtraVT.getScalarSizeInBits();
3041 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3042 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3043 Node->getOperand(0), ShiftCst);
3044 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3045 Results.push_back(Tmp1);
3046 break;
3047 }
3048 case ISD::FP_ROUND_INREG: {
3049 // The only way we can lower this is to turn it into a TRUNCSTORE,
3050 // EXTLOAD pair, targeting a temporary location (a stack slot).
3051
3052 // NOTE: there is a choice here between constantly creating new stack
3053 // slots and always reusing the same one. We currently always create
3054 // new ones, as reuse may inhibit scheduling.
3055 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3056 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
3057 Node->getValueType(0), dl);
3058 Results.push_back(Tmp1);
3059 break;
3060 }
3061 case ISD::SINT_TO_FP:
3062 case ISD::UINT_TO_FP:
3063 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
3064 Node->getOperand(0), Node->getValueType(0), dl);
3065 Results.push_back(Tmp1);
3066 break;
3067 case ISD::FP_TO_SINT:
3068 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3069 Results.push_back(Tmp1);
3070 break;
3071 case ISD::FP_TO_UINT: {
3072 SDValue True, False;
3073 EVT VT = Node->getOperand(0).getValueType();
3074 EVT NVT = Node->getValueType(0);
3075 APFloat apf(DAG.EVTToAPFloatSemantics(VT),
3076 APInt::getNullValue(VT.getSizeInBits()));
3077 APInt x = APInt::getSignMask(NVT.getSizeInBits());
3078 (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
3079 Tmp1 = DAG.getConstantFP(apf, dl, VT);
3080 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT),
3081 Node->getOperand(0),
3082 Tmp1, ISD::SETLT);
3083 True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
3084 // TODO: Should any fast-math-flags be set for the FSUB?
3085 False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
3086 DAG.getNode(ISD::FSUB, dl, VT,
3087 Node->getOperand(0), Tmp1));
3088 False = DAG.getNode(ISD::XOR, dl, NVT, False,
3089 DAG.getConstant(x, dl, NVT));
3090 Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False);
3091 Results.push_back(Tmp1);
3092 break;
3093 }
3094 case ISD::VAARG:
3095 Results.push_back(DAG.expandVAArg(Node));
3096 Results.push_back(Results[0].getValue(1));
3097 break;
3098 case ISD::VACOPY:
3099 Results.push_back(DAG.expandVACopy(Node));
3100 break;
3101 case ISD::EXTRACT_VECTOR_ELT:
3102 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
3103 // This must be an access of the only element. Return it.
3104 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3105 Node->getOperand(0));
3106 else
3107 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3108 Results.push_back(Tmp1);
3109 break;
3110 case ISD::EXTRACT_SUBVECTOR:
3111 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3112 break;
3113 case ISD::INSERT_SUBVECTOR:
3114 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3115 break;
3116 case ISD::CONCAT_VECTORS:
3117 Results.push_back(ExpandVectorBuildThroughStack(Node));
3118 break;
3119 case ISD::SCALAR_TO_VECTOR:
3120 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3121 break;
3122 case ISD::INSERT_VECTOR_ELT:
3123 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3124 Node->getOperand(1),
3125 Node->getOperand(2), dl));
3126 break;
3127 case ISD::VECTOR_SHUFFLE: {
3128 SmallVector<int, 32> NewMask;
3129 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3130
3131 EVT VT = Node->getValueType(0);
3132 EVT EltVT = VT.getVectorElementType();
3133 SDValue Op0 = Node->getOperand(0);
3134 SDValue Op1 = Node->getOperand(1);
3135 if (!TLI.isTypeLegal(EltVT)) {
3136 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3137
3138 // BUILD_VECTOR operands are allowed to be wider than the element type.
3139 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3140 // it.
3141 if (NewEltVT.bitsLT(EltVT)) {
3142 // Convert shuffle node.
3143 // If original node was v4i64 and the new EltVT is i32,
3144 // cast operands to v8i32 and re-build the mask.
3145
3146 // Calculate new VT, the size of the new VT should be equal to original.
3147 EVT NewVT =
3148 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3149 VT.getSizeInBits() / NewEltVT.getSizeInBits());
3150 assert(NewVT.bitsEq(VT))(static_cast <bool> (NewVT.bitsEq(VT)) ? void (0) : __assert_fail
("NewVT.bitsEq(VT)", "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3150, __extension__ __PRETTY_FUNCTION__))
;
3151
3152 // cast operands to new VT
3153 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3154 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3155
3156 // Convert the shuffle mask
3157 unsigned int factor =
3158 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3159
3160 // EltVT gets smaller
3161 assert(factor > 0)(static_cast <bool> (factor > 0) ? void (0) : __assert_fail
("factor > 0", "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3161, __extension__ __PRETTY_FUNCTION__))
;
3162
3163 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3164 if (Mask[i] < 0) {
3165 for (unsigned fi = 0; fi < factor; ++fi)
3166 NewMask.push_back(Mask[i]);
3167 }
3168 else {
3169 for (unsigned fi = 0; fi < factor; ++fi)
3170 NewMask.push_back(Mask[i]*factor+fi);
3171 }
3172 }
3173 Mask = NewMask;
3174 VT = NewVT;
3175 }
3176 EltVT = NewEltVT;
3177 }
3178 unsigned NumElems = VT.getVectorNumElements();
3179 SmallVector<SDValue, 16> Ops;
3180 for (unsigned i = 0; i != NumElems; ++i) {
3181 if (Mask[i] < 0) {
3182 Ops.push_back(DAG.getUNDEF(EltVT));
3183 continue;
3184 }
3185 unsigned Idx = Mask[i];
3186 if (Idx < NumElems)
3187 Ops.push_back(DAG.getNode(
3188 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3189 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3190 else
3191 Ops.push_back(DAG.getNode(
3192 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3193 DAG.getConstant(Idx - NumElems, dl,
3194 TLI.getVectorIdxTy(DAG.getDataLayout()))));
3195 }
3196
3197 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3198 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3199 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3200 Results.push_back(Tmp1);
3201 break;
3202 }
3203 case ISD::EXTRACT_ELEMENT: {
3204 EVT OpTy = Node->getOperand(0).getValueType();
3205 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3206 // 1 -> Hi
3207 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3208 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3209 TLI.getShiftAmountTy(
3210 Node->getOperand(0).getValueType(),
3211 DAG.getDataLayout())));
3212 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3213 } else {
3214 // 0 -> Lo
3215 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3216 Node->getOperand(0));
3217 }
3218 Results.push_back(Tmp1);
3219 break;
3220 }
3221 case ISD::STACKSAVE:
3222 // Expand to CopyFromReg if the target set
3223 // StackPointerRegisterToSaveRestore.
3224 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3225 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3226 Node->getValueType(0)));
3227 Results.push_back(Results[0].getValue(1));
3228 } else {
3229 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3230 Results.push_back(Node->getOperand(0));
3231 }
3232 break;
3233 case ISD::STACKRESTORE:
3234 // Expand to CopyToReg if the target set
3235 // StackPointerRegisterToSaveRestore.
3236 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3237 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3238 Node->getOperand(1)));
3239 } else {
3240 Results.push_back(Node->getOperand(0));
3241 }
3242 break;
3243 case ISD::GET_DYNAMIC_AREA_OFFSET:
3244 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3245 Results.push_back(Results[0].getValue(0));
3246 break;
3247 case ISD::FCOPYSIGN:
3248 Results.push_back(ExpandFCOPYSIGN(Node));
3249 break;
3250 case ISD::FNEG:
3251 // Expand Y = FNEG(X) -> Y = SUB -0.0, X
3252 Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3253 // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3254 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3255 Node->getOperand(0));
3256 Results.push_back(Tmp1);
3257 break;
3258 case ISD::FABS:
3259 Results.push_back(ExpandFABS(Node));
3260 break;
3261 case ISD::SMIN:
3262 case ISD::SMAX:
3263 case ISD::UMIN:
3264 case ISD::UMAX: {
3265 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3266 ISD::CondCode Pred;
3267 switch (Node->getOpcode()) {
3268 default: llvm_unreachable("How did we get here?")::llvm::llvm_unreachable_internal("How did we get here?", "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3268)
;
3269 case ISD::SMAX: Pred = ISD::SETGT; break;
3270 case ISD::SMIN: Pred = ISD::SETLT; break;
3271 case ISD::UMAX: Pred = ISD::SETUGT; break;
3272 case ISD::UMIN: Pred = ISD::SETULT; break;
3273 }
3274 Tmp1 = Node->getOperand(0);
3275 Tmp2 = Node->getOperand(1);
3276 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3277 Results.push_back(Tmp1);
3278 break;
3279 }
3280
3281 case ISD::FSIN:
3282 case ISD::FCOS: {
3283 EVT VT = Node->getValueType(0);
3284 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3285 // fcos which share the same operand and both are used.
3286 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3287 isSinCosLibcallAvailable(Node, TLI))
3288 && useSinCos(Node)) {
3289 SDVTList VTs = DAG.getVTList(VT, VT);
3290 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3291 if (Node->getOpcode() == ISD::FCOS)
3292 Tmp1 = Tmp1.getValue(1);
3293 Results.push_back(Tmp1);
3294 }
3295 break;
3296 }
3297 case ISD::FMAD:
3298 llvm_unreachable("Illegal fmad should never be formed")::llvm::llvm_unreachable_internal("Illegal fmad should never be formed"
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3298)
;
3299
3300 case ISD::FP16_TO_FP:
3301 if (Node->getValueType(0) != MVT::f32) {
3302 // We can extend to types bigger than f32 in two steps without changing
3303 // the result. Since "f16 -> f32" is much more commonly available, give
3304 // CodeGen the option of emitting that before resorting to a libcall.
3305 SDValue Res =
3306 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3307 Results.push_back(
3308 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3309 }
3310 break;
3311 case ISD::FP_TO_FP16:
3312 DEBUG(dbgs() << "Legalizing FP_TO_FP16\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing FP_TO_FP16\n";
} } while (false)
;
3313 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3314 SDValue Op = Node->getOperand(0);
3315 MVT SVT = Op.getSimpleValueType();
3316 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3317 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3318 // Under fastmath, we can expand this node into a fround followed by
3319 // a float-half conversion.
3320 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3321 DAG.getIntPtrConstant(0, dl));
3322 Results.push_back(
3323 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3324 }
3325 }
3326 break;
3327 case ISD::ConstantFP: {
3328 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3329 // Check to see if this FP immediate is already legal.
3330 // If this is a legal constant, turn it into a TargetConstantFP node.
3331 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
3332 Results.push_back(ExpandConstantFP(CFP, true));
3333 break;
3334 }
3335 case ISD::Constant: {
3336 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3337 Results.push_back(ExpandConstant(CP));
3338 break;
3339 }
3340 case ISD::FSUB: {
3341 EVT VT = Node->getValueType(0);
3342 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3343 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3344 const SDNodeFlags Flags = Node->getFlags();
3345 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3346 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3347 Results.push_back(Tmp1);
3348 }
3349 break;
3350 }
3351 case ISD::SUB: {
3352 EVT VT = Node->getValueType(0);
3353 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&(static_cast <bool> (TLI.isOperationLegalOrCustom(ISD::
ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT
) && "Don't know how to expand this subtraction!") ? void
(0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3355, __extension__ __PRETTY_FUNCTION__))
3354 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&(static_cast <bool> (TLI.isOperationLegalOrCustom(ISD::
ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT
) && "Don't know how to expand this subtraction!") ? void
(0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3355, __extension__ __PRETTY_FUNCTION__))
3355 "Don't know how to expand this subtraction!")(static_cast <bool> (TLI.isOperationLegalOrCustom(ISD::
ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT
) && "Don't know how to expand this subtraction!") ? void
(0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3355, __extension__ __PRETTY_FUNCTION__))
;
3356 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3357 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3358 VT));
3359 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3360 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3361 break;
3362 }
3363 case ISD::UREM:
3364 case ISD::SREM: {
3365 EVT VT = Node->getValueType(0);
3366 bool isSigned = Node->getOpcode() == ISD::SREM;
3367 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3368 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3369 Tmp2 = Node->getOperand(0);
3370 Tmp3 = Node->getOperand(1);
3371 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3372 SDVTList VTs = DAG.getVTList(VT, VT);
3373 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3374 Results.push_back(Tmp1);
3375 } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3376 // X % Y -> X-X/Y*Y
3377 Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3378 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3379 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3380 Results.push_back(Tmp1);
3381 }
3382 break;
3383 }
3384 case ISD::UDIV:
3385 case ISD::SDIV: {
3386 bool isSigned = Node->getOpcode() == ISD::SDIV;
3387 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3388 EVT VT = Node->getValueType(0);
3389 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3390 SDVTList VTs = DAG.getVTList(VT, VT);
3391 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3392 Node->getOperand(1));
3393 Results.push_back(Tmp1);
3394 }
3395 break;
3396 }
3397 case ISD::MULHU:
3398 case ISD::MULHS: {
3399 unsigned ExpandOpcode =
3400 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3401 EVT VT = Node->getValueType(0);
3402 SDVTList VTs = DAG.getVTList(VT, VT);
3403
3404 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3405 Node->getOperand(1));
3406 Results.push_back(Tmp1.getValue(1));
3407 break;
3408 }
3409 case ISD::UMUL_LOHI:
3410 case ISD::SMUL_LOHI: {
3411 SDValue LHS = Node->getOperand(0);
3412 SDValue RHS = Node->getOperand(1);
3413 MVT VT = LHS.getSimpleValueType();
3414 unsigned MULHOpcode =
3415 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3416
3417 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3418 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3419 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3420 break;
3421 }
3422
3423 SmallVector<SDValue, 4> Halves;
3424 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3425 assert(TLI.isTypeLegal(HalfType))(static_cast <bool> (TLI.isTypeLegal(HalfType)) ? void (
0) : __assert_fail ("TLI.isTypeLegal(HalfType)", "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3425, __extension__ __PRETTY_FUNCTION__))
;
3426 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
3427 HalfType, DAG,
3428 TargetLowering::MulExpansionKind::Always)) {
3429 for (unsigned i = 0; i < 2; ++i) {
3430 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3431 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3432 SDValue Shift = DAG.getConstant(
3433 HalfType.getScalarSizeInBits(), dl,
3434 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3435 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3436 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3437 }
3438 break;
3439 }
3440 break;
3441 }
3442 case ISD::MUL: {
3443 EVT VT = Node->getValueType(0);
3444 SDVTList VTs = DAG.getVTList(VT, VT);
3445 // See if multiply or divide can be lowered using two-result operations.
3446 // We just need the low half of the multiply; try both the signed
3447 // and unsigned forms. If the target supports both SMUL_LOHI and
3448 // UMUL_LOHI, form a preference by checking which forms of plain
3449 // MULH it supports.
3450 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3451 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3452 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3453 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3454 unsigned OpToUse = 0;
3455 if (HasSMUL_LOHI && !HasMULHS) {
3456 OpToUse = ISD::SMUL_LOHI;
3457 } else if (HasUMUL_LOHI && !HasMULHU) {
3458 OpToUse = ISD::UMUL_LOHI;
3459 } else if (HasSMUL_LOHI) {
3460 OpToUse = ISD::SMUL_LOHI;
3461 } else if (HasUMUL_LOHI) {
3462 OpToUse = ISD::UMUL_LOHI;
3463 }
3464 if (OpToUse) {
3465 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3466 Node->getOperand(1)));
3467 break;
3468 }
3469
3470 SDValue Lo, Hi;
3471 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3472 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3473 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3474 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3475 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3476 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3477 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3478 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3479 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3480 SDValue Shift =
3481 DAG.getConstant(HalfType.getSizeInBits(), dl,
3482 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3483 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3484 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3485 }
3486 break;
3487 }
3488 case ISD::SADDO:
3489 case ISD::SSUBO: {
3490 SDValue LHS = Node->getOperand(0);
3491 SDValue RHS = Node->getOperand(1);
3492 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3493 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3494 LHS, RHS);
3495 Results.push_back(Sum);
3496 EVT ResultType = Node->getValueType(1);
3497 EVT OType = getSetCCResultType(Node->getValueType(0));
3498
3499 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3500
3501 // LHSSign -> LHS >= 0
3502 // RHSSign -> RHS >= 0
3503 // SumSign -> Sum >= 0
3504 //
3505 // Add:
3506 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3507 // Sub:
3508 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3509 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3510 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3511 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3512 Node->getOpcode() == ISD::SADDO ?
3513 ISD::SETEQ : ISD::SETNE);
3514
3515 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3516 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3517
3518 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3519 Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
3520 break;
3521 }
3522 case ISD::UADDO:
3523 case ISD::USUBO: {
3524 SDValue LHS = Node->getOperand(0);
3525 SDValue RHS = Node->getOperand(1);
3526 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ?
3527 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3528 LHS, RHS);
3529 Results.push_back(Sum);
3530
3531 EVT ResultType = Node->getValueType(1);
3532 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3533 ISD::CondCode CC
3534 = Node->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
3535 SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3536
3537 Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
3538 break;
3539 }
3540 case ISD::UMULO:
3541 case ISD::SMULO: {
3542 EVT VT = Node->getValueType(0);
3543 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
3544 SDValue LHS = Node->getOperand(0);
3545 SDValue RHS = Node->getOperand(1);
3546 SDValue BottomHalf;
3547 SDValue TopHalf;
3548 static const unsigned Ops[2][3] =
3549 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
3550 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
3551 bool isSigned = Node->getOpcode() == ISD::SMULO;
3552 if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
3553 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3554 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
3555 } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
3556 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
3557 RHS);
3558 TopHalf = BottomHalf.getValue(1);
3559 } else if (TLI.isTypeLegal(WideVT)) {
3560 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
3561 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
3562 Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3563 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3564 DAG.getIntPtrConstant(0, dl));
3565 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3566 DAG.getIntPtrConstant(1, dl));
3567 } else {
3568 // We can fall back to a libcall with an illegal type for the MUL if we
3569 // have a libcall big enough.
3570 // Also, we can fall back to a division in some cases, but that's a big
3571 // performance hit in the general case.
3572 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3573 if (WideVT == MVT::i16)
3574 LC = RTLIB::MUL_I16;
3575 else if (WideVT == MVT::i32)
3576 LC = RTLIB::MUL_I32;
3577 else if (WideVT == MVT::i64)
3578 LC = RTLIB::MUL_I64;
3579 else if (WideVT == MVT::i128)
3580 LC = RTLIB::MUL_I128;
3581 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Cannot expand this operation!") ? void (0) : __assert_fail (
"LC != RTLIB::UNKNOWN_LIBCALL && \"Cannot expand this operation!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3581, __extension__ __PRETTY_FUNCTION__))
;
3582
3583 SDValue HiLHS;
3584 SDValue HiRHS;
3585 if (isSigned) {
3586 // The high part is obtained by SRA'ing all but one of the bits of low
3587 // part.
3588 unsigned LoSize = VT.getSizeInBits();
3589 HiLHS =
3590 DAG.getNode(ISD::SRA, dl, VT, LHS,
3591 DAG.getConstant(LoSize - 1, dl,
3592 TLI.getPointerTy(DAG.getDataLayout())));
3593 HiRHS =
3594 DAG.getNode(ISD::SRA, dl, VT, RHS,
3595 DAG.getConstant(LoSize - 1, dl,
3596 TLI.getPointerTy(DAG.getDataLayout())));
3597 } else {
3598 HiLHS = DAG.getConstant(0, dl, VT);
3599 HiRHS = DAG.getConstant(0, dl, VT);
3600 }
3601
3602 // Here we're passing the 2 arguments explicitly as 4 arguments that are
3603 // pre-lowered to the correct types. This all depends upon WideVT not
3604 // being a legal type for the architecture and thus has to be split to
3605 // two arguments.
3606 SDValue Ret;
3607 if(DAG.getDataLayout().isLittleEndian()) {
3608 // Halves of WideVT are packed into registers in different order
3609 // depending on platform endianness. This is usually handled by
3610 // the C calling convention, but we can't defer to it in
3611 // the legalizer.
3612 SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
3613 Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3614 } else {
3615 SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
3616 Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3617 }
3618 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&(static_cast <bool> (Ret.getOpcode() == ISD::MERGE_VALUES
&& "Ret value is a collection of constituent nodes holding result."
) ? void (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3619, __extension__ __PRETTY_FUNCTION__))
3619 "Ret value is a collection of constituent nodes holding result.")(static_cast <bool> (Ret.getOpcode() == ISD::MERGE_VALUES
&& "Ret value is a collection of constituent nodes holding result."
) ? void (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3619, __extension__ __PRETTY_FUNCTION__))
;
3620 BottomHalf = Ret.getOperand(0);
3621 TopHalf = Ret.getOperand(1);
3622 }
3623
3624 if (isSigned) {
3625 Tmp1 = DAG.getConstant(
3626 VT.getSizeInBits() - 1, dl,
3627 TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
3628 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3629 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
3630 ISD::SETNE);
3631 } else {
3632 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
3633 DAG.getConstant(0, dl, VT), ISD::SETNE);
3634 }
3635
3636 // Truncate the result if SetCC returns a larger type than needed.
3637 EVT RType = Node->getValueType(1);
3638 if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
3639 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
3640
3641 assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&(static_cast <bool> (RType.getSizeInBits() == TopHalf.getValueSizeInBits
() && "Unexpected result type for S/UMULO legalization"
) ? void (0) : __assert_fail ("RType.getSizeInBits() == TopHalf.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3642, __extension__ __PRETTY_FUNCTION__))
3642 "Unexpected result type for S/UMULO legalization")(static_cast <bool> (RType.getSizeInBits() == TopHalf.getValueSizeInBits
() && "Unexpected result type for S/UMULO legalization"
) ? void (0) : __assert_fail ("RType.getSizeInBits() == TopHalf.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3642, __extension__ __PRETTY_FUNCTION__))
;
3643
3644 Results.push_back(BottomHalf);
3645 Results.push_back(TopHalf);
3646 break;
3647 }
3648 case ISD::BUILD_PAIR: {
3649 EVT PairTy = Node->getValueType(0);
3650 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3651 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3652 Tmp2 = DAG.getNode(
3653 ISD::SHL, dl, PairTy, Tmp2,
3654 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3655 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3656 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3657 break;
3658 }
3659 case ISD::SELECT:
3660 Tmp1 = Node->getOperand(0);
3661 Tmp2 = Node->getOperand(1);
3662 Tmp3 = Node->getOperand(2);
3663 if (Tmp1.getOpcode() == ISD::SETCC) {
3664 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3665 Tmp2, Tmp3,
3666 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3667 } else {
3668 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3669 DAG.getConstant(0, dl, Tmp1.getValueType()),
3670 Tmp2, Tmp3, ISD::SETNE);
3671 }
3672 Results.push_back(Tmp1);
3673 break;
3674 case ISD::BR_JT: {
3675 SDValue Chain = Node->getOperand(0);
3676 SDValue Table = Node->getOperand(1);
3677 SDValue Index = Node->getOperand(2);
3678
3679 const DataLayout &TD = DAG.getDataLayout();
3680 EVT PTy = TLI.getPointerTy(TD);
3681
3682 unsigned EntrySize =
3683 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3684
3685 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3686 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3687 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3688 Index, Table);
3689
3690 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3691 SDValue LD = DAG.getExtLoad(
3692 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3693 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3694 Addr = LD;
3695 if (TLI.isJumpTableRelative()) {
3696 // For PIC, the sequence is:
3697 // BRIND(load(Jumptable + index) + RelocBase)
3698 // RelocBase can be JumpTable, GOT or some sort of global base.
3699 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3700 TLI.getPICJumpTableRelocBase(Table, DAG));
3701 }
3702 Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr);
3703 Results.push_back(Tmp1);
3704 break;
3705 }
3706 case ISD::BRCOND:
3707 // Expand brcond's setcc into its constituent parts and create a BR_CC
3708 // Node.
3709 Tmp1 = Node->getOperand(0);
3710 Tmp2 = Node->getOperand(1);
3711 if (Tmp2.getOpcode() == ISD::SETCC) {
3712 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3713 Tmp1, Tmp2.getOperand(2),
3714 Tmp2.getOperand(0), Tmp2.getOperand(1),
3715 Node->getOperand(2));
3716 } else {
3717 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3718 if (Tmp2.isUndef() ||
3719 (Tmp2.getOpcode() == ISD::AND &&
3720 isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3721 dyn_cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3722 Tmp3 = Tmp2;
3723 else
3724 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3725 DAG.getConstant(1, dl, Tmp2.getValueType()));
3726 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3727 DAG.getCondCode(ISD::SETNE), Tmp3,
3728 DAG.getConstant(0, dl, Tmp3.getValueType()),
3729 Node->getOperand(2));
3730 }
3731 Results.push_back(Tmp1);
3732 break;
3733 case ISD::SETCC: {
3734 Tmp1 = Node->getOperand(0);
3735 Tmp2 = Node->getOperand(1);
3736 Tmp3 = Node->getOperand(2);
3737 bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3738 Tmp3, NeedInvert, dl);
3739
3740 if (Legalized) {
3741 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3742 // condition code, create a new SETCC node.
3743 if (Tmp3.getNode())
3744 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3745 Tmp1, Tmp2, Tmp3);
3746
3747 // If we expanded the SETCC by inverting the condition code, then wrap
3748 // the existing SETCC in a NOT to restore the intended condition.
3749 if (NeedInvert)
3750 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3751
3752 Results.push_back(Tmp1);
3753 break;
3754 }
3755
3756 // Otherwise, SETCC for the given comparison type must be completely
3757 // illegal; expand it into a SELECT_CC.
3758 EVT VT = Node->getValueType(0);
3759 int TrueValue;
3760 switch (TLI.getBooleanContents(Tmp1->getValueType(0))) {
3761 case TargetLowering::ZeroOrOneBooleanContent:
3762 case TargetLowering::UndefinedBooleanContent:
3763 TrueValue = 1;
3764 break;
3765 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3766 TrueValue = -1;
3767 break;
3768 }
3769 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3770 DAG.getConstant(TrueValue, dl, VT),
3771 DAG.getConstant(0, dl, VT),
3772 Tmp3);
3773 Results.push_back(Tmp1);
3774 break;
3775 }
3776 case ISD::SELECT_CC: {
3777 Tmp1 = Node->getOperand(0); // LHS
3778 Tmp2 = Node->getOperand(1); // RHS
3779 Tmp3 = Node->getOperand(2); // True
3780 Tmp4 = Node->getOperand(3); // False
3781 EVT VT = Node->getValueType(0);
3782 SDValue CC = Node->getOperand(4);
3783 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3784
3785 if (TLI.isCondCodeLegal(CCOp, Tmp1.getSimpleValueType())) {
3786 // If the condition code is legal, then we need to expand this
3787 // node using SETCC and SELECT.
3788 EVT CmpVT = Tmp1.getValueType();
3789 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&(static_cast <bool> (!TLI.isOperationExpand(ISD::SELECT
, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? void (0) : __assert_fail ("!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3791, __extension__ __PRETTY_FUNCTION__))
3790 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "(static_cast <bool> (!TLI.isOperationExpand(ISD::SELECT
, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? void (0) : __assert_fail ("!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3791, __extension__ __PRETTY_FUNCTION__))
3791 "expanded.")(static_cast <bool> (!TLI.isOperationExpand(ISD::SELECT
, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? void (0) : __assert_fail ("!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3791, __extension__ __PRETTY_FUNCTION__))
;
3792 EVT CCVT =
3793 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
3794 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
3795 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3796 break;
3797 }
3798
3799 // SELECT_CC is legal, so the condition code must not be.
3800 bool Legalized = false;
3801 // Try to legalize by inverting the condition. This is for targets that
3802 // might support an ordered version of a condition, but not the unordered
3803 // version (or vice versa).
3804 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3805 Tmp1.getValueType().isInteger());
3806 if (TLI.isCondCodeLegal(InvCC, Tmp1.getSimpleValueType())) {
3807 // Use the new condition code and swap true and false
3808 Legalized = true;
3809 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3810 } else {
3811 // If The inverse is not legal, then try to swap the arguments using
3812 // the inverse condition code.
3813 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3814 if (TLI.isCondCodeLegal(SwapInvCC, Tmp1.getSimpleValueType())) {
3815 // The swapped inverse condition is legal, so swap true and false,
3816 // lhs and rhs.
3817 Legalized = true;
3818 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3819 }
3820 }
3821
3822 if (!Legalized) {
3823 Legalized = LegalizeSetCCCondCode(
3824 getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3825 dl);
3826
3827 assert(Legalized && "Can't legalize SELECT_CC with legal condition!")(static_cast <bool> (Legalized && "Can't legalize SELECT_CC with legal condition!"
) ? void (0) : __assert_fail ("Legalized && \"Can't legalize SELECT_CC with legal condition!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3827, __extension__ __PRETTY_FUNCTION__))
;
3828
3829 // If we expanded the SETCC by inverting the condition code, then swap
3830 // the True/False operands to match.
3831 if (NeedInvert)
3832 std::swap(Tmp3, Tmp4);
3833
3834 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3835 // condition code, create a new SELECT_CC node.
3836 if (CC.getNode()) {
3837 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3838 Tmp1, Tmp2, Tmp3, Tmp4, CC);
3839 } else {
3840 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3841 CC = DAG.getCondCode(ISD::SETNE);
3842 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3843 Tmp2, Tmp3, Tmp4, CC);
3844 }
3845 }
3846 Results.push_back(Tmp1);
3847 break;
3848 }
3849 case ISD::BR_CC: {
3850 Tmp1 = Node->getOperand(0); // Chain
3851 Tmp2 = Node->getOperand(2); // LHS
3852 Tmp3 = Node->getOperand(3); // RHS
3853 Tmp4 = Node->getOperand(1); // CC
3854
3855 bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
3856 Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
3857 (void)Legalized;
3858 assert(Legalized && "Can't legalize BR_CC with legal condition!")(static_cast <bool> (Legalized && "Can't legalize BR_CC with legal condition!"
) ? void (0) : __assert_fail ("Legalized && \"Can't legalize BR_CC with legal condition!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3858, __extension__ __PRETTY_FUNCTION__))
;
3859
3860 // If we expanded the SETCC by inverting the condition code, then wrap
3861 // the existing SETCC in a NOT to restore the intended condition.
3862 if (NeedInvert)
3863 Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
3864
3865 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3866 // node.
3867 if (Tmp4.getNode()) {
3868 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3869 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3870 } else {
3871 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3872 Tmp4 = DAG.getCondCode(ISD::SETNE);
3873 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3874 Tmp2, Tmp3, Node->getOperand(4));
3875 }
3876 Results.push_back(Tmp1);
3877 break;
3878 }
3879 case ISD::BUILD_VECTOR:
3880 Results.push_back(ExpandBUILD_VECTOR(Node));
3881 break;
3882 case ISD::SRA:
3883 case ISD::SRL:
3884 case ISD::SHL: {
3885 // Scalarize vector SRA/SRL/SHL.
3886 EVT VT = Node->getValueType(0);
3887 assert(VT.isVector() && "Unable to legalize non-vector shift")(static_cast <bool> (VT.isVector() && "Unable to legalize non-vector shift"
) ? void (0) : __assert_fail ("VT.isVector() && \"Unable to legalize non-vector shift\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3887, __extension__ __PRETTY_FUNCTION__))
;
3888 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal")(static_cast <bool> (TLI.isTypeLegal(VT.getScalarType()
)&& "Element type must be legal") ? void (0) : __assert_fail
("TLI.isTypeLegal(VT.getScalarType())&& \"Element type must be legal\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3888, __extension__ __PRETTY_FUNCTION__))
;
3889 unsigned NumElem = VT.getVectorNumElements();
3890
3891 SmallVector<SDValue, 8> Scalars;
3892 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3893 SDValue Ex = DAG.getNode(
3894 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3895 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3896 SDValue Sh = DAG.getNode(
3897 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3898 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3899 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3900 VT.getScalarType(), Ex, Sh));
3901 }
3902
3903 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3904 ReplaceNode(SDValue(Node, 0), Result);
3905 break;
3906 }
3907 case ISD::GLOBAL_OFFSET_TABLE:
3908 case ISD::GlobalAddress:
3909 case ISD::GlobalTLSAddress:
3910 case ISD::ExternalSymbol:
3911 case ISD::ConstantPool:
3912 case ISD::JumpTable:
3913 case ISD::INTRINSIC_W_CHAIN:
3914 case ISD::INTRINSIC_WO_CHAIN:
3915 case ISD::INTRINSIC_VOID:
3916 // FIXME: Custom lowering for these operations shouldn't return null!
3917 break;
3918 }
3919
3920 // Replace the original node with the legalized result.
3921 if (Results.empty()) {
3922 DEBUG(dbgs() << "Cannot expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Cannot expand node\n"; } }
while (false)
;
3923 return false;
3924 }
3925
3926 DEBUG(dbgs() << "Succesfully expanded node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Succesfully expanded node\n"
; } } while (false)
;
3927 ReplaceNode(Node, Results.data());
3928 return true;
3929}
3930
3931void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3932 DEBUG(dbgs() << "Trying to convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to convert node to libcall\n"
; } } while (false)
;
3933 SmallVector<SDValue, 8> Results;
3934 SDLoc dl(Node);
3935 unsigned Opc = Node->getOpcode();
3936 switch (Opc) {
3937 case ISD::ATOMIC_FENCE: {
3938 // If the target didn't lower this, lower it to '__sync_synchronize()' call
3939 // FIXME: handle "fence singlethread" more efficiently.
3940 TargetLowering::ArgListTy Args;
3941
3942 TargetLowering::CallLoweringInfo CLI(DAG);
3943 CLI.setDebugLoc(dl)
3944 .setChain(Node->getOperand(0))
3945 .setLibCallee(
3946 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3947 DAG.getExternalSymbol("__sync_synchronize",
3948 TLI.getPointerTy(DAG.getDataLayout())),
3949 std::move(Args));
3950
3951 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3952
3953 Results.push_back(CallResult.second);
3954 break;
3955 }
3956 // By default, atomic intrinsics are marked Legal and lowered. Targets
3957 // which don't support them directly, however, may want libcalls, in which
3958 // case they mark them Expand, and we get here.
3959 case ISD::ATOMIC_SWAP:
3960 case ISD::ATOMIC_LOAD_ADD:
3961 case ISD::ATOMIC_LOAD_SUB:
3962 case ISD::ATOMIC_LOAD_AND:
3963 case ISD::ATOMIC_LOAD_OR:
3964 case ISD::ATOMIC_LOAD_XOR:
3965 case ISD::ATOMIC_LOAD_NAND:
3966 case ISD::ATOMIC_LOAD_MIN:
3967 case ISD::ATOMIC_LOAD_MAX:
3968 case ISD::ATOMIC_LOAD_UMIN:
3969 case ISD::ATOMIC_LOAD_UMAX:
3970 case ISD::ATOMIC_CMP_SWAP: {
3971 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3972 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
3973 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected atomic op or value type!") ? void (0) : __assert_fail
("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected atomic op or value type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3973, __extension__ __PRETTY_FUNCTION__))
;
3974
3975 std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
3976 Results.push_back(Tmp.first);
3977 Results.push_back(Tmp.second);
3978 break;
3979 }
3980 case ISD::TRAP: {
3981 // If this operation is not supported, lower it to 'abort()' call
3982 TargetLowering::ArgListTy Args;
3983 TargetLowering::CallLoweringInfo CLI(DAG);
3984 CLI.setDebugLoc(dl)
3985 .setChain(Node->getOperand(0))
3986 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3987 DAG.getExternalSymbol(
3988 "abort", TLI.getPointerTy(DAG.getDataLayout())),
3989 std::move(Args));
3990 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3991
3992 Results.push_back(CallResult.second);
3993 break;
3994 }
3995 case ISD::FMINNUM:
3996 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3997 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3998 RTLIB::FMIN_PPCF128));
3999 break;
4000 case ISD::FMAXNUM:
4001 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4002 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4003 RTLIB::FMAX_PPCF128));
4004 break;
4005 case ISD::FSQRT:
4006 case ISD::STRICT_FSQRT:
4007 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4008 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4009 RTLIB::SQRT_PPCF128));
4010 break;
4011 case ISD::FSIN:
4012 case ISD::STRICT_FSIN:
4013 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4014 RTLIB::SIN_F80, RTLIB::SIN_F128,
4015 RTLIB::SIN_PPCF128));
4016 break;
4017 case ISD::FCOS:
4018 case ISD::STRICT_FCOS:
4019 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4020 RTLIB::COS_F80, RTLIB::COS_F128,
4021 RTLIB::COS_PPCF128));
4022 break;
4023 case ISD::FSINCOS:
4024 // Expand into sincos libcall.
4025 ExpandSinCosLibCall(Node, Results);
4026 break;
4027 case ISD::FLOG:
4028 case ISD::STRICT_FLOG:
4029 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
4030 RTLIB::LOG_F80, RTLIB::LOG_F128,
4031 RTLIB::LOG_PPCF128));
4032 break;
4033 case ISD::FLOG2:
4034 case ISD::STRICT_FLOG2:
4035 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
4036 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
4037 RTLIB::LOG2_PPCF128));
4038 break;
4039 case ISD::FLOG10:
4040 case ISD::STRICT_FLOG10:
4041 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
4042 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
4043 RTLIB::LOG10_PPCF128));
4044 break;
4045 case ISD::FEXP:
4046 case ISD::STRICT_FEXP:
4047 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
4048 RTLIB::EXP_F80, RTLIB::EXP_F128,
4049 RTLIB::EXP_PPCF128));
4050 break;
4051 case ISD::FEXP2:
4052 case ISD::STRICT_FEXP2:
4053 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
4054 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
4055 RTLIB::EXP2_PPCF128));
4056 break;
4057 case ISD::FTRUNC:
4058 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4059 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4060 RTLIB::TRUNC_PPCF128));
4061 break;
4062 case ISD::FFLOOR:
4063 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4064 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4065 RTLIB::FLOOR_PPCF128));
4066 break;
4067 case ISD::FCEIL:
4068 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4069 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4070 RTLIB::CEIL_PPCF128));
4071 break;
4072 case ISD::FRINT:
4073 case ISD::STRICT_FRINT:
4074 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4075 RTLIB::RINT_F80, RTLIB::RINT_F128,
4076 RTLIB::RINT_PPCF128));
4077 break;
4078 case ISD::FNEARBYINT:
4079 case ISD::STRICT_FNEARBYINT:
4080 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4081 RTLIB::NEARBYINT_F64,
4082 RTLIB::NEARBYINT_F80,
4083 RTLIB::NEARBYINT_F128,
4084 RTLIB::NEARBYINT_PPCF128));
4085 break;
4086 case ISD::FROUND:
4087 Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4088 RTLIB::ROUND_F64,
4089 RTLIB::ROUND_F80,
4090 RTLIB::ROUND_F128,
4091 RTLIB::ROUND_PPCF128));
4092 break;
4093 case ISD::FPOWI:
4094 case ISD::STRICT_FPOWI:
4095 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
4096 RTLIB::POWI_F80, RTLIB::POWI_F128,
4097 RTLIB::POWI_PPCF128));
4098 break;
4099 case ISD::FPOW:
4100 case ISD::STRICT_FPOW:
4101 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
4102 RTLIB::POW_F80, RTLIB::POW_F128,
4103 RTLIB::POW_PPCF128));
4104 break;
4105 case ISD::FDIV:
4106 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4107 RTLIB::DIV_F80, RTLIB::DIV_F128,
4108 RTLIB::DIV_PPCF128));
4109 break;
4110 case ISD::FREM:
4111 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4112 RTLIB::REM_F80, RTLIB::REM_F128,
4113 RTLIB::REM_PPCF128));
4114 break;
4115 case ISD::FMA:
4116 case ISD::STRICT_FMA:
4117 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4118 RTLIB::FMA_F80, RTLIB::FMA_F128,
4119 RTLIB::FMA_PPCF128));
4120 break;
4121 case ISD::FADD:
4122 Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4123 RTLIB::ADD_F80, RTLIB::ADD_F128,
4124 RTLIB::ADD_PPCF128));
4125 break;
4126 case ISD::FMUL:
4127 Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4128 RTLIB::MUL_F80, RTLIB::MUL_F128,
4129 RTLIB::MUL_PPCF128));
4130 break;
4131 case ISD::FP16_TO_FP:
4132 if (Node->getValueType(0) == MVT::f32) {
4133 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4134 }
4135 break;
4136 case ISD::FP_TO_FP16: {
4137 RTLIB::Libcall LC =
4138 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4139 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_fp16") ? void (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unable to expand fp_to_fp16\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4139, __extension__ __PRETTY_FUNCTION__))
;
4140 Results.push_back(ExpandLibCall(LC, Node, false));
4141 break;
4142 }
4143 case ISD::FSUB:
4144 Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4145 RTLIB::SUB_F80, RTLIB::SUB_F128,
4146 RTLIB::SUB_PPCF128));
4147 break;
4148 case ISD::SREM:
4149 Results.push_back(ExpandIntLibCall(Node, true,
4150 RTLIB::SREM_I8,
4151 RTLIB::SREM_I16, RTLIB::SREM_I32,
4152 RTLIB::SREM_I64, RTLIB::SREM_I128));
4153 break;
4154 case ISD::UREM:
4155 Results.push_back(ExpandIntLibCall(Node, false,
4156 RTLIB::UREM_I8,
4157 RTLIB::UREM_I16, RTLIB::UREM_I32,
4158 RTLIB::UREM_I64, RTLIB::UREM_I128));
4159 break;
4160 case ISD::SDIV:
4161 Results.push_back(ExpandIntLibCall(Node, true,
4162 RTLIB::SDIV_I8,
4163 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4164 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4165 break;
4166 case ISD::UDIV:
4167 Results.push_back(ExpandIntLibCall(Node, false,
4168 RTLIB::UDIV_I8,
4169 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4170 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4171 break;
4172 case ISD::SDIVREM:
4173 case ISD::UDIVREM:
4174 // Expand into divrem libcall
4175 ExpandDivRemLibCall(Node, Results);
4176 break;
4177 case ISD::MUL:
4178 Results.push_back(ExpandIntLibCall(Node, false,
4179 RTLIB::MUL_I8,
4180 RTLIB::MUL_I16, RTLIB::MUL_I32,
4181 RTLIB::MUL_I64, RTLIB::MUL_I128));
4182 break;
4183 }
4184
4185 // Replace the original node with the legalized result.
4186 if (!Results.empty()) {
4187 DEBUG(dbgs() << "Successfully converted node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully converted node to libcall\n"
; } } while (false)
;
4188 ReplaceNode(Node, Results.data());
4189 } else
4190 DEBUG(dbgs() << "Could not convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not convert node to libcall\n"
; } } while (false)
;
4191}
4192
4193// Determine the vector type to use in place of an original scalar element when
4194// promoting equally sized vectors.
4195static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4196 MVT EltVT, MVT NewEltVT) {
4197 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4198 MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4199 assert(TLI.isTypeLegal(MidVT) && "unexpected")(static_cast <bool> (TLI.isTypeLegal(MidVT) && "unexpected"
) ? void (0) : __assert_fail ("TLI.isTypeLegal(MidVT) && \"unexpected\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4199, __extension__ __PRETTY_FUNCTION__))
;
4200 return MidVT;
4201}
4202
4203void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4204 DEBUG(dbgs() << "Trying to promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to promote node\n"
; } } while (false)
;
4205 SmallVector<SDValue, 8> Results;
4206 MVT OVT = Node->getSimpleValueType(0);
4207 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4208 Node->getOpcode() == ISD::SINT_TO_FP ||
4209 Node->getOpcode() == ISD::SETCC ||
4210 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4211 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4212 OVT = Node->getOperand(0).getSimpleValueType();
4213 }
4214 if (Node->getOpcode() == ISD::BR_CC)
4215 OVT = Node->getOperand(2).getSimpleValueType();
4216 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4217 SDLoc dl(Node);
4218 SDValue Tmp1, Tmp2, Tmp3;
4219 switch (Node->getOpcode()) {
4220 case ISD::CTTZ:
4221 case ISD::CTTZ_ZERO_UNDEF:
4222 case ISD::CTLZ:
4223 case ISD::CTLZ_ZERO_UNDEF:
4224 case ISD::CTPOP:
4225 // Zero extend the argument.
4226 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4227 if (Node->getOpcode() == ISD::CTTZ) {
4228 // The count is the same in the promoted type except if the original
4229 // value was zero. This can be handled by setting the bit just off
4230 // the top of the original type.
4231 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4232 OVT.getSizeInBits());
4233 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4234 DAG.getConstant(TopBit, dl, NVT));
4235 }
4236 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4237 // already the correct result.
4238 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4239 if (Node->getOpcode() == ISD::CTLZ ||
4240 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4241 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4242 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4243 DAG.getConstant(NVT.getSizeInBits() -
4244 OVT.getSizeInBits(), dl, NVT));
4245 }
4246 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4247 break;
4248 case ISD::BITREVERSE:
4249 case ISD::BSWAP: {
4250 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4251 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4252 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4253 Tmp1 = DAG.getNode(
4254 ISD::SRL, dl, NVT, Tmp1,
4255 DAG.getConstant(DiffBits, dl,
4256 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4257 Results.push_back(Tmp1);
4258 break;
4259 }
4260 case ISD::FP_TO_UINT:
4261 case ISD::FP_TO_SINT:
4262 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4263 Node->getOpcode() == ISD::FP_TO_SINT, dl);
4264 Results.push_back(Tmp1);
4265 break;
4266 case ISD::UINT_TO_FP:
4267 case ISD::SINT_TO_FP:
4268 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4269 Node->getOpcode() == ISD::SINT_TO_FP, dl);
4270 Results.push_back(Tmp1);
4271 break;
4272 case ISD::VAARG: {
4273 SDValue Chain = Node->getOperand(0); // Get the chain.
4274 SDValue Ptr = Node->getOperand(1); // Get the pointer.
4275
4276 unsigned TruncOp;
4277 if (OVT.isVector()) {
4278 TruncOp = ISD::BITCAST;
4279 } else {
4280 assert(OVT.isInteger()(static_cast <bool> (OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? void (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4281, __extension__ __PRETTY_FUNCTION__))
4281 && "VAARG promotion is supported only for vectors or integer types")(static_cast <bool> (OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? void (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4281, __extension__ __PRETTY_FUNCTION__))
;
4282 TruncOp = ISD::TRUNCATE;
4283 }
4284
4285 // Perform the larger operation, then convert back
4286 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4287 Node->getConstantOperandVal(3));
4288 Chain = Tmp1.getValue(1);
4289
4290 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4291
4292 // Modified the chain result - switch anything that used the old chain to
4293 // use the new one.
4294 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4295 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4296 if (UpdatedNodes) {
4297 UpdatedNodes->insert(Tmp2.getNode());
4298 UpdatedNodes->insert(Chain.getNode());
4299 }
4300 ReplacedNode(Node);
4301 break;
4302 }
4303 case ISD::MUL:
4304 case ISD::SDIV:
4305 case ISD::SREM:
4306 case ISD::UDIV:
4307 case ISD::UREM:
4308 case ISD::AND:
4309 case ISD::OR:
4310 case ISD::XOR: {
4311 unsigned ExtOp, TruncOp;
4312 if (OVT.isVector()) {
4313 ExtOp = ISD::BITCAST;
4314 TruncOp = ISD::BITCAST;
4315 } else {
4316 assert(OVT.isInteger() && "Cannot promote logic operation")(static_cast <bool> (OVT.isInteger() && "Cannot promote logic operation"
) ? void (0) : __assert_fail ("OVT.isInteger() && \"Cannot promote logic operation\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4316, __extension__ __PRETTY_FUNCTION__))
;
4317
4318 switch (Node->getOpcode()) {
4319 default:
4320 ExtOp = ISD::ANY_EXTEND;
4321 break;
4322 case ISD::SDIV:
4323 case ISD::SREM:
4324 ExtOp = ISD::SIGN_EXTEND;
4325 break;
4326 case ISD::UDIV:
4327 case ISD::UREM:
4328 ExtOp = ISD::ZERO_EXTEND;
4329 break;
4330 }
4331 TruncOp = ISD::TRUNCATE;
4332 }
4333 // Promote each of the values to the new type.
4334 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4335 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4336 // Perform the larger operation, then convert back
4337 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4338 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4339 break;
4340 }
4341 case ISD::UMUL_LOHI:
4342 case ISD::SMUL_LOHI: {
4343 // Promote to a multiply in a wider integer type.
4344 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4345 : ISD::SIGN_EXTEND;
4346 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4347 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4348 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4349
4350 auto &DL = DAG.getDataLayout();
4351 unsigned OriginalSize = OVT.getScalarSizeInBits();
4352 Tmp2 = DAG.getNode(
4353 ISD::SRL, dl, NVT, Tmp1,
4354 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4355 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4356 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4357 break;
4358 }
4359 case ISD::SELECT: {
4360 unsigned ExtOp, TruncOp;
4361 if (Node->getValueType(0).isVector() ||
4362 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4363 ExtOp = ISD::BITCAST;
4364 TruncOp = ISD::BITCAST;
4365 } else if (Node->getValueType(0).isInteger()) {
4366 ExtOp = ISD::ANY_EXTEND;
4367 TruncOp = ISD::TRUNCATE;
4368 } else {
4369 ExtOp = ISD::FP_EXTEND;
4370 TruncOp = ISD::FP_ROUND;
4371 }
4372 Tmp1 = Node->getOperand(0);
4373 // Promote each of the values to the new type.
4374 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4375 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4376 // Perform the larger operation, then round down.
4377 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4378 if (TruncOp != ISD::FP_ROUND)
4379 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4380 else
4381 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4382 DAG.getIntPtrConstant(0, dl));
4383 Results.push_back(Tmp1);
4384 break;
4385 }
4386 case ISD::VECTOR_SHUFFLE: {
4387 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4388
4389 // Cast the two input vectors.
4390 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4391 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4392
4393 // Convert the shuffle mask to the right # elements.
4394 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4395 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4396 Results.push_back(Tmp1);
4397 break;
4398 }
4399 case ISD::SETCC: {
4400 unsigned ExtOp = ISD::FP_EXTEND;
4401 if (NVT.isInteger()) {
4402 ISD::CondCode CCCode =
4403 cast<CondCodeSDNode>(Node->getOperand(2))->get();
4404 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4405 }
4406 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4407 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4408 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
4409 Tmp1, Tmp2, Node->getOperand(2)));
4410 break;
4411 }
4412 case ISD::BR_CC: {
4413 unsigned ExtOp = ISD::FP_EXTEND;
4414 if (NVT.isInteger()) {
4415 ISD::CondCode CCCode =
4416 cast<CondCodeSDNode>(Node->getOperand(1))->get();
4417 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4418 }
4419 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4420 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4421 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4422 Node->getOperand(0), Node->getOperand(1),
4423 Tmp1, Tmp2, Node->getOperand(4)));
4424 break;
4425 }
4426 case ISD::FADD:
4427 case ISD::FSUB:
4428 case ISD::FMUL:
4429 case ISD::FDIV:
4430 case ISD::FREM:
4431 case ISD::FMINNUM:
4432 case ISD::FMAXNUM:
4433 case ISD::FPOW:
4434 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4435 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4436 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4437 Node->getFlags());
4438 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4439 Tmp3, DAG.getIntPtrConstant(0, dl)));
4440 break;
4441 case ISD::FMA:
4442 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4443 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4444 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4445 Results.push_back(
4446 DAG.getNode(ISD::FP_ROUND, dl, OVT,
4447 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4448 DAG.getIntPtrConstant(0, dl)));
4449 break;
4450 case ISD::FCOPYSIGN:
4451 case ISD::FPOWI: {
4452 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4453 Tmp2 = Node->getOperand(1);
4454 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4455
4456 // fcopysign doesn't change anything but the sign bit, so
4457 // (fp_round (fcopysign (fpext a), b))
4458 // is as precise as
4459 // (fp_round (fpext a))
4460 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4461 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4462 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4463 Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4464 break;
4465 }
4466 case ISD::FFLOOR:
4467 case ISD::FCEIL:
4468 case ISD::FRINT:
4469 case ISD::FNEARBYINT:
4470 case ISD::FROUND:
4471 case ISD::FTRUNC:
4472 case ISD::FNEG:
4473 case ISD::FSQRT:
4474 case ISD::FSIN:
4475 case ISD::FCOS:
4476 case ISD::FLOG:
4477 case ISD::FLOG2:
4478 case ISD::FLOG10:
4479 case ISD::FABS:
4480 case ISD::FEXP:
4481 case ISD::FEXP2:
4482 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4483 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4484 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4485 Tmp2, DAG.getIntPtrConstant(0, dl)));
4486 break;
4487 case ISD::BUILD_VECTOR: {
4488 MVT EltVT = OVT.getVectorElementType();
4489 MVT NewEltVT = NVT.getVectorElementType();
4490
4491 // Handle bitcasts to a different vector type with the same total bit size
4492 //
4493 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4494 // =>
4495 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4496
4497 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for build_vector"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4498, __extension__ __PRETTY_FUNCTION__))
4498 "Invalid promote type for build_vector")(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for build_vector"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4498, __extension__ __PRETTY_FUNCTION__))
;
4499 assert(NewEltVT.bitsLT(EltVT) && "not handled")(static_cast <bool> (NewEltVT.bitsLT(EltVT) && "not handled"
) ? void (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4499, __extension__ __PRETTY_FUNCTION__))
;
4500
4501 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4502
4503 SmallVector<SDValue, 8> NewOps;
4504 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4505 SDValue Op = Node->getOperand(I);
4506 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4507 }
4508
4509 SDLoc SL(Node);
4510 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4511 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4512 Results.push_back(CvtVec);
4513 break;
4514 }
4515 case ISD::EXTRACT_VECTOR_ELT: {
4516 MVT EltVT = OVT.getVectorElementType();
4517 MVT NewEltVT = NVT.getVectorElementType();
4518
4519 // Handle bitcasts to a different vector type with the same total bit size.
4520 //
4521 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4522 // =>
4523 // v4i32:castx = bitcast x:v2i64
4524 //
4525 // i64 = bitcast
4526 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4527 // (i32 (extract_vector_elt castx, (2 * y + 1)))
4528 //
4529
4530 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for extract_vector_elt"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4531, __extension__ __PRETTY_FUNCTION__))
4531 "Invalid promote type for extract_vector_elt")(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for extract_vector_elt"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4531, __extension__ __PRETTY_FUNCTION__))
;
4532 assert(NewEltVT.bitsLT(EltVT) && "not handled")(static_cast <bool> (NewEltVT.bitsLT(EltVT) && "not handled"
) ? void (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4532, __extension__ __PRETTY_FUNCTION__))
;
4533
4534 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4535 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4536
4537 SDValue Idx = Node->getOperand(1);
4538 EVT IdxVT = Idx.getValueType();
4539 SDLoc SL(Node);
4540 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4541 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4542
4543 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4544
4545 SmallVector<SDValue, 8> NewOps;
4546 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4547 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4548 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4549
4550 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4551 CastVec, TmpIdx);
4552 NewOps.push_back(Elt);
4553 }
4554
4555 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4556 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4557 break;
4558 }
4559 case ISD::INSERT_VECTOR_ELT: {
4560 MVT EltVT = OVT.getVectorElementType();
4561 MVT NewEltVT = NVT.getVectorElementType();
4562
4563 // Handle bitcasts to a different vector type with the same total bit size
4564 //
4565 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4566 // =>
4567 // v4i32:castx = bitcast x:v2i64
4568 // v2i32:casty = bitcast y:i64
4569 //
4570 // v2i64 = bitcast
4571 // (v4i32 insert_vector_elt
4572 // (v4i32 insert_vector_elt v4i32:castx,
4573 // (extract_vector_elt casty, 0), 2 * z),
4574 // (extract_vector_elt casty, 1), (2 * z + 1))
4575
4576 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for insert_vector_elt"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4577, __extension__ __PRETTY_FUNCTION__))
4577 "Invalid promote type for insert_vector_elt")(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for insert_vector_elt"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4577, __extension__ __PRETTY_FUNCTION__))
;
4578 assert(NewEltVT.bitsLT(EltVT) && "not handled")(static_cast <bool> (NewEltVT.bitsLT(EltVT) && "not handled"
) ? void (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-6.0~svn319613/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4578, __extension__ __PRETTY_FUNCTION__))
;
4579
4580 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4581 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4582
4583 SDValue Val = Node->getOperand(1);
4584 SDValue Idx = Node->getOperand(2);
4585 EVT IdxVT = Idx.getValueType();
4586 SDLoc SL(Node);
4587
4588 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4589 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4590
4591 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4592 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4593
4594 SDValue NewVec = CastVec;
4595 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4596 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4597 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4598
4599 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4600 CastVal, IdxOffset);
4601
4602 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4603 NewVec, Elt, InEltIdx);
4604 }
4605
4606 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4607 break;
4608 }
4609 case ISD::SCALAR_TO_VECTOR: {
4610 MVT EltVT = OVT.getVectorElementType();
4611 MVT NewEltVT = NVT.getVectorElementType();
4612
4613 // Handle bitcasts to different vector type with the same total bit size.
4614 //
4615 // e.g. v2i64 = scalar_to_vector x:i64
4616 // =>
4617 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4618 //
4619
4620 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4621 SDValue Val = Node->getOperand(0);
4622 SDLoc SL(Node);
4623
4624 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4625 SDValue Undef = DAG.getUNDEF(MidVT);
4626
4627 SmallVector<SDValue, 8> NewElts;
4628 NewElts.push_back(CastVal);
4629 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4630 NewElts.push_back(Undef);
4631
4632 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4633 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4634 Results.push_back(CvtVec);
4635 break;
4636 }
4637 }
4638
4639 // Replace the original node with the legalized result.
4640 if (!Results.empty()) {
4641 DEBUG(dbgs() << "Successfully promoted node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully promoted node\n"
; } } while (false)
;
4642 ReplaceNode(Node, Results.data());
4643 } else
4644 DEBUG(dbgs() << "Could not promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not promote node\n"
; } } while (false)
;
4645}
4646
4647/// This is the entry point for the file.
4648void SelectionDAG::Legalize() {
4649 AssignTopologicalOrder();
4650
4651 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4652 // Use a delete listener to remove nodes which were deleted during
4653 // legalization from LegalizeNodes. This is needed to handle the situation
4654 // where a new node is allocated by the object pool to the same address of a
4655 // previously deleted node.
4656 DAGNodeDeletedListener DeleteListener(
4657 *this,
4658 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4659
4660 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4661
4662 // Visit all the nodes. We start in topological order, so that we see
4663 // nodes with their original operands intact. Legalization can produce
4664 // new nodes which may themselves need to be legalized. Iterate until all
4665 // nodes have been legalized.
4666 while (true) {
4667 bool AnyLegalized = false;
4668 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4669 --NI;
4670
4671 SDNode *N = &*NI;
4672 if (N->use_empty() && N != getRoot().getNode()) {
4673 ++NI;
4674 DeleteNode(N);
4675 continue;
4676 }
4677
4678 if (LegalizedNodes.insert(N).second) {
4679 AnyLegalized = true;
4680 Legalizer.LegalizeOp(N);
4681
4682 if (N->use_empty() && N != getRoot().getNode()) {
4683 ++NI;
4684 DeleteNode(N);
4685 }
4686 }
4687 }
4688 if (!AnyLegalized)
4689 break;
4690
4691 }
4692
4693 // Remove dead nodes now.
4694 RemoveDeadNodes();
4695}
4696
4697bool SelectionDAG::LegalizeOp(SDNode *N,
4698 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4699 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4700 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4701
4702 // Directly insert the node in question, and legalize it. This will recurse
4703 // as needed through operands.
4704 LegalizedNodes.insert(N);
4705 Legalizer.LegalizeOp(N);
4706
4707 return LegalizedNodes.count(N);
4708}