Bug Summary

File:lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
Warning:line 760, column 29
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizeDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-eagerly-assume -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-7/lib/clang/7.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/include -I /build/llvm-toolchain-snapshot-7~svn338205/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/x86_64-linux-gnu/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/x86_64-linux-gnu/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/c++/8/backward -internal-isystem /usr/include/clang/7.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-7/lib/clang/7.0.0/include -internal-externc-isystem /usr/lib/gcc/x86_64-linux-gnu/8/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/lib/CodeGen/SelectionDAG -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-07-29-043837-17923-1 -x c++ /build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp -faddrsig
1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the SelectionDAG::Legalize method.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ADT/APFloat.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/CodeGen/ISDOpcodes.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineMemOperand.h"
25#include "llvm/CodeGen/RuntimeLibcalls.h"
26#include "llvm/CodeGen/SelectionDAG.h"
27#include "llvm/CodeGen/SelectionDAGNodes.h"
28#include "llvm/CodeGen/TargetFrameLowering.h"
29#include "llvm/CodeGen/TargetLowering.h"
30#include "llvm/CodeGen/TargetSubtargetInfo.h"
31#include "llvm/CodeGen/ValueTypes.h"
32#include "llvm/IR/CallingConv.h"
33#include "llvm/IR/Constants.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/DerivedTypes.h"
36#include "llvm/IR/Function.h"
37#include "llvm/IR/Metadata.h"
38#include "llvm/IR/Type.h"
39#include "llvm/Support/Casting.h"
40#include "llvm/Support/Compiler.h"
41#include "llvm/Support/Debug.h"
42#include "llvm/Support/ErrorHandling.h"
43#include "llvm/Support/MachineValueType.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/raw_ostream.h"
46#include "llvm/Target/TargetMachine.h"
47#include "llvm/Target/TargetOptions.h"
48#include <algorithm>
49#include <cassert>
50#include <cstdint>
51#include <tuple>
52#include <utility>
53
54using namespace llvm;
55
56#define DEBUG_TYPE"legalizedag" "legalizedag"
57
58namespace {
59
60/// Keeps track of state when getting the sign of a floating-point value as an
61/// integer.
62struct FloatSignAsInt {
63 EVT FloatVT;
64 SDValue Chain;
65 SDValue FloatPtr;
66 SDValue IntPtr;
67 MachinePointerInfo IntPointerInfo;
68 MachinePointerInfo FloatPointerInfo;
69 SDValue IntValue;
70 APInt SignMask;
71 uint8_t SignBit;
72};
73
74//===----------------------------------------------------------------------===//
75/// This takes an arbitrary SelectionDAG as input and
76/// hacks on it until the target machine can handle it. This involves
77/// eliminating value sizes the machine cannot handle (promoting small sizes to
78/// large sizes or splitting up large values into small values) as well as
79/// eliminating operations the machine cannot handle.
80///
81/// This code also does a small amount of optimization and recognition of idioms
82/// as part of its processing. For example, if a target does not support a
83/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
84/// will attempt merge setcc and brc instructions into brcc's.
85class SelectionDAGLegalize {
86 const TargetMachine &TM;
87 const TargetLowering &TLI;
88 SelectionDAG &DAG;
89
90 /// The set of nodes which have already been legalized. We hold a
91 /// reference to it in order to update as necessary on node deletion.
92 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93
94 /// A set of all the nodes updated during legalization.
95 SmallSetVector<SDNode *, 16> *UpdatedNodes;
96
97 EVT getSetCCResultType(EVT VT) const {
98 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
99 }
100
101 // Libcall insertion helpers.
102
103public:
104 SelectionDAGLegalize(SelectionDAG &DAG,
105 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
106 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
107 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
108 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109
110 /// Legalizes the given operation.
111 void LegalizeOp(SDNode *Node);
112
113private:
114 SDValue OptimizeFloatStore(StoreSDNode *ST);
115
116 void LegalizeLoadOps(SDNode *Node);
117 void LegalizeStoreOps(SDNode *Node);
118
119 /// Some targets cannot handle a variable
120 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
121 /// is necessary to spill the vector being inserted into to memory, perform
122 /// the insert there, and then read the result back.
123 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
124 const SDLoc &dl);
125 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
126 const SDLoc &dl);
127
128 /// Return a vector shuffle operation which
129 /// performs the same shuffe in terms of order or result bytes, but on a type
130 /// whose vector element type is narrower than the original shuffle type.
131 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
132 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133 SDValue N1, SDValue N2,
134 ArrayRef<int> Mask) const;
135
136 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
137 bool &NeedInvert, const SDLoc &dl);
138
139 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
140 SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
141 unsigned NumOps, bool isSigned, const SDLoc &dl);
142
143 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
144 SDNode *Node, bool isSigned);
145 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
146 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
147 RTLIB::Libcall Call_F128,
148 RTLIB::Libcall Call_PPCF128);
149 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
150 RTLIB::Libcall Call_I8,
151 RTLIB::Libcall Call_I16,
152 RTLIB::Libcall Call_I32,
153 RTLIB::Libcall Call_I64,
154 RTLIB::Libcall Call_I128);
155 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157
158 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
159 const SDLoc &dl);
160 SDValue ExpandBUILD_VECTOR(SDNode *Node);
161 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
162 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
163 SmallVectorImpl<SDValue> &Results);
164 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
165 SDValue Value) const;
166 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
167 SDValue NewIntValue) const;
168 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
169 SDValue ExpandFABS(SDNode *Node) const;
170 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, EVT DestVT,
171 const SDLoc &dl);
172 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
173 const SDLoc &dl);
174 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
175 const SDLoc &dl);
176
177 SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
178 SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
179 SDValue ExpandBitCount(unsigned Opc, SDValue Op, const SDLoc &dl);
180
181 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
182 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
183 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
184
185 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
186 SDValue ExpandConstant(ConstantSDNode *CP);
187
188 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
189 bool ExpandNode(SDNode *Node);
190 void ConvertNodeToLibcall(SDNode *Node);
191 void PromoteNode(SDNode *Node);
192
193public:
194 // Node replacement helpers
195
196 void ReplacedNode(SDNode *N) {
197 LegalizedNodes.erase(N);
198 if (UpdatedNodes)
199 UpdatedNodes->insert(N);
200 }
201
202 void ReplaceNode(SDNode *Old, SDNode *New) {
203 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
204 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
205
206 assert(Old->getNumValues() == New->getNumValues() &&(static_cast <bool> (Old->getNumValues() == New->
getNumValues() && "Replacing one node with another that produces a different number "
"of values!") ? void (0) : __assert_fail ("Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 208, __extension__ __PRETTY_FUNCTION__))
207 "Replacing one node with another that produces a different number "(static_cast <bool> (Old->getNumValues() == New->
getNumValues() && "Replacing one node with another that produces a different number "
"of values!") ? void (0) : __assert_fail ("Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 208, __extension__ __PRETTY_FUNCTION__))
208 "of values!")(static_cast <bool> (Old->getNumValues() == New->
getNumValues() && "Replacing one node with another that produces a different number "
"of values!") ? void (0) : __assert_fail ("Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 208, __extension__ __PRETTY_FUNCTION__))
;
209 DAG.ReplaceAllUsesWith(Old, New);
210 if (UpdatedNodes)
211 UpdatedNodes->insert(New);
212 ReplacedNode(Old);
213 }
214
215 void ReplaceNode(SDValue Old, SDValue New) {
216 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
217 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
218
219 DAG.ReplaceAllUsesWith(Old, New);
220 if (UpdatedNodes)
221 UpdatedNodes->insert(New.getNode());
222 ReplacedNode(Old.getNode());
223 }
224
225 void ReplaceNode(SDNode *Old, const SDValue *New) {
226 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); } } while (false)
;
227
228 DAG.ReplaceAllUsesWith(Old, New);
229 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
230 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
231 New[i]->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
;
232 if (UpdatedNodes)
233 UpdatedNodes->insert(New[i].getNode());
234 }
235 ReplacedNode(Old);
236 }
237};
238
239} // end anonymous namespace
240
241/// Return a vector shuffle operation which
242/// performs the same shuffe in terms of order or result bytes, but on a type
243/// whose vector element type is narrower than the original shuffle type.
244/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
245SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
246 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
247 ArrayRef<int> Mask) const {
248 unsigned NumMaskElts = VT.getVectorNumElements();
249 unsigned NumDestElts = NVT.getVectorNumElements();
250 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
251
252 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!")(static_cast <bool> (NumEltsGrowth && "Cannot promote to vector type with fewer elts!"
) ? void (0) : __assert_fail ("NumEltsGrowth && \"Cannot promote to vector type with fewer elts!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 252, __extension__ __PRETTY_FUNCTION__))
;
253
254 if (NumEltsGrowth == 1)
255 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
256
257 SmallVector<int, 8> NewMask;
258 for (unsigned i = 0; i != NumMaskElts; ++i) {
259 int Idx = Mask[i];
260 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
261 if (Idx < 0)
262 NewMask.push_back(-1);
263 else
264 NewMask.push_back(Idx * NumEltsGrowth + j);
265 }
266 }
267 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?")(static_cast <bool> (NewMask.size() == NumDestElts &&
"Non-integer NumEltsGrowth?") ? void (0) : __assert_fail ("NewMask.size() == NumDestElts && \"Non-integer NumEltsGrowth?\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 267, __extension__ __PRETTY_FUNCTION__))
;
268 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?")(static_cast <bool> (TLI.isShuffleMaskLegal(NewMask, NVT
) && "Shuffle not legal?") ? void (0) : __assert_fail
("TLI.isShuffleMaskLegal(NewMask, NVT) && \"Shuffle not legal?\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 268, __extension__ __PRETTY_FUNCTION__))
;
269 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
270}
271
272/// Expands the ConstantFP node to an integer constant or
273/// a load from the constant pool.
274SDValue
275SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
276 bool Extend = false;
277 SDLoc dl(CFP);
278
279 // If a FP immediate is precise when represented as a float and if the
280 // target can do an extending load from float to double, we put it into
281 // the constant pool as a float, even if it's is statically typed as a
282 // double. This shrinks FP constants and canonicalizes them for targets where
283 // an FP extending load is the same cost as a normal load (such as on the x87
284 // fp stack or PPC FP unit).
285 EVT VT = CFP->getValueType(0);
286 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
287 if (!UseCP) {
288 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion")(static_cast <bool> ((VT == MVT::f64 || VT == MVT::f32)
&& "Invalid type expansion") ? void (0) : __assert_fail
("(VT == MVT::f64 || VT == MVT::f32) && \"Invalid type expansion\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 288, __extension__ __PRETTY_FUNCTION__))
;
289 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
290 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
291 }
292
293 APFloat APF = CFP->getValueAPF();
294 EVT OrigVT = VT;
295 EVT SVT = VT;
296
297 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
298 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
299 if (!APF.isSignaling()) {
300 while (SVT != MVT::f32 && SVT != MVT::f16) {
301 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
302 if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
303 // Only do this if the target has a native EXTLOAD instruction from
304 // smaller type.
305 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
306 TLI.ShouldShrinkFPConstant(OrigVT)) {
307 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
308 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
309 VT = SVT;
310 Extend = true;
311 }
312 }
313 }
314
315 SDValue CPIdx =
316 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
317 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
318 if (Extend) {
319 SDValue Result = DAG.getExtLoad(
320 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
321 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
322 Alignment);
323 return Result;
324 }
325 SDValue Result = DAG.getLoad(
326 OrigVT, dl, DAG.getEntryNode(), CPIdx,
327 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
328 return Result;
329}
330
331/// Expands the Constant node to a load from the constant pool.
332SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
333 SDLoc dl(CP);
334 EVT VT = CP->getValueType(0);
335 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
336 TLI.getPointerTy(DAG.getDataLayout()));
337 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
338 SDValue Result = DAG.getLoad(
339 VT, dl, DAG.getEntryNode(), CPIdx,
340 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
341 return Result;
342}
343
344/// Some target cannot handle a variable insertion index for the
345/// INSERT_VECTOR_ELT instruction. In this case, it
346/// is necessary to spill the vector being inserted into to memory, perform
347/// the insert there, and then read the result back.
348SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
349 SDValue Val,
350 SDValue Idx,
351 const SDLoc &dl) {
352 SDValue Tmp1 = Vec;
353 SDValue Tmp2 = Val;
354 SDValue Tmp3 = Idx;
355
356 // If the target doesn't support this, we have to spill the input vector
357 // to a temporary stack slot, update the element, then reload it. This is
358 // badness. We could also load the value into a vector register (either
359 // with a "move to register" or "extload into register" instruction, then
360 // permute it into place, if the idx is a constant and if the idx is
361 // supported by the target.
362 EVT VT = Tmp1.getValueType();
363 EVT EltVT = VT.getVectorElementType();
364 SDValue StackPtr = DAG.CreateStackTemporary(VT);
365
366 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
367
368 // Store the vector.
369 SDValue Ch = DAG.getStore(
370 DAG.getEntryNode(), dl, Tmp1, StackPtr,
371 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
372
373 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
374
375 // Store the scalar value.
376 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
377 // Load the updated vector.
378 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
379 DAG.getMachineFunction(), SPFI));
380}
381
382SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
383 SDValue Idx,
384 const SDLoc &dl) {
385 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
386 // SCALAR_TO_VECTOR requires that the type of the value being inserted
387 // match the element type of the vector being created, except for
388 // integers in which case the inserted value can be over width.
389 EVT EltVT = Vec.getValueType().getVectorElementType();
390 if (Val.getValueType() == EltVT ||
391 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
392 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
393 Vec.getValueType(), Val);
394
395 unsigned NumElts = Vec.getValueType().getVectorNumElements();
396 // We generate a shuffle of InVec and ScVec, so the shuffle mask
397 // should be 0,1,2,3,4,5... with the appropriate element replaced with
398 // elt 0 of the RHS.
399 SmallVector<int, 8> ShufOps;
400 for (unsigned i = 0; i != NumElts; ++i)
401 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
402
403 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
404 }
405 }
406 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
407}
408
409SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
410 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Optimizing float store operations\n"
; } } while (false)
;
411 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
412 // FIXME: We shouldn't do this for TargetConstantFP's.
413 // FIXME: move this to the DAG Combiner! Note that we can't regress due
414 // to phase ordering between legalized code and the dag combiner. This
415 // probably means that we need to integrate dag combiner and legalizer
416 // together.
417 // We generally can't do this one for long doubles.
418 SDValue Chain = ST->getChain();
419 SDValue Ptr = ST->getBasePtr();
420 unsigned Alignment = ST->getAlignment();
421 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
422 AAMDNodes AAInfo = ST->getAAInfo();
423 SDLoc dl(ST);
424 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
425 if (CFP->getValueType(0) == MVT::f32 &&
426 TLI.isTypeLegal(MVT::i32)) {
427 SDValue Con = DAG.getConstant(CFP->getValueAPF().
428 bitcastToAPInt().zextOrTrunc(32),
429 SDLoc(CFP), MVT::i32);
430 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
431 MMOFlags, AAInfo);
432 }
433
434 if (CFP->getValueType(0) == MVT::f64) {
435 // If this target supports 64-bit registers, do a single 64-bit store.
436 if (TLI.isTypeLegal(MVT::i64)) {
437 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
438 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
439 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
440 Alignment, MMOFlags, AAInfo);
441 }
442
443 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
444 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
445 // stores. If the target supports neither 32- nor 64-bits, this
446 // xform is certainly not worth it.
447 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
448 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
449 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
450 if (DAG.getDataLayout().isBigEndian())
451 std::swap(Lo, Hi);
452
453 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
454 MMOFlags, AAInfo);
455 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
456 DAG.getConstant(4, dl, Ptr.getValueType()));
457 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
458 ST->getPointerInfo().getWithOffset(4),
459 MinAlign(Alignment, 4U), MMOFlags, AAInfo);
460
461 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
462 }
463 }
464 }
465 return SDValue(nullptr, 0);
466}
467
468void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
469 StoreSDNode *ST = cast<StoreSDNode>(Node);
470 SDValue Chain = ST->getChain();
471 SDValue Ptr = ST->getBasePtr();
472 SDLoc dl(Node);
473
474 unsigned Alignment = ST->getAlignment();
475 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
476 AAMDNodes AAInfo = ST->getAAInfo();
477
478 if (!ST->isTruncatingStore()) {
479 LLVM_DEBUG(dbgs() << "Legalizing store operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing store operation\n"
; } } while (false)
;
480 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
481 ReplaceNode(ST, OptStore);
482 return;
483 }
484
485 SDValue Value = ST->getValue();
486 MVT VT = Value.getSimpleValueType();
487 switch (TLI.getOperationAction(ISD::STORE, VT)) {
488 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 488)
;
489 case TargetLowering::Legal: {
490 // If this is an unaligned store and the target doesn't support it,
491 // expand it.
492 EVT MemVT = ST->getMemoryVT();
493 unsigned AS = ST->getAddressSpace();
494 unsigned Align = ST->getAlignment();
495 const DataLayout &DL = DAG.getDataLayout();
496 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
497 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Expanding unsupported unaligned store\n"
; } } while (false)
;
498 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
499 ReplaceNode(SDValue(ST, 0), Result);
500 } else
501 LLVM_DEBUG(dbgs() << "Legal store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legal store\n"; } } while
(false)
;
502 break;
503 }
504 case TargetLowering::Custom: {
505 LLVM_DEBUG(dbgs() << "Trying custom lowering\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying custom lowering\n"
; } } while (false)
;
506 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
507 if (Res && Res != SDValue(Node, 0))
508 ReplaceNode(SDValue(Node, 0), Res);
509 return;
510 }
511 case TargetLowering::Promote: {
512 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
513 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&(static_cast <bool> (NVT.getSizeInBits() == VT.getSizeInBits
() && "Can only promote stores to same size type") ? void
(0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 514, __extension__ __PRETTY_FUNCTION__))
514 "Can only promote stores to same size type")(static_cast <bool> (NVT.getSizeInBits() == VT.getSizeInBits
() && "Can only promote stores to same size type") ? void
(0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 514, __extension__ __PRETTY_FUNCTION__))
;
515 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
516 SDValue Result =
517 DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
518 Alignment, MMOFlags, AAInfo);
519 ReplaceNode(SDValue(Node, 0), Result);
520 break;
521 }
522 }
523 return;
524 }
525
526 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing truncating store operations\n"
; } } while (false)
;
527 SDValue Value = ST->getValue();
528 EVT StVT = ST->getMemoryVT();
529 unsigned StWidth = StVT.getSizeInBits();
530 auto &DL = DAG.getDataLayout();
531
532 if (StWidth != StVT.getStoreSizeInBits()) {
533 // Promote to a byte-sized store with upper bits zero if not
534 // storing an integral number of bytes. For example, promote
535 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
536 EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
537 StVT.getStoreSizeInBits());
538 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
539 SDValue Result =
540 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
541 Alignment, MMOFlags, AAInfo);
542 ReplaceNode(SDValue(Node, 0), Result);
543 } else if (StWidth & (StWidth - 1)) {
544 // If not storing a power-of-2 number of bits, expand as two stores.
545 assert(!StVT.isVector() && "Unsupported truncstore!")(static_cast <bool> (!StVT.isVector() && "Unsupported truncstore!"
) ? void (0) : __assert_fail ("!StVT.isVector() && \"Unsupported truncstore!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 545, __extension__ __PRETTY_FUNCTION__))
;
546 unsigned RoundWidth = 1 << Log2_32(StWidth);
547 assert(RoundWidth < StWidth)(static_cast <bool> (RoundWidth < StWidth) ? void (0
) : __assert_fail ("RoundWidth < StWidth", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 547, __extension__ __PRETTY_FUNCTION__))
;
548 unsigned ExtraWidth = StWidth - RoundWidth;
549 assert(ExtraWidth < RoundWidth)(static_cast <bool> (ExtraWidth < RoundWidth) ? void
(0) : __assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 549, __extension__ __PRETTY_FUNCTION__))
;
550 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&(static_cast <bool> (!(RoundWidth % 8) && !(ExtraWidth
% 8) && "Store size not an integral number of bytes!"
) ? void (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 551, __extension__ __PRETTY_FUNCTION__))
551 "Store size not an integral number of bytes!")(static_cast <bool> (!(RoundWidth % 8) && !(ExtraWidth
% 8) && "Store size not an integral number of bytes!"
) ? void (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 551, __extension__ __PRETTY_FUNCTION__))
;
552 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
553 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
554 SDValue Lo, Hi;
555 unsigned IncrementSize;
556
557 if (DL.isLittleEndian()) {
558 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
559 // Store the bottom RoundWidth bits.
560 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
561 RoundVT, Alignment, MMOFlags, AAInfo);
562
563 // Store the remaining ExtraWidth bits.
564 IncrementSize = RoundWidth / 8;
565 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
566 DAG.getConstant(IncrementSize, dl,
567 Ptr.getValueType()));
568 Hi = DAG.getNode(
569 ISD::SRL, dl, Value.getValueType(), Value,
570 DAG.getConstant(RoundWidth, dl,
571 TLI.getShiftAmountTy(Value.getValueType(), DL)));
572 Hi = DAG.getTruncStore(
573 Chain, dl, Hi, Ptr,
574 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
575 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
576 } else {
577 // Big endian - avoid unaligned stores.
578 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
579 // Store the top RoundWidth bits.
580 Hi = DAG.getNode(
581 ISD::SRL, dl, Value.getValueType(), Value,
582 DAG.getConstant(ExtraWidth, dl,
583 TLI.getShiftAmountTy(Value.getValueType(), DL)));
584 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
585 RoundVT, Alignment, MMOFlags, AAInfo);
586
587 // Store the remaining ExtraWidth bits.
588 IncrementSize = RoundWidth / 8;
589 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
590 DAG.getConstant(IncrementSize, dl,
591 Ptr.getValueType()));
592 Lo = DAG.getTruncStore(
593 Chain, dl, Value, Ptr,
594 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
595 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
596 }
597
598 // The order of the stores doesn't matter.
599 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
600 ReplaceNode(SDValue(Node, 0), Result);
601 } else {
602 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
603 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 603)
;
604 case TargetLowering::Legal: {
605 EVT MemVT = ST->getMemoryVT();
606 unsigned AS = ST->getAddressSpace();
607 unsigned Align = ST->getAlignment();
608 // If this is an unaligned store and the target doesn't support it,
609 // expand it.
610 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
611 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
612 ReplaceNode(SDValue(ST, 0), Result);
613 }
614 break;
615 }
616 case TargetLowering::Custom: {
617 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
618 if (Res && Res != SDValue(Node, 0))
619 ReplaceNode(SDValue(Node, 0), Res);
620 return;
621 }
622 case TargetLowering::Expand:
623 assert(!StVT.isVector() &&(static_cast <bool> (!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? void (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 624, __extension__ __PRETTY_FUNCTION__))
624 "Vector Stores are handled in LegalizeVectorOps")(static_cast <bool> (!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? void (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 624, __extension__ __PRETTY_FUNCTION__))
;
625
626 SDValue Result;
627
628 // TRUNCSTORE:i16 i32 -> STORE i16
629 if (TLI.isTypeLegal(StVT)) {
630 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
631 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
632 Alignment, MMOFlags, AAInfo);
633 } else {
634 // The in-memory type isn't legal. Truncate to the type it would promote
635 // to, and then do a truncstore.
636 Value = DAG.getNode(ISD::TRUNCATE, dl,
637 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
638 Value);
639 Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
640 StVT, Alignment, MMOFlags, AAInfo);
641 }
642
643 ReplaceNode(SDValue(Node, 0), Result);
644 break;
645 }
646 }
647}
648
649void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
650 LoadSDNode *LD = cast<LoadSDNode>(Node);
651 SDValue Chain = LD->getChain(); // The chain.
652 SDValue Ptr = LD->getBasePtr(); // The base pointer.
653 SDValue Value; // The value returned by the load op.
654 SDLoc dl(Node);
655
656 ISD::LoadExtType ExtType = LD->getExtensionType();
657 if (ExtType == ISD::NON_EXTLOAD) {
1
Assuming 'ExtType' is not equal to NON_EXTLOAD
2
Taking false branch
658 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing non-extending load operation\n"
; } } while (false)
;
659 MVT VT = Node->getSimpleValueType(0);
660 SDValue RVal = SDValue(Node, 0);
661 SDValue RChain = SDValue(Node, 1);
662
663 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
664 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 664)
;
665 case TargetLowering::Legal: {
666 EVT MemVT = LD->getMemoryVT();
667 unsigned AS = LD->getAddressSpace();
668 unsigned Align = LD->getAlignment();
669 const DataLayout &DL = DAG.getDataLayout();
670 // If this is an unaligned load and the target doesn't support it,
671 // expand it.
672 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
673 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
674 }
675 break;
676 }
677 case TargetLowering::Custom:
678 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
679 RVal = Res;
680 RChain = Res.getValue(1);
681 }
682 break;
683
684 case TargetLowering::Promote: {
685 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
686 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&(static_cast <bool> (NVT.getSizeInBits() == VT.getSizeInBits
() && "Can only promote loads to same size type") ? void
(0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 687, __extension__ __PRETTY_FUNCTION__))
687 "Can only promote loads to same size type")(static_cast <bool> (NVT.getSizeInBits() == VT.getSizeInBits
() && "Can only promote loads to same size type") ? void
(0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 687, __extension__ __PRETTY_FUNCTION__))
;
688
689 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
690 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
691 RChain = Res.getValue(1);
692 break;
693 }
694 }
695 if (RChain.getNode() != Node) {
696 assert(RVal.getNode() != Node && "Load must be completely replaced")(static_cast <bool> (RVal.getNode() != Node && "Load must be completely replaced"
) ? void (0) : __assert_fail ("RVal.getNode() != Node && \"Load must be completely replaced\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 696, __extension__ __PRETTY_FUNCTION__))
;
697 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
698 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
699 if (UpdatedNodes) {
700 UpdatedNodes->insert(RVal.getNode());
701 UpdatedNodes->insert(RChain.getNode());
702 }
703 ReplacedNode(Node);
704 }
705 return;
706 }
707
708 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing extending load operation\n"
; } } while (false)
;
709 EVT SrcVT = LD->getMemoryVT();
710 unsigned SrcWidth = SrcVT.getSizeInBits();
711 unsigned Alignment = LD->getAlignment();
712 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
713 AAMDNodes AAInfo = LD->getAAInfo();
714
715 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
3
Assuming the condition is false
4
Taking false branch
716 // Some targets pretend to have an i1 loading operation, and actually
717 // load an i8. This trick is correct for ZEXTLOAD because the top 7
718 // bits are guaranteed to be zero; it helps the optimizers understand
719 // that these bits are zero. It is also useful for EXTLOAD, since it
720 // tells the optimizers that those bits are undefined. It would be
721 // nice to have an effective generic way of getting these benefits...
722 // Until such a way is found, don't insist on promoting i1 here.
723 (SrcVT != MVT::i1 ||
724 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
725 TargetLowering::Promote)) {
726 // Promote to a byte-sized load if not loading an integral number of
727 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
728 unsigned NewWidth = SrcVT.getStoreSizeInBits();
729 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
730 SDValue Ch;
731
732 // The extra bits are guaranteed to be zero, since we stored them that
733 // way. A zext load from NVT thus automatically gives zext from SrcVT.
734
735 ISD::LoadExtType NewExtType =
736 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
737
738 SDValue Result =
739 DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
740 LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
741
742 Ch = Result.getValue(1); // The chain.
743
744 if (ExtType == ISD::SEXTLOAD)
745 // Having the top bits zero doesn't help when sign extending.
746 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
747 Result.getValueType(),
748 Result, DAG.getValueType(SrcVT));
749 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
750 // All the top bits are guaranteed to be zero - inform the optimizers.
751 Result = DAG.getNode(ISD::AssertZext, dl,
752 Result.getValueType(), Result,
753 DAG.getValueType(SrcVT));
754
755 Value = Result;
756 Chain = Ch;
757 } else if (SrcWidth & (SrcWidth - 1)) {
5
Assuming the condition is true
6
Taking true branch
758 // If not loading a power-of-2 number of bits, expand as two loads.
759 assert(!SrcVT.isVector() && "Unsupported extload!")(static_cast <bool> (!SrcVT.isVector() && "Unsupported extload!"
) ? void (0) : __assert_fail ("!SrcVT.isVector() && \"Unsupported extload!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 759, __extension__ __PRETTY_FUNCTION__))
;
760 unsigned RoundWidth = 1 << Log2_32(SrcWidth);
7
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'
761 assert(RoundWidth < SrcWidth)(static_cast <bool> (RoundWidth < SrcWidth) ? void (
0) : __assert_fail ("RoundWidth < SrcWidth", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 761, __extension__ __PRETTY_FUNCTION__))
;
762 unsigned ExtraWidth = SrcWidth - RoundWidth;
763 assert(ExtraWidth < RoundWidth)(static_cast <bool> (ExtraWidth < RoundWidth) ? void
(0) : __assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 763, __extension__ __PRETTY_FUNCTION__))
;
764 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&(static_cast <bool> (!(RoundWidth % 8) && !(ExtraWidth
% 8) && "Load size not an integral number of bytes!"
) ? void (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 765, __extension__ __PRETTY_FUNCTION__))
765 "Load size not an integral number of bytes!")(static_cast <bool> (!(RoundWidth % 8) && !(ExtraWidth
% 8) && "Load size not an integral number of bytes!"
) ? void (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 765, __extension__ __PRETTY_FUNCTION__))
;
766 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
767 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
768 SDValue Lo, Hi, Ch;
769 unsigned IncrementSize;
770 auto &DL = DAG.getDataLayout();
771
772 if (DL.isLittleEndian()) {
773 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
774 // Load the bottom RoundWidth bits.
775 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
776 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
777 AAInfo);
778
779 // Load the remaining ExtraWidth bits.
780 IncrementSize = RoundWidth / 8;
781 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
782 DAG.getConstant(IncrementSize, dl,
783 Ptr.getValueType()));
784 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
785 LD->getPointerInfo().getWithOffset(IncrementSize),
786 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
787 AAInfo);
788
789 // Build a factor node to remember that this load is independent of
790 // the other one.
791 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
792 Hi.getValue(1));
793
794 // Move the top bits to the right place.
795 Hi = DAG.getNode(
796 ISD::SHL, dl, Hi.getValueType(), Hi,
797 DAG.getConstant(RoundWidth, dl,
798 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
799
800 // Join the hi and lo parts.
801 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
802 } else {
803 // Big endian - avoid unaligned loads.
804 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
805 // Load the top RoundWidth bits.
806 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
807 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
808 AAInfo);
809
810 // Load the remaining ExtraWidth bits.
811 IncrementSize = RoundWidth / 8;
812 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
813 DAG.getConstant(IncrementSize, dl,
814 Ptr.getValueType()));
815 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
816 LD->getPointerInfo().getWithOffset(IncrementSize),
817 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
818 AAInfo);
819
820 // Build a factor node to remember that this load is independent of
821 // the other one.
822 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
823 Hi.getValue(1));
824
825 // Move the top bits to the right place.
826 Hi = DAG.getNode(
827 ISD::SHL, dl, Hi.getValueType(), Hi,
828 DAG.getConstant(ExtraWidth, dl,
829 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
830
831 // Join the hi and lo parts.
832 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
833 }
834
835 Chain = Ch;
836 } else {
837 bool isCustom = false;
838 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
839 SrcVT.getSimpleVT())) {
840 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 840)
;
841 case TargetLowering::Custom:
842 isCustom = true;
843 LLVM_FALLTHROUGH[[clang::fallthrough]];
844 case TargetLowering::Legal:
845 Value = SDValue(Node, 0);
846 Chain = SDValue(Node, 1);
847
848 if (isCustom) {
849 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
850 Value = Res;
851 Chain = Res.getValue(1);
852 }
853 } else {
854 // If this is an unaligned load and the target doesn't support it,
855 // expand it.
856 EVT MemVT = LD->getMemoryVT();
857 unsigned AS = LD->getAddressSpace();
858 unsigned Align = LD->getAlignment();
859 const DataLayout &DL = DAG.getDataLayout();
860 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
861 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
862 }
863 }
864 break;
865
866 case TargetLowering::Expand: {
867 EVT DestVT = Node->getValueType(0);
868 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
869 // If the source type is not legal, see if there is a legal extload to
870 // an intermediate type that we can then extend further.
871 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
872 if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
873 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
874 // If we are loading a legal type, this is a non-extload followed by a
875 // full extend.
876 ISD::LoadExtType MidExtType =
877 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
878
879 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
880 SrcVT, LD->getMemOperand());
881 unsigned ExtendOp =
882 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
883 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
884 Chain = Load.getValue(1);
885 break;
886 }
887
888 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
889 // normal undefined upper bits behavior to allow using an in-reg extend
890 // with the illegal FP type, so load as an integer and do the
891 // from-integer conversion.
892 if (SrcVT.getScalarType() == MVT::f16) {
893 EVT ISrcVT = SrcVT.changeTypeToInteger();
894 EVT IDestVT = DestVT.changeTypeToInteger();
895 EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
896
897 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
898 Chain, Ptr, ISrcVT,
899 LD->getMemOperand());
900 Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
901 Chain = Result.getValue(1);
902 break;
903 }
904 }
905
906 assert(!SrcVT.isVector() &&(static_cast <bool> (!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? void (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 907, __extension__ __PRETTY_FUNCTION__))
907 "Vector Loads are handled in LegalizeVectorOps")(static_cast <bool> (!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? void (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 907, __extension__ __PRETTY_FUNCTION__))
;
908
909 // FIXME: This does not work for vectors on most targets. Sign-
910 // and zero-extend operations are currently folded into extending
911 // loads, whether they are legal or not, and then we end up here
912 // without any support for legalizing them.
913 assert(ExtType != ISD::EXTLOAD &&(static_cast <bool> (ExtType != ISD::EXTLOAD &&
"EXTLOAD should always be supported!") ? void (0) : __assert_fail
("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 914, __extension__ __PRETTY_FUNCTION__))
914 "EXTLOAD should always be supported!")(static_cast <bool> (ExtType != ISD::EXTLOAD &&
"EXTLOAD should always be supported!") ? void (0) : __assert_fail
("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 914, __extension__ __PRETTY_FUNCTION__))
;
915 // Turn the unsupported load into an EXTLOAD followed by an
916 // explicit zero/sign extend inreg.
917 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
918 Node->getValueType(0),
919 Chain, Ptr, SrcVT,
920 LD->getMemOperand());
921 SDValue ValRes;
922 if (ExtType == ISD::SEXTLOAD)
923 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
924 Result.getValueType(),
925 Result, DAG.getValueType(SrcVT));
926 else
927 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
928 Value = ValRes;
929 Chain = Result.getValue(1);
930 break;
931 }
932 }
933 }
934
935 // Since loads produce two values, make sure to remember that we legalized
936 // both of them.
937 if (Chain.getNode() != Node) {
938 assert(Value.getNode() != Node && "Load must be completely replaced")(static_cast <bool> (Value.getNode() != Node &&
"Load must be completely replaced") ? void (0) : __assert_fail
("Value.getNode() != Node && \"Load must be completely replaced\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 938, __extension__ __PRETTY_FUNCTION__))
;
939 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
940 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
941 if (UpdatedNodes) {
942 UpdatedNodes->insert(Value.getNode());
943 UpdatedNodes->insert(Chain.getNode());
944 }
945 ReplacedNode(Node);
946 }
947}
948
949/// Return a legal replacement for the given operation, with all legal operands.
950void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
951 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "\nLegalizing: "; Node->
dump(&DAG); } } while (false)
;
952
953 // Allow illegal target nodes and illegal registers.
954 if (Node->getOpcode() == ISD::TargetConstant ||
955 Node->getOpcode() == ISD::Register)
956 return;
957
958#ifndef NDEBUG
959 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
960 assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Node->getValueType(i)) == TargetLowering::TypeLegal ||
TLI.isTypeLegal(Node->getValueType(i))) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 963, __extension__ __PRETTY_FUNCTION__))
961 TargetLowering::TypeLegal ||(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Node->getValueType(i)) == TargetLowering::TypeLegal ||
TLI.isTypeLegal(Node->getValueType(i))) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 963, __extension__ __PRETTY_FUNCTION__))
962 TLI.isTypeLegal(Node->getValueType(i))) &&(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Node->getValueType(i)) == TargetLowering::TypeLegal ||
TLI.isTypeLegal(Node->getValueType(i))) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 963, __extension__ __PRETTY_FUNCTION__))
963 "Unexpected illegal type!")(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Node->getValueType(i)) == TargetLowering::TypeLegal ||
TLI.isTypeLegal(Node->getValueType(i))) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 963, __extension__ __PRETTY_FUNCTION__))
;
964
965 for (const SDValue &Op : Node->op_values())
966 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 971, __extension__ __PRETTY_FUNCTION__))
967 TargetLowering::TypeLegal ||(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 971, __extension__ __PRETTY_FUNCTION__))
968 TLI.isTypeLegal(Op.getValueType()) ||(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 971, __extension__ __PRETTY_FUNCTION__))
969 Op.getOpcode() == ISD::TargetConstant ||(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 971, __extension__ __PRETTY_FUNCTION__))
970 Op.getOpcode() == ISD::Register) &&(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 971, __extension__ __PRETTY_FUNCTION__))
971 "Unexpected illegal type!")(static_cast <bool> ((TLI.getTypeAction(*DAG.getContext
(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal
(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"
) ? void (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 971, __extension__ __PRETTY_FUNCTION__))
;
972#endif
973
974 // Figure out the correct action; the way to query this varies by opcode
975 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
976 bool SimpleFinishLegalizing = true;
977 switch (Node->getOpcode()) {
978 case ISD::INTRINSIC_W_CHAIN:
979 case ISD::INTRINSIC_WO_CHAIN:
980 case ISD::INTRINSIC_VOID:
981 case ISD::STACKSAVE:
982 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
983 break;
984 case ISD::GET_DYNAMIC_AREA_OFFSET:
985 Action = TLI.getOperationAction(Node->getOpcode(),
986 Node->getValueType(0));
987 break;
988 case ISD::VAARG:
989 Action = TLI.getOperationAction(Node->getOpcode(),
990 Node->getValueType(0));
991 if (Action != TargetLowering::Promote)
992 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
993 break;
994 case ISD::FP_TO_FP16:
995 case ISD::SINT_TO_FP:
996 case ISD::UINT_TO_FP:
997 case ISD::EXTRACT_VECTOR_ELT:
998 Action = TLI.getOperationAction(Node->getOpcode(),
999 Node->getOperand(0).getValueType());
1000 break;
1001 case ISD::FP_ROUND_INREG:
1002 case ISD::SIGN_EXTEND_INREG: {
1003 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1004 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1005 break;
1006 }
1007 case ISD::ATOMIC_STORE:
1008 Action = TLI.getOperationAction(Node->getOpcode(),
1009 Node->getOperand(2).getValueType());
1010 break;
1011 case ISD::SELECT_CC:
1012 case ISD::SETCC:
1013 case ISD::BR_CC: {
1014 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1015 Node->getOpcode() == ISD::SETCC ? 2 : 1;
1016 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
1017 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1018 ISD::CondCode CCCode =
1019 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1020 Action = TLI.getCondCodeAction(CCCode, OpVT);
1021 if (Action == TargetLowering::Legal) {
1022 if (Node->getOpcode() == ISD::SELECT_CC)
1023 Action = TLI.getOperationAction(Node->getOpcode(),
1024 Node->getValueType(0));
1025 else
1026 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1027 }
1028 break;
1029 }
1030 case ISD::LOAD:
1031 case ISD::STORE:
1032 // FIXME: Model these properly. LOAD and STORE are complicated, and
1033 // STORE expects the unlegalized operand in some cases.
1034 SimpleFinishLegalizing = false;
1035 break;
1036 case ISD::CALLSEQ_START:
1037 case ISD::CALLSEQ_END:
1038 // FIXME: This shouldn't be necessary. These nodes have special properties
1039 // dealing with the recursive nature of legalization. Removing this
1040 // special case should be done as part of making LegalizeDAG non-recursive.
1041 SimpleFinishLegalizing = false;
1042 break;
1043 case ISD::EXTRACT_ELEMENT:
1044 case ISD::FLT_ROUNDS_:
1045 case ISD::MERGE_VALUES:
1046 case ISD::EH_RETURN:
1047 case ISD::FRAME_TO_ARGS_OFFSET:
1048 case ISD::EH_DWARF_CFA:
1049 case ISD::EH_SJLJ_SETJMP:
1050 case ISD::EH_SJLJ_LONGJMP:
1051 case ISD::EH_SJLJ_SETUP_DISPATCH:
1052 // These operations lie about being legal: when they claim to be legal,
1053 // they should actually be expanded.
1054 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1055 if (Action == TargetLowering::Legal)
1056 Action = TargetLowering::Expand;
1057 break;
1058 case ISD::INIT_TRAMPOLINE:
1059 case ISD::ADJUST_TRAMPOLINE:
1060 case ISD::FRAMEADDR:
1061 case ISD::RETURNADDR:
1062 case ISD::ADDROFRETURNADDR:
1063 // These operations lie about being legal: when they claim to be legal,
1064 // they should actually be custom-lowered.
1065 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1066 if (Action == TargetLowering::Legal)
1067 Action = TargetLowering::Custom;
1068 break;
1069 case ISD::READCYCLECOUNTER:
1070 // READCYCLECOUNTER returns an i64, even if type legalization might have
1071 // expanded that to several smaller types.
1072 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1073 break;
1074 case ISD::READ_REGISTER:
1075 case ISD::WRITE_REGISTER:
1076 // Named register is legal in the DAG, but blocked by register name
1077 // selection if not implemented by target (to chose the correct register)
1078 // They'll be converted to Copy(To/From)Reg.
1079 Action = TargetLowering::Legal;
1080 break;
1081 case ISD::DEBUGTRAP:
1082 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1083 if (Action == TargetLowering::Expand) {
1084 // replace ISD::DEBUGTRAP with ISD::TRAP
1085 SDValue NewVal;
1086 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1087 Node->getOperand(0));
1088 ReplaceNode(Node, NewVal.getNode());
1089 LegalizeOp(NewVal.getNode());
1090 return;
1091 }
1092 break;
1093 case ISD::STRICT_FADD:
1094 case ISD::STRICT_FSUB:
1095 case ISD::STRICT_FMUL:
1096 case ISD::STRICT_FDIV:
1097 case ISD::STRICT_FSQRT:
1098 case ISD::STRICT_FMA:
1099 case ISD::STRICT_FPOW:
1100 case ISD::STRICT_FPOWI:
1101 case ISD::STRICT_FSIN:
1102 case ISD::STRICT_FCOS:
1103 case ISD::STRICT_FEXP:
1104 case ISD::STRICT_FEXP2:
1105 case ISD::STRICT_FLOG:
1106 case ISD::STRICT_FLOG10:
1107 case ISD::STRICT_FLOG2:
1108 case ISD::STRICT_FRINT:
1109 case ISD::STRICT_FNEARBYINT:
1110 // These pseudo-ops get legalized as if they were their non-strict
1111 // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
1112 // is also legal, but if ISD::FSQRT requires expansion then so does
1113 // ISD::STRICT_FSQRT.
1114 Action = TLI.getStrictFPOperationAction(Node->getOpcode(),
1115 Node->getValueType(0));
1116 break;
1117 default:
1118 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1119 Action = TargetLowering::Legal;
1120 } else {
1121 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1122 }
1123 break;
1124 }
1125
1126 if (SimpleFinishLegalizing) {
1127 SDNode *NewNode = Node;
1128 switch (Node->getOpcode()) {
1129 default: break;
1130 case ISD::SHL:
1131 case ISD::SRL:
1132 case ISD::SRA:
1133 case ISD::ROTL:
1134 case ISD::ROTR: {
1135 // Legalizing shifts/rotates requires adjusting the shift amount
1136 // to the appropriate width.
1137 SDValue Op0 = Node->getOperand(0);
1138 SDValue Op1 = Node->getOperand(1);
1139 if (!Op1.getValueType().isVector()) {
1140 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1141 // The getShiftAmountOperand() may create a new operand node or
1142 // return the existing one. If new operand is created we need
1143 // to update the parent node.
1144 // Do not try to legalize SAO here! It will be automatically legalized
1145 // in the next round.
1146 if (SAO != Op1)
1147 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1148 }
1149 }
1150 break;
1151 case ISD::SRL_PARTS:
1152 case ISD::SRA_PARTS:
1153 case ISD::SHL_PARTS: {
1154 // Legalizing shifts/rotates requires adjusting the shift amount
1155 // to the appropriate width.
1156 SDValue Op0 = Node->getOperand(0);
1157 SDValue Op1 = Node->getOperand(1);
1158 SDValue Op2 = Node->getOperand(2);
1159 if (!Op2.getValueType().isVector()) {
1160 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1161 // The getShiftAmountOperand() may create a new operand node or
1162 // return the existing one. If new operand is created we need
1163 // to update the parent node.
1164 if (SAO != Op2)
1165 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1166 }
1167 break;
1168 }
1169 }
1170
1171 if (NewNode != Node) {
1172 ReplaceNode(Node, NewNode);
1173 Node = NewNode;
1174 }
1175 switch (Action) {
1176 case TargetLowering::Legal:
1177 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legal node: nothing to do\n"
; } } while (false)
;
1178 return;
1179 case TargetLowering::Custom:
1180 LLVM_DEBUG(dbgs() << "Trying custom legalization\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying custom legalization\n"
; } } while (false)
;
1181 // FIXME: The handling for custom lowering with multiple results is
1182 // a complete mess.
1183 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1184 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1185 return;
1186
1187 if (Node->getNumValues() == 1) {
1188 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully custom legalized node\n"
; } } while (false)
;
1189 // We can just directly replace this node with the lowered value.
1190 ReplaceNode(SDValue(Node, 0), Res);
1191 return;
1192 }
1193
1194 SmallVector<SDValue, 8> ResultVals;
1195 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1196 ResultVals.push_back(Res.getValue(i));
1197 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully custom legalized node\n"
; } } while (false)
;
1198 ReplaceNode(Node, ResultVals.data());
1199 return;
1200 }
1201 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not custom legalize node\n"
; } } while (false)
;
1202 LLVM_FALLTHROUGH[[clang::fallthrough]];
1203 case TargetLowering::Expand:
1204 if (ExpandNode(Node))
1205 return;
1206 LLVM_FALLTHROUGH[[clang::fallthrough]];
1207 case TargetLowering::LibCall:
1208 ConvertNodeToLibcall(Node);
1209 return;
1210 case TargetLowering::Promote:
1211 PromoteNode(Node);
1212 return;
1213 }
1214 }
1215
1216 switch (Node->getOpcode()) {
1217 default:
1218#ifndef NDEBUG
1219 dbgs() << "NODE: ";
1220 Node->dump( &DAG);
1221 dbgs() << "\n";
1222#endif
1223 llvm_unreachable("Do not know how to legalize this operator!")::llvm::llvm_unreachable_internal("Do not know how to legalize this operator!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1223)
;
1224
1225 case ISD::CALLSEQ_START:
1226 case ISD::CALLSEQ_END:
1227 break;
1228 case ISD::LOAD:
1229 return LegalizeLoadOps(Node);
1230 case ISD::STORE:
1231 return LegalizeStoreOps(Node);
1232 }
1233}
1234
1235SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1236 SDValue Vec = Op.getOperand(0);
1237 SDValue Idx = Op.getOperand(1);
1238 SDLoc dl(Op);
1239
1240 // Before we generate a new store to a temporary stack slot, see if there is
1241 // already one that we can use. There often is because when we scalarize
1242 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1243 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1244 // the vector. If all are expanded here, we don't want one store per vector
1245 // element.
1246
1247 // Caches for hasPredecessorHelper
1248 SmallPtrSet<const SDNode *, 32> Visited;
1249 SmallVector<const SDNode *, 16> Worklist;
1250 Worklist.push_back(Idx.getNode());
1251 SDValue StackPtr, Ch;
1252 for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1253 UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1254 SDNode *User = *UI;
1255 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1256 if (ST->isIndexed() || ST->isTruncatingStore() ||
1257 ST->getValue() != Vec)
1258 continue;
1259
1260 // Make sure that nothing else could have stored into the destination of
1261 // this store.
1262 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1263 continue;
1264
1265 // If the index is dependent on the store we will introduce a cycle when
1266 // creating the load (the load uses the index, and by replacing the chain
1267 // we will make the index dependent on the load). Also, the store might be
1268 // dependent on the extractelement and introduce a cycle when creating
1269 // the load.
1270 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1271 ST->hasPredecessor(Op.getNode()))
1272 continue;
1273
1274 StackPtr = ST->getBasePtr();
1275 Ch = SDValue(ST, 0);
1276 break;
1277 }
1278 }
1279
1280 EVT VecVT = Vec.getValueType();
1281
1282 if (!Ch.getNode()) {
1283 // Store the value to a temporary stack slot, then LOAD the returned part.
1284 StackPtr = DAG.CreateStackTemporary(VecVT);
1285 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1286 MachinePointerInfo());
1287 }
1288
1289 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1290
1291 SDValue NewLoad;
1292
1293 if (Op.getValueType().isVector())
1294 NewLoad =
1295 DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1296 else
1297 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1298 MachinePointerInfo(),
1299 VecVT.getVectorElementType());
1300
1301 // Replace the chain going out of the store, by the one out of the load.
1302 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1303
1304 // We introduced a cycle though, so update the loads operands, making sure
1305 // to use the original store's chain as an incoming chain.
1306 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1307 NewLoad->op_end());
1308 NewLoadOperands[0] = Ch;
1309 NewLoad =
1310 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1311 return NewLoad;
1312}
1313
1314SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1315 assert(Op.getValueType().isVector() && "Non-vector insert subvector!")(static_cast <bool> (Op.getValueType().isVector() &&
"Non-vector insert subvector!") ? void (0) : __assert_fail (
"Op.getValueType().isVector() && \"Non-vector insert subvector!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1315, __extension__ __PRETTY_FUNCTION__))
;
1316
1317 SDValue Vec = Op.getOperand(0);
1318 SDValue Part = Op.getOperand(1);
1319 SDValue Idx = Op.getOperand(2);
1320 SDLoc dl(Op);
1321
1322 // Store the value to a temporary stack slot, then LOAD the returned part.
1323 EVT VecVT = Vec.getValueType();
1324 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1325 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1326 MachinePointerInfo PtrInfo =
1327 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1328
1329 // First store the whole vector.
1330 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1331
1332 // Then store the inserted part.
1333 SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1334
1335 // Store the subvector.
1336 Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
1337
1338 // Finally, load the updated vector.
1339 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1340}
1341
1342SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1343 // We can't handle this case efficiently. Allocate a sufficiently
1344 // aligned object on the stack, store each element into it, then load
1345 // the result as a vector.
1346 // Create the stack frame object.
1347 EVT VT = Node->getValueType(0);
1348 EVT EltVT = VT.getVectorElementType();
1349 SDLoc dl(Node);
1350 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1351 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1352 MachinePointerInfo PtrInfo =
1353 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1354
1355 // Emit a store of each element to the stack slot.
1356 SmallVector<SDValue, 8> Stores;
1357 unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1358 // Store (in the right endianness) the elements to memory.
1359 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1360 // Ignore undef elements.
1361 if (Node->getOperand(i).isUndef()) continue;
1362
1363 unsigned Offset = TypeByteSize*i;
1364
1365 SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1366 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1367
1368 // If the destination vector element type is narrower than the source
1369 // element type, only store the bits necessary.
1370 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1371 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1372 Node->getOperand(i), Idx,
1373 PtrInfo.getWithOffset(Offset), EltVT));
1374 } else
1375 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1376 Idx, PtrInfo.getWithOffset(Offset)));
1377 }
1378
1379 SDValue StoreChain;
1380 if (!Stores.empty()) // Not all undef elements?
1381 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1382 else
1383 StoreChain = DAG.getEntryNode();
1384
1385 // Result is a load from the stack slot.
1386 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1387}
1388
1389/// Bitcast a floating-point value to an integer value. Only bitcast the part
1390/// containing the sign bit if the target has no integer value capable of
1391/// holding all bits of the floating-point value.
1392void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1393 const SDLoc &DL,
1394 SDValue Value) const {
1395 EVT FloatVT = Value.getValueType();
1396 unsigned NumBits = FloatVT.getSizeInBits();
1397 State.FloatVT = FloatVT;
1398 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1399 // Convert to an integer of the same size.
1400 if (TLI.isTypeLegal(IVT)) {
1401 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1402 State.SignMask = APInt::getSignMask(NumBits);
1403 State.SignBit = NumBits - 1;
1404 return;
1405 }
1406
1407 auto &DataLayout = DAG.getDataLayout();
1408 // Store the float to memory, then load the sign part out as an integer.
1409 MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1410 // First create a temporary that is aligned for both the load and store.
1411 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1412 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1413 // Then store the float to it.
1414 State.FloatPtr = StackPtr;
1415 MachineFunction &MF = DAG.getMachineFunction();
1416 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1417 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1418 State.FloatPointerInfo);
1419
1420 SDValue IntPtr;
1421 if (DataLayout.isBigEndian()) {
1422 assert(FloatVT.isByteSized() && "Unsupported floating point type!")(static_cast <bool> (FloatVT.isByteSized() && "Unsupported floating point type!"
) ? void (0) : __assert_fail ("FloatVT.isByteSized() && \"Unsupported floating point type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1422, __extension__ __PRETTY_FUNCTION__))
;
1423 // Load out a legal integer with the same sign bit as the float.
1424 IntPtr = StackPtr;
1425 State.IntPointerInfo = State.FloatPointerInfo;
1426 } else {
1427 // Advance the pointer so that the loaded byte will contain the sign bit.
1428 unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
1429 IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
1430 DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
1431 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1432 ByteOffset);
1433 }
1434
1435 State.IntPtr = IntPtr;
1436 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1437 State.IntPointerInfo, MVT::i8);
1438 State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
1439 State.SignBit = 7;
1440}
1441
1442/// Replace the integer value produced by getSignAsIntValue() with a new value
1443/// and cast the result back to a floating-point type.
1444SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1445 const SDLoc &DL,
1446 SDValue NewIntValue) const {
1447 if (!State.Chain)
1448 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1449
1450 // Override the part containing the sign bit in the value stored on the stack.
1451 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1452 State.IntPointerInfo, MVT::i8);
1453 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1454 State.FloatPointerInfo);
1455}
1456
1457SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1458 SDLoc DL(Node);
1459 SDValue Mag = Node->getOperand(0);
1460 SDValue Sign = Node->getOperand(1);
1461
1462 // Get sign bit into an integer value.
1463 FloatSignAsInt SignAsInt;
1464 getSignAsIntValue(SignAsInt, DL, Sign);
1465
1466 EVT IntVT = SignAsInt.IntValue.getValueType();
1467 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1468 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1469 SignMask);
1470
1471 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1472 EVT FloatVT = Mag.getValueType();
1473 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1474 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1475 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1476 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1477 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1478 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1479 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1480 }
1481
1482 // Transform Mag value to integer, and clear the sign bit.
1483 FloatSignAsInt MagAsInt;
1484 getSignAsIntValue(MagAsInt, DL, Mag);
1485 EVT MagVT = MagAsInt.IntValue.getValueType();
1486 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1487 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1488 ClearSignMask);
1489
1490 // Get the signbit at the right position for MagAsInt.
1491 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1492 if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
1493 if (ShiftAmount > 0) {
1494 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, IntVT);
1495 SignBit = DAG.getNode(ISD::SRL, DL, IntVT, SignBit, ShiftCnst);
1496 } else if (ShiftAmount < 0) {
1497 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, IntVT);
1498 SignBit = DAG.getNode(ISD::SHL, DL, IntVT, SignBit, ShiftCnst);
1499 }
1500 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1501 } else if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
1502 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1503 if (ShiftAmount > 0) {
1504 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, MagVT);
1505 SignBit = DAG.getNode(ISD::SRL, DL, MagVT, SignBit, ShiftCnst);
1506 } else if (ShiftAmount < 0) {
1507 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, MagVT);
1508 SignBit = DAG.getNode(ISD::SHL, DL, MagVT, SignBit, ShiftCnst);
1509 }
1510 }
1511
1512 // Store the part with the modified sign and convert back to float.
1513 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1514 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1515}
1516
1517SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1518 SDLoc DL(Node);
1519 SDValue Value = Node->getOperand(0);
1520
1521 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1522 EVT FloatVT = Value.getValueType();
1523 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1524 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1525 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1526 }
1527
1528 // Transform value to integer, clear the sign bit and transform back.
1529 FloatSignAsInt ValueAsInt;
1530 getSignAsIntValue(ValueAsInt, DL, Value);
1531 EVT IntVT = ValueAsInt.IntValue.getValueType();
1532 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1533 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1534 ClearSignMask);
1535 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1536}
1537
1538void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1539 SmallVectorImpl<SDValue> &Results) {
1540 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1541 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"(static_cast <bool> (SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? void (0) :
__assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1542, __extension__ __PRETTY_FUNCTION__))
1542 " not tell us which reg is the stack pointer!")(static_cast <bool> (SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? void (0) :
__assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1542, __extension__ __PRETTY_FUNCTION__))
;
1543 SDLoc dl(Node);
1544 EVT VT = Node->getValueType(0);
1545 SDValue Tmp1 = SDValue(Node, 0);
1546 SDValue Tmp2 = SDValue(Node, 1);
1547 SDValue Tmp3 = Node->getOperand(2);
1548 SDValue Chain = Tmp1.getOperand(0);
1549
1550 // Chain the dynamic stack allocation so that it doesn't modify the stack
1551 // pointer when other instructions are using the stack.
1552 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1553
1554 SDValue Size = Tmp2.getOperand(1);
1555 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1556 Chain = SP.getValue(1);
1557 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1558 unsigned StackAlign =
1559 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1560 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1561 if (Align > StackAlign)
1562 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1563 DAG.getConstant(-(uint64_t)Align, dl, VT));
1564 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1565
1566 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1567 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1568
1569 Results.push_back(Tmp1);
1570 Results.push_back(Tmp2);
1571}
1572
1573/// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1574/// target.
1575///
1576/// If the SETCC has been legalized using AND / OR, then the legalized node
1577/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1578/// will be set to false.
1579///
1580/// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1581/// then the values of LHS and RHS will be swapped, CC will be set to the
1582/// new condition, and NeedInvert will be set to false.
1583///
1584/// If the SETCC has been legalized using the inverse condcode, then LHS and
1585/// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1586/// will be set to true. The caller must invert the result of the SETCC with
1587/// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1588/// of a true/false result.
1589///
1590/// \returns true if the SetCC has been legalized, false if it hasn't.
1591bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1592 SDValue &RHS, SDValue &CC,
1593 bool &NeedInvert,
1594 const SDLoc &dl) {
1595 MVT OpVT = LHS.getSimpleValueType();
1596 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1597 NeedInvert = false;
1598 bool NeedSwap = false;
1599 switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1600 default: llvm_unreachable("Unknown condition code action!")::llvm::llvm_unreachable_internal("Unknown condition code action!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1600)
;
1601 case TargetLowering::Legal:
1602 // Nothing to do.
1603 break;
1604 case TargetLowering::Expand: {
1605 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
1606 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1607 std::swap(LHS, RHS);
1608 CC = DAG.getCondCode(InvCC);
1609 return true;
1610 }
1611 // Swapping operands didn't work. Try inverting the condition.
1612 InvCC = getSetCCInverse(CCCode, OpVT.isInteger());
1613 if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1614 // If inverting the condition is not enough, try swapping operands
1615 // on top of it.
1616 InvCC = ISD::getSetCCSwappedOperands(InvCC);
1617 NeedSwap = true;
1618 }
1619 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1620 CC = DAG.getCondCode(InvCC);
1621 NeedInvert = true;
1622 if (NeedSwap)
1623 std::swap(LHS, RHS);
1624 return true;
1625 }
1626
1627 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1628 unsigned Opc = 0;
1629 switch (CCCode) {
1630 default: llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1630)
;
1631 case ISD::SETO:
1632 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT)(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETOEQ, OpVT
) && "If SETO is expanded, SETOEQ must be legal!") ? void
(0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1633, __extension__ __PRETTY_FUNCTION__))
1633 && "If SETO is expanded, SETOEQ must be legal!")(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETOEQ, OpVT
) && "If SETO is expanded, SETOEQ must be legal!") ? void
(0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1633, __extension__ __PRETTY_FUNCTION__))
;
1634 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1635 case ISD::SETUO:
1636 assert(TLI.isCondCodeLegal(ISD::SETUNE, OpVT)(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETUNE, OpVT
) && "If SETUO is expanded, SETUNE must be legal!") ?
void (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1637, __extension__ __PRETTY_FUNCTION__))
1637 && "If SETUO is expanded, SETUNE must be legal!")(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETUNE, OpVT
) && "If SETUO is expanded, SETUNE must be legal!") ?
void (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1637, __extension__ __PRETTY_FUNCTION__))
;
1638 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break;
1639 case ISD::SETOEQ:
1640 case ISD::SETOGT:
1641 case ISD::SETOGE:
1642 case ISD::SETOLT:
1643 case ISD::SETOLE:
1644 case ISD::SETONE:
1645 case ISD::SETUEQ:
1646 case ISD::SETUNE:
1647 case ISD::SETUGT:
1648 case ISD::SETUGE:
1649 case ISD::SETULT:
1650 case ISD::SETULE:
1651 // If we are floating point, assign and break, otherwise fall through.
1652 if (!OpVT.isInteger()) {
1653 // We can use the 4th bit to tell if we are the unordered
1654 // or ordered version of the opcode.
1655 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
1656 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
1657 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1658 break;
1659 }
1660 // Fallthrough if we are unsigned integer.
1661 LLVM_FALLTHROUGH[[clang::fallthrough]];
1662 case ISD::SETLE:
1663 case ISD::SETGT:
1664 case ISD::SETGE:
1665 case ISD::SETLT:
1666 case ISD::SETNE:
1667 case ISD::SETEQ:
1668 // If all combinations of inverting the condition and swapping operands
1669 // didn't work then we have no means to expand the condition.
1670 llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1670)
;
1671 }
1672
1673 SDValue SetCC1, SetCC2;
1674 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
1675 // If we aren't the ordered or unorder operation,
1676 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1677 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1678 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1679 } else {
1680 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1681 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1682 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1683 }
1684 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1685 RHS = SDValue();
1686 CC = SDValue();
1687 return true;
1688 }
1689 }
1690 return false;
1691}
1692
1693/// Emit a store/load combination to the stack. This stores
1694/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1695/// a load from the stack slot to DestVT, extending it if needed.
1696/// The resultant code need not be legal.
1697SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1698 EVT DestVT, const SDLoc &dl) {
1699 // Create the stack frame object.
1700 unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1701 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1702 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1703
1704 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1705 int SPFI = StackPtrFI->getIndex();
1706 MachinePointerInfo PtrInfo =
1707 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1708
1709 unsigned SrcSize = SrcOp.getValueSizeInBits();
1710 unsigned SlotSize = SlotVT.getSizeInBits();
1711 unsigned DestSize = DestVT.getSizeInBits();
1712 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1713 unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1714
1715 // Emit a store to the stack slot. Use a truncstore if the input value is
1716 // later than DestVT.
1717 SDValue Store;
1718
1719 if (SrcSize > SlotSize)
1720 Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
1721 SlotVT, SrcAlign);
1722 else {
1723 assert(SrcSize == SlotSize && "Invalid store")(static_cast <bool> (SrcSize == SlotSize && "Invalid store"
) ? void (0) : __assert_fail ("SrcSize == SlotSize && \"Invalid store\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1723, __extension__ __PRETTY_FUNCTION__))
;
1724 Store =
1725 DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1726 }
1727
1728 // Result is a load from the stack slot.
1729 if (SlotSize == DestSize)
1730 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1731
1732 assert(SlotSize < DestSize && "Unknown extension!")(static_cast <bool> (SlotSize < DestSize && "Unknown extension!"
) ? void (0) : __assert_fail ("SlotSize < DestSize && \"Unknown extension!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1732, __extension__ __PRETTY_FUNCTION__))
;
1733 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1734 DestAlign);
1735}
1736
1737SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1738 SDLoc dl(Node);
1739 // Create a vector sized/aligned stack slot, store the value to element #0,
1740 // then load the whole vector back out.
1741 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1742
1743 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1744 int SPFI = StackPtrFI->getIndex();
1745
1746 SDValue Ch = DAG.getTruncStore(
1747 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1748 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1749 Node->getValueType(0).getVectorElementType());
1750 return DAG.getLoad(
1751 Node->getValueType(0), dl, Ch, StackPtr,
1752 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1753}
1754
1755static bool
1756ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1757 const TargetLowering &TLI, SDValue &Res) {
1758 unsigned NumElems = Node->getNumOperands();
1759 SDLoc dl(Node);
1760 EVT VT = Node->getValueType(0);
1761
1762 // Try to group the scalars into pairs, shuffle the pairs together, then
1763 // shuffle the pairs of pairs together, etc. until the vector has
1764 // been built. This will work only if all of the necessary shuffle masks
1765 // are legal.
1766
1767 // We do this in two phases; first to check the legality of the shuffles,
1768 // and next, assuming that all shuffles are legal, to create the new nodes.
1769 for (int Phase = 0; Phase < 2; ++Phase) {
1770 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1771 NewIntermedVals;
1772 for (unsigned i = 0; i < NumElems; ++i) {
1773 SDValue V = Node->getOperand(i);
1774 if (V.isUndef())
1775 continue;
1776
1777 SDValue Vec;
1778 if (Phase)
1779 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1780 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1781 }
1782
1783 while (IntermedVals.size() > 2) {
1784 NewIntermedVals.clear();
1785 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1786 // This vector and the next vector are shuffled together (simply to
1787 // append the one to the other).
1788 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1789
1790 SmallVector<int, 16> FinalIndices;
1791 FinalIndices.reserve(IntermedVals[i].second.size() +
1792 IntermedVals[i+1].second.size());
1793
1794 int k = 0;
1795 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1796 ++j, ++k) {
1797 ShuffleVec[k] = j;
1798 FinalIndices.push_back(IntermedVals[i].second[j]);
1799 }
1800 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1801 ++j, ++k) {
1802 ShuffleVec[k] = NumElems + j;
1803 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1804 }
1805
1806 SDValue Shuffle;
1807 if (Phase)
1808 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1809 IntermedVals[i+1].first,
1810 ShuffleVec);
1811 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1812 return false;
1813 NewIntermedVals.push_back(
1814 std::make_pair(Shuffle, std::move(FinalIndices)));
1815 }
1816
1817 // If we had an odd number of defined values, then append the last
1818 // element to the array of new vectors.
1819 if ((IntermedVals.size() & 1) != 0)
1820 NewIntermedVals.push_back(IntermedVals.back());
1821
1822 IntermedVals.swap(NewIntermedVals);
1823 }
1824
1825 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&(static_cast <bool> (IntermedVals.size() <= 2 &&
IntermedVals.size() > 0 && "Invalid number of intermediate vectors"
) ? void (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1826, __extension__ __PRETTY_FUNCTION__))
1826 "Invalid number of intermediate vectors")(static_cast <bool> (IntermedVals.size() <= 2 &&
IntermedVals.size() > 0 && "Invalid number of intermediate vectors"
) ? void (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1826, __extension__ __PRETTY_FUNCTION__))
;
1827 SDValue Vec1 = IntermedVals[0].first;
1828 SDValue Vec2;
1829 if (IntermedVals.size() > 1)
1830 Vec2 = IntermedVals[1].first;
1831 else if (Phase)
1832 Vec2 = DAG.getUNDEF(VT);
1833
1834 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1835 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1836 ShuffleVec[IntermedVals[0].second[i]] = i;
1837 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1838 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1839
1840 if (Phase)
1841 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1842 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1843 return false;
1844 }
1845
1846 return true;
1847}
1848
1849/// Expand a BUILD_VECTOR node on targets that don't
1850/// support the operation, but do support the resultant vector type.
1851SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1852 unsigned NumElems = Node->getNumOperands();
1853 SDValue Value1, Value2;
1854 SDLoc dl(Node);
1855 EVT VT = Node->getValueType(0);
1856 EVT OpVT = Node->getOperand(0).getValueType();
1857 EVT EltVT = VT.getVectorElementType();
1858
1859 // If the only non-undef value is the low element, turn this into a
1860 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1861 bool isOnlyLowElement = true;
1862 bool MoreThanTwoValues = false;
1863 bool isConstant = true;
1864 for (unsigned i = 0; i < NumElems; ++i) {
1865 SDValue V = Node->getOperand(i);
1866 if (V.isUndef())
1867 continue;
1868 if (i > 0)
1869 isOnlyLowElement = false;
1870 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1871 isConstant = false;
1872
1873 if (!Value1.getNode()) {
1874 Value1 = V;
1875 } else if (!Value2.getNode()) {
1876 if (V != Value1)
1877 Value2 = V;
1878 } else if (V != Value1 && V != Value2) {
1879 MoreThanTwoValues = true;
1880 }
1881 }
1882
1883 if (!Value1.getNode())
1884 return DAG.getUNDEF(VT);
1885
1886 if (isOnlyLowElement)
1887 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1888
1889 // If all elements are constants, create a load from the constant pool.
1890 if (isConstant) {
1891 SmallVector<Constant*, 16> CV;
1892 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1893 if (ConstantFPSDNode *V =
1894 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1895 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1896 } else if (ConstantSDNode *V =
1897 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1898 if (OpVT==EltVT)
1899 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1900 else {
1901 // If OpVT and EltVT don't match, EltVT is not legal and the
1902 // element values have been promoted/truncated earlier. Undo this;
1903 // we don't want a v16i8 to become a v16i32 for example.
1904 const ConstantInt *CI = V->getConstantIntValue();
1905 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1906 CI->getZExtValue()));
1907 }
1908 } else {
1909 assert(Node->getOperand(i).isUndef())(static_cast <bool> (Node->getOperand(i).isUndef()) ?
void (0) : __assert_fail ("Node->getOperand(i).isUndef()"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1909, __extension__ __PRETTY_FUNCTION__))
;
1910 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1911 CV.push_back(UndefValue::get(OpNTy));
1912 }
1913 }
1914 Constant *CP = ConstantVector::get(CV);
1915 SDValue CPIdx =
1916 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1917 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1918 return DAG.getLoad(
1919 VT, dl, DAG.getEntryNode(), CPIdx,
1920 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1921 Alignment);
1922 }
1923
1924 SmallSet<SDValue, 16> DefinedValues;
1925 for (unsigned i = 0; i < NumElems; ++i) {
1926 if (Node->getOperand(i).isUndef())
1927 continue;
1928 DefinedValues.insert(Node->getOperand(i));
1929 }
1930
1931 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1932 if (!MoreThanTwoValues) {
1933 SmallVector<int, 8> ShuffleVec(NumElems, -1);
1934 for (unsigned i = 0; i < NumElems; ++i) {
1935 SDValue V = Node->getOperand(i);
1936 if (V.isUndef())
1937 continue;
1938 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1939 }
1940 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1941 // Get the splatted value into the low element of a vector register.
1942 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1943 SDValue Vec2;
1944 if (Value2.getNode())
1945 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1946 else
1947 Vec2 = DAG.getUNDEF(VT);
1948
1949 // Return shuffle(LowValVec, undef, <0,0,0,0>)
1950 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1951 }
1952 } else {
1953 SDValue Res;
1954 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1955 return Res;
1956 }
1957 }
1958
1959 // Otherwise, we can't handle this case efficiently.
1960 return ExpandVectorBuildThroughStack(Node);
1961}
1962
1963// Expand a node into a call to a libcall. If the result value
1964// does not fit into a register, return the lo part and set the hi part to the
1965// by-reg argument. If it does fit into a single register, return the result
1966// and leave the Hi part unset.
1967SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
1968 bool isSigned) {
1969 TargetLowering::ArgListTy Args;
1970 TargetLowering::ArgListEntry Entry;
1971 for (const SDValue &Op : Node->op_values()) {
1972 EVT ArgVT = Op.getValueType();
1973 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1974 Entry.Node = Op;
1975 Entry.Ty = ArgTy;
1976 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
1977 Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
1978 Args.push_back(Entry);
1979 }
1980 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
1981 TLI.getPointerTy(DAG.getDataLayout()));
1982
1983 EVT RetVT = Node->getValueType(0);
1984 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
1985
1986 // By default, the input chain to this libcall is the entry node of the
1987 // function. If the libcall is going to be emitted as a tail call then
1988 // TLI.isUsedByReturnOnly will change it to the right chain if the return
1989 // node which is being folded has a non-entry input chain.
1990 SDValue InChain = DAG.getEntryNode();
1991
1992 // isTailCall may be true since the callee does not reference caller stack
1993 // frame. Check if it's in the right position and that the return types match.
1994 SDValue TCChain = InChain;
1995 const Function &F = DAG.getMachineFunction().getFunction();
1996 bool isTailCall =
1997 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
1998 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
1999 if (isTailCall)
2000 InChain = TCChain;
2001
2002 TargetLowering::CallLoweringInfo CLI(DAG);
2003 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2004 CLI.setDebugLoc(SDLoc(Node))
2005 .setChain(InChain)
2006 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2007 std::move(Args))
2008 .setTailCall(isTailCall)
2009 .setSExtResult(signExtend)
2010 .setZExtResult(!signExtend)
2011 .setIsPostTypeLegalization(true);
2012
2013 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2014
2015 if (!CallInfo.second.getNode()) {
2016 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Created tailcall: "; DAG.
getRoot().dump(); } } while (false)
;
2017 // It's a tailcall, return the chain (which is the DAG root).
2018 return DAG.getRoot();
2019 }
2020
2021 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Created libcall: "; CallInfo
.first.dump(); } } while (false)
;
2022 return CallInfo.first;
2023}
2024
2025/// Generate a libcall taking the given operands as arguments
2026/// and returning a result of type RetVT.
2027SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
2028 const SDValue *Ops, unsigned NumOps,
2029 bool isSigned, const SDLoc &dl) {
2030 TargetLowering::ArgListTy Args;
2031 Args.reserve(NumOps);
2032
2033 TargetLowering::ArgListEntry Entry;
2034 for (unsigned i = 0; i != NumOps; ++i) {
2035 Entry.Node = Ops[i];
2036 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
2037 Entry.IsSExt = isSigned;
2038 Entry.IsZExt = !isSigned;
2039 Args.push_back(Entry);
2040 }
2041 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2042 TLI.getPointerTy(DAG.getDataLayout()));
2043
2044 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2045
2046 TargetLowering::CallLoweringInfo CLI(DAG);
2047 CLI.setDebugLoc(dl)
2048 .setChain(DAG.getEntryNode())
2049 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2050 std::move(Args))
2051 .setSExtResult(isSigned)
2052 .setZExtResult(!isSigned)
2053 .setIsPostTypeLegalization(true);
2054
2055 std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
2056
2057 return CallInfo.first;
2058}
2059
2060// Expand a node into a call to a libcall. Similar to
2061// ExpandLibCall except that the first operand is the in-chain.
2062std::pair<SDValue, SDValue>
2063SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2064 SDNode *Node,
2065 bool isSigned) {
2066 SDValue InChain = Node->getOperand(0);
2067
2068 TargetLowering::ArgListTy Args;
2069 TargetLowering::ArgListEntry Entry;
2070 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
2071 EVT ArgVT = Node->getOperand(i).getValueType();
2072 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2073 Entry.Node = Node->getOperand(i);
2074 Entry.Ty = ArgTy;
2075 Entry.IsSExt = isSigned;
2076 Entry.IsZExt = !isSigned;
2077 Args.push_back(Entry);
2078 }
2079 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2080 TLI.getPointerTy(DAG.getDataLayout()));
2081
2082 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2083
2084 TargetLowering::CallLoweringInfo CLI(DAG);
2085 CLI.setDebugLoc(SDLoc(Node))
2086 .setChain(InChain)
2087 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2088 std::move(Args))
2089 .setSExtResult(isSigned)
2090 .setZExtResult(!isSigned);
2091
2092 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2093
2094 return CallInfo;
2095}
2096
2097SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2098 RTLIB::Libcall Call_F32,
2099 RTLIB::Libcall Call_F64,
2100 RTLIB::Libcall Call_F80,
2101 RTLIB::Libcall Call_F128,
2102 RTLIB::Libcall Call_PPCF128) {
2103 if (Node->isStrictFPOpcode())
2104 Node = DAG.mutateStrictFPToFP(Node);
2105
2106 RTLIB::Libcall LC;
2107 switch (Node->getSimpleValueType(0).SimpleTy) {
2108 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2108)
;
2109 case MVT::f32: LC = Call_F32; break;
2110 case MVT::f64: LC = Call_F64; break;
2111 case MVT::f80: LC = Call_F80; break;
2112 case MVT::f128: LC = Call_F128; break;
2113 case MVT::ppcf128: LC = Call_PPCF128; break;
2114 }
2115 return ExpandLibCall(LC, Node, false);
2116}
2117
2118SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2119 RTLIB::Libcall Call_I8,
2120 RTLIB::Libcall Call_I16,
2121 RTLIB::Libcall Call_I32,
2122 RTLIB::Libcall Call_I64,
2123 RTLIB::Libcall Call_I128) {
2124 RTLIB::Libcall LC;
2125 switch (Node->getSimpleValueType(0).SimpleTy) {
2126 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2126)
;
2127 case MVT::i8: LC = Call_I8; break;
2128 case MVT::i16: LC = Call_I16; break;
2129 case MVT::i32: LC = Call_I32; break;
2130 case MVT::i64: LC = Call_I64; break;
2131 case MVT::i128: LC = Call_I128; break;
2132 }
2133 return ExpandLibCall(LC, Node, isSigned);
2134}
2135
2136/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2137void
2138SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2139 SmallVectorImpl<SDValue> &Results) {
2140 unsigned Opcode = Node->getOpcode();
2141 bool isSigned = Opcode == ISD::SDIVREM;
2142
2143 RTLIB::Libcall LC;
2144 switch (Node->getSimpleValueType(0).SimpleTy) {
2145 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2145)
;
2146 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2147 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2148 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2149 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2150 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2151 }
2152
2153 // The input chain to this libcall is the entry node of the function.
2154 // Legalizing the call will automatically add the previous call to the
2155 // dependence.
2156 SDValue InChain = DAG.getEntryNode();
2157
2158 EVT RetVT = Node->getValueType(0);
2159 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2160
2161 TargetLowering::ArgListTy Args;
2162 TargetLowering::ArgListEntry Entry;
2163 for (const SDValue &Op : Node->op_values()) {
2164 EVT ArgVT = Op.getValueType();
2165 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2166 Entry.Node = Op;
2167 Entry.Ty = ArgTy;
2168 Entry.IsSExt = isSigned;
2169 Entry.IsZExt = !isSigned;
2170 Args.push_back(Entry);
2171 }
2172
2173 // Also pass the return address of the remainder.
2174 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2175 Entry.Node = FIPtr;
2176 Entry.Ty = RetTy->getPointerTo();
2177 Entry.IsSExt = isSigned;
2178 Entry.IsZExt = !isSigned;
2179 Args.push_back(Entry);
2180
2181 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2182 TLI.getPointerTy(DAG.getDataLayout()));
2183
2184 SDLoc dl(Node);
2185 TargetLowering::CallLoweringInfo CLI(DAG);
2186 CLI.setDebugLoc(dl)
2187 .setChain(InChain)
2188 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2189 std::move(Args))
2190 .setSExtResult(isSigned)
2191 .setZExtResult(!isSigned);
2192
2193 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2194
2195 // Remainder is loaded back from the stack frame.
2196 SDValue Rem =
2197 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2198 Results.push_back(CallInfo.first);
2199 Results.push_back(Rem);
2200}
2201
2202/// Return true if sincos libcall is available.
2203static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2204 RTLIB::Libcall LC;
2205 switch (Node->getSimpleValueType(0).SimpleTy) {
2206 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2206)
;
2207 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2208 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2209 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2210 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2211 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2212 }
2213 return TLI.getLibcallName(LC) != nullptr;
2214}
2215
2216/// Only issue sincos libcall if both sin and cos are needed.
2217static bool useSinCos(SDNode *Node) {
2218 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2219 ? ISD::FCOS : ISD::FSIN;
2220
2221 SDValue Op0 = Node->getOperand(0);
2222 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2223 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2224 SDNode *User = *UI;
2225 if (User == Node)
2226 continue;
2227 // The other user might have been turned into sincos already.
2228 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2229 return true;
2230 }
2231 return false;
2232}
2233
2234/// Issue libcalls to sincos to compute sin / cos pairs.
2235void
2236SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2237 SmallVectorImpl<SDValue> &Results) {
2238 RTLIB::Libcall LC;
2239 switch (Node->getSimpleValueType(0).SimpleTy) {
2240 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2240)
;
2241 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2242 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2243 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2244 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2245 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2246 }
2247
2248 // The input chain to this libcall is the entry node of the function.
2249 // Legalizing the call will automatically add the previous call to the
2250 // dependence.
2251 SDValue InChain = DAG.getEntryNode();
2252
2253 EVT RetVT = Node->getValueType(0);
2254 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2255
2256 TargetLowering::ArgListTy Args;
2257 TargetLowering::ArgListEntry Entry;
2258
2259 // Pass the argument.
2260 Entry.Node = Node->getOperand(0);
2261 Entry.Ty = RetTy;
2262 Entry.IsSExt = false;
2263 Entry.IsZExt = false;
2264 Args.push_back(Entry);
2265
2266 // Pass the return address of sin.
2267 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2268 Entry.Node = SinPtr;
2269 Entry.Ty = RetTy->getPointerTo();
2270 Entry.IsSExt = false;
2271 Entry.IsZExt = false;
2272 Args.push_back(Entry);
2273
2274 // Also pass the return address of the cos.
2275 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2276 Entry.Node = CosPtr;
2277 Entry.Ty = RetTy->getPointerTo();
2278 Entry.IsSExt = false;
2279 Entry.IsZExt = false;
2280 Args.push_back(Entry);
2281
2282 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2283 TLI.getPointerTy(DAG.getDataLayout()));
2284
2285 SDLoc dl(Node);
2286 TargetLowering::CallLoweringInfo CLI(DAG);
2287 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2288 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2289 std::move(Args));
2290
2291 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2292
2293 Results.push_back(
2294 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2295 Results.push_back(
2296 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2297}
2298
2299/// This function is responsible for legalizing a
2300/// INT_TO_FP operation of the specified operand when the target requests that
2301/// we expand it. At this point, we know that the result and operand types are
2302/// legal for the target.
2303SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2304 EVT DestVT,
2305 const SDLoc &dl) {
2306 // TODO: Should any fast-math-flags be set for the created nodes?
2307 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing INT_TO_FP\n"; }
} while (false)
;
2308 if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
2309 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double "
"expansion\n"; } } while (false)
2310 "expansion\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double "
"expansion\n"; } } while (false)
;
2311
2312 // Get the stack frame index of a 8 byte buffer.
2313 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2314
2315 // word offset constant for Hi/Lo address computation
2316 SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2317 StackSlot.getValueType());
2318 // set up Hi and Lo (into buffer) address based on endian
2319 SDValue Hi = StackSlot;
2320 SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2321 StackSlot, WordOff);
2322 if (DAG.getDataLayout().isLittleEndian())
2323 std::swap(Hi, Lo);
2324
2325 // if signed map to unsigned space
2326 SDValue Op0Mapped;
2327 if (isSigned) {
2328 // constant used to invert sign bit (signed to unsigned mapping)
2329 SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2330 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2331 } else {
2332 Op0Mapped = Op0;
2333 }
2334 // store the lo of the constructed double - based on integer input
2335 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2336 MachinePointerInfo());
2337 // initial hi portion of constructed double
2338 SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2339 // store the hi of the constructed double - biased exponent
2340 SDValue Store2 =
2341 DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
2342 // load the constructed double
2343 SDValue Load =
2344 DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
2345 // FP constant to bias correct the final result
2346 SDValue Bias = DAG.getConstantFP(isSigned ?
2347 BitsToDouble(0x4330000080000000ULL) :
2348 BitsToDouble(0x4330000000000000ULL),
2349 dl, MVT::f64);
2350 // subtract the bias
2351 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2352 // final result
2353 SDValue Result;
2354 // handle final rounding
2355 if (DestVT == MVT::f64) {
2356 // do nothing
2357 Result = Sub;
2358 } else if (DestVT.bitsLT(MVT::f64)) {
2359 Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
2360 DAG.getIntPtrConstant(0, dl));
2361 } else if (DestVT.bitsGT(MVT::f64)) {
2362 Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
2363 }
2364 return Result;
2365 }
2366 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet")(static_cast <bool> (!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"
) ? void (0) : __assert_fail ("!isSigned && \"Legalize cannot Expand SINT_TO_FP for i64 yet\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2366, __extension__ __PRETTY_FUNCTION__))
;
2367 // Code below here assumes !isSigned without checking again.
2368
2369 // Implementation of unsigned i64 to f64 following the algorithm in
2370 // __floatundidf in compiler_rt. This implementation has the advantage
2371 // of performing rounding correctly, both in the default rounding mode
2372 // and in all alternate rounding modes.
2373 // TODO: Generalize this for use with other types.
2374 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) {
2375 LLVM_DEBUG(dbgs() << "Converting unsigned i64 to f64\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Converting unsigned i64 to f64\n"
; } } while (false)
;
2376 SDValue TwoP52 =
2377 DAG.getConstant(UINT64_C(0x4330000000000000)0x4330000000000000UL, dl, MVT::i64);
2378 SDValue TwoP84PlusTwoP52 =
2379 DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)0x4530000000100000UL), dl,
2380 MVT::f64);
2381 SDValue TwoP84 =
2382 DAG.getConstant(UINT64_C(0x4530000000000000)0x4530000000000000UL, dl, MVT::i64);
2383
2384 SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32);
2385 SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
2386 DAG.getConstant(32, dl, MVT::i64));
2387 SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
2388 SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
2389 SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
2390 SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr);
2391 SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt,
2392 TwoP84PlusTwoP52);
2393 return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub);
2394 }
2395
2396 // TODO: Generalize this for use with other types.
2397 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) {
2398 LLVM_DEBUG(dbgs() << "Converting unsigned i64 to f32\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Converting unsigned i64 to f32\n"
; } } while (false)
;
2399 // For unsigned conversions, convert them to signed conversions using the
2400 // algorithm from the x86_64 __floatundidf in compiler_rt.
2401 if (!isSigned) {
2402 SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
2403
2404 SDValue ShiftConst = DAG.getConstant(
2405 1, dl, TLI.getShiftAmountTy(Op0.getValueType(), DAG.getDataLayout()));
2406 SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
2407 SDValue AndConst = DAG.getConstant(1, dl, MVT::i64);
2408 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
2409 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
2410
2411 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
2412 SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
2413
2414 // TODO: This really should be implemented using a branch rather than a
2415 // select. We happen to get lucky and machinesink does the right
2416 // thing most of the time. This would be a good candidate for a
2417 //pseudo-op, or, even better, for whole-function isel.
2418 SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
2419 Op0, DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
2420 return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast);
2421 }
2422
2423 // Otherwise, implement the fully general conversion.
2424
2425 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2426 DAG.getConstant(UINT64_C(0xfffffffffffff800)0xfffffffffffff800UL, dl, MVT::i64));
2427 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
2428 DAG.getConstant(UINT64_C(0x800)0x800UL, dl, MVT::i64));
2429 SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2430 DAG.getConstant(UINT64_C(0x7ff)0x7ffUL, dl, MVT::i64));
2431 SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), And2,
2432 DAG.getConstant(UINT64_C(0)0UL, dl, MVT::i64),
2433 ISD::SETNE);
2434 SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0);
2435 SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), Op0,
2436 DAG.getConstant(UINT64_C(0x0020000000000000)0x0020000000000000UL, dl,
2437 MVT::i64),
2438 ISD::SETUGE);
2439 SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0);
2440 EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType(), DAG.getDataLayout());
2441
2442 SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
2443 DAG.getConstant(32, dl, SHVT));
2444 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
2445 SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
2446 SDValue TwoP32 =
2447 DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)0x41f0000000000000UL), dl,
2448 MVT::f64);
2449 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
2450 SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
2451 SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
2452 SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
2453 return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
2454 DAG.getIntPtrConstant(0, dl));
2455 }
2456
2457 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2458
2459 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()),
2460 Op0,
2461 DAG.getConstant(0, dl, Op0.getValueType()),
2462 ISD::SETLT);
2463 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2464 Four = DAG.getIntPtrConstant(4, dl);
2465 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2466 SignSet, Four, Zero);
2467
2468 // If the sign bit of the integer is set, the large number will be treated
2469 // as a negative number. To counteract this, the dynamic code adds an
2470 // offset depending on the data type.
2471 uint64_t FF;
2472 switch (Op0.getSimpleValueType().SimpleTy) {
2473 default: llvm_unreachable("Unsupported integer type!")::llvm::llvm_unreachable_internal("Unsupported integer type!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2473)
;
2474 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2475 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2476 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2477 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2478 }
2479 if (DAG.getDataLayout().isLittleEndian())
2480 FF <<= 32;
2481 Constant *FudgeFactor = ConstantInt::get(
2482 Type::getInt64Ty(*DAG.getContext()), FF);
2483
2484 SDValue CPIdx =
2485 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2486 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2487 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2488 Alignment = std::min(Alignment, 4u);
2489 SDValue FudgeInReg;
2490 if (DestVT == MVT::f32)
2491 FudgeInReg = DAG.getLoad(
2492 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2493 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2494 Alignment);
2495 else {
2496 SDValue Load = DAG.getExtLoad(
2497 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2498 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2499 Alignment);
2500 HandleSDNode Handle(Load);
2501 LegalizeOp(Load.getNode());
2502 FudgeInReg = Handle.getValue();
2503 }
2504
2505 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2506}
2507
2508/// This function is responsible for legalizing a
2509/// *INT_TO_FP operation of the specified operand when the target requests that
2510/// we promote it. At this point, we know that the result and operand types are
2511/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2512/// operation that takes a larger input.
2513SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2514 bool isSigned,
2515 const SDLoc &dl) {
2516 // First step, figure out the appropriate *INT_TO_FP operation to use.
2517 EVT NewInTy = LegalOp.getValueType();
2518
2519 unsigned OpToUse = 0;
2520
2521 // Scan for the appropriate larger type to use.
2522 while (true) {
2523 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2524 assert(NewInTy.isInteger() && "Ran out of possibilities!")(static_cast <bool> (NewInTy.isInteger() && "Ran out of possibilities!"
) ? void (0) : __assert_fail ("NewInTy.isInteger() && \"Ran out of possibilities!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2524, __extension__ __PRETTY_FUNCTION__))
;
2525
2526 // If the target supports SINT_TO_FP of this type, use it.
2527 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2528 OpToUse = ISD::SINT_TO_FP;
2529 break;
2530 }
2531 if (isSigned) continue;
2532
2533 // If the target supports UINT_TO_FP of this type, use it.
2534 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2535 OpToUse = ISD::UINT_TO_FP;
2536 break;
2537 }
2538
2539 // Otherwise, try a larger type.
2540 }
2541
2542 // Okay, we found the operation and type to use. Zero extend our input to the
2543 // desired type then run the operation on it.
2544 return DAG.getNode(OpToUse, dl, DestVT,
2545 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2546 dl, NewInTy, LegalOp));
2547}
2548
2549/// This function is responsible for legalizing a
2550/// FP_TO_*INT operation of the specified operand when the target requests that
2551/// we promote it. At this point, we know that the result and operand types are
2552/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2553/// operation that returns a larger result.
2554SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2555 bool isSigned,
2556 const SDLoc &dl) {
2557 // First step, figure out the appropriate FP_TO*INT operation to use.
2558 EVT NewOutTy = DestVT;
2559
2560 unsigned OpToUse = 0;
2561
2562 // Scan for the appropriate larger type to use.
2563 while (true) {
2564 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2565 assert(NewOutTy.isInteger() && "Ran out of possibilities!")(static_cast <bool> (NewOutTy.isInteger() && "Ran out of possibilities!"
) ? void (0) : __assert_fail ("NewOutTy.isInteger() && \"Ran out of possibilities!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2565, __extension__ __PRETTY_FUNCTION__))
;
2566
2567 // A larger signed type can hold all unsigned values of the requested type,
2568 // so using FP_TO_SINT is valid
2569 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2570 OpToUse = ISD::FP_TO_SINT;
2571 break;
2572 }
2573
2574 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2575 if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2576 OpToUse = ISD::FP_TO_UINT;
2577 break;
2578 }
2579
2580 // Otherwise, try a larger type.
2581 }
2582
2583 // Okay, we found the operation and type to use.
2584 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2585
2586 // Truncate the result of the extended FP_TO_*INT operation to the desired
2587 // size.
2588 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2589}
2590
2591/// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2592SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2593 EVT VT = Op.getValueType();
2594 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2595 unsigned Sz = VT.getScalarSizeInBits();
2596
2597 SDValue Tmp, Tmp2, Tmp3;
2598
2599 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2600 // and finally the i1 pairs.
2601 // TODO: We can easily support i4/i2 legal types if any target ever does.
2602 if (Sz >= 8 && isPowerOf2_32(Sz)) {
2603 // Create the masks - repeating the pattern every byte.
2604 APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0);
2605 APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0);
2606 for (unsigned J = 0; J != Sz; J += 8) {
2607 MaskHi4 = MaskHi4 | (0xF0ull << J);
2608 MaskLo4 = MaskLo4 | (0x0Full << J);
2609 MaskHi2 = MaskHi2 | (0xCCull << J);
2610 MaskLo2 = MaskLo2 | (0x33ull << J);
2611 MaskHi1 = MaskHi1 | (0xAAull << J);
2612 MaskLo1 = MaskLo1 | (0x55ull << J);
2613 }
2614
2615 // BSWAP if the type is wider than a single byte.
2616 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
2617
2618 // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2619 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
2620 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
2621 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, VT));
2622 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, VT));
2623 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2624
2625 // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2626 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
2627 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
2628 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, VT));
2629 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, VT));
2630 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2631
2632 // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2633 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
2634 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
2635 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, VT));
2636 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, VT));
2637 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2638 return Tmp;
2639 }
2640
2641 Tmp = DAG.getConstant(0, dl, VT);
2642 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
2643 if (I < J)
2644 Tmp2 =
2645 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
2646 else
2647 Tmp2 =
2648 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
2649
2650 APInt Shift(Sz, 1);
2651 Shift <<= J;
2652 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
2653 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
2654 }
2655
2656 return Tmp;
2657}
2658
2659/// Open code the operations for BSWAP of the specified operation.
2660SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2661 EVT VT = Op.getValueType();
2662 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2663 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2664 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
2665 default: llvm_unreachable("Unhandled Expand type in BSWAP!")::llvm::llvm_unreachable_internal("Unhandled Expand type in BSWAP!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2665)
;
2666 case MVT::i16:
2667 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2668 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2669 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2670 case MVT::i32:
2671 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2672 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2673 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2674 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2675 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2676 DAG.getConstant(0xFF0000, dl, VT));
2677 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2678 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2679 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2680 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2681 case MVT::i64:
2682 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2683 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2684 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2685 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2686 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2687 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2688 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2689 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2690 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2691 DAG.getConstant(255ULL<<48, dl, VT));
2692 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2693 DAG.getConstant(255ULL<<40, dl, VT));
2694 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2695 DAG.getConstant(255ULL<<32, dl, VT));
2696 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2697 DAG.getConstant(255ULL<<24, dl, VT));
2698 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2699 DAG.getConstant(255ULL<<16, dl, VT));
2700 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2701 DAG.getConstant(255ULL<<8 , dl, VT));
2702 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2703 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2704 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2705 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2706 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2707 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2708 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2709 }
2710}
2711
2712/// Expand the specified bitcount instruction into operations.
2713SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
2714 const SDLoc &dl) {
2715 switch (Opc) {
2716 default: llvm_unreachable("Cannot expand this yet!")::llvm::llvm_unreachable_internal("Cannot expand this yet!", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2716)
;
2717 case ISD::CTPOP: {
2718 EVT VT = Op.getValueType();
2719 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2720 unsigned Len = VT.getSizeInBits();
2721
2722 assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 &&(static_cast <bool> (VT.isInteger() && Len <=
128 && Len % 8 == 0 && "CTPOP not implemented for this type."
) ? void (0) : __assert_fail ("VT.isInteger() && Len <= 128 && Len % 8 == 0 && \"CTPOP not implemented for this type.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2723, __extension__ __PRETTY_FUNCTION__))
2723 "CTPOP not implemented for this type.")(static_cast <bool> (VT.isInteger() && Len <=
128 && Len % 8 == 0 && "CTPOP not implemented for this type."
) ? void (0) : __assert_fail ("VT.isInteger() && Len <= 128 && Len % 8 == 0 && \"CTPOP not implemented for this type.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2723, __extension__ __PRETTY_FUNCTION__))
;
2724
2725 // This is the "best" algorithm from
2726 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
2727
2728 SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)),
2729 dl, VT);
2730 SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)),
2731 dl, VT);
2732 SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)),
2733 dl, VT);
2734 SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)),
2735 dl, VT);
2736
2737 // v = v - ((v >> 1) & 0x55555555...)
2738 Op = DAG.getNode(ISD::SUB, dl, VT, Op,
2739 DAG.getNode(ISD::AND, dl, VT,
2740 DAG.getNode(ISD::SRL, dl, VT, Op,
2741 DAG.getConstant(1, dl, ShVT)),
2742 Mask55));
2743 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
2744 Op = DAG.getNode(ISD::ADD, dl, VT,
2745 DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
2746 DAG.getNode(ISD::AND, dl, VT,
2747 DAG.getNode(ISD::SRL, dl, VT, Op,
2748 DAG.getConstant(2, dl, ShVT)),
2749 Mask33));
2750 // v = (v + (v >> 4)) & 0x0F0F0F0F...
2751 Op = DAG.getNode(ISD::AND, dl, VT,
2752 DAG.getNode(ISD::ADD, dl, VT, Op,
2753 DAG.getNode(ISD::SRL, dl, VT, Op,
2754 DAG.getConstant(4, dl, ShVT))),
2755 Mask0F);
2756 // v = (v * 0x01010101...) >> (Len - 8)
2757 Op = DAG.getNode(ISD::SRL, dl, VT,
2758 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
2759 DAG.getConstant(Len - 8, dl, ShVT));
2760
2761 return Op;
2762 }
2763 case ISD::CTLZ_ZERO_UNDEF:
2764 // This trivially expands to CTLZ.
2765 return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op);
2766 case ISD::CTLZ: {
2767 EVT VT = Op.getValueType();
2768 unsigned Len = VT.getSizeInBits();
2769
2770 if (TLI.isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
2771 EVT SetCCVT = getSetCCResultType(VT);
2772 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
2773 SDValue Zero = DAG.getConstant(0, dl, VT);
2774 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
2775 return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
2776 DAG.getConstant(Len, dl, VT), CTLZ);
2777 }
2778
2779 // for now, we do this:
2780 // x = x | (x >> 1);
2781 // x = x | (x >> 2);
2782 // ...
2783 // x = x | (x >>16);
2784 // x = x | (x >>32); // for 64-bit input
2785 // return popcount(~x);
2786 //
2787 // Ref: "Hacker's Delight" by Henry Warren
2788 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2789 for (unsigned i = 0; (1U << i) <= (Len / 2); ++i) {
2790 SDValue Tmp3 = DAG.getConstant(1ULL << i, dl, ShVT);
2791 Op = DAG.getNode(ISD::OR, dl, VT, Op,
2792 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
2793 }
2794 Op = DAG.getNOT(dl, Op, VT);
2795 return DAG.getNode(ISD::CTPOP, dl, VT, Op);
2796 }
2797 case ISD::CTTZ_ZERO_UNDEF:
2798 // This trivially expands to CTTZ.
2799 return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op);
2800 case ISD::CTTZ: {
2801 EVT VT = Op.getValueType();
2802 unsigned Len = VT.getSizeInBits();
2803
2804 if (TLI.isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
2805 EVT SetCCVT = getSetCCResultType(VT);
2806 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
2807 SDValue Zero = DAG.getConstant(0, dl, VT);
2808 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
2809 return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
2810 DAG.getConstant(Len, dl, VT), CTTZ);
2811 }
2812
2813 // for now, we use: { return popcount(~x & (x - 1)); }
2814 // unless the target has ctlz but not ctpop, in which case we use:
2815 // { return 32 - nlz(~x & (x-1)); }
2816 // Ref: "Hacker's Delight" by Henry Warren
2817 SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
2818 DAG.getNOT(dl, Op, VT),
2819 DAG.getNode(ISD::SUB, dl, VT, Op,
2820 DAG.getConstant(1, dl, VT)));
2821 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
2822 if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
2823 TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
2824 return DAG.getNode(ISD::SUB, dl, VT,
2825 DAG.getConstant(VT.getSizeInBits(), dl, VT),
2826 DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
2827 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
2828 }
2829 }
2830}
2831
2832bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2833 LLVM_DEBUG(dbgs() << "Trying to expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to expand node\n";
} } while (false)
;
2834 SmallVector<SDValue, 8> Results;
2835 SDLoc dl(Node);
2836 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2837 bool NeedInvert;
2838 switch (Node->getOpcode()) {
2839 case ISD::CTPOP:
2840 case ISD::CTLZ:
2841 case ISD::CTLZ_ZERO_UNDEF:
2842 case ISD::CTTZ:
2843 case ISD::CTTZ_ZERO_UNDEF:
2844 Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
2845 Results.push_back(Tmp1);
2846 break;
2847 case ISD::BITREVERSE:
2848 Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
2849 break;
2850 case ISD::BSWAP:
2851 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2852 break;
2853 case ISD::FRAMEADDR:
2854 case ISD::RETURNADDR:
2855 case ISD::FRAME_TO_ARGS_OFFSET:
2856 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2857 break;
2858 case ISD::EH_DWARF_CFA: {
2859 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2860 TLI.getPointerTy(DAG.getDataLayout()));
2861 SDValue Offset = DAG.getNode(ISD::ADD, dl,
2862 CfaArg.getValueType(),
2863 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2864 CfaArg.getValueType()),
2865 CfaArg);
2866 SDValue FA = DAG.getNode(
2867 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2868 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2869 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2870 FA, Offset));
2871 break;
2872 }
2873 case ISD::FLT_ROUNDS_:
2874 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2875 break;
2876 case ISD::EH_RETURN:
2877 case ISD::EH_LABEL:
2878 case ISD::PREFETCH:
2879 case ISD::VAEND:
2880 case ISD::EH_SJLJ_LONGJMP:
2881 // If the target didn't expand these, there's nothing to do, so just
2882 // preserve the chain and be done.
2883 Results.push_back(Node->getOperand(0));
2884 break;
2885 case ISD::READCYCLECOUNTER:
2886 // If the target didn't expand this, just return 'zero' and preserve the
2887 // chain.
2888 Results.append(Node->getNumValues() - 1,
2889 DAG.getConstant(0, dl, Node->getValueType(0)));
2890 Results.push_back(Node->getOperand(0));
2891 break;
2892 case ISD::EH_SJLJ_SETJMP:
2893 // If the target didn't expand this, just return 'zero' and preserve the
2894 // chain.
2895 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2896 Results.push_back(Node->getOperand(0));
2897 break;
2898 case ISD::ATOMIC_LOAD: {
2899 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2900 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2901 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2902 SDValue Swap = DAG.getAtomicCmpSwap(
2903 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2904 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2905 cast<AtomicSDNode>(Node)->getMemOperand());
2906 Results.push_back(Swap.getValue(0));
2907 Results.push_back(Swap.getValue(1));
2908 break;
2909 }
2910 case ISD::ATOMIC_STORE: {
2911 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2912 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2913 cast<AtomicSDNode>(Node)->getMemoryVT(),
2914 Node->getOperand(0),
2915 Node->getOperand(1), Node->getOperand(2),
2916 cast<AtomicSDNode>(Node)->getMemOperand());
2917 Results.push_back(Swap.getValue(1));
2918 break;
2919 }
2920 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2921 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2922 // splits out the success value as a comparison. Expanding the resulting
2923 // ATOMIC_CMP_SWAP will produce a libcall.
2924 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2925 SDValue Res = DAG.getAtomicCmpSwap(
2926 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2927 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2928 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2929
2930 SDValue ExtRes = Res;
2931 SDValue LHS = Res;
2932 SDValue RHS = Node->getOperand(1);
2933
2934 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2935 EVT OuterType = Node->getValueType(0);
2936 switch (TLI.getExtendForAtomicOps()) {
2937 case ISD::SIGN_EXTEND:
2938 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2939 DAG.getValueType(AtomicType));
2940 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2941 Node->getOperand(2), DAG.getValueType(AtomicType));
2942 ExtRes = LHS;
2943 break;
2944 case ISD::ZERO_EXTEND:
2945 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2946 DAG.getValueType(AtomicType));
2947 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2948 ExtRes = LHS;
2949 break;
2950 case ISD::ANY_EXTEND:
2951 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2952 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2953 break;
2954 default:
2955 llvm_unreachable("Invalid atomic op extension")::llvm::llvm_unreachable_internal("Invalid atomic op extension"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2955)
;
2956 }
2957
2958 SDValue Success =
2959 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2960
2961 Results.push_back(ExtRes.getValue(0));
2962 Results.push_back(Success);
2963 Results.push_back(Res.getValue(1));
2964 break;
2965 }
2966 case ISD::DYNAMIC_STACKALLOC:
2967 ExpandDYNAMIC_STACKALLOC(Node, Results);
2968 break;
2969 case ISD::MERGE_VALUES:
2970 for (unsigned i = 0; i < Node->getNumValues(); i++)
2971 Results.push_back(Node->getOperand(i));
2972 break;
2973 case ISD::UNDEF: {
2974 EVT VT = Node->getValueType(0);
2975 if (VT.isInteger())
2976 Results.push_back(DAG.getConstant(0, dl, VT));
2977 else {
2978 assert(VT.isFloatingPoint() && "Unknown value type!")(static_cast <bool> (VT.isFloatingPoint() && "Unknown value type!"
) ? void (0) : __assert_fail ("VT.isFloatingPoint() && \"Unknown value type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2978, __extension__ __PRETTY_FUNCTION__))
;
2979 Results.push_back(DAG.getConstantFP(0, dl, VT));
2980 }
2981 break;
2982 }
2983 case ISD::FP_ROUND:
2984 case ISD::BITCAST:
2985 Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2986 Node->getValueType(0), dl);
2987 Results.push_back(Tmp1);
2988 break;
2989 case ISD::FP_EXTEND:
2990 Tmp1 = EmitStackConvert(Node->getOperand(0),
2991 Node->getOperand(0).getValueType(),
2992 Node->getValueType(0), dl);
2993 Results.push_back(Tmp1);
2994 break;
2995 case ISD::SIGN_EXTEND_INREG: {
2996 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2997 EVT VT = Node->getValueType(0);
2998
2999 // An in-register sign-extend of a boolean is a negation:
3000 // 'true' (1) sign-extended is -1.
3001 // 'false' (0) sign-extended is 0.
3002 // However, we must mask the high bits of the source operand because the
3003 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3004
3005 // TODO: Do this for vectors too?
3006 if (ExtraVT.getSizeInBits() == 1) {
3007 SDValue One = DAG.getConstant(1, dl, VT);
3008 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
3009 SDValue Zero = DAG.getConstant(0, dl, VT);
3010 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
3011 Results.push_back(Neg);
3012 break;
3013 }
3014
3015 // NOTE: we could fall back on load/store here too for targets without
3016 // SRA. However, it is doubtful that any exist.
3017 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3018 unsigned BitsDiff = VT.getScalarSizeInBits() -
3019 ExtraVT.getScalarSizeInBits();
3020 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3021 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3022 Node->getOperand(0), ShiftCst);
3023 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3024 Results.push_back(Tmp1);
3025 break;
3026 }
3027 case ISD::FP_ROUND_INREG: {
3028 // The only way we can lower this is to turn it into a TRUNCSTORE,
3029 // EXTLOAD pair, targeting a temporary location (a stack slot).
3030
3031 // NOTE: there is a choice here between constantly creating new stack
3032 // slots and always reusing the same one. We currently always create
3033 // new ones, as reuse may inhibit scheduling.
3034 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3035 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
3036 Node->getValueType(0), dl);
3037 Results.push_back(Tmp1);
3038 break;
3039 }
3040 case ISD::SINT_TO_FP:
3041 case ISD::UINT_TO_FP:
3042 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
3043 Node->getOperand(0), Node->getValueType(0), dl);
3044 Results.push_back(Tmp1);
3045 break;
3046 case ISD::FP_TO_SINT:
3047 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3048 Results.push_back(Tmp1);
3049 break;
3050 case ISD::FP_TO_UINT: {
3051 SDValue True, False;
3052 EVT VT = Node->getOperand(0).getValueType();
3053 EVT NVT = Node->getValueType(0);
3054 APFloat apf(DAG.EVTToAPFloatSemantics(VT),
3055 APInt::getNullValue(VT.getSizeInBits()));
3056 APInt x = APInt::getSignMask(NVT.getSizeInBits());
3057 (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
3058 Tmp1 = DAG.getConstantFP(apf, dl, VT);
3059 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT),
3060 Node->getOperand(0),
3061 Tmp1, ISD::SETLT);
3062 True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
3063 // TODO: Should any fast-math-flags be set for the FSUB?
3064 False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
3065 DAG.getNode(ISD::FSUB, dl, VT,
3066 Node->getOperand(0), Tmp1));
3067 False = DAG.getNode(ISD::XOR, dl, NVT, False,
3068 DAG.getConstant(x, dl, NVT));
3069 Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False);
3070 Results.push_back(Tmp1);
3071 break;
3072 }
3073 case ISD::VAARG:
3074 Results.push_back(DAG.expandVAArg(Node));
3075 Results.push_back(Results[0].getValue(1));
3076 break;
3077 case ISD::VACOPY:
3078 Results.push_back(DAG.expandVACopy(Node));
3079 break;
3080 case ISD::EXTRACT_VECTOR_ELT:
3081 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
3082 // This must be an access of the only element. Return it.
3083 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3084 Node->getOperand(0));
3085 else
3086 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3087 Results.push_back(Tmp1);
3088 break;
3089 case ISD::EXTRACT_SUBVECTOR:
3090 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3091 break;
3092 case ISD::INSERT_SUBVECTOR:
3093 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3094 break;
3095 case ISD::CONCAT_VECTORS:
3096 Results.push_back(ExpandVectorBuildThroughStack(Node));
3097 break;
3098 case ISD::SCALAR_TO_VECTOR:
3099 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3100 break;
3101 case ISD::INSERT_VECTOR_ELT:
3102 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3103 Node->getOperand(1),
3104 Node->getOperand(2), dl));
3105 break;
3106 case ISD::VECTOR_SHUFFLE: {
3107 SmallVector<int, 32> NewMask;
3108 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3109
3110 EVT VT = Node->getValueType(0);
3111 EVT EltVT = VT.getVectorElementType();
3112 SDValue Op0 = Node->getOperand(0);
3113 SDValue Op1 = Node->getOperand(1);
3114 if (!TLI.isTypeLegal(EltVT)) {
3115 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3116
3117 // BUILD_VECTOR operands are allowed to be wider than the element type.
3118 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3119 // it.
3120 if (NewEltVT.bitsLT(EltVT)) {
3121 // Convert shuffle node.
3122 // If original node was v4i64 and the new EltVT is i32,
3123 // cast operands to v8i32 and re-build the mask.
3124
3125 // Calculate new VT, the size of the new VT should be equal to original.
3126 EVT NewVT =
3127 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3128 VT.getSizeInBits() / NewEltVT.getSizeInBits());
3129 assert(NewVT.bitsEq(VT))(static_cast <bool> (NewVT.bitsEq(VT)) ? void (0) : __assert_fail
("NewVT.bitsEq(VT)", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3129, __extension__ __PRETTY_FUNCTION__))
;
3130
3131 // cast operands to new VT
3132 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3133 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3134
3135 // Convert the shuffle mask
3136 unsigned int factor =
3137 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3138
3139 // EltVT gets smaller
3140 assert(factor > 0)(static_cast <bool> (factor > 0) ? void (0) : __assert_fail
("factor > 0", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3140, __extension__ __PRETTY_FUNCTION__))
;
3141
3142 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3143 if (Mask[i] < 0) {
3144 for (unsigned fi = 0; fi < factor; ++fi)
3145 NewMask.push_back(Mask[i]);
3146 }
3147 else {
3148 for (unsigned fi = 0; fi < factor; ++fi)
3149 NewMask.push_back(Mask[i]*factor+fi);
3150 }
3151 }
3152 Mask = NewMask;
3153 VT = NewVT;
3154 }
3155 EltVT = NewEltVT;
3156 }
3157 unsigned NumElems = VT.getVectorNumElements();
3158 SmallVector<SDValue, 16> Ops;
3159 for (unsigned i = 0; i != NumElems; ++i) {
3160 if (Mask[i] < 0) {
3161 Ops.push_back(DAG.getUNDEF(EltVT));
3162 continue;
3163 }
3164 unsigned Idx = Mask[i];
3165 if (Idx < NumElems)
3166 Ops.push_back(DAG.getNode(
3167 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3168 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3169 else
3170 Ops.push_back(DAG.getNode(
3171 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3172 DAG.getConstant(Idx - NumElems, dl,
3173 TLI.getVectorIdxTy(DAG.getDataLayout()))));
3174 }
3175
3176 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3177 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3178 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3179 Results.push_back(Tmp1);
3180 break;
3181 }
3182 case ISD::EXTRACT_ELEMENT: {
3183 EVT OpTy = Node->getOperand(0).getValueType();
3184 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3185 // 1 -> Hi
3186 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3187 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3188 TLI.getShiftAmountTy(
3189 Node->getOperand(0).getValueType(),
3190 DAG.getDataLayout())));
3191 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3192 } else {
3193 // 0 -> Lo
3194 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3195 Node->getOperand(0));
3196 }
3197 Results.push_back(Tmp1);
3198 break;
3199 }
3200 case ISD::STACKSAVE:
3201 // Expand to CopyFromReg if the target set
3202 // StackPointerRegisterToSaveRestore.
3203 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3204 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3205 Node->getValueType(0)));
3206 Results.push_back(Results[0].getValue(1));
3207 } else {
3208 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3209 Results.push_back(Node->getOperand(0));
3210 }
3211 break;
3212 case ISD::STACKRESTORE:
3213 // Expand to CopyToReg if the target set
3214 // StackPointerRegisterToSaveRestore.
3215 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3216 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3217 Node->getOperand(1)));
3218 } else {
3219 Results.push_back(Node->getOperand(0));
3220 }
3221 break;
3222 case ISD::GET_DYNAMIC_AREA_OFFSET:
3223 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3224 Results.push_back(Results[0].getValue(0));
3225 break;
3226 case ISD::FCOPYSIGN:
3227 Results.push_back(ExpandFCOPYSIGN(Node));
3228 break;
3229 case ISD::FNEG:
3230 // Expand Y = FNEG(X) -> Y = SUB -0.0, X
3231 Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3232 // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3233 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3234 Node->getOperand(0));
3235 Results.push_back(Tmp1);
3236 break;
3237 case ISD::FABS:
3238 Results.push_back(ExpandFABS(Node));
3239 break;
3240 case ISD::SMIN:
3241 case ISD::SMAX:
3242 case ISD::UMIN:
3243 case ISD::UMAX: {
3244 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3245 ISD::CondCode Pred;
3246 switch (Node->getOpcode()) {
3247 default: llvm_unreachable("How did we get here?")::llvm::llvm_unreachable_internal("How did we get here?", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3247)
;
3248 case ISD::SMAX: Pred = ISD::SETGT; break;
3249 case ISD::SMIN: Pred = ISD::SETLT; break;
3250 case ISD::UMAX: Pred = ISD::SETUGT; break;
3251 case ISD::UMIN: Pred = ISD::SETULT; break;
3252 }
3253 Tmp1 = Node->getOperand(0);
3254 Tmp2 = Node->getOperand(1);
3255 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3256 Results.push_back(Tmp1);
3257 break;
3258 }
3259
3260 case ISD::FSIN:
3261 case ISD::FCOS: {
3262 EVT VT = Node->getValueType(0);
3263 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3264 // fcos which share the same operand and both are used.
3265 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3266 isSinCosLibcallAvailable(Node, TLI))
3267 && useSinCos(Node)) {
3268 SDVTList VTs = DAG.getVTList(VT, VT);
3269 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3270 if (Node->getOpcode() == ISD::FCOS)
3271 Tmp1 = Tmp1.getValue(1);
3272 Results.push_back(Tmp1);
3273 }
3274 break;
3275 }
3276 case ISD::FMAD:
3277 llvm_unreachable("Illegal fmad should never be formed")::llvm::llvm_unreachable_internal("Illegal fmad should never be formed"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3277)
;
3278
3279 case ISD::FP16_TO_FP:
3280 if (Node->getValueType(0) != MVT::f32) {
3281 // We can extend to types bigger than f32 in two steps without changing
3282 // the result. Since "f16 -> f32" is much more commonly available, give
3283 // CodeGen the option of emitting that before resorting to a libcall.
3284 SDValue Res =
3285 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3286 Results.push_back(
3287 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3288 }
3289 break;
3290 case ISD::FP_TO_FP16:
3291 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing FP_TO_FP16\n";
} } while (false)
;
3292 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3293 SDValue Op = Node->getOperand(0);
3294 MVT SVT = Op.getSimpleValueType();
3295 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3296 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3297 // Under fastmath, we can expand this node into a fround followed by
3298 // a float-half conversion.
3299 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3300 DAG.getIntPtrConstant(0, dl));
3301 Results.push_back(
3302 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3303 }
3304 }
3305 break;
3306 case ISD::ConstantFP: {
3307 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3308 // Check to see if this FP immediate is already legal.
3309 // If this is a legal constant, turn it into a TargetConstantFP node.
3310 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
3311 Results.push_back(ExpandConstantFP(CFP, true));
3312 break;
3313 }
3314 case ISD::Constant: {
3315 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3316 Results.push_back(ExpandConstant(CP));
3317 break;
3318 }
3319 case ISD::FSUB: {
3320 EVT VT = Node->getValueType(0);
3321 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3322 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3323 const SDNodeFlags Flags = Node->getFlags();
3324 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3325 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3326 Results.push_back(Tmp1);
3327 }
3328 break;
3329 }
3330 case ISD::SUB: {
3331 EVT VT = Node->getValueType(0);
3332 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&(static_cast <bool> (TLI.isOperationLegalOrCustom(ISD::
ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT
) && "Don't know how to expand this subtraction!") ? void
(0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3334, __extension__ __PRETTY_FUNCTION__))
3333 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&(static_cast <bool> (TLI.isOperationLegalOrCustom(ISD::
ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT
) && "Don't know how to expand this subtraction!") ? void
(0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3334, __extension__ __PRETTY_FUNCTION__))
3334 "Don't know how to expand this subtraction!")(static_cast <bool> (TLI.isOperationLegalOrCustom(ISD::
ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT
) && "Don't know how to expand this subtraction!") ? void
(0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3334, __extension__ __PRETTY_FUNCTION__))
;
3335 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3336 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3337 VT));
3338 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3339 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3340 break;
3341 }
3342 case ISD::UREM:
3343 case ISD::SREM: {
3344 EVT VT = Node->getValueType(0);
3345 bool isSigned = Node->getOpcode() == ISD::SREM;
3346 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3347 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3348 Tmp2 = Node->getOperand(0);
3349 Tmp3 = Node->getOperand(1);
3350 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3351 SDVTList VTs = DAG.getVTList(VT, VT);
3352 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3353 Results.push_back(Tmp1);
3354 } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3355 // X % Y -> X-X/Y*Y
3356 Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3357 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3358 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3359 Results.push_back(Tmp1);
3360 }
3361 break;
3362 }
3363 case ISD::UDIV:
3364 case ISD::SDIV: {
3365 bool isSigned = Node->getOpcode() == ISD::SDIV;
3366 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3367 EVT VT = Node->getValueType(0);
3368 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3369 SDVTList VTs = DAG.getVTList(VT, VT);
3370 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3371 Node->getOperand(1));
3372 Results.push_back(Tmp1);
3373 }
3374 break;
3375 }
3376 case ISD::MULHU:
3377 case ISD::MULHS: {
3378 unsigned ExpandOpcode =
3379 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3380 EVT VT = Node->getValueType(0);
3381 SDVTList VTs = DAG.getVTList(VT, VT);
3382
3383 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3384 Node->getOperand(1));
3385 Results.push_back(Tmp1.getValue(1));
3386 break;
3387 }
3388 case ISD::UMUL_LOHI:
3389 case ISD::SMUL_LOHI: {
3390 SDValue LHS = Node->getOperand(0);
3391 SDValue RHS = Node->getOperand(1);
3392 MVT VT = LHS.getSimpleValueType();
3393 unsigned MULHOpcode =
3394 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3395
3396 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3397 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3398 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3399 break;
3400 }
3401
3402 SmallVector<SDValue, 4> Halves;
3403 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3404 assert(TLI.isTypeLegal(HalfType))(static_cast <bool> (TLI.isTypeLegal(HalfType)) ? void (
0) : __assert_fail ("TLI.isTypeLegal(HalfType)", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3404, __extension__ __PRETTY_FUNCTION__))
;
3405 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
3406 HalfType, DAG,
3407 TargetLowering::MulExpansionKind::Always)) {
3408 for (unsigned i = 0; i < 2; ++i) {
3409 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3410 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3411 SDValue Shift = DAG.getConstant(
3412 HalfType.getScalarSizeInBits(), dl,
3413 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3414 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3415 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3416 }
3417 break;
3418 }
3419 break;
3420 }
3421 case ISD::MUL: {
3422 EVT VT = Node->getValueType(0);
3423 SDVTList VTs = DAG.getVTList(VT, VT);
3424 // See if multiply or divide can be lowered using two-result operations.
3425 // We just need the low half of the multiply; try both the signed
3426 // and unsigned forms. If the target supports both SMUL_LOHI and
3427 // UMUL_LOHI, form a preference by checking which forms of plain
3428 // MULH it supports.
3429 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3430 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3431 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3432 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3433 unsigned OpToUse = 0;
3434 if (HasSMUL_LOHI && !HasMULHS) {
3435 OpToUse = ISD::SMUL_LOHI;
3436 } else if (HasUMUL_LOHI && !HasMULHU) {
3437 OpToUse = ISD::UMUL_LOHI;
3438 } else if (HasSMUL_LOHI) {
3439 OpToUse = ISD::SMUL_LOHI;
3440 } else if (HasUMUL_LOHI) {
3441 OpToUse = ISD::UMUL_LOHI;
3442 }
3443 if (OpToUse) {
3444 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3445 Node->getOperand(1)));
3446 break;
3447 }
3448
3449 SDValue Lo, Hi;
3450 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3451 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3452 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3453 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3454 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3455 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3456 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3457 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3458 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3459 SDValue Shift =
3460 DAG.getConstant(HalfType.getSizeInBits(), dl,
3461 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3462 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3463 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3464 }
3465 break;
3466 }
3467 case ISD::SADDO:
3468 case ISD::SSUBO: {
3469 SDValue LHS = Node->getOperand(0);
3470 SDValue RHS = Node->getOperand(1);
3471 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3472 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3473 LHS, RHS);
3474 Results.push_back(Sum);
3475 EVT ResultType = Node->getValueType(1);
3476 EVT OType = getSetCCResultType(Node->getValueType(0));
3477
3478 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3479
3480 // LHSSign -> LHS >= 0
3481 // RHSSign -> RHS >= 0
3482 // SumSign -> Sum >= 0
3483 //
3484 // Add:
3485 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3486 // Sub:
3487 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3488 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3489 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3490 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3491 Node->getOpcode() == ISD::SADDO ?
3492 ISD::SETEQ : ISD::SETNE);
3493
3494 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3495 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3496
3497 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3498 Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
3499 break;
3500 }
3501 case ISD::UADDO:
3502 case ISD::USUBO: {
3503 SDValue LHS = Node->getOperand(0);
3504 SDValue RHS = Node->getOperand(1);
3505 bool IsAdd = Node->getOpcode() == ISD::UADDO;
3506 // If ADD/SUBCARRY is legal, use that instead.
3507 unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;
3508 if (TLI.isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
3509 SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
3510 SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
3511 { LHS, RHS, CarryIn });
3512 Results.push_back(SDValue(NodeCarry.getNode(), 0));
3513 Results.push_back(SDValue(NodeCarry.getNode(), 1));
3514 break;
3515 }
3516
3517 SDValue Sum = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
3518 LHS.getValueType(), LHS, RHS);
3519 Results.push_back(Sum);
3520
3521 EVT ResultType = Node->getValueType(1);
3522 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3523 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3524 SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3525
3526 Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
3527 break;
3528 }
3529 case ISD::UMULO:
3530 case ISD::SMULO: {
3531 EVT VT = Node->getValueType(0);
3532 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
3533 SDValue LHS = Node->getOperand(0);
3534 SDValue RHS = Node->getOperand(1);
3535 SDValue BottomHalf;
3536 SDValue TopHalf;
3537 static const unsigned Ops[2][3] =
3538 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
3539 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
3540 bool isSigned = Node->getOpcode() == ISD::SMULO;
3541 if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
3542 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3543 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
3544 } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
3545 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
3546 RHS);
3547 TopHalf = BottomHalf.getValue(1);
3548 } else if (TLI.isTypeLegal(WideVT)) {
3549 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
3550 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
3551 Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3552 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3553 DAG.getIntPtrConstant(0, dl));
3554 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3555 DAG.getIntPtrConstant(1, dl));
3556 } else {
3557 // We can fall back to a libcall with an illegal type for the MUL if we
3558 // have a libcall big enough.
3559 // Also, we can fall back to a division in some cases, but that's a big
3560 // performance hit in the general case.
3561 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3562 if (WideVT == MVT::i16)
3563 LC = RTLIB::MUL_I16;
3564 else if (WideVT == MVT::i32)
3565 LC = RTLIB::MUL_I32;
3566 else if (WideVT == MVT::i64)
3567 LC = RTLIB::MUL_I64;
3568 else if (WideVT == MVT::i128)
3569 LC = RTLIB::MUL_I128;
3570 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Cannot expand this operation!") ? void (0) : __assert_fail (
"LC != RTLIB::UNKNOWN_LIBCALL && \"Cannot expand this operation!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3570, __extension__ __PRETTY_FUNCTION__))
;
3571
3572 SDValue HiLHS;
3573 SDValue HiRHS;
3574 if (isSigned) {
3575 // The high part is obtained by SRA'ing all but one of the bits of low
3576 // part.
3577 unsigned LoSize = VT.getSizeInBits();
3578 HiLHS =
3579 DAG.getNode(ISD::SRA, dl, VT, LHS,
3580 DAG.getConstant(LoSize - 1, dl,
3581 TLI.getPointerTy(DAG.getDataLayout())));
3582 HiRHS =
3583 DAG.getNode(ISD::SRA, dl, VT, RHS,
3584 DAG.getConstant(LoSize - 1, dl,
3585 TLI.getPointerTy(DAG.getDataLayout())));
3586 } else {
3587 HiLHS = DAG.getConstant(0, dl, VT);
3588 HiRHS = DAG.getConstant(0, dl, VT);
3589 }
3590
3591 // Here we're passing the 2 arguments explicitly as 4 arguments that are
3592 // pre-lowered to the correct types. This all depends upon WideVT not
3593 // being a legal type for the architecture and thus has to be split to
3594 // two arguments.
3595 SDValue Ret;
3596 if(DAG.getDataLayout().isLittleEndian()) {
3597 // Halves of WideVT are packed into registers in different order
3598 // depending on platform endianness. This is usually handled by
3599 // the C calling convention, but we can't defer to it in
3600 // the legalizer.
3601 SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
3602 Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3603 } else {
3604 SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
3605 Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3606 }
3607 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&(static_cast <bool> (Ret.getOpcode() == ISD::MERGE_VALUES
&& "Ret value is a collection of constituent nodes holding result."
) ? void (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3608, __extension__ __PRETTY_FUNCTION__))
3608 "Ret value is a collection of constituent nodes holding result.")(static_cast <bool> (Ret.getOpcode() == ISD::MERGE_VALUES
&& "Ret value is a collection of constituent nodes holding result."
) ? void (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3608, __extension__ __PRETTY_FUNCTION__))
;
3609 BottomHalf = Ret.getOperand(0);
3610 TopHalf = Ret.getOperand(1);
3611 }
3612
3613 if (isSigned) {
3614 Tmp1 = DAG.getConstant(
3615 VT.getSizeInBits() - 1, dl,
3616 TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
3617 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3618 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
3619 ISD::SETNE);
3620 } else {
3621 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
3622 DAG.getConstant(0, dl, VT), ISD::SETNE);
3623 }
3624
3625 // Truncate the result if SetCC returns a larger type than needed.
3626 EVT RType = Node->getValueType(1);
3627 if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
3628 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
3629
3630 assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&(static_cast <bool> (RType.getSizeInBits() == TopHalf.getValueSizeInBits
() && "Unexpected result type for S/UMULO legalization"
) ? void (0) : __assert_fail ("RType.getSizeInBits() == TopHalf.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3631, __extension__ __PRETTY_FUNCTION__))
3631 "Unexpected result type for S/UMULO legalization")(static_cast <bool> (RType.getSizeInBits() == TopHalf.getValueSizeInBits
() && "Unexpected result type for S/UMULO legalization"
) ? void (0) : __assert_fail ("RType.getSizeInBits() == TopHalf.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3631, __extension__ __PRETTY_FUNCTION__))
;
3632
3633 Results.push_back(BottomHalf);
3634 Results.push_back(TopHalf);
3635 break;
3636 }
3637 case ISD::BUILD_PAIR: {
3638 EVT PairTy = Node->getValueType(0);
3639 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3640 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3641 Tmp2 = DAG.getNode(
3642 ISD::SHL, dl, PairTy, Tmp2,
3643 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3644 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3645 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3646 break;
3647 }
3648 case ISD::SELECT:
3649 Tmp1 = Node->getOperand(0);
3650 Tmp2 = Node->getOperand(1);
3651 Tmp3 = Node->getOperand(2);
3652 if (Tmp1.getOpcode() == ISD::SETCC) {
3653 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3654 Tmp2, Tmp3,
3655 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3656 } else {
3657 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3658 DAG.getConstant(0, dl, Tmp1.getValueType()),
3659 Tmp2, Tmp3, ISD::SETNE);
3660 }
3661 Results.push_back(Tmp1);
3662 break;
3663 case ISD::BR_JT: {
3664 SDValue Chain = Node->getOperand(0);
3665 SDValue Table = Node->getOperand(1);
3666 SDValue Index = Node->getOperand(2);
3667
3668 const DataLayout &TD = DAG.getDataLayout();
3669 EVT PTy = TLI.getPointerTy(TD);
3670
3671 unsigned EntrySize =
3672 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3673
3674 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3675 // This transformation needs to be done here since otherwise the MIPS
3676 // backend will end up emitting a three instruction multiply sequence
3677 // instead of a single shift and MSP430 will call a runtime function.
3678 if (llvm::isPowerOf2_32(EntrySize))
3679 Index = DAG.getNode(
3680 ISD::SHL, dl, Index.getValueType(), Index,
3681 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3682 else
3683 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3684 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3685 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3686 Index, Table);
3687
3688 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3689 SDValue LD = DAG.getExtLoad(
3690 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3691 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3692 Addr = LD;
3693 if (TLI.isJumpTableRelative()) {
3694 // For PIC, the sequence is:
3695 // BRIND(load(Jumptable + index) + RelocBase)
3696 // RelocBase can be JumpTable, GOT or some sort of global base.
3697 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3698 TLI.getPICJumpTableRelocBase(Table, DAG));
3699 }
3700
3701 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG);
3702 Results.push_back(Tmp1);
3703 break;
3704 }
3705 case ISD::BRCOND:
3706 // Expand brcond's setcc into its constituent parts and create a BR_CC
3707 // Node.
3708 Tmp1 = Node->getOperand(0);
3709 Tmp2 = Node->getOperand(1);
3710 if (Tmp2.getOpcode() == ISD::SETCC) {
3711 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3712 Tmp1, Tmp2.getOperand(2),
3713 Tmp2.getOperand(0), Tmp2.getOperand(1),
3714 Node->getOperand(2));
3715 } else {
3716 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3717 if (Tmp2.isUndef() ||
3718 (Tmp2.getOpcode() == ISD::AND &&
3719 isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3720 cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3721 Tmp3 = Tmp2;
3722 else
3723 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3724 DAG.getConstant(1, dl, Tmp2.getValueType()));
3725 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3726 DAG.getCondCode(ISD::SETNE), Tmp3,
3727 DAG.getConstant(0, dl, Tmp3.getValueType()),
3728 Node->getOperand(2));
3729 }
3730 Results.push_back(Tmp1);
3731 break;
3732 case ISD::SETCC: {
3733 Tmp1 = Node->getOperand(0);
3734 Tmp2 = Node->getOperand(1);
3735 Tmp3 = Node->getOperand(2);
3736 bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3737 Tmp3, NeedInvert, dl);
3738
3739 if (Legalized) {
3740 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3741 // condition code, create a new SETCC node.
3742 if (Tmp3.getNode())
3743 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3744 Tmp1, Tmp2, Tmp3);
3745
3746 // If we expanded the SETCC by inverting the condition code, then wrap
3747 // the existing SETCC in a NOT to restore the intended condition.
3748 if (NeedInvert)
3749 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3750
3751 Results.push_back(Tmp1);
3752 break;
3753 }
3754
3755 // Otherwise, SETCC for the given comparison type must be completely
3756 // illegal; expand it into a SELECT_CC.
3757 EVT VT = Node->getValueType(0);
3758 int TrueValue;
3759 switch (TLI.getBooleanContents(Tmp1.getValueType())) {
3760 case TargetLowering::ZeroOrOneBooleanContent:
3761 case TargetLowering::UndefinedBooleanContent:
3762 TrueValue = 1;
3763 break;
3764 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3765 TrueValue = -1;
3766 break;
3767 }
3768 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3769 DAG.getConstant(TrueValue, dl, VT),
3770 DAG.getConstant(0, dl, VT),
3771 Tmp3);
3772 Results.push_back(Tmp1);
3773 break;
3774 }
3775 case ISD::SELECT_CC: {
3776 Tmp1 = Node->getOperand(0); // LHS
3777 Tmp2 = Node->getOperand(1); // RHS
3778 Tmp3 = Node->getOperand(2); // True
3779 Tmp4 = Node->getOperand(3); // False
3780 EVT VT = Node->getValueType(0);
3781 SDValue CC = Node->getOperand(4);
3782 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3783
3784 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
3785 // If the condition code is legal, then we need to expand this
3786 // node using SETCC and SELECT.
3787 EVT CmpVT = Tmp1.getValueType();
3788 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&(static_cast <bool> (!TLI.isOperationExpand(ISD::SELECT
, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? void (0) : __assert_fail ("!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3790, __extension__ __PRETTY_FUNCTION__))
3789 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "(static_cast <bool> (!TLI.isOperationExpand(ISD::SELECT
, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? void (0) : __assert_fail ("!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3790, __extension__ __PRETTY_FUNCTION__))
3790 "expanded.")(static_cast <bool> (!TLI.isOperationExpand(ISD::SELECT
, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? void (0) : __assert_fail ("!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3790, __extension__ __PRETTY_FUNCTION__))
;
3791 EVT CCVT =
3792 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
3793 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
3794 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3795 break;
3796 }
3797
3798 // SELECT_CC is legal, so the condition code must not be.
3799 bool Legalized = false;
3800 // Try to legalize by inverting the condition. This is for targets that
3801 // might support an ordered version of a condition, but not the unordered
3802 // version (or vice versa).
3803 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3804 Tmp1.getValueType().isInteger());
3805 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
3806 // Use the new condition code and swap true and false
3807 Legalized = true;
3808 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3809 } else {
3810 // If The inverse is not legal, then try to swap the arguments using
3811 // the inverse condition code.
3812 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3813 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
3814 // The swapped inverse condition is legal, so swap true and false,
3815 // lhs and rhs.
3816 Legalized = true;
3817 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3818 }
3819 }
3820
3821 if (!Legalized) {
3822 Legalized = LegalizeSetCCCondCode(
3823 getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3824 dl);
3825
3826 assert(Legalized && "Can't legalize SELECT_CC with legal condition!")(static_cast <bool> (Legalized && "Can't legalize SELECT_CC with legal condition!"
) ? void (0) : __assert_fail ("Legalized && \"Can't legalize SELECT_CC with legal condition!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3826, __extension__ __PRETTY_FUNCTION__))
;
3827
3828 // If we expanded the SETCC by inverting the condition code, then swap
3829 // the True/False operands to match.
3830 if (NeedInvert)
3831 std::swap(Tmp3, Tmp4);
3832
3833 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3834 // condition code, create a new SELECT_CC node.
3835 if (CC.getNode()) {
3836 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3837 Tmp1, Tmp2, Tmp3, Tmp4, CC);
3838 } else {
3839 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3840 CC = DAG.getCondCode(ISD::SETNE);
3841 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3842 Tmp2, Tmp3, Tmp4, CC);
3843 }
3844 }
3845 Results.push_back(Tmp1);
3846 break;
3847 }
3848 case ISD::BR_CC: {
3849 Tmp1 = Node->getOperand(0); // Chain
3850 Tmp2 = Node->getOperand(2); // LHS
3851 Tmp3 = Node->getOperand(3); // RHS
3852 Tmp4 = Node->getOperand(1); // CC
3853
3854 bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
3855 Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
3856 (void)Legalized;
3857 assert(Legalized && "Can't legalize BR_CC with legal condition!")(static_cast <bool> (Legalized && "Can't legalize BR_CC with legal condition!"
) ? void (0) : __assert_fail ("Legalized && \"Can't legalize BR_CC with legal condition!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3857, __extension__ __PRETTY_FUNCTION__))
;
3858
3859 // If we expanded the SETCC by inverting the condition code, then wrap
3860 // the existing SETCC in a NOT to restore the intended condition.
3861 if (NeedInvert)
3862 Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
3863
3864 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3865 // node.
3866 if (Tmp4.getNode()) {
3867 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3868 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3869 } else {
3870 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3871 Tmp4 = DAG.getCondCode(ISD::SETNE);
3872 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3873 Tmp2, Tmp3, Node->getOperand(4));
3874 }
3875 Results.push_back(Tmp1);
3876 break;
3877 }
3878 case ISD::BUILD_VECTOR:
3879 Results.push_back(ExpandBUILD_VECTOR(Node));
3880 break;
3881 case ISD::SRA:
3882 case ISD::SRL:
3883 case ISD::SHL: {
3884 // Scalarize vector SRA/SRL/SHL.
3885 EVT VT = Node->getValueType(0);
3886 assert(VT.isVector() && "Unable to legalize non-vector shift")(static_cast <bool> (VT.isVector() && "Unable to legalize non-vector shift"
) ? void (0) : __assert_fail ("VT.isVector() && \"Unable to legalize non-vector shift\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3886, __extension__ __PRETTY_FUNCTION__))
;
3887 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal")(static_cast <bool> (TLI.isTypeLegal(VT.getScalarType()
)&& "Element type must be legal") ? void (0) : __assert_fail
("TLI.isTypeLegal(VT.getScalarType())&& \"Element type must be legal\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3887, __extension__ __PRETTY_FUNCTION__))
;
3888 unsigned NumElem = VT.getVectorNumElements();
3889
3890 SmallVector<SDValue, 8> Scalars;
3891 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3892 SDValue Ex = DAG.getNode(
3893 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3894 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3895 SDValue Sh = DAG.getNode(
3896 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3897 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3898 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3899 VT.getScalarType(), Ex, Sh));
3900 }
3901
3902 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3903 ReplaceNode(SDValue(Node, 0), Result);
3904 break;
3905 }
3906 case ISD::ROTL:
3907 case ISD::ROTR: {
3908 bool IsLeft = Node->getOpcode() == ISD::ROTL;
3909 SDValue Op0 = Node->getOperand(0), Op1 = Node->getOperand(1);
3910 EVT ResVT = Node->getValueType(0);
3911 EVT OpVT = Op0.getValueType();
3912 assert(OpVT == ResVT &&(static_cast <bool> (OpVT == ResVT && "The result and the operand types of rotate should match"
) ? void (0) : __assert_fail ("OpVT == ResVT && \"The result and the operand types of rotate should match\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3913, __extension__ __PRETTY_FUNCTION__))
3913 "The result and the operand types of rotate should match")(static_cast <bool> (OpVT == ResVT && "The result and the operand types of rotate should match"
) ? void (0) : __assert_fail ("OpVT == ResVT && \"The result and the operand types of rotate should match\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3913, __extension__ __PRETTY_FUNCTION__))
;
3914 EVT ShVT = Op1.getValueType();
3915 SDValue Width = DAG.getConstant(OpVT.getScalarSizeInBits(), dl, ShVT);
3916
3917 // If a rotate in the other direction is legal, use it.
3918 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
3919 if (TLI.isOperationLegal(RevRot, ResVT)) {
3920 SDValue Sub = DAG.getNode(ISD::SUB, dl, ShVT, Width, Op1);
3921 Results.push_back(DAG.getNode(RevRot, dl, ResVT, Op0, Sub));
3922 break;
3923 }
3924
3925 // Otherwise,
3926 // (rotl x, c) -> (or (shl x, (and c, w-1)), (srl x, (and w-c, w-1)))
3927 // (rotr x, c) -> (or (srl x, (and c, w-1)), (shl x, (and w-c, w-1)))
3928 //
3929 assert(isPowerOf2_32(OpVT.getScalarSizeInBits()) &&(static_cast <bool> (isPowerOf2_32(OpVT.getScalarSizeInBits
()) && "Expecting the type bitwidth to be a power of 2"
) ? void (0) : __assert_fail ("isPowerOf2_32(OpVT.getScalarSizeInBits()) && \"Expecting the type bitwidth to be a power of 2\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3930, __extension__ __PRETTY_FUNCTION__))
3930 "Expecting the type bitwidth to be a power of 2")(static_cast <bool> (isPowerOf2_32(OpVT.getScalarSizeInBits
()) && "Expecting the type bitwidth to be a power of 2"
) ? void (0) : __assert_fail ("isPowerOf2_32(OpVT.getScalarSizeInBits()) && \"Expecting the type bitwidth to be a power of 2\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3930, __extension__ __PRETTY_FUNCTION__))
;
3931 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
3932 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
3933 SDValue Width1 = DAG.getNode(ISD::SUB, dl, ShVT,
3934 Width, DAG.getConstant(1, dl, ShVT));
3935 SDValue NegOp1 = DAG.getNode(ISD::SUB, dl, ShVT, Width, Op1);
3936 SDValue And0 = DAG.getNode(ISD::AND, dl, ShVT, Op1, Width1);
3937 SDValue And1 = DAG.getNode(ISD::AND, dl, ShVT, NegOp1, Width1);
3938
3939 SDValue Or = DAG.getNode(ISD::OR, dl, ResVT,
3940 DAG.getNode(ShOpc, dl, ResVT, Op0, And0),
3941 DAG.getNode(HsOpc, dl, ResVT, Op0, And1));
3942 Results.push_back(Or);
3943 break;
3944 }
3945
3946 case ISD::GLOBAL_OFFSET_TABLE:
3947 case ISD::GlobalAddress:
3948 case ISD::GlobalTLSAddress:
3949 case ISD::ExternalSymbol:
3950 case ISD::ConstantPool:
3951 case ISD::JumpTable:
3952 case ISD::INTRINSIC_W_CHAIN:
3953 case ISD::INTRINSIC_WO_CHAIN:
3954 case ISD::INTRINSIC_VOID:
3955 // FIXME: Custom lowering for these operations shouldn't return null!
3956 break;
3957 }
3958
3959 // Replace the original node with the legalized result.
3960 if (Results.empty()) {
3961 LLVM_DEBUG(dbgs() << "Cannot expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Cannot expand node\n"; } }
while (false)
;
3962 return false;
3963 }
3964
3965 LLVM_DEBUG(dbgs() << "Succesfully expanded node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Succesfully expanded node\n"
; } } while (false)
;
3966 ReplaceNode(Node, Results.data());
3967 return true;
3968}
3969
3970void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3971 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to convert node to libcall\n"
; } } while (false)
;
3972 SmallVector<SDValue, 8> Results;
3973 SDLoc dl(Node);
3974 // FIXME: Check flags on the node to see if we can use a finite call.
3975 bool CanUseFiniteLibCall = TM.Options.NoInfsFPMath && TM.Options.NoNaNsFPMath;
3976 unsigned Opc = Node->getOpcode();
3977 switch (Opc) {
3978 case ISD::ATOMIC_FENCE: {
3979 // If the target didn't lower this, lower it to '__sync_synchronize()' call
3980 // FIXME: handle "fence singlethread" more efficiently.
3981 TargetLowering::ArgListTy Args;
3982
3983 TargetLowering::CallLoweringInfo CLI(DAG);
3984 CLI.setDebugLoc(dl)
3985 .setChain(Node->getOperand(0))
3986 .setLibCallee(
3987 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3988 DAG.getExternalSymbol("__sync_synchronize",
3989 TLI.getPointerTy(DAG.getDataLayout())),
3990 std::move(Args));
3991
3992 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3993
3994 Results.push_back(CallResult.second);
3995 break;
3996 }
3997 // By default, atomic intrinsics are marked Legal and lowered. Targets
3998 // which don't support them directly, however, may want libcalls, in which
3999 // case they mark them Expand, and we get here.
4000 case ISD::ATOMIC_SWAP:
4001 case ISD::ATOMIC_LOAD_ADD:
4002 case ISD::ATOMIC_LOAD_SUB:
4003 case ISD::ATOMIC_LOAD_AND:
4004 case ISD::ATOMIC_LOAD_CLR:
4005 case ISD::ATOMIC_LOAD_OR:
4006 case ISD::ATOMIC_LOAD_XOR:
4007 case ISD::ATOMIC_LOAD_NAND:
4008 case ISD::ATOMIC_LOAD_MIN:
4009 case ISD::ATOMIC_LOAD_MAX:
4010 case ISD::ATOMIC_LOAD_UMIN:
4011 case ISD::ATOMIC_LOAD_UMAX:
4012 case ISD::ATOMIC_CMP_SWAP: {
4013 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
4014 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
4015 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected atomic op or value type!") ? void (0) : __assert_fail
("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected atomic op or value type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4015, __extension__ __PRETTY_FUNCTION__))
;
4016
4017 std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
4018 Results.push_back(Tmp.first);
4019 Results.push_back(Tmp.second);
4020 break;
4021 }
4022 case ISD::TRAP: {
4023 // If this operation is not supported, lower it to 'abort()' call
4024 TargetLowering::ArgListTy Args;
4025 TargetLowering::CallLoweringInfo CLI(DAG);
4026 CLI.setDebugLoc(dl)
4027 .setChain(Node->getOperand(0))
4028 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4029 DAG.getExternalSymbol(
4030 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4031 std::move(Args));
4032 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4033
4034 Results.push_back(CallResult.second);
4035 break;
4036 }
4037 case ISD::FMINNUM:
4038 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4039 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4040 RTLIB::FMIN_PPCF128));
4041 break;
4042 case ISD::FMAXNUM:
4043 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4044 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4045 RTLIB::FMAX_PPCF128));
4046 break;
4047 case ISD::FSQRT:
4048 case ISD::STRICT_FSQRT:
4049 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4050 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4051 RTLIB::SQRT_PPCF128));
4052 break;
4053 case ISD::FSIN:
4054 case ISD::STRICT_FSIN:
4055 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4056 RTLIB::SIN_F80, RTLIB::SIN_F128,
4057 RTLIB::SIN_PPCF128));
4058 break;
4059 case ISD::FCOS:
4060 case ISD::STRICT_FCOS:
4061 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4062 RTLIB::COS_F80, RTLIB::COS_F128,
4063 RTLIB::COS_PPCF128));
4064 break;
4065 case ISD::FSINCOS:
4066 // Expand into sincos libcall.
4067 ExpandSinCosLibCall(Node, Results);
4068 break;
4069 case ISD::FLOG:
4070 case ISD::STRICT_FLOG:
4071 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log_finite))
4072 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_FINITE_F32,
4073 RTLIB::LOG_FINITE_F64,
4074 RTLIB::LOG_FINITE_F80,
4075 RTLIB::LOG_FINITE_F128,
4076 RTLIB::LOG_FINITE_PPCF128));
4077 else
4078 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
4079 RTLIB::LOG_F80, RTLIB::LOG_F128,
4080 RTLIB::LOG_PPCF128));
4081 break;
4082 case ISD::FLOG2:
4083 case ISD::STRICT_FLOG2:
4084 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log2_finite))
4085 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_FINITE_F32,
4086 RTLIB::LOG2_FINITE_F64,
4087 RTLIB::LOG2_FINITE_F80,
4088 RTLIB::LOG2_FINITE_F128,
4089 RTLIB::LOG2_FINITE_PPCF128));
4090 else
4091 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
4092 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
4093 RTLIB::LOG2_PPCF128));
4094 break;
4095 case ISD::FLOG10:
4096 case ISD::STRICT_FLOG10:
4097 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log10_finite))
4098 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_FINITE_F32,
4099 RTLIB::LOG10_FINITE_F64,
4100 RTLIB::LOG10_FINITE_F80,
4101 RTLIB::LOG10_FINITE_F128,
4102 RTLIB::LOG10_FINITE_PPCF128));
4103 else
4104 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
4105 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
4106 RTLIB::LOG10_PPCF128));
4107 break;
4108 case ISD::FEXP:
4109 case ISD::STRICT_FEXP:
4110 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp_finite))
4111 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_FINITE_F32,
4112 RTLIB::EXP_FINITE_F64,
4113 RTLIB::EXP_FINITE_F80,
4114 RTLIB::EXP_FINITE_F128,
4115 RTLIB::EXP_FINITE_PPCF128));
4116 else
4117 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
4118 RTLIB::EXP_F80, RTLIB::EXP_F128,
4119 RTLIB::EXP_PPCF128));
4120 break;
4121 case ISD::FEXP2:
4122 case ISD::STRICT_FEXP2:
4123 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp2_finite))
4124 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_FINITE_F32,
4125 RTLIB::EXP2_FINITE_F64,
4126 RTLIB::EXP2_FINITE_F80,
4127 RTLIB::EXP2_FINITE_F128,
4128 RTLIB::EXP2_FINITE_PPCF128));
4129 else
4130 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
4131 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
4132 RTLIB::EXP2_PPCF128));
4133 break;
4134 case ISD::FTRUNC:
4135 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4136 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4137 RTLIB::TRUNC_PPCF128));
4138 break;
4139 case ISD::FFLOOR:
4140 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4141 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4142 RTLIB::FLOOR_PPCF128));
4143 break;
4144 case ISD::FCEIL:
4145 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4146 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4147 RTLIB::CEIL_PPCF128));
4148 break;
4149 case ISD::FRINT:
4150 case ISD::STRICT_FRINT:
4151 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4152 RTLIB::RINT_F80, RTLIB::RINT_F128,
4153 RTLIB::RINT_PPCF128));
4154 break;
4155 case ISD::FNEARBYINT:
4156 case ISD::STRICT_FNEARBYINT:
4157 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4158 RTLIB::NEARBYINT_F64,
4159 RTLIB::NEARBYINT_F80,
4160 RTLIB::NEARBYINT_F128,
4161 RTLIB::NEARBYINT_PPCF128));
4162 break;
4163 case ISD::FROUND:
4164 Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4165 RTLIB::ROUND_F64,
4166 RTLIB::ROUND_F80,
4167 RTLIB::ROUND_F128,
4168 RTLIB::ROUND_PPCF128));
4169 break;
4170 case ISD::FPOWI:
4171 case ISD::STRICT_FPOWI:
4172 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
4173 RTLIB::POWI_F80, RTLIB::POWI_F128,
4174 RTLIB::POWI_PPCF128));
4175 break;
4176 case ISD::FPOW:
4177 case ISD::STRICT_FPOW:
4178 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_pow_finite))
4179 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_FINITE_F32,
4180 RTLIB::POW_FINITE_F64,
4181 RTLIB::POW_FINITE_F80,
4182 RTLIB::POW_FINITE_F128,
4183 RTLIB::POW_FINITE_PPCF128));
4184 else
4185 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
4186 RTLIB::POW_F80, RTLIB::POW_F128,
4187 RTLIB::POW_PPCF128));
4188 break;
4189 case ISD::FDIV:
4190 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4191 RTLIB::DIV_F80, RTLIB::DIV_F128,
4192 RTLIB::DIV_PPCF128));
4193 break;
4194 case ISD::FREM:
4195 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4196 RTLIB::REM_F80, RTLIB::REM_F128,
4197 RTLIB::REM_PPCF128));
4198 break;
4199 case ISD::FMA:
4200 case ISD::STRICT_FMA:
4201 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4202 RTLIB::FMA_F80, RTLIB::FMA_F128,
4203 RTLIB::FMA_PPCF128));
4204 break;
4205 case ISD::FADD:
4206 Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4207 RTLIB::ADD_F80, RTLIB::ADD_F128,
4208 RTLIB::ADD_PPCF128));
4209 break;
4210 case ISD::FMUL:
4211 Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4212 RTLIB::MUL_F80, RTLIB::MUL_F128,
4213 RTLIB::MUL_PPCF128));
4214 break;
4215 case ISD::FP16_TO_FP:
4216 if (Node->getValueType(0) == MVT::f32) {
4217 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4218 }
4219 break;
4220 case ISD::FP_TO_FP16: {
4221 RTLIB::Libcall LC =
4222 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4223 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_fp16") ? void (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unable to expand fp_to_fp16\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4223, __extension__ __PRETTY_FUNCTION__))
;
4224 Results.push_back(ExpandLibCall(LC, Node, false));
4225 break;
4226 }
4227 case ISD::FSUB:
4228 Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4229 RTLIB::SUB_F80, RTLIB::SUB_F128,
4230 RTLIB::SUB_PPCF128));
4231 break;
4232 case ISD::SREM:
4233 Results.push_back(ExpandIntLibCall(Node, true,
4234 RTLIB::SREM_I8,
4235 RTLIB::SREM_I16, RTLIB::SREM_I32,
4236 RTLIB::SREM_I64, RTLIB::SREM_I128));
4237 break;
4238 case ISD::UREM:
4239 Results.push_back(ExpandIntLibCall(Node, false,
4240 RTLIB::UREM_I8,
4241 RTLIB::UREM_I16, RTLIB::UREM_I32,
4242 RTLIB::UREM_I64, RTLIB::UREM_I128));
4243 break;
4244 case ISD::SDIV:
4245 Results.push_back(ExpandIntLibCall(Node, true,
4246 RTLIB::SDIV_I8,
4247 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4248 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4249 break;
4250 case ISD::UDIV:
4251 Results.push_back(ExpandIntLibCall(Node, false,
4252 RTLIB::UDIV_I8,
4253 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4254 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4255 break;
4256 case ISD::SDIVREM:
4257 case ISD::UDIVREM:
4258 // Expand into divrem libcall
4259 ExpandDivRemLibCall(Node, Results);
4260 break;
4261 case ISD::MUL:
4262 Results.push_back(ExpandIntLibCall(Node, false,
4263 RTLIB::MUL_I8,
4264 RTLIB::MUL_I16, RTLIB::MUL_I32,
4265 RTLIB::MUL_I64, RTLIB::MUL_I128));
4266 break;
4267 }
4268
4269 // Replace the original node with the legalized result.
4270 if (!Results.empty()) {
4271 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully converted node to libcall\n"
; } } while (false)
;
4272 ReplaceNode(Node, Results.data());
4273 } else
4274 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not convert node to libcall\n"
; } } while (false)
;
4275}
4276
4277// Determine the vector type to use in place of an original scalar element when
4278// promoting equally sized vectors.
4279static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4280 MVT EltVT, MVT NewEltVT) {
4281 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4282 MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4283 assert(TLI.isTypeLegal(MidVT) && "unexpected")(static_cast <bool> (TLI.isTypeLegal(MidVT) && "unexpected"
) ? void (0) : __assert_fail ("TLI.isTypeLegal(MidVT) && \"unexpected\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4283, __extension__ __PRETTY_FUNCTION__))
;
4284 return MidVT;
4285}
4286
4287void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4288 LLVM_DEBUG(dbgs() << "Trying to promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to promote node\n"
; } } while (false)
;
4289 SmallVector<SDValue, 8> Results;
4290 MVT OVT = Node->getSimpleValueType(0);
4291 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4292 Node->getOpcode() == ISD::SINT_TO_FP ||
4293 Node->getOpcode() == ISD::SETCC ||
4294 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4295 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4296 OVT = Node->getOperand(0).getSimpleValueType();
4297 }
4298 if (Node->getOpcode() == ISD::BR_CC)
4299 OVT = Node->getOperand(2).getSimpleValueType();
4300 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4301 SDLoc dl(Node);
4302 SDValue Tmp1, Tmp2, Tmp3;
4303 switch (Node->getOpcode()) {
4304 case ISD::CTTZ:
4305 case ISD::CTTZ_ZERO_UNDEF:
4306 case ISD::CTLZ:
4307 case ISD::CTLZ_ZERO_UNDEF:
4308 case ISD::CTPOP:
4309 // Zero extend the argument.
4310 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4311 if (Node->getOpcode() == ISD::CTTZ) {
4312 // The count is the same in the promoted type except if the original
4313 // value was zero. This can be handled by setting the bit just off
4314 // the top of the original type.
4315 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4316 OVT.getSizeInBits());
4317 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4318 DAG.getConstant(TopBit, dl, NVT));
4319 }
4320 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4321 // already the correct result.
4322 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4323 if (Node->getOpcode() == ISD::CTLZ ||
4324 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4325 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4326 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4327 DAG.getConstant(NVT.getSizeInBits() -
4328 OVT.getSizeInBits(), dl, NVT));
4329 }
4330 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4331 break;
4332 case ISD::BITREVERSE:
4333 case ISD::BSWAP: {
4334 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4335 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4336 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4337 Tmp1 = DAG.getNode(
4338 ISD::SRL, dl, NVT, Tmp1,
4339 DAG.getConstant(DiffBits, dl,
4340 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4341
4342 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4343 break;
4344 }
4345 case ISD::FP_TO_UINT:
4346 case ISD::FP_TO_SINT:
4347 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4348 Node->getOpcode() == ISD::FP_TO_SINT, dl);
4349 Results.push_back(Tmp1);
4350 break;
4351 case ISD::UINT_TO_FP:
4352 case ISD::SINT_TO_FP:
4353 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4354 Node->getOpcode() == ISD::SINT_TO_FP, dl);
4355 Results.push_back(Tmp1);
4356 break;
4357 case ISD::VAARG: {
4358 SDValue Chain = Node->getOperand(0); // Get the chain.
4359 SDValue Ptr = Node->getOperand(1); // Get the pointer.
4360
4361 unsigned TruncOp;
4362 if (OVT.isVector()) {
4363 TruncOp = ISD::BITCAST;
4364 } else {
4365 assert(OVT.isInteger()(static_cast <bool> (OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? void (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4366, __extension__ __PRETTY_FUNCTION__))
4366 && "VAARG promotion is supported only for vectors or integer types")(static_cast <bool> (OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? void (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4366, __extension__ __PRETTY_FUNCTION__))
;
4367 TruncOp = ISD::TRUNCATE;
4368 }
4369
4370 // Perform the larger operation, then convert back
4371 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4372 Node->getConstantOperandVal(3));
4373 Chain = Tmp1.getValue(1);
4374
4375 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4376
4377 // Modified the chain result - switch anything that used the old chain to
4378 // use the new one.
4379 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4380 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4381 if (UpdatedNodes) {
4382 UpdatedNodes->insert(Tmp2.getNode());
4383 UpdatedNodes->insert(Chain.getNode());
4384 }
4385 ReplacedNode(Node);
4386 break;
4387 }
4388 case ISD::MUL:
4389 case ISD::SDIV:
4390 case ISD::SREM:
4391 case ISD::UDIV:
4392 case ISD::UREM:
4393 case ISD::AND:
4394 case ISD::OR:
4395 case ISD::XOR: {
4396 unsigned ExtOp, TruncOp;
4397 if (OVT.isVector()) {
4398 ExtOp = ISD::BITCAST;
4399 TruncOp = ISD::BITCAST;
4400 } else {
4401 assert(OVT.isInteger() && "Cannot promote logic operation")(static_cast <bool> (OVT.isInteger() && "Cannot promote logic operation"
) ? void (0) : __assert_fail ("OVT.isInteger() && \"Cannot promote logic operation\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4401, __extension__ __PRETTY_FUNCTION__))
;
4402
4403 switch (Node->getOpcode()) {
4404 default:
4405 ExtOp = ISD::ANY_EXTEND;
4406 break;
4407 case ISD::SDIV:
4408 case ISD::SREM:
4409 ExtOp = ISD::SIGN_EXTEND;
4410 break;
4411 case ISD::UDIV:
4412 case ISD::UREM:
4413 ExtOp = ISD::ZERO_EXTEND;
4414 break;
4415 }
4416 TruncOp = ISD::TRUNCATE;
4417 }
4418 // Promote each of the values to the new type.
4419 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4420 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4421 // Perform the larger operation, then convert back
4422 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4423 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4424 break;
4425 }
4426 case ISD::UMUL_LOHI:
4427 case ISD::SMUL_LOHI: {
4428 // Promote to a multiply in a wider integer type.
4429 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4430 : ISD::SIGN_EXTEND;
4431 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4432 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4433 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4434
4435 auto &DL = DAG.getDataLayout();
4436 unsigned OriginalSize = OVT.getScalarSizeInBits();
4437 Tmp2 = DAG.getNode(
4438 ISD::SRL, dl, NVT, Tmp1,
4439 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4440 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4441 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4442 break;
4443 }
4444 case ISD::SELECT: {
4445 unsigned ExtOp, TruncOp;
4446 if (Node->getValueType(0).isVector() ||
4447 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4448 ExtOp = ISD::BITCAST;
4449 TruncOp = ISD::BITCAST;
4450 } else if (Node->getValueType(0).isInteger()) {
4451 ExtOp = ISD::ANY_EXTEND;
4452 TruncOp = ISD::TRUNCATE;
4453 } else {
4454 ExtOp = ISD::FP_EXTEND;
4455 TruncOp = ISD::FP_ROUND;
4456 }
4457 Tmp1 = Node->getOperand(0);
4458 // Promote each of the values to the new type.
4459 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4460 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4461 // Perform the larger operation, then round down.
4462 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4463 if (TruncOp != ISD::FP_ROUND)
4464 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4465 else
4466 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4467 DAG.getIntPtrConstant(0, dl));
4468 Results.push_back(Tmp1);
4469 break;
4470 }
4471 case ISD::VECTOR_SHUFFLE: {
4472 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4473
4474 // Cast the two input vectors.
4475 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4476 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4477
4478 // Convert the shuffle mask to the right # elements.
4479 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4480 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4481 Results.push_back(Tmp1);
4482 break;
4483 }
4484 case ISD::SETCC: {
4485 unsigned ExtOp = ISD::FP_EXTEND;
4486 if (NVT.isInteger()) {
4487 ISD::CondCode CCCode =
4488 cast<CondCodeSDNode>(Node->getOperand(2))->get();
4489 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4490 }
4491 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4492 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4493 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
4494 Tmp1, Tmp2, Node->getOperand(2)));
4495 break;
4496 }
4497 case ISD::BR_CC: {
4498 unsigned ExtOp = ISD::FP_EXTEND;
4499 if (NVT.isInteger()) {
4500 ISD::CondCode CCCode =
4501 cast<CondCodeSDNode>(Node->getOperand(1))->get();
4502 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4503 }
4504 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4505 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4506 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4507 Node->getOperand(0), Node->getOperand(1),
4508 Tmp1, Tmp2, Node->getOperand(4)));
4509 break;
4510 }
4511 case ISD::FADD:
4512 case ISD::FSUB:
4513 case ISD::FMUL:
4514 case ISD::FDIV:
4515 case ISD::FREM:
4516 case ISD::FMINNUM:
4517 case ISD::FMAXNUM:
4518 case ISD::FPOW:
4519 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4520 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4521 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4522 Node->getFlags());
4523 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4524 Tmp3, DAG.getIntPtrConstant(0, dl)));
4525 break;
4526 case ISD::FMA:
4527 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4528 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4529 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4530 Results.push_back(
4531 DAG.getNode(ISD::FP_ROUND, dl, OVT,
4532 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4533 DAG.getIntPtrConstant(0, dl)));
4534 break;
4535 case ISD::FCOPYSIGN:
4536 case ISD::FPOWI: {
4537 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4538 Tmp2 = Node->getOperand(1);
4539 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4540
4541 // fcopysign doesn't change anything but the sign bit, so
4542 // (fp_round (fcopysign (fpext a), b))
4543 // is as precise as
4544 // (fp_round (fpext a))
4545 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4546 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4547 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4548 Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4549 break;
4550 }
4551 case ISD::FFLOOR:
4552 case ISD::FCEIL:
4553 case ISD::FRINT:
4554 case ISD::FNEARBYINT:
4555 case ISD::FROUND:
4556 case ISD::FTRUNC:
4557 case ISD::FNEG:
4558 case ISD::FSQRT:
4559 case ISD::FSIN:
4560 case ISD::FCOS:
4561 case ISD::FLOG:
4562 case ISD::FLOG2:
4563 case ISD::FLOG10:
4564 case ISD::FABS:
4565 case ISD::FEXP:
4566 case ISD::FEXP2:
4567 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4568 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4569 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4570 Tmp2, DAG.getIntPtrConstant(0, dl)));
4571 break;
4572 case ISD::BUILD_VECTOR: {
4573 MVT EltVT = OVT.getVectorElementType();
4574 MVT NewEltVT = NVT.getVectorElementType();
4575
4576 // Handle bitcasts to a different vector type with the same total bit size
4577 //
4578 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4579 // =>
4580 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4581
4582 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for build_vector"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4583, __extension__ __PRETTY_FUNCTION__))
4583 "Invalid promote type for build_vector")(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for build_vector"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4583, __extension__ __PRETTY_FUNCTION__))
;
4584 assert(NewEltVT.bitsLT(EltVT) && "not handled")(static_cast <bool> (NewEltVT.bitsLT(EltVT) && "not handled"
) ? void (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4584, __extension__ __PRETTY_FUNCTION__))
;
4585
4586 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4587
4588 SmallVector<SDValue, 8> NewOps;
4589 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4590 SDValue Op = Node->getOperand(I);
4591 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4592 }
4593
4594 SDLoc SL(Node);
4595 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4596 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4597 Results.push_back(CvtVec);
4598 break;
4599 }
4600 case ISD::EXTRACT_VECTOR_ELT: {
4601 MVT EltVT = OVT.getVectorElementType();
4602 MVT NewEltVT = NVT.getVectorElementType();
4603
4604 // Handle bitcasts to a different vector type with the same total bit size.
4605 //
4606 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4607 // =>
4608 // v4i32:castx = bitcast x:v2i64
4609 //
4610 // i64 = bitcast
4611 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4612 // (i32 (extract_vector_elt castx, (2 * y + 1)))
4613 //
4614
4615 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for extract_vector_elt"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4616, __extension__ __PRETTY_FUNCTION__))
4616 "Invalid promote type for extract_vector_elt")(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for extract_vector_elt"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4616, __extension__ __PRETTY_FUNCTION__))
;
4617 assert(NewEltVT.bitsLT(EltVT) && "not handled")(static_cast <bool> (NewEltVT.bitsLT(EltVT) && "not handled"
) ? void (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4617, __extension__ __PRETTY_FUNCTION__))
;
4618
4619 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4620 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4621
4622 SDValue Idx = Node->getOperand(1);
4623 EVT IdxVT = Idx.getValueType();
4624 SDLoc SL(Node);
4625 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4626 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4627
4628 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4629
4630 SmallVector<SDValue, 8> NewOps;
4631 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4632 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4633 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4634
4635 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4636 CastVec, TmpIdx);
4637 NewOps.push_back(Elt);
4638 }
4639
4640 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4641 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4642 break;
4643 }
4644 case ISD::INSERT_VECTOR_ELT: {
4645 MVT EltVT = OVT.getVectorElementType();
4646 MVT NewEltVT = NVT.getVectorElementType();
4647
4648 // Handle bitcasts to a different vector type with the same total bit size
4649 //
4650 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4651 // =>
4652 // v4i32:castx = bitcast x:v2i64
4653 // v2i32:casty = bitcast y:i64
4654 //
4655 // v2i64 = bitcast
4656 // (v4i32 insert_vector_elt
4657 // (v4i32 insert_vector_elt v4i32:castx,
4658 // (extract_vector_elt casty, 0), 2 * z),
4659 // (extract_vector_elt casty, 1), (2 * z + 1))
4660
4661 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for insert_vector_elt"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4662, __extension__ __PRETTY_FUNCTION__))
4662 "Invalid promote type for insert_vector_elt")(static_cast <bool> (NVT.isVector() && OVT.getSizeInBits
() == NVT.getSizeInBits() && "Invalid promote type for insert_vector_elt"
) ? void (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4662, __extension__ __PRETTY_FUNCTION__))
;
4663 assert(NewEltVT.bitsLT(EltVT) && "not handled")(static_cast <bool> (NewEltVT.bitsLT(EltVT) && "not handled"
) ? void (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4663, __extension__ __PRETTY_FUNCTION__))
;
4664
4665 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4666 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4667
4668 SDValue Val = Node->getOperand(1);
4669 SDValue Idx = Node->getOperand(2);
4670 EVT IdxVT = Idx.getValueType();
4671 SDLoc SL(Node);
4672
4673 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4674 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4675
4676 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4677 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4678
4679 SDValue NewVec = CastVec;
4680 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4681 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4682 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4683
4684 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4685 CastVal, IdxOffset);
4686
4687 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4688 NewVec, Elt, InEltIdx);
4689 }
4690
4691 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4692 break;
4693 }
4694 case ISD::SCALAR_TO_VECTOR: {
4695 MVT EltVT = OVT.getVectorElementType();
4696 MVT NewEltVT = NVT.getVectorElementType();
4697
4698 // Handle bitcasts to different vector type with the same total bit size.
4699 //
4700 // e.g. v2i64 = scalar_to_vector x:i64
4701 // =>
4702 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4703 //
4704
4705 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4706 SDValue Val = Node->getOperand(0);
4707 SDLoc SL(Node);
4708
4709 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4710 SDValue Undef = DAG.getUNDEF(MidVT);
4711
4712 SmallVector<SDValue, 8> NewElts;
4713 NewElts.push_back(CastVal);
4714 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4715 NewElts.push_back(Undef);
4716
4717 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4718 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4719 Results.push_back(CvtVec);
4720 break;
4721 }
4722 }
4723
4724 // Replace the original node with the legalized result.
4725 if (!Results.empty()) {
4726 LLVM_DEBUG(dbgs() << "Successfully promoted node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully promoted node\n"
; } } while (false)
;
4727 ReplaceNode(Node, Results.data());
4728 } else
4729 LLVM_DEBUG(dbgs() << "Could not promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not promote node\n"
; } } while (false)
;
4730}
4731
4732/// This is the entry point for the file.
4733void SelectionDAG::Legalize() {
4734 AssignTopologicalOrder();
4735
4736 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4737 // Use a delete listener to remove nodes which were deleted during
4738 // legalization from LegalizeNodes. This is needed to handle the situation
4739 // where a new node is allocated by the object pool to the same address of a
4740 // previously deleted node.
4741 DAGNodeDeletedListener DeleteListener(
4742 *this,
4743 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4744
4745 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4746
4747 // Visit all the nodes. We start in topological order, so that we see
4748 // nodes with their original operands intact. Legalization can produce
4749 // new nodes which may themselves need to be legalized. Iterate until all
4750 // nodes have been legalized.
4751 while (true) {
4752 bool AnyLegalized = false;
4753 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4754 --NI;
4755
4756 SDNode *N = &*NI;
4757 if (N->use_empty() && N != getRoot().getNode()) {
4758 ++NI;
4759 DeleteNode(N);
4760 continue;
4761 }
4762
4763 if (LegalizedNodes.insert(N).second) {
4764 AnyLegalized = true;
4765 Legalizer.LegalizeOp(N);
4766
4767 if (N->use_empty() && N != getRoot().getNode()) {
4768 ++NI;
4769 DeleteNode(N);
4770 }
4771 }
4772 }
4773 if (!AnyLegalized)
4774 break;
4775
4776 }
4777
4778 // Remove dead nodes now.
4779 RemoveDeadNodes();
4780}
4781
4782bool SelectionDAG::LegalizeOp(SDNode *N,
4783 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4784 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4785 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4786
4787 // Directly insert the node in question, and legalize it. This will recurse
4788 // as needed through operands.
4789 LegalizedNodes.insert(N);
4790 Legalizer.LegalizeOp(N);
4791
4792 return LegalizedNodes.count(N);
4793}