Bug Summary

File:lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
Warning:line 545, column 29
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizeDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-config-compatibility-mode=true -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-8/lib/clang/8.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-8~svn348900/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-8~svn348900/build-llvm/include -I /build/llvm-toolchain-snapshot-8~svn348900/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/8.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-8/lib/clang/8.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-command-line-argument -Wno-unknown-warning-option -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-8~svn348900/build-llvm/lib/CodeGen/SelectionDAG -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-12-12-042652-12204-1 -x c++ /build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp -faddrsig

/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the SelectionDAG::Legalize method.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ADT/APFloat.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/CodeGen/ISDOpcodes.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineMemOperand.h"
25#include "llvm/CodeGen/RuntimeLibcalls.h"
26#include "llvm/CodeGen/SelectionDAG.h"
27#include "llvm/CodeGen/SelectionDAGNodes.h"
28#include "llvm/CodeGen/TargetFrameLowering.h"
29#include "llvm/CodeGen/TargetLowering.h"
30#include "llvm/CodeGen/TargetSubtargetInfo.h"
31#include "llvm/CodeGen/ValueTypes.h"
32#include "llvm/IR/CallingConv.h"
33#include "llvm/IR/Constants.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/DerivedTypes.h"
36#include "llvm/IR/Function.h"
37#include "llvm/IR/Metadata.h"
38#include "llvm/IR/Type.h"
39#include "llvm/Support/Casting.h"
40#include "llvm/Support/Compiler.h"
41#include "llvm/Support/Debug.h"
42#include "llvm/Support/ErrorHandling.h"
43#include "llvm/Support/MachineValueType.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/raw_ostream.h"
46#include "llvm/Target/TargetMachine.h"
47#include "llvm/Target/TargetOptions.h"
48#include <algorithm>
49#include <cassert>
50#include <cstdint>
51#include <tuple>
52#include <utility>
53
54using namespace llvm;
55
56#define DEBUG_TYPE"legalizedag" "legalizedag"
57
58namespace {
59
60/// Keeps track of state when getting the sign of a floating-point value as an
61/// integer.
62struct FloatSignAsInt {
63 EVT FloatVT;
64 SDValue Chain;
65 SDValue FloatPtr;
66 SDValue IntPtr;
67 MachinePointerInfo IntPointerInfo;
68 MachinePointerInfo FloatPointerInfo;
69 SDValue IntValue;
70 APInt SignMask;
71 uint8_t SignBit;
72};
73
74//===----------------------------------------------------------------------===//
75/// This takes an arbitrary SelectionDAG as input and
76/// hacks on it until the target machine can handle it. This involves
77/// eliminating value sizes the machine cannot handle (promoting small sizes to
78/// large sizes or splitting up large values into small values) as well as
79/// eliminating operations the machine cannot handle.
80///
81/// This code also does a small amount of optimization and recognition of idioms
82/// as part of its processing. For example, if a target does not support a
83/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
84/// will attempt merge setcc and brc instructions into brcc's.
85class SelectionDAGLegalize {
86 const TargetMachine &TM;
87 const TargetLowering &TLI;
88 SelectionDAG &DAG;
89
90 /// The set of nodes which have already been legalized. We hold a
91 /// reference to it in order to update as necessary on node deletion.
92 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93
94 /// A set of all the nodes updated during legalization.
95 SmallSetVector<SDNode *, 16> *UpdatedNodes;
96
97 EVT getSetCCResultType(EVT VT) const {
98 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
99 }
100
101 // Libcall insertion helpers.
102
103public:
104 SelectionDAGLegalize(SelectionDAG &DAG,
105 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
106 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
107 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
108 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109
110 /// Legalizes the given operation.
111 void LegalizeOp(SDNode *Node);
112
113private:
114 SDValue OptimizeFloatStore(StoreSDNode *ST);
115
116 void LegalizeLoadOps(SDNode *Node);
117 void LegalizeStoreOps(SDNode *Node);
118
119 /// Some targets cannot handle a variable
120 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
121 /// is necessary to spill the vector being inserted into to memory, perform
122 /// the insert there, and then read the result back.
123 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
124 const SDLoc &dl);
125 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
126 const SDLoc &dl);
127
128 /// Return a vector shuffle operation which
129 /// performs the same shuffe in terms of order or result bytes, but on a type
130 /// whose vector element type is narrower than the original shuffle type.
131 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
132 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133 SDValue N1, SDValue N2,
134 ArrayRef<int> Mask) const;
135
136 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
137 bool &NeedInvert, const SDLoc &dl);
138
139 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
140 SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
141 unsigned NumOps, bool isSigned, const SDLoc &dl);
142
143 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
144 SDNode *Node, bool isSigned);
145 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
146 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
147 RTLIB::Libcall Call_F128,
148 RTLIB::Libcall Call_PPCF128);
149 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
150 RTLIB::Libcall Call_I8,
151 RTLIB::Libcall Call_I16,
152 RTLIB::Libcall Call_I32,
153 RTLIB::Libcall Call_I64,
154 RTLIB::Libcall Call_I128);
155 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157
158 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
159 const SDLoc &dl);
160 SDValue ExpandBUILD_VECTOR(SDNode *Node);
161 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
162 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
163 SmallVectorImpl<SDValue> &Results);
164 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
165 SDValue Value) const;
166 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
167 SDValue NewIntValue) const;
168 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
169 SDValue ExpandFABS(SDNode *Node) const;
170 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, EVT DestVT,
171 const SDLoc &dl);
172 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
173 const SDLoc &dl);
174 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
175 const SDLoc &dl);
176
177 SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
178 SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
179
180 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
181 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
182 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
183
184 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
185 SDValue ExpandConstant(ConstantSDNode *CP);
186
187 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
188 bool ExpandNode(SDNode *Node);
189 void ConvertNodeToLibcall(SDNode *Node);
190 void PromoteNode(SDNode *Node);
191
192public:
193 // Node replacement helpers
194
195 void ReplacedNode(SDNode *N) {
196 LegalizedNodes.erase(N);
197 if (UpdatedNodes)
198 UpdatedNodes->insert(N);
199 }
200
201 void ReplaceNode(SDNode *Old, SDNode *New) {
202 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
203 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
204
205 assert(Old->getNumValues() == New->getNumValues() &&((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 207, __PRETTY_FUNCTION__))
206 "Replacing one node with another that produces a different number "((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 207, __PRETTY_FUNCTION__))
207 "of values!")((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 207, __PRETTY_FUNCTION__))
;
208 DAG.ReplaceAllUsesWith(Old, New);
209 if (UpdatedNodes)
210 UpdatedNodes->insert(New);
211 ReplacedNode(Old);
212 }
213
214 void ReplaceNode(SDValue Old, SDValue New) {
215 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
216 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
217
218 DAG.ReplaceAllUsesWith(Old, New);
219 if (UpdatedNodes)
220 UpdatedNodes->insert(New.getNode());
221 ReplacedNode(Old.getNode());
222 }
223
224 void ReplaceNode(SDNode *Old, const SDValue *New) {
225 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); } } while (false)
;
226
227 DAG.ReplaceAllUsesWith(Old, New);
228 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
229 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
230 New[i]->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
;
231 if (UpdatedNodes)
232 UpdatedNodes->insert(New[i].getNode());
233 }
234 ReplacedNode(Old);
235 }
236};
237
238} // end anonymous namespace
239
240/// Return a vector shuffle operation which
241/// performs the same shuffle in terms of order or result bytes, but on a type
242/// whose vector element type is narrower than the original shuffle type.
243/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
244SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
245 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
246 ArrayRef<int> Mask) const {
247 unsigned NumMaskElts = VT.getVectorNumElements();
248 unsigned NumDestElts = NVT.getVectorNumElements();
249 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
250
251 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!")((NumEltsGrowth && "Cannot promote to vector type with fewer elts!"
) ? static_cast<void> (0) : __assert_fail ("NumEltsGrowth && \"Cannot promote to vector type with fewer elts!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 251, __PRETTY_FUNCTION__))
;
252
253 if (NumEltsGrowth == 1)
254 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
255
256 SmallVector<int, 8> NewMask;
257 for (unsigned i = 0; i != NumMaskElts; ++i) {
258 int Idx = Mask[i];
259 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
260 if (Idx < 0)
261 NewMask.push_back(-1);
262 else
263 NewMask.push_back(Idx * NumEltsGrowth + j);
264 }
265 }
266 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?")((NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"
) ? static_cast<void> (0) : __assert_fail ("NewMask.size() == NumDestElts && \"Non-integer NumEltsGrowth?\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 266, __PRETTY_FUNCTION__))
;
267 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?")((TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"
) ? static_cast<void> (0) : __assert_fail ("TLI.isShuffleMaskLegal(NewMask, NVT) && \"Shuffle not legal?\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 267, __PRETTY_FUNCTION__))
;
268 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
269}
270
271/// Expands the ConstantFP node to an integer constant or
272/// a load from the constant pool.
273SDValue
274SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
275 bool Extend = false;
276 SDLoc dl(CFP);
277
278 // If a FP immediate is precise when represented as a float and if the
279 // target can do an extending load from float to double, we put it into
280 // the constant pool as a float, even if it's is statically typed as a
281 // double. This shrinks FP constants and canonicalizes them for targets where
282 // an FP extending load is the same cost as a normal load (such as on the x87
283 // fp stack or PPC FP unit).
284 EVT VT = CFP->getValueType(0);
285 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
286 if (!UseCP) {
287 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion")(((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::f64 || VT == MVT::f32) && \"Invalid type expansion\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 287, __PRETTY_FUNCTION__))
;
288 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
289 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
290 }
291
292 APFloat APF = CFP->getValueAPF();
293 EVT OrigVT = VT;
294 EVT SVT = VT;
295
296 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
297 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
298 if (!APF.isSignaling()) {
299 while (SVT != MVT::f32 && SVT != MVT::f16) {
300 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
301 if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
302 // Only do this if the target has a native EXTLOAD instruction from
303 // smaller type.
304 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
305 TLI.ShouldShrinkFPConstant(OrigVT)) {
306 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
307 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
308 VT = SVT;
309 Extend = true;
310 }
311 }
312 }
313
314 SDValue CPIdx =
315 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
316 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
317 if (Extend) {
318 SDValue Result = DAG.getExtLoad(
319 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
320 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
321 Alignment);
322 return Result;
323 }
324 SDValue Result = DAG.getLoad(
325 OrigVT, dl, DAG.getEntryNode(), CPIdx,
326 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
327 return Result;
328}
329
330/// Expands the Constant node to a load from the constant pool.
331SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
332 SDLoc dl(CP);
333 EVT VT = CP->getValueType(0);
334 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
335 TLI.getPointerTy(DAG.getDataLayout()));
336 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
337 SDValue Result = DAG.getLoad(
338 VT, dl, DAG.getEntryNode(), CPIdx,
339 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
340 return Result;
341}
342
343/// Some target cannot handle a variable insertion index for the
344/// INSERT_VECTOR_ELT instruction. In this case, it
345/// is necessary to spill the vector being inserted into to memory, perform
346/// the insert there, and then read the result back.
347SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
348 SDValue Val,
349 SDValue Idx,
350 const SDLoc &dl) {
351 SDValue Tmp1 = Vec;
352 SDValue Tmp2 = Val;
353 SDValue Tmp3 = Idx;
354
355 // If the target doesn't support this, we have to spill the input vector
356 // to a temporary stack slot, update the element, then reload it. This is
357 // badness. We could also load the value into a vector register (either
358 // with a "move to register" or "extload into register" instruction, then
359 // permute it into place, if the idx is a constant and if the idx is
360 // supported by the target.
361 EVT VT = Tmp1.getValueType();
362 EVT EltVT = VT.getVectorElementType();
363 SDValue StackPtr = DAG.CreateStackTemporary(VT);
364
365 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
366
367 // Store the vector.
368 SDValue Ch = DAG.getStore(
369 DAG.getEntryNode(), dl, Tmp1, StackPtr,
370 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
371
372 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
373
374 // Store the scalar value.
375 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
376 // Load the updated vector.
377 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
378 DAG.getMachineFunction(), SPFI));
379}
380
381SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
382 SDValue Idx,
383 const SDLoc &dl) {
384 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
385 // SCALAR_TO_VECTOR requires that the type of the value being inserted
386 // match the element type of the vector being created, except for
387 // integers in which case the inserted value can be over width.
388 EVT EltVT = Vec.getValueType().getVectorElementType();
389 if (Val.getValueType() == EltVT ||
390 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
391 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
392 Vec.getValueType(), Val);
393
394 unsigned NumElts = Vec.getValueType().getVectorNumElements();
395 // We generate a shuffle of InVec and ScVec, so the shuffle mask
396 // should be 0,1,2,3,4,5... with the appropriate element replaced with
397 // elt 0 of the RHS.
398 SmallVector<int, 8> ShufOps;
399 for (unsigned i = 0; i != NumElts; ++i)
400 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
401
402 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
403 }
404 }
405 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
406}
407
408SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
409 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Optimizing float store operations\n"
; } } while (false)
;
410 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
411 // FIXME: We shouldn't do this for TargetConstantFP's.
412 // FIXME: move this to the DAG Combiner! Note that we can't regress due
413 // to phase ordering between legalized code and the dag combiner. This
414 // probably means that we need to integrate dag combiner and legalizer
415 // together.
416 // We generally can't do this one for long doubles.
417 SDValue Chain = ST->getChain();
418 SDValue Ptr = ST->getBasePtr();
419 unsigned Alignment = ST->getAlignment();
420 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
421 AAMDNodes AAInfo = ST->getAAInfo();
422 SDLoc dl(ST);
423 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
424 if (CFP->getValueType(0) == MVT::f32 &&
425 TLI.isTypeLegal(MVT::i32)) {
426 SDValue Con = DAG.getConstant(CFP->getValueAPF().
427 bitcastToAPInt().zextOrTrunc(32),
428 SDLoc(CFP), MVT::i32);
429 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
430 MMOFlags, AAInfo);
431 }
432
433 if (CFP->getValueType(0) == MVT::f64) {
434 // If this target supports 64-bit registers, do a single 64-bit store.
435 if (TLI.isTypeLegal(MVT::i64)) {
436 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
437 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
438 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
439 Alignment, MMOFlags, AAInfo);
440 }
441
442 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
443 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
444 // stores. If the target supports neither 32- nor 64-bits, this
445 // xform is certainly not worth it.
446 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
447 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
448 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
449 if (DAG.getDataLayout().isBigEndian())
450 std::swap(Lo, Hi);
451
452 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
453 MMOFlags, AAInfo);
454 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
455 DAG.getConstant(4, dl, Ptr.getValueType()));
456 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
457 ST->getPointerInfo().getWithOffset(4),
458 MinAlign(Alignment, 4U), MMOFlags, AAInfo);
459
460 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
461 }
462 }
463 }
464 return SDValue(nullptr, 0);
465}
466
467void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
468 StoreSDNode *ST = cast<StoreSDNode>(Node);
469 SDValue Chain = ST->getChain();
470 SDValue Ptr = ST->getBasePtr();
471 SDLoc dl(Node);
472
473 unsigned Alignment = ST->getAlignment();
474 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
475 AAMDNodes AAInfo = ST->getAAInfo();
476
477 if (!ST->isTruncatingStore()) {
13
Assuming the condition is false
14
Taking false branch
478 LLVM_DEBUG(dbgs() << "Legalizing store operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing store operation\n"
; } } while (false)
;
479 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
480 ReplaceNode(ST, OptStore);
481 return;
482 }
483
484 SDValue Value = ST->getValue();
485 MVT VT = Value.getSimpleValueType();
486 switch (TLI.getOperationAction(ISD::STORE, VT)) {
487 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 487)
;
488 case TargetLowering::Legal: {
489 // If this is an unaligned store and the target doesn't support it,
490 // expand it.
491 EVT MemVT = ST->getMemoryVT();
492 unsigned AS = ST->getAddressSpace();
493 unsigned Align = ST->getAlignment();
494 const DataLayout &DL = DAG.getDataLayout();
495 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
496 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Expanding unsupported unaligned store\n"
; } } while (false)
;
497 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
498 ReplaceNode(SDValue(ST, 0), Result);
499 } else
500 LLVM_DEBUG(dbgs() << "Legal store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legal store\n"; } } while
(false)
;
501 break;
502 }
503 case TargetLowering::Custom: {
504 LLVM_DEBUG(dbgs() << "Trying custom lowering\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying custom lowering\n"
; } } while (false)
;
505 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
506 if (Res && Res != SDValue(Node, 0))
507 ReplaceNode(SDValue(Node, 0), Res);
508 return;
509 }
510 case TargetLowering::Promote: {
511 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
512 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote stores to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 513, __PRETTY_FUNCTION__))
513 "Can only promote stores to same size type")((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote stores to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 513, __PRETTY_FUNCTION__))
;
514 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
515 SDValue Result =
516 DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
517 Alignment, MMOFlags, AAInfo);
518 ReplaceNode(SDValue(Node, 0), Result);
519 break;
520 }
521 }
522 return;
523 }
524
525 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing truncating store operations\n"
; } } while (false)
;
15
Assuming 'DebugFlag' is 0
16
Loop condition is false. Exiting loop
526 SDValue Value = ST->getValue();
527 EVT StVT = ST->getMemoryVT();
528 unsigned StWidth = StVT.getSizeInBits();
529 auto &DL = DAG.getDataLayout();
530
531 if (StWidth != StVT.getStoreSizeInBits()) {
17
Assuming the condition is false
18
Taking false branch
532 // Promote to a byte-sized store with upper bits zero if not
533 // storing an integral number of bytes. For example, promote
534 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
535 EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
536 StVT.getStoreSizeInBits());
537 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
538 SDValue Result =
539 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
540 Alignment, MMOFlags, AAInfo);
541 ReplaceNode(SDValue(Node, 0), Result);
542 } else if (StWidth & (StWidth - 1)) {
19
Assuming the condition is true
20
Taking true branch
543 // If not storing a power-of-2 number of bits, expand as two stores.
544 assert(!StVT.isVector() && "Unsupported truncstore!")((!StVT.isVector() && "Unsupported truncstore!") ? static_cast
<void> (0) : __assert_fail ("!StVT.isVector() && \"Unsupported truncstore!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 544, __PRETTY_FUNCTION__))
;
21
Assuming the condition is true
22
'?' condition is true
545 unsigned RoundWidth = 1 << Log2_32(StWidth);
23
Calling 'Log2_32'
25
Returning from 'Log2_32'
26
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'
546 assert(RoundWidth < StWidth)((RoundWidth < StWidth) ? static_cast<void> (0) : __assert_fail
("RoundWidth < StWidth", "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 546, __PRETTY_FUNCTION__))
;
547 unsigned ExtraWidth = StWidth - RoundWidth;
548 assert(ExtraWidth < RoundWidth)((ExtraWidth < RoundWidth) ? static_cast<void> (0) :
__assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 548, __PRETTY_FUNCTION__))
;
549 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Store size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 550, __PRETTY_FUNCTION__))
550 "Store size not an integral number of bytes!")((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Store size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 550, __PRETTY_FUNCTION__))
;
551 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
552 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
553 SDValue Lo, Hi;
554 unsigned IncrementSize;
555
556 if (DL.isLittleEndian()) {
557 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
558 // Store the bottom RoundWidth bits.
559 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
560 RoundVT, Alignment, MMOFlags, AAInfo);
561
562 // Store the remaining ExtraWidth bits.
563 IncrementSize = RoundWidth / 8;
564 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
565 DAG.getConstant(IncrementSize, dl,
566 Ptr.getValueType()));
567 Hi = DAG.getNode(
568 ISD::SRL, dl, Value.getValueType(), Value,
569 DAG.getConstant(RoundWidth, dl,
570 TLI.getShiftAmountTy(Value.getValueType(), DL)));
571 Hi = DAG.getTruncStore(
572 Chain, dl, Hi, Ptr,
573 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
574 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
575 } else {
576 // Big endian - avoid unaligned stores.
577 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
578 // Store the top RoundWidth bits.
579 Hi = DAG.getNode(
580 ISD::SRL, dl, Value.getValueType(), Value,
581 DAG.getConstant(ExtraWidth, dl,
582 TLI.getShiftAmountTy(Value.getValueType(), DL)));
583 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
584 RoundVT, Alignment, MMOFlags, AAInfo);
585
586 // Store the remaining ExtraWidth bits.
587 IncrementSize = RoundWidth / 8;
588 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
589 DAG.getConstant(IncrementSize, dl,
590 Ptr.getValueType()));
591 Lo = DAG.getTruncStore(
592 Chain, dl, Value, Ptr,
593 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
594 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
595 }
596
597 // The order of the stores doesn't matter.
598 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
599 ReplaceNode(SDValue(Node, 0), Result);
600 } else {
601 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
602 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 602)
;
603 case TargetLowering::Legal: {
604 EVT MemVT = ST->getMemoryVT();
605 unsigned AS = ST->getAddressSpace();
606 unsigned Align = ST->getAlignment();
607 // If this is an unaligned store and the target doesn't support it,
608 // expand it.
609 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
610 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
611 ReplaceNode(SDValue(ST, 0), Result);
612 }
613 break;
614 }
615 case TargetLowering::Custom: {
616 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
617 if (Res && Res != SDValue(Node, 0))
618 ReplaceNode(SDValue(Node, 0), Res);
619 return;
620 }
621 case TargetLowering::Expand:
622 assert(!StVT.isVector() &&((!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 623, __PRETTY_FUNCTION__))
623 "Vector Stores are handled in LegalizeVectorOps")((!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 623, __PRETTY_FUNCTION__))
;
624
625 SDValue Result;
626
627 // TRUNCSTORE:i16 i32 -> STORE i16
628 if (TLI.isTypeLegal(StVT)) {
629 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
630 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
631 Alignment, MMOFlags, AAInfo);
632 } else {
633 // The in-memory type isn't legal. Truncate to the type it would promote
634 // to, and then do a truncstore.
635 Value = DAG.getNode(ISD::TRUNCATE, dl,
636 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
637 Value);
638 Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
639 StVT, Alignment, MMOFlags, AAInfo);
640 }
641
642 ReplaceNode(SDValue(Node, 0), Result);
643 break;
644 }
645 }
646}
647
648void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
649 LoadSDNode *LD = cast<LoadSDNode>(Node);
650 SDValue Chain = LD->getChain(); // The chain.
651 SDValue Ptr = LD->getBasePtr(); // The base pointer.
652 SDValue Value; // The value returned by the load op.
653 SDLoc dl(Node);
654
655 ISD::LoadExtType ExtType = LD->getExtensionType();
656 if (ExtType == ISD::NON_EXTLOAD) {
657 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing non-extending load operation\n"
; } } while (false)
;
658 MVT VT = Node->getSimpleValueType(0);
659 SDValue RVal = SDValue(Node, 0);
660 SDValue RChain = SDValue(Node, 1);
661
662 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
663 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 663)
;
664 case TargetLowering::Legal: {
665 EVT MemVT = LD->getMemoryVT();
666 unsigned AS = LD->getAddressSpace();
667 unsigned Align = LD->getAlignment();
668 const DataLayout &DL = DAG.getDataLayout();
669 // If this is an unaligned load and the target doesn't support it,
670 // expand it.
671 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
672 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
673 }
674 break;
675 }
676 case TargetLowering::Custom:
677 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
678 RVal = Res;
679 RChain = Res.getValue(1);
680 }
681 break;
682
683 case TargetLowering::Promote: {
684 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
685 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote loads to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 686, __PRETTY_FUNCTION__))
686 "Can only promote loads to same size type")((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote loads to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 686, __PRETTY_FUNCTION__))
;
687
688 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
689 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
690 RChain = Res.getValue(1);
691 break;
692 }
693 }
694 if (RChain.getNode() != Node) {
695 assert(RVal.getNode() != Node && "Load must be completely replaced")((RVal.getNode() != Node && "Load must be completely replaced"
) ? static_cast<void> (0) : __assert_fail ("RVal.getNode() != Node && \"Load must be completely replaced\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 695, __PRETTY_FUNCTION__))
;
696 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
697 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
698 if (UpdatedNodes) {
699 UpdatedNodes->insert(RVal.getNode());
700 UpdatedNodes->insert(RChain.getNode());
701 }
702 ReplacedNode(Node);
703 }
704 return;
705 }
706
707 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing extending load operation\n"
; } } while (false)
;
708 EVT SrcVT = LD->getMemoryVT();
709 unsigned SrcWidth = SrcVT.getSizeInBits();
710 unsigned Alignment = LD->getAlignment();
711 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
712 AAMDNodes AAInfo = LD->getAAInfo();
713
714 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
715 // Some targets pretend to have an i1 loading operation, and actually
716 // load an i8. This trick is correct for ZEXTLOAD because the top 7
717 // bits are guaranteed to be zero; it helps the optimizers understand
718 // that these bits are zero. It is also useful for EXTLOAD, since it
719 // tells the optimizers that those bits are undefined. It would be
720 // nice to have an effective generic way of getting these benefits...
721 // Until such a way is found, don't insist on promoting i1 here.
722 (SrcVT != MVT::i1 ||
723 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
724 TargetLowering::Promote)) {
725 // Promote to a byte-sized load if not loading an integral number of
726 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
727 unsigned NewWidth = SrcVT.getStoreSizeInBits();
728 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
729 SDValue Ch;
730
731 // The extra bits are guaranteed to be zero, since we stored them that
732 // way. A zext load from NVT thus automatically gives zext from SrcVT.
733
734 ISD::LoadExtType NewExtType =
735 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
736
737 SDValue Result =
738 DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
739 LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
740
741 Ch = Result.getValue(1); // The chain.
742
743 if (ExtType == ISD::SEXTLOAD)
744 // Having the top bits zero doesn't help when sign extending.
745 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
746 Result.getValueType(),
747 Result, DAG.getValueType(SrcVT));
748 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
749 // All the top bits are guaranteed to be zero - inform the optimizers.
750 Result = DAG.getNode(ISD::AssertZext, dl,
751 Result.getValueType(), Result,
752 DAG.getValueType(SrcVT));
753
754 Value = Result;
755 Chain = Ch;
756 } else if (SrcWidth & (SrcWidth - 1)) {
757 // If not loading a power-of-2 number of bits, expand as two loads.
758 assert(!SrcVT.isVector() && "Unsupported extload!")((!SrcVT.isVector() && "Unsupported extload!") ? static_cast
<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Unsupported extload!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 758, __PRETTY_FUNCTION__))
;
759 unsigned RoundWidth = 1 << Log2_32(SrcWidth);
760 assert(RoundWidth < SrcWidth)((RoundWidth < SrcWidth) ? static_cast<void> (0) : __assert_fail
("RoundWidth < SrcWidth", "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 760, __PRETTY_FUNCTION__))
;
761 unsigned ExtraWidth = SrcWidth - RoundWidth;
762 assert(ExtraWidth < RoundWidth)((ExtraWidth < RoundWidth) ? static_cast<void> (0) :
__assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 762, __PRETTY_FUNCTION__))
;
763 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Load size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 764, __PRETTY_FUNCTION__))
764 "Load size not an integral number of bytes!")((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Load size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 764, __PRETTY_FUNCTION__))
;
765 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
766 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
767 SDValue Lo, Hi, Ch;
768 unsigned IncrementSize;
769 auto &DL = DAG.getDataLayout();
770
771 if (DL.isLittleEndian()) {
772 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
773 // Load the bottom RoundWidth bits.
774 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
775 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
776 AAInfo);
777
778 // Load the remaining ExtraWidth bits.
779 IncrementSize = RoundWidth / 8;
780 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
781 DAG.getConstant(IncrementSize, dl,
782 Ptr.getValueType()));
783 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
784 LD->getPointerInfo().getWithOffset(IncrementSize),
785 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
786 AAInfo);
787
788 // Build a factor node to remember that this load is independent of
789 // the other one.
790 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
791 Hi.getValue(1));
792
793 // Move the top bits to the right place.
794 Hi = DAG.getNode(
795 ISD::SHL, dl, Hi.getValueType(), Hi,
796 DAG.getConstant(RoundWidth, dl,
797 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
798
799 // Join the hi and lo parts.
800 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
801 } else {
802 // Big endian - avoid unaligned loads.
803 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
804 // Load the top RoundWidth bits.
805 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
806 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
807 AAInfo);
808
809 // Load the remaining ExtraWidth bits.
810 IncrementSize = RoundWidth / 8;
811 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
812 DAG.getConstant(IncrementSize, dl,
813 Ptr.getValueType()));
814 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
815 LD->getPointerInfo().getWithOffset(IncrementSize),
816 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
817 AAInfo);
818
819 // Build a factor node to remember that this load is independent of
820 // the other one.
821 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
822 Hi.getValue(1));
823
824 // Move the top bits to the right place.
825 Hi = DAG.getNode(
826 ISD::SHL, dl, Hi.getValueType(), Hi,
827 DAG.getConstant(ExtraWidth, dl,
828 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
829
830 // Join the hi and lo parts.
831 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
832 }
833
834 Chain = Ch;
835 } else {
836 bool isCustom = false;
837 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
838 SrcVT.getSimpleVT())) {
839 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 839)
;
840 case TargetLowering::Custom:
841 isCustom = true;
842 LLVM_FALLTHROUGH[[clang::fallthrough]];
843 case TargetLowering::Legal:
844 Value = SDValue(Node, 0);
845 Chain = SDValue(Node, 1);
846
847 if (isCustom) {
848 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
849 Value = Res;
850 Chain = Res.getValue(1);
851 }
852 } else {
853 // If this is an unaligned load and the target doesn't support it,
854 // expand it.
855 EVT MemVT = LD->getMemoryVT();
856 unsigned AS = LD->getAddressSpace();
857 unsigned Align = LD->getAlignment();
858 const DataLayout &DL = DAG.getDataLayout();
859 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
860 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
861 }
862 }
863 break;
864
865 case TargetLowering::Expand: {
866 EVT DestVT = Node->getValueType(0);
867 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
868 // If the source type is not legal, see if there is a legal extload to
869 // an intermediate type that we can then extend further.
870 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
871 if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
872 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
873 // If we are loading a legal type, this is a non-extload followed by a
874 // full extend.
875 ISD::LoadExtType MidExtType =
876 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
877
878 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
879 SrcVT, LD->getMemOperand());
880 unsigned ExtendOp =
881 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
882 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
883 Chain = Load.getValue(1);
884 break;
885 }
886
887 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
888 // normal undefined upper bits behavior to allow using an in-reg extend
889 // with the illegal FP type, so load as an integer and do the
890 // from-integer conversion.
891 if (SrcVT.getScalarType() == MVT::f16) {
892 EVT ISrcVT = SrcVT.changeTypeToInteger();
893 EVT IDestVT = DestVT.changeTypeToInteger();
894 EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
895
896 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
897 Chain, Ptr, ISrcVT,
898 LD->getMemOperand());
899 Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
900 Chain = Result.getValue(1);
901 break;
902 }
903 }
904
905 assert(!SrcVT.isVector() &&((!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 906, __PRETTY_FUNCTION__))
906 "Vector Loads are handled in LegalizeVectorOps")((!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 906, __PRETTY_FUNCTION__))
;
907
908 // FIXME: This does not work for vectors on most targets. Sign-
909 // and zero-extend operations are currently folded into extending
910 // loads, whether they are legal or not, and then we end up here
911 // without any support for legalizing them.
912 assert(ExtType != ISD::EXTLOAD &&((ExtType != ISD::EXTLOAD && "EXTLOAD should always be supported!"
) ? static_cast<void> (0) : __assert_fail ("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 913, __PRETTY_FUNCTION__))
913 "EXTLOAD should always be supported!")((ExtType != ISD::EXTLOAD && "EXTLOAD should always be supported!"
) ? static_cast<void> (0) : __assert_fail ("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 913, __PRETTY_FUNCTION__))
;
914 // Turn the unsupported load into an EXTLOAD followed by an
915 // explicit zero/sign extend inreg.
916 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
917 Node->getValueType(0),
918 Chain, Ptr, SrcVT,
919 LD->getMemOperand());
920 SDValue ValRes;
921 if (ExtType == ISD::SEXTLOAD)
922 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
923 Result.getValueType(),
924 Result, DAG.getValueType(SrcVT));
925 else
926 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
927 Value = ValRes;
928 Chain = Result.getValue(1);
929 break;
930 }
931 }
932 }
933
934 // Since loads produce two values, make sure to remember that we legalized
935 // both of them.
936 if (Chain.getNode() != Node) {
937 assert(Value.getNode() != Node && "Load must be completely replaced")((Value.getNode() != Node && "Load must be completely replaced"
) ? static_cast<void> (0) : __assert_fail ("Value.getNode() != Node && \"Load must be completely replaced\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 937, __PRETTY_FUNCTION__))
;
938 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
939 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
940 if (UpdatedNodes) {
941 UpdatedNodes->insert(Value.getNode());
942 UpdatedNodes->insert(Chain.getNode());
943 }
944 ReplacedNode(Node);
945 }
946}
947
948/// Return a legal replacement for the given operation, with all legal operands.
949void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
950 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "\nLegalizing: "; Node->
dump(&DAG); } } while (false)
;
1
Assuming 'DebugFlag' is 0
2
Loop condition is false. Exiting loop
951
952 // Allow illegal target nodes and illegal registers.
953 if (Node->getOpcode() == ISD::TargetConstant ||
3
Assuming the condition is false
5
Taking false branch
954 Node->getOpcode() == ISD::Register)
4
Assuming the condition is false
955 return;
956
957#ifndef NDEBUG
958 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
6
Assuming 'i' is equal to 'e'
7
Loop condition is false. Execution continues on line 964
959 assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 962, __PRETTY_FUNCTION__))
960 TargetLowering::TypeLegal ||(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 962, __PRETTY_FUNCTION__))
961 TLI.isTypeLegal(Node->getValueType(i))) &&(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 962, __PRETTY_FUNCTION__))
962 "Unexpected illegal type!")(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 962, __PRETTY_FUNCTION__))
;
963
964 for (const SDValue &Op : Node->op_values())
965 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
966 TargetLowering::TypeLegal ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
967 TLI.isTypeLegal(Op.getValueType()) ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
968 Op.getOpcode() == ISD::TargetConstant ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
969 Op.getOpcode() == ISD::Register) &&(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
970 "Unexpected illegal type!")(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
;
971#endif
972
973 // Figure out the correct action; the way to query this varies by opcode
974 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
975 bool SimpleFinishLegalizing = true;
976 switch (Node->getOpcode()) {
8
Control jumps to 'case STORE:' at line 1030
977 case ISD::INTRINSIC_W_CHAIN:
978 case ISD::INTRINSIC_WO_CHAIN:
979 case ISD::INTRINSIC_VOID:
980 case ISD::STACKSAVE:
981 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
982 break;
983 case ISD::GET_DYNAMIC_AREA_OFFSET:
984 Action = TLI.getOperationAction(Node->getOpcode(),
985 Node->getValueType(0));
986 break;
987 case ISD::VAARG:
988 Action = TLI.getOperationAction(Node->getOpcode(),
989 Node->getValueType(0));
990 if (Action != TargetLowering::Promote)
991 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
992 break;
993 case ISD::FP_TO_FP16:
994 case ISD::SINT_TO_FP:
995 case ISD::UINT_TO_FP:
996 case ISD::EXTRACT_VECTOR_ELT:
997 Action = TLI.getOperationAction(Node->getOpcode(),
998 Node->getOperand(0).getValueType());
999 break;
1000 case ISD::FP_ROUND_INREG:
1001 case ISD::SIGN_EXTEND_INREG: {
1002 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1003 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1004 break;
1005 }
1006 case ISD::ATOMIC_STORE:
1007 Action = TLI.getOperationAction(Node->getOpcode(),
1008 Node->getOperand(2).getValueType());
1009 break;
1010 case ISD::SELECT_CC:
1011 case ISD::SETCC:
1012 case ISD::BR_CC: {
1013 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1014 Node->getOpcode() == ISD::SETCC ? 2 : 1;
1015 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
1016 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1017 ISD::CondCode CCCode =
1018 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1019 Action = TLI.getCondCodeAction(CCCode, OpVT);
1020 if (Action == TargetLowering::Legal) {
1021 if (Node->getOpcode() == ISD::SELECT_CC)
1022 Action = TLI.getOperationAction(Node->getOpcode(),
1023 Node->getValueType(0));
1024 else
1025 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1026 }
1027 break;
1028 }
1029 case ISD::LOAD:
1030 case ISD::STORE:
1031 // FIXME: Model these properly. LOAD and STORE are complicated, and
1032 // STORE expects the unlegalized operand in some cases.
1033 SimpleFinishLegalizing = false;
1034 break;
9
Execution continues on line 1148
1035 case ISD::CALLSEQ_START:
1036 case ISD::CALLSEQ_END:
1037 // FIXME: This shouldn't be necessary. These nodes have special properties
1038 // dealing with the recursive nature of legalization. Removing this
1039 // special case should be done as part of making LegalizeDAG non-recursive.
1040 SimpleFinishLegalizing = false;
1041 break;
1042 case ISD::EXTRACT_ELEMENT:
1043 case ISD::FLT_ROUNDS_:
1044 case ISD::MERGE_VALUES:
1045 case ISD::EH_RETURN:
1046 case ISD::FRAME_TO_ARGS_OFFSET:
1047 case ISD::EH_DWARF_CFA:
1048 case ISD::EH_SJLJ_SETJMP:
1049 case ISD::EH_SJLJ_LONGJMP:
1050 case ISD::EH_SJLJ_SETUP_DISPATCH:
1051 // These operations lie about being legal: when they claim to be legal,
1052 // they should actually be expanded.
1053 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1054 if (Action == TargetLowering::Legal)
1055 Action = TargetLowering::Expand;
1056 break;
1057 case ISD::INIT_TRAMPOLINE:
1058 case ISD::ADJUST_TRAMPOLINE:
1059 case ISD::FRAMEADDR:
1060 case ISD::RETURNADDR:
1061 case ISD::ADDROFRETURNADDR:
1062 case ISD::SPONENTRY:
1063 // These operations lie about being legal: when they claim to be legal,
1064 // they should actually be custom-lowered.
1065 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1066 if (Action == TargetLowering::Legal)
1067 Action = TargetLowering::Custom;
1068 break;
1069 case ISD::READCYCLECOUNTER:
1070 // READCYCLECOUNTER returns an i64, even if type legalization might have
1071 // expanded that to several smaller types.
1072 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1073 break;
1074 case ISD::READ_REGISTER:
1075 case ISD::WRITE_REGISTER:
1076 // Named register is legal in the DAG, but blocked by register name
1077 // selection if not implemented by target (to chose the correct register)
1078 // They'll be converted to Copy(To/From)Reg.
1079 Action = TargetLowering::Legal;
1080 break;
1081 case ISD::DEBUGTRAP:
1082 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1083 if (Action == TargetLowering::Expand) {
1084 // replace ISD::DEBUGTRAP with ISD::TRAP
1085 SDValue NewVal;
1086 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1087 Node->getOperand(0));
1088 ReplaceNode(Node, NewVal.getNode());
1089 LegalizeOp(NewVal.getNode());
1090 return;
1091 }
1092 break;
1093 case ISD::STRICT_FADD:
1094 case ISD::STRICT_FSUB:
1095 case ISD::STRICT_FMUL:
1096 case ISD::STRICT_FDIV:
1097 case ISD::STRICT_FREM:
1098 case ISD::STRICT_FSQRT:
1099 case ISD::STRICT_FMA:
1100 case ISD::STRICT_FPOW:
1101 case ISD::STRICT_FPOWI:
1102 case ISD::STRICT_FSIN:
1103 case ISD::STRICT_FCOS:
1104 case ISD::STRICT_FEXP:
1105 case ISD::STRICT_FEXP2:
1106 case ISD::STRICT_FLOG:
1107 case ISD::STRICT_FLOG10:
1108 case ISD::STRICT_FLOG2:
1109 case ISD::STRICT_FRINT:
1110 case ISD::STRICT_FNEARBYINT:
1111 case ISD::STRICT_FMAXNUM:
1112 case ISD::STRICT_FMINNUM:
1113 case ISD::STRICT_FCEIL:
1114 case ISD::STRICT_FFLOOR:
1115 case ISD::STRICT_FROUND:
1116 case ISD::STRICT_FTRUNC:
1117 // These pseudo-ops get legalized as if they were their non-strict
1118 // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
1119 // is also legal, but if ISD::FSQRT requires expansion then so does
1120 // ISD::STRICT_FSQRT.
1121 Action = TLI.getStrictFPOperationAction(Node->getOpcode(),
1122 Node->getValueType(0));
1123 break;
1124 case ISD::SADDSAT:
1125 case ISD::UADDSAT:
1126 case ISD::SSUBSAT:
1127 case ISD::USUBSAT: {
1128 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1129 break;
1130 }
1131 case ISD::MSCATTER:
1132 Action = TLI.getOperationAction(Node->getOpcode(),
1133 cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1134 break;
1135 case ISD::MSTORE:
1136 Action = TLI.getOperationAction(Node->getOpcode(),
1137 cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1138 break;
1139 default:
1140 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1141 Action = TargetLowering::Legal;
1142 } else {
1143 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1144 }
1145 break;
1146 }
1147
1148 if (SimpleFinishLegalizing) {
10
Taking false branch
1149 SDNode *NewNode = Node;
1150 switch (Node->getOpcode()) {
1151 default: break;
1152 case ISD::SHL:
1153 case ISD::SRL:
1154 case ISD::SRA:
1155 case ISD::ROTL:
1156 case ISD::ROTR: {
1157 // Legalizing shifts/rotates requires adjusting the shift amount
1158 // to the appropriate width.
1159 SDValue Op0 = Node->getOperand(0);
1160 SDValue Op1 = Node->getOperand(1);
1161 if (!Op1.getValueType().isVector()) {
1162 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1163 // The getShiftAmountOperand() may create a new operand node or
1164 // return the existing one. If new operand is created we need
1165 // to update the parent node.
1166 // Do not try to legalize SAO here! It will be automatically legalized
1167 // in the next round.
1168 if (SAO != Op1)
1169 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1170 }
1171 }
1172 break;
1173 case ISD::FSHL:
1174 case ISD::FSHR:
1175 case ISD::SRL_PARTS:
1176 case ISD::SRA_PARTS:
1177 case ISD::SHL_PARTS: {
1178 // Legalizing shifts/rotates requires adjusting the shift amount
1179 // to the appropriate width.
1180 SDValue Op0 = Node->getOperand(0);
1181 SDValue Op1 = Node->getOperand(1);
1182 SDValue Op2 = Node->getOperand(2);
1183 if (!Op2.getValueType().isVector()) {
1184 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1185 // The getShiftAmountOperand() may create a new operand node or
1186 // return the existing one. If new operand is created we need
1187 // to update the parent node.
1188 if (SAO != Op2)
1189 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1190 }
1191 break;
1192 }
1193 }
1194
1195 if (NewNode != Node) {
1196 ReplaceNode(Node, NewNode);
1197 Node = NewNode;
1198 }
1199 switch (Action) {
1200 case TargetLowering::Legal:
1201 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legal node: nothing to do\n"
; } } while (false)
;
1202 return;
1203 case TargetLowering::Custom:
1204 LLVM_DEBUG(dbgs() << "Trying custom legalization\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying custom legalization\n"
; } } while (false)
;
1205 // FIXME: The handling for custom lowering with multiple results is
1206 // a complete mess.
1207 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1208 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1209 return;
1210
1211 if (Node->getNumValues() == 1) {
1212 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully custom legalized node\n"
; } } while (false)
;
1213 // We can just directly replace this node with the lowered value.
1214 ReplaceNode(SDValue(Node, 0), Res);
1215 return;
1216 }
1217
1218 SmallVector<SDValue, 8> ResultVals;
1219 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1220 ResultVals.push_back(Res.getValue(i));
1221 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully custom legalized node\n"
; } } while (false)
;
1222 ReplaceNode(Node, ResultVals.data());
1223 return;
1224 }
1225 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not custom legalize node\n"
; } } while (false)
;
1226 LLVM_FALLTHROUGH[[clang::fallthrough]];
1227 case TargetLowering::Expand:
1228 if (ExpandNode(Node))
1229 return;
1230 LLVM_FALLTHROUGH[[clang::fallthrough]];
1231 case TargetLowering::LibCall:
1232 ConvertNodeToLibcall(Node);
1233 return;
1234 case TargetLowering::Promote:
1235 PromoteNode(Node);
1236 return;
1237 }
1238 }
1239
1240 switch (Node->getOpcode()) {
11
Control jumps to 'case STORE:' at line 1254
1241 default:
1242#ifndef NDEBUG
1243 dbgs() << "NODE: ";
1244 Node->dump( &DAG);
1245 dbgs() << "\n";
1246#endif
1247 llvm_unreachable("Do not know how to legalize this operator!")::llvm::llvm_unreachable_internal("Do not know how to legalize this operator!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1247)
;
1248
1249 case ISD::CALLSEQ_START:
1250 case ISD::CALLSEQ_END:
1251 break;
1252 case ISD::LOAD:
1253 return LegalizeLoadOps(Node);
1254 case ISD::STORE:
1255 return LegalizeStoreOps(Node);
12
Calling 'SelectionDAGLegalize::LegalizeStoreOps'
1256 }
1257}
1258
1259SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1260 SDValue Vec = Op.getOperand(0);
1261 SDValue Idx = Op.getOperand(1);
1262 SDLoc dl(Op);
1263
1264 // Before we generate a new store to a temporary stack slot, see if there is
1265 // already one that we can use. There often is because when we scalarize
1266 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1267 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1268 // the vector. If all are expanded here, we don't want one store per vector
1269 // element.
1270
1271 // Caches for hasPredecessorHelper
1272 SmallPtrSet<const SDNode *, 32> Visited;
1273 SmallVector<const SDNode *, 16> Worklist;
1274 Visited.insert(Op.getNode());
1275 Worklist.push_back(Idx.getNode());
1276 SDValue StackPtr, Ch;
1277 for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1278 UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1279 SDNode *User = *UI;
1280 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1281 if (ST->isIndexed() || ST->isTruncatingStore() ||
1282 ST->getValue() != Vec)
1283 continue;
1284
1285 // Make sure that nothing else could have stored into the destination of
1286 // this store.
1287 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1288 continue;
1289
1290 // If the index is dependent on the store we will introduce a cycle when
1291 // creating the load (the load uses the index, and by replacing the chain
1292 // we will make the index dependent on the load). Also, the store might be
1293 // dependent on the extractelement and introduce a cycle when creating
1294 // the load.
1295 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1296 ST->hasPredecessor(Op.getNode()))
1297 continue;
1298
1299 StackPtr = ST->getBasePtr();
1300 Ch = SDValue(ST, 0);
1301 break;
1302 }
1303 }
1304
1305 EVT VecVT = Vec.getValueType();
1306
1307 if (!Ch.getNode()) {
1308 // Store the value to a temporary stack slot, then LOAD the returned part.
1309 StackPtr = DAG.CreateStackTemporary(VecVT);
1310 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1311 MachinePointerInfo());
1312 }
1313
1314 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1315
1316 SDValue NewLoad;
1317
1318 if (Op.getValueType().isVector())
1319 NewLoad =
1320 DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1321 else
1322 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1323 MachinePointerInfo(),
1324 VecVT.getVectorElementType());
1325
1326 // Replace the chain going out of the store, by the one out of the load.
1327 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1328
1329 // We introduced a cycle though, so update the loads operands, making sure
1330 // to use the original store's chain as an incoming chain.
1331 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1332 NewLoad->op_end());
1333 NewLoadOperands[0] = Ch;
1334 NewLoad =
1335 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1336 return NewLoad;
1337}
1338
1339SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1340 assert(Op.getValueType().isVector() && "Non-vector insert subvector!")((Op.getValueType().isVector() && "Non-vector insert subvector!"
) ? static_cast<void> (0) : __assert_fail ("Op.getValueType().isVector() && \"Non-vector insert subvector!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1340, __PRETTY_FUNCTION__))
;
1341
1342 SDValue Vec = Op.getOperand(0);
1343 SDValue Part = Op.getOperand(1);
1344 SDValue Idx = Op.getOperand(2);
1345 SDLoc dl(Op);
1346
1347 // Store the value to a temporary stack slot, then LOAD the returned part.
1348 EVT VecVT = Vec.getValueType();
1349 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1350 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1351 MachinePointerInfo PtrInfo =
1352 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1353
1354 // First store the whole vector.
1355 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1356
1357 // Then store the inserted part.
1358 SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1359
1360 // Store the subvector.
1361 Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
1362
1363 // Finally, load the updated vector.
1364 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1365}
1366
1367SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1368 // We can't handle this case efficiently. Allocate a sufficiently
1369 // aligned object on the stack, store each element into it, then load
1370 // the result as a vector.
1371 // Create the stack frame object.
1372 EVT VT = Node->getValueType(0);
1373 EVT EltVT = VT.getVectorElementType();
1374 SDLoc dl(Node);
1375 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1376 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1377 MachinePointerInfo PtrInfo =
1378 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1379
1380 // Emit a store of each element to the stack slot.
1381 SmallVector<SDValue, 8> Stores;
1382 unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1383 // Store (in the right endianness) the elements to memory.
1384 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1385 // Ignore undef elements.
1386 if (Node->getOperand(i).isUndef()) continue;
1387
1388 unsigned Offset = TypeByteSize*i;
1389
1390 SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1391 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1392
1393 // If the destination vector element type is narrower than the source
1394 // element type, only store the bits necessary.
1395 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1396 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1397 Node->getOperand(i), Idx,
1398 PtrInfo.getWithOffset(Offset), EltVT));
1399 } else
1400 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1401 Idx, PtrInfo.getWithOffset(Offset)));
1402 }
1403
1404 SDValue StoreChain;
1405 if (!Stores.empty()) // Not all undef elements?
1406 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1407 else
1408 StoreChain = DAG.getEntryNode();
1409
1410 // Result is a load from the stack slot.
1411 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1412}
1413
1414/// Bitcast a floating-point value to an integer value. Only bitcast the part
1415/// containing the sign bit if the target has no integer value capable of
1416/// holding all bits of the floating-point value.
1417void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1418 const SDLoc &DL,
1419 SDValue Value) const {
1420 EVT FloatVT = Value.getValueType();
1421 unsigned NumBits = FloatVT.getSizeInBits();
1422 State.FloatVT = FloatVT;
1423 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1424 // Convert to an integer of the same size.
1425 if (TLI.isTypeLegal(IVT)) {
1426 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1427 State.SignMask = APInt::getSignMask(NumBits);
1428 State.SignBit = NumBits - 1;
1429 return;
1430 }
1431
1432 auto &DataLayout = DAG.getDataLayout();
1433 // Store the float to memory, then load the sign part out as an integer.
1434 MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1435 // First create a temporary that is aligned for both the load and store.
1436 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1437 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1438 // Then store the float to it.
1439 State.FloatPtr = StackPtr;
1440 MachineFunction &MF = DAG.getMachineFunction();
1441 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1442 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1443 State.FloatPointerInfo);
1444
1445 SDValue IntPtr;
1446 if (DataLayout.isBigEndian()) {
1447 assert(FloatVT.isByteSized() && "Unsupported floating point type!")((FloatVT.isByteSized() && "Unsupported floating point type!"
) ? static_cast<void> (0) : __assert_fail ("FloatVT.isByteSized() && \"Unsupported floating point type!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1447, __PRETTY_FUNCTION__))
;
1448 // Load out a legal integer with the same sign bit as the float.
1449 IntPtr = StackPtr;
1450 State.IntPointerInfo = State.FloatPointerInfo;
1451 } else {
1452 // Advance the pointer so that the loaded byte will contain the sign bit.
1453 unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
1454 IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
1455 DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
1456 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1457 ByteOffset);
1458 }
1459
1460 State.IntPtr = IntPtr;
1461 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1462 State.IntPointerInfo, MVT::i8);
1463 State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
1464 State.SignBit = 7;
1465}
1466
1467/// Replace the integer value produced by getSignAsIntValue() with a new value
1468/// and cast the result back to a floating-point type.
1469SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1470 const SDLoc &DL,
1471 SDValue NewIntValue) const {
1472 if (!State.Chain)
1473 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1474
1475 // Override the part containing the sign bit in the value stored on the stack.
1476 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1477 State.IntPointerInfo, MVT::i8);
1478 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1479 State.FloatPointerInfo);
1480}
1481
1482SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1483 SDLoc DL(Node);
1484 SDValue Mag = Node->getOperand(0);
1485 SDValue Sign = Node->getOperand(1);
1486
1487 // Get sign bit into an integer value.
1488 FloatSignAsInt SignAsInt;
1489 getSignAsIntValue(SignAsInt, DL, Sign);
1490
1491 EVT IntVT = SignAsInt.IntValue.getValueType();
1492 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1493 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1494 SignMask);
1495
1496 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1497 EVT FloatVT = Mag.getValueType();
1498 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1499 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1500 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1501 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1502 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1503 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1504 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1505 }
1506
1507 // Transform Mag value to integer, and clear the sign bit.
1508 FloatSignAsInt MagAsInt;
1509 getSignAsIntValue(MagAsInt, DL, Mag);
1510 EVT MagVT = MagAsInt.IntValue.getValueType();
1511 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1512 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1513 ClearSignMask);
1514
1515 // Get the signbit at the right position for MagAsInt.
1516 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1517 EVT ShiftVT = IntVT;
1518 if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
1519 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1520 ShiftVT = MagVT;
1521 }
1522 if (ShiftAmount > 0) {
1523 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1524 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1525 } else if (ShiftAmount < 0) {
1526 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1527 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1528 }
1529 if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
1530 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1531 }
1532
1533 // Store the part with the modified sign and convert back to float.
1534 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1535 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1536}
1537
1538SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1539 SDLoc DL(Node);
1540 SDValue Value = Node->getOperand(0);
1541
1542 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1543 EVT FloatVT = Value.getValueType();
1544 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1545 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1546 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1547 }
1548
1549 // Transform value to integer, clear the sign bit and transform back.
1550 FloatSignAsInt ValueAsInt;
1551 getSignAsIntValue(ValueAsInt, DL, Value);
1552 EVT IntVT = ValueAsInt.IntValue.getValueType();
1553 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1554 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1555 ClearSignMask);
1556 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1557}
1558
1559void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1560 SmallVectorImpl<SDValue> &Results) {
1561 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1562 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"((SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? static_cast
<void> (0) : __assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1563, __PRETTY_FUNCTION__))
1563 " not tell us which reg is the stack pointer!")((SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? static_cast
<void> (0) : __assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1563, __PRETTY_FUNCTION__))
;
1564 SDLoc dl(Node);
1565 EVT VT = Node->getValueType(0);
1566 SDValue Tmp1 = SDValue(Node, 0);
1567 SDValue Tmp2 = SDValue(Node, 1);
1568 SDValue Tmp3 = Node->getOperand(2);
1569 SDValue Chain = Tmp1.getOperand(0);
1570
1571 // Chain the dynamic stack allocation so that it doesn't modify the stack
1572 // pointer when other instructions are using the stack.
1573 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1574
1575 SDValue Size = Tmp2.getOperand(1);
1576 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1577 Chain = SP.getValue(1);
1578 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1579 unsigned StackAlign =
1580 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1581 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1582 if (Align > StackAlign)
1583 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1584 DAG.getConstant(-(uint64_t)Align, dl, VT));
1585 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1586
1587 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1588 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1589
1590 Results.push_back(Tmp1);
1591 Results.push_back(Tmp2);
1592}
1593
1594/// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1595/// target.
1596///
1597/// If the SETCC has been legalized using AND / OR, then the legalized node
1598/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1599/// will be set to false.
1600///
1601/// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1602/// then the values of LHS and RHS will be swapped, CC will be set to the
1603/// new condition, and NeedInvert will be set to false.
1604///
1605/// If the SETCC has been legalized using the inverse condcode, then LHS and
1606/// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1607/// will be set to true. The caller must invert the result of the SETCC with
1608/// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1609/// of a true/false result.
1610///
1611/// \returns true if the SetCC has been legalized, false if it hasn't.
1612bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1613 SDValue &RHS, SDValue &CC,
1614 bool &NeedInvert,
1615 const SDLoc &dl) {
1616 MVT OpVT = LHS.getSimpleValueType();
1617 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1618 NeedInvert = false;
1619 bool NeedSwap = false;
1620 switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1621 default: llvm_unreachable("Unknown condition code action!")::llvm::llvm_unreachable_internal("Unknown condition code action!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1621)
;
1622 case TargetLowering::Legal:
1623 // Nothing to do.
1624 break;
1625 case TargetLowering::Expand: {
1626 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
1627 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1628 std::swap(LHS, RHS);
1629 CC = DAG.getCondCode(InvCC);
1630 return true;
1631 }
1632 // Swapping operands didn't work. Try inverting the condition.
1633 InvCC = getSetCCInverse(CCCode, OpVT.isInteger());
1634 if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1635 // If inverting the condition is not enough, try swapping operands
1636 // on top of it.
1637 InvCC = ISD::getSetCCSwappedOperands(InvCC);
1638 NeedSwap = true;
1639 }
1640 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1641 CC = DAG.getCondCode(InvCC);
1642 NeedInvert = true;
1643 if (NeedSwap)
1644 std::swap(LHS, RHS);
1645 return true;
1646 }
1647
1648 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1649 unsigned Opc = 0;
1650 switch (CCCode) {
1651 default: llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1651)
;
1652 case ISD::SETO:
1653 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT)((TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && "If SETO is expanded, SETOEQ must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1654, __PRETTY_FUNCTION__))
1654 && "If SETO is expanded, SETOEQ must be legal!")((TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && "If SETO is expanded, SETOEQ must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1654, __PRETTY_FUNCTION__))
;
1655 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1656 case ISD::SETUO:
1657 assert(TLI.isCondCodeLegal(ISD::SETUNE, OpVT)((TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && "If SETUO is expanded, SETUNE must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1658, __PRETTY_FUNCTION__))
1658 && "If SETUO is expanded, SETUNE must be legal!")((TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && "If SETUO is expanded, SETUNE must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1658, __PRETTY_FUNCTION__))
;
1659 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break;
1660 case ISD::SETOEQ:
1661 case ISD::SETOGT:
1662 case ISD::SETOGE:
1663 case ISD::SETOLT:
1664 case ISD::SETOLE:
1665 case ISD::SETONE:
1666 case ISD::SETUEQ:
1667 case ISD::SETUNE:
1668 case ISD::SETUGT:
1669 case ISD::SETUGE:
1670 case ISD::SETULT:
1671 case ISD::SETULE:
1672 // If we are floating point, assign and break, otherwise fall through.
1673 if (!OpVT.isInteger()) {
1674 // We can use the 4th bit to tell if we are the unordered
1675 // or ordered version of the opcode.
1676 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
1677 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
1678 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1679 break;
1680 }
1681 // Fallthrough if we are unsigned integer.
1682 LLVM_FALLTHROUGH[[clang::fallthrough]];
1683 case ISD::SETLE:
1684 case ISD::SETGT:
1685 case ISD::SETGE:
1686 case ISD::SETLT:
1687 case ISD::SETNE:
1688 case ISD::SETEQ:
1689 // If all combinations of inverting the condition and swapping operands
1690 // didn't work then we have no means to expand the condition.
1691 llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1691)
;
1692 }
1693
1694 SDValue SetCC1, SetCC2;
1695 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
1696 // If we aren't the ordered or unorder operation,
1697 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1698 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1699 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1700 } else {
1701 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1702 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1703 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1704 }
1705 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1706 RHS = SDValue();
1707 CC = SDValue();
1708 return true;
1709 }
1710 }
1711 return false;
1712}
1713
1714/// Emit a store/load combination to the stack. This stores
1715/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1716/// a load from the stack slot to DestVT, extending it if needed.
1717/// The resultant code need not be legal.
1718SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1719 EVT DestVT, const SDLoc &dl) {
1720 // Create the stack frame object.
1721 unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1722 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1723 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1724
1725 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1726 int SPFI = StackPtrFI->getIndex();
1727 MachinePointerInfo PtrInfo =
1728 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1729
1730 unsigned SrcSize = SrcOp.getValueSizeInBits();
1731 unsigned SlotSize = SlotVT.getSizeInBits();
1732 unsigned DestSize = DestVT.getSizeInBits();
1733 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1734 unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1735
1736 // Emit a store to the stack slot. Use a truncstore if the input value is
1737 // later than DestVT.
1738 SDValue Store;
1739
1740 if (SrcSize > SlotSize)
1741 Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
1742 SlotVT, SrcAlign);
1743 else {
1744 assert(SrcSize == SlotSize && "Invalid store")((SrcSize == SlotSize && "Invalid store") ? static_cast
<void> (0) : __assert_fail ("SrcSize == SlotSize && \"Invalid store\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1744, __PRETTY_FUNCTION__))
;
1745 Store =
1746 DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1747 }
1748
1749 // Result is a load from the stack slot.
1750 if (SlotSize == DestSize)
1751 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1752
1753 assert(SlotSize < DestSize && "Unknown extension!")((SlotSize < DestSize && "Unknown extension!") ? static_cast
<void> (0) : __assert_fail ("SlotSize < DestSize && \"Unknown extension!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1753, __PRETTY_FUNCTION__))
;
1754 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1755 DestAlign);
1756}
1757
1758SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1759 SDLoc dl(Node);
1760 // Create a vector sized/aligned stack slot, store the value to element #0,
1761 // then load the whole vector back out.
1762 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1763
1764 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1765 int SPFI = StackPtrFI->getIndex();
1766
1767 SDValue Ch = DAG.getTruncStore(
1768 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1769 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1770 Node->getValueType(0).getVectorElementType());
1771 return DAG.getLoad(
1772 Node->getValueType(0), dl, Ch, StackPtr,
1773 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1774}
1775
1776static bool
1777ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1778 const TargetLowering &TLI, SDValue &Res) {
1779 unsigned NumElems = Node->getNumOperands();
1780 SDLoc dl(Node);
1781 EVT VT = Node->getValueType(0);
1782
1783 // Try to group the scalars into pairs, shuffle the pairs together, then
1784 // shuffle the pairs of pairs together, etc. until the vector has
1785 // been built. This will work only if all of the necessary shuffle masks
1786 // are legal.
1787
1788 // We do this in two phases; first to check the legality of the shuffles,
1789 // and next, assuming that all shuffles are legal, to create the new nodes.
1790 for (int Phase = 0; Phase < 2; ++Phase) {
1791 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1792 NewIntermedVals;
1793 for (unsigned i = 0; i < NumElems; ++i) {
1794 SDValue V = Node->getOperand(i);
1795 if (V.isUndef())
1796 continue;
1797
1798 SDValue Vec;
1799 if (Phase)
1800 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1801 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1802 }
1803
1804 while (IntermedVals.size() > 2) {
1805 NewIntermedVals.clear();
1806 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1807 // This vector and the next vector are shuffled together (simply to
1808 // append the one to the other).
1809 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1810
1811 SmallVector<int, 16> FinalIndices;
1812 FinalIndices.reserve(IntermedVals[i].second.size() +
1813 IntermedVals[i+1].second.size());
1814
1815 int k = 0;
1816 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1817 ++j, ++k) {
1818 ShuffleVec[k] = j;
1819 FinalIndices.push_back(IntermedVals[i].second[j]);
1820 }
1821 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1822 ++j, ++k) {
1823 ShuffleVec[k] = NumElems + j;
1824 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1825 }
1826
1827 SDValue Shuffle;
1828 if (Phase)
1829 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1830 IntermedVals[i+1].first,
1831 ShuffleVec);
1832 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1833 return false;
1834 NewIntermedVals.push_back(
1835 std::make_pair(Shuffle, std::move(FinalIndices)));
1836 }
1837
1838 // If we had an odd number of defined values, then append the last
1839 // element to the array of new vectors.
1840 if ((IntermedVals.size() & 1) != 0)
1841 NewIntermedVals.push_back(IntermedVals.back());
1842
1843 IntermedVals.swap(NewIntermedVals);
1844 }
1845
1846 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&((IntermedVals.size() <= 2 && IntermedVals.size() >
0 && "Invalid number of intermediate vectors") ? static_cast
<void> (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1847, __PRETTY_FUNCTION__))
1847 "Invalid number of intermediate vectors")((IntermedVals.size() <= 2 && IntermedVals.size() >
0 && "Invalid number of intermediate vectors") ? static_cast
<void> (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1847, __PRETTY_FUNCTION__))
;
1848 SDValue Vec1 = IntermedVals[0].first;
1849 SDValue Vec2;
1850 if (IntermedVals.size() > 1)
1851 Vec2 = IntermedVals[1].first;
1852 else if (Phase)
1853 Vec2 = DAG.getUNDEF(VT);
1854
1855 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1856 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1857 ShuffleVec[IntermedVals[0].second[i]] = i;
1858 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1859 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1860
1861 if (Phase)
1862 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1863 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1864 return false;
1865 }
1866
1867 return true;
1868}
1869
1870/// Expand a BUILD_VECTOR node on targets that don't
1871/// support the operation, but do support the resultant vector type.
1872SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1873 unsigned NumElems = Node->getNumOperands();
1874 SDValue Value1, Value2;
1875 SDLoc dl(Node);
1876 EVT VT = Node->getValueType(0);
1877 EVT OpVT = Node->getOperand(0).getValueType();
1878 EVT EltVT = VT.getVectorElementType();
1879
1880 // If the only non-undef value is the low element, turn this into a
1881 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1882 bool isOnlyLowElement = true;
1883 bool MoreThanTwoValues = false;
1884 bool isConstant = true;
1885 for (unsigned i = 0; i < NumElems; ++i) {
1886 SDValue V = Node->getOperand(i);
1887 if (V.isUndef())
1888 continue;
1889 if (i > 0)
1890 isOnlyLowElement = false;
1891 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1892 isConstant = false;
1893
1894 if (!Value1.getNode()) {
1895 Value1 = V;
1896 } else if (!Value2.getNode()) {
1897 if (V != Value1)
1898 Value2 = V;
1899 } else if (V != Value1 && V != Value2) {
1900 MoreThanTwoValues = true;
1901 }
1902 }
1903
1904 if (!Value1.getNode())
1905 return DAG.getUNDEF(VT);
1906
1907 if (isOnlyLowElement)
1908 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1909
1910 // If all elements are constants, create a load from the constant pool.
1911 if (isConstant) {
1912 SmallVector<Constant*, 16> CV;
1913 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1914 if (ConstantFPSDNode *V =
1915 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1916 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1917 } else if (ConstantSDNode *V =
1918 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1919 if (OpVT==EltVT)
1920 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1921 else {
1922 // If OpVT and EltVT don't match, EltVT is not legal and the
1923 // element values have been promoted/truncated earlier. Undo this;
1924 // we don't want a v16i8 to become a v16i32 for example.
1925 const ConstantInt *CI = V->getConstantIntValue();
1926 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1927 CI->getZExtValue()));
1928 }
1929 } else {
1930 assert(Node->getOperand(i).isUndef())((Node->getOperand(i).isUndef()) ? static_cast<void>
(0) : __assert_fail ("Node->getOperand(i).isUndef()", "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1930, __PRETTY_FUNCTION__))
;
1931 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1932 CV.push_back(UndefValue::get(OpNTy));
1933 }
1934 }
1935 Constant *CP = ConstantVector::get(CV);
1936 SDValue CPIdx =
1937 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1938 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1939 return DAG.getLoad(
1940 VT, dl, DAG.getEntryNode(), CPIdx,
1941 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1942 Alignment);
1943 }
1944
1945 SmallSet<SDValue, 16> DefinedValues;
1946 for (unsigned i = 0; i < NumElems; ++i) {
1947 if (Node->getOperand(i).isUndef())
1948 continue;
1949 DefinedValues.insert(Node->getOperand(i));
1950 }
1951
1952 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1953 if (!MoreThanTwoValues) {
1954 SmallVector<int, 8> ShuffleVec(NumElems, -1);
1955 for (unsigned i = 0; i < NumElems; ++i) {
1956 SDValue V = Node->getOperand(i);
1957 if (V.isUndef())
1958 continue;
1959 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1960 }
1961 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1962 // Get the splatted value into the low element of a vector register.
1963 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1964 SDValue Vec2;
1965 if (Value2.getNode())
1966 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1967 else
1968 Vec2 = DAG.getUNDEF(VT);
1969
1970 // Return shuffle(LowValVec, undef, <0,0,0,0>)
1971 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1972 }
1973 } else {
1974 SDValue Res;
1975 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1976 return Res;
1977 }
1978 }
1979
1980 // Otherwise, we can't handle this case efficiently.
1981 return ExpandVectorBuildThroughStack(Node);
1982}
1983
1984// Expand a node into a call to a libcall. If the result value
1985// does not fit into a register, return the lo part and set the hi part to the
1986// by-reg argument. If it does fit into a single register, return the result
1987// and leave the Hi part unset.
1988SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
1989 bool isSigned) {
1990 TargetLowering::ArgListTy Args;
1991 TargetLowering::ArgListEntry Entry;
1992 for (const SDValue &Op : Node->op_values()) {
1993 EVT ArgVT = Op.getValueType();
1994 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1995 Entry.Node = Op;
1996 Entry.Ty = ArgTy;
1997 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
1998 Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
1999 Args.push_back(Entry);
2000 }
2001 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2002 TLI.getPointerTy(DAG.getDataLayout()));
2003
2004 EVT RetVT = Node->getValueType(0);
2005 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2006
2007 // By default, the input chain to this libcall is the entry node of the
2008 // function. If the libcall is going to be emitted as a tail call then
2009 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2010 // node which is being folded has a non-entry input chain.
2011 SDValue InChain = DAG.getEntryNode();
2012
2013 // isTailCall may be true since the callee does not reference caller stack
2014 // frame. Check if it's in the right position and that the return types match.
2015 SDValue TCChain = InChain;
2016 const Function &F = DAG.getMachineFunction().getFunction();
2017 bool isTailCall =
2018 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2019 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2020 if (isTailCall)
2021 InChain = TCChain;
2022
2023 TargetLowering::CallLoweringInfo CLI(DAG);
2024 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2025 CLI.setDebugLoc(SDLoc(Node))
2026 .setChain(InChain)
2027 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2028 std::move(Args))
2029 .setTailCall(isTailCall)
2030 .setSExtResult(signExtend)
2031 .setZExtResult(!signExtend)
2032 .setIsPostTypeLegalization(true);
2033
2034 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2035
2036 if (!CallInfo.second.getNode()) {
2037 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Created tailcall: "; DAG.
getRoot().dump(); } } while (false)
;
2038 // It's a tailcall, return the chain (which is the DAG root).
2039 return DAG.getRoot();
2040 }
2041
2042 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Created libcall: "; CallInfo
.first.dump(); } } while (false)
;
2043 return CallInfo.first;
2044}
2045
2046/// Generate a libcall taking the given operands as arguments
2047/// and returning a result of type RetVT.
2048SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
2049 const SDValue *Ops, unsigned NumOps,
2050 bool isSigned, const SDLoc &dl) {
2051 TargetLowering::ArgListTy Args;
2052 Args.reserve(NumOps);
2053
2054 TargetLowering::ArgListEntry Entry;
2055 for (unsigned i = 0; i != NumOps; ++i) {
2056 Entry.Node = Ops[i];
2057 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
2058 Entry.IsSExt = isSigned;
2059 Entry.IsZExt = !isSigned;
2060 Args.push_back(Entry);
2061 }
2062 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2063 TLI.getPointerTy(DAG.getDataLayout()));
2064
2065 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2066
2067 TargetLowering::CallLoweringInfo CLI(DAG);
2068 CLI.setDebugLoc(dl)
2069 .setChain(DAG.getEntryNode())
2070 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2071 std::move(Args))
2072 .setSExtResult(isSigned)
2073 .setZExtResult(!isSigned)
2074 .setIsPostTypeLegalization(true);
2075
2076 std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
2077
2078 return CallInfo.first;
2079}
2080
2081// Expand a node into a call to a libcall. Similar to
2082// ExpandLibCall except that the first operand is the in-chain.
2083std::pair<SDValue, SDValue>
2084SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2085 SDNode *Node,
2086 bool isSigned) {
2087 SDValue InChain = Node->getOperand(0);
2088
2089 TargetLowering::ArgListTy Args;
2090 TargetLowering::ArgListEntry Entry;
2091 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
2092 EVT ArgVT = Node->getOperand(i).getValueType();
2093 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2094 Entry.Node = Node->getOperand(i);
2095 Entry.Ty = ArgTy;
2096 Entry.IsSExt = isSigned;
2097 Entry.IsZExt = !isSigned;
2098 Args.push_back(Entry);
2099 }
2100 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2101 TLI.getPointerTy(DAG.getDataLayout()));
2102
2103 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2104
2105 TargetLowering::CallLoweringInfo CLI(DAG);
2106 CLI.setDebugLoc(SDLoc(Node))
2107 .setChain(InChain)
2108 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2109 std::move(Args))
2110 .setSExtResult(isSigned)
2111 .setZExtResult(!isSigned);
2112
2113 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2114
2115 return CallInfo;
2116}
2117
2118SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2119 RTLIB::Libcall Call_F32,
2120 RTLIB::Libcall Call_F64,
2121 RTLIB::Libcall Call_F80,
2122 RTLIB::Libcall Call_F128,
2123 RTLIB::Libcall Call_PPCF128) {
2124 if (Node->isStrictFPOpcode())
2125 Node = DAG.mutateStrictFPToFP(Node);
2126
2127 RTLIB::Libcall LC;
2128 switch (Node->getSimpleValueType(0).SimpleTy) {
2129 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2129)
;
2130 case MVT::f32: LC = Call_F32; break;
2131 case MVT::f64: LC = Call_F64; break;
2132 case MVT::f80: LC = Call_F80; break;
2133 case MVT::f128: LC = Call_F128; break;
2134 case MVT::ppcf128: LC = Call_PPCF128; break;
2135 }
2136 return ExpandLibCall(LC, Node, false);
2137}
2138
2139SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2140 RTLIB::Libcall Call_I8,
2141 RTLIB::Libcall Call_I16,
2142 RTLIB::Libcall Call_I32,
2143 RTLIB::Libcall Call_I64,
2144 RTLIB::Libcall Call_I128) {
2145 RTLIB::Libcall LC;
2146 switch (Node->getSimpleValueType(0).SimpleTy) {
2147 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2147)
;
2148 case MVT::i8: LC = Call_I8; break;
2149 case MVT::i16: LC = Call_I16; break;
2150 case MVT::i32: LC = Call_I32; break;
2151 case MVT::i64: LC = Call_I64; break;
2152 case MVT::i128: LC = Call_I128; break;
2153 }
2154 return ExpandLibCall(LC, Node, isSigned);
2155}
2156
2157/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2158void
2159SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2160 SmallVectorImpl<SDValue> &Results) {
2161 unsigned Opcode = Node->getOpcode();
2162 bool isSigned = Opcode == ISD::SDIVREM;
2163
2164 RTLIB::Libcall LC;
2165 switch (Node->getSimpleValueType(0).SimpleTy) {
2166 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2166)
;
2167 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2168 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2169 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2170 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2171 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2172 }
2173
2174 // The input chain to this libcall is the entry node of the function.
2175 // Legalizing the call will automatically add the previous call to the
2176 // dependence.
2177 SDValue InChain = DAG.getEntryNode();
2178
2179 EVT RetVT = Node->getValueType(0);
2180 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2181
2182 TargetLowering::ArgListTy Args;
2183 TargetLowering::ArgListEntry Entry;
2184 for (const SDValue &Op : Node->op_values()) {
2185 EVT ArgVT = Op.getValueType();
2186 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2187 Entry.Node = Op;
2188 Entry.Ty = ArgTy;
2189 Entry.IsSExt = isSigned;
2190 Entry.IsZExt = !isSigned;
2191 Args.push_back(Entry);
2192 }
2193
2194 // Also pass the return address of the remainder.
2195 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2196 Entry.Node = FIPtr;
2197 Entry.Ty = RetTy->getPointerTo();
2198 Entry.IsSExt = isSigned;
2199 Entry.IsZExt = !isSigned;
2200 Args.push_back(Entry);
2201
2202 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2203 TLI.getPointerTy(DAG.getDataLayout()));
2204
2205 SDLoc dl(Node);
2206 TargetLowering::CallLoweringInfo CLI(DAG);
2207 CLI.setDebugLoc(dl)
2208 .setChain(InChain)
2209 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2210 std::move(Args))
2211 .setSExtResult(isSigned)
2212 .setZExtResult(!isSigned);
2213
2214 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2215
2216 // Remainder is loaded back from the stack frame.
2217 SDValue Rem =
2218 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2219 Results.push_back(CallInfo.first);
2220 Results.push_back(Rem);
2221}
2222
2223/// Return true if sincos libcall is available.
2224static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2225 RTLIB::Libcall LC;
2226 switch (Node->getSimpleValueType(0).SimpleTy) {
2227 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2227)
;
2228 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2229 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2230 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2231 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2232 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2233 }
2234 return TLI.getLibcallName(LC) != nullptr;
2235}
2236
2237/// Only issue sincos libcall if both sin and cos are needed.
2238static bool useSinCos(SDNode *Node) {
2239 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2240 ? ISD::FCOS : ISD::FSIN;
2241
2242 SDValue Op0 = Node->getOperand(0);
2243 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2244 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2245 SDNode *User = *UI;
2246 if (User == Node)
2247 continue;
2248 // The other user might have been turned into sincos already.
2249 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2250 return true;
2251 }
2252 return false;
2253}
2254
2255/// Issue libcalls to sincos to compute sin / cos pairs.
2256void
2257SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2258 SmallVectorImpl<SDValue> &Results) {
2259 RTLIB::Libcall LC;
2260 switch (Node->getSimpleValueType(0).SimpleTy) {
2261 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2261)
;
2262 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2263 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2264 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2265 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2266 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2267 }
2268
2269 // The input chain to this libcall is the entry node of the function.
2270 // Legalizing the call will automatically add the previous call to the
2271 // dependence.
2272 SDValue InChain = DAG.getEntryNode();
2273
2274 EVT RetVT = Node->getValueType(0);
2275 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2276
2277 TargetLowering::ArgListTy Args;
2278 TargetLowering::ArgListEntry Entry;
2279
2280 // Pass the argument.
2281 Entry.Node = Node->getOperand(0);
2282 Entry.Ty = RetTy;
2283 Entry.IsSExt = false;
2284 Entry.IsZExt = false;
2285 Args.push_back(Entry);
2286
2287 // Pass the return address of sin.
2288 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2289 Entry.Node = SinPtr;
2290 Entry.Ty = RetTy->getPointerTo();
2291 Entry.IsSExt = false;
2292 Entry.IsZExt = false;
2293 Args.push_back(Entry);
2294
2295 // Also pass the return address of the cos.
2296 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2297 Entry.Node = CosPtr;
2298 Entry.Ty = RetTy->getPointerTo();
2299 Entry.IsSExt = false;
2300 Entry.IsZExt = false;
2301 Args.push_back(Entry);
2302
2303 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2304 TLI.getPointerTy(DAG.getDataLayout()));
2305
2306 SDLoc dl(Node);
2307 TargetLowering::CallLoweringInfo CLI(DAG);
2308 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2309 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2310 std::move(Args));
2311
2312 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2313
2314 Results.push_back(
2315 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2316 Results.push_back(
2317 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2318}
2319
2320/// This function is responsible for legalizing a
2321/// INT_TO_FP operation of the specified operand when the target requests that
2322/// we expand it. At this point, we know that the result and operand types are
2323/// legal for the target.
2324SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2325 EVT DestVT,
2326 const SDLoc &dl) {
2327 EVT SrcVT = Op0.getValueType();
2328
2329 // TODO: Should any fast-math-flags be set for the created nodes?
2330 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing INT_TO_FP\n"; }
} while (false)
;
2331 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
2332 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double "
"expansion\n"; } } while (false)
2333 "expansion\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double "
"expansion\n"; } } while (false)
;
2334
2335 // Get the stack frame index of a 8 byte buffer.
2336 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2337
2338 // word offset constant for Hi/Lo address computation
2339 SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2340 StackSlot.getValueType());
2341 // set up Hi and Lo (into buffer) address based on endian
2342 SDValue Hi = StackSlot;
2343 SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2344 StackSlot, WordOff);
2345 if (DAG.getDataLayout().isLittleEndian())
2346 std::swap(Hi, Lo);
2347
2348 // if signed map to unsigned space
2349 SDValue Op0Mapped;
2350 if (isSigned) {
2351 // constant used to invert sign bit (signed to unsigned mapping)
2352 SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2353 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2354 } else {
2355 Op0Mapped = Op0;
2356 }
2357 // store the lo of the constructed double - based on integer input
2358 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2359 MachinePointerInfo());
2360 // initial hi portion of constructed double
2361 SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2362 // store the hi of the constructed double - biased exponent
2363 SDValue Store2 =
2364 DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
2365 // load the constructed double
2366 SDValue Load =
2367 DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
2368 // FP constant to bias correct the final result
2369 SDValue Bias = DAG.getConstantFP(isSigned ?
2370 BitsToDouble(0x4330000080000000ULL) :
2371 BitsToDouble(0x4330000000000000ULL),
2372 dl, MVT::f64);
2373 // subtract the bias
2374 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2375 // final result
2376 SDValue Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2377 return Result;
2378 }
2379 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet")((!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"
) ? static_cast<void> (0) : __assert_fail ("!isSigned && \"Legalize cannot Expand SINT_TO_FP for i64 yet\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2379, __PRETTY_FUNCTION__))
;
2380 // Code below here assumes !isSigned without checking again.
2381
2382 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2383
2384 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2385 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2386 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2387 Four = DAG.getIntPtrConstant(4, dl);
2388 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2389 SignSet, Four, Zero);
2390
2391 // If the sign bit of the integer is set, the large number will be treated
2392 // as a negative number. To counteract this, the dynamic code adds an
2393 // offset depending on the data type.
2394 uint64_t FF;
2395 switch (SrcVT.getSimpleVT().SimpleTy) {
2396 default: llvm_unreachable("Unsupported integer type!")::llvm::llvm_unreachable_internal("Unsupported integer type!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2396)
;
2397 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2398 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2399 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2400 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2401 }
2402 if (DAG.getDataLayout().isLittleEndian())
2403 FF <<= 32;
2404 Constant *FudgeFactor = ConstantInt::get(
2405 Type::getInt64Ty(*DAG.getContext()), FF);
2406
2407 SDValue CPIdx =
2408 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2409 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2410 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2411 Alignment = std::min(Alignment, 4u);
2412 SDValue FudgeInReg;
2413 if (DestVT == MVT::f32)
2414 FudgeInReg = DAG.getLoad(
2415 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2416 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2417 Alignment);
2418 else {
2419 SDValue Load = DAG.getExtLoad(
2420 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2421 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2422 Alignment);
2423 HandleSDNode Handle(Load);
2424 LegalizeOp(Load.getNode());
2425 FudgeInReg = Handle.getValue();
2426 }
2427
2428 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2429}
2430
2431/// This function is responsible for legalizing a
2432/// *INT_TO_FP operation of the specified operand when the target requests that
2433/// we promote it. At this point, we know that the result and operand types are
2434/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2435/// operation that takes a larger input.
2436SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2437 bool isSigned,
2438 const SDLoc &dl) {
2439 // First step, figure out the appropriate *INT_TO_FP operation to use.
2440 EVT NewInTy = LegalOp.getValueType();
2441
2442 unsigned OpToUse = 0;
2443
2444 // Scan for the appropriate larger type to use.
2445 while (true) {
2446 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2447 assert(NewInTy.isInteger() && "Ran out of possibilities!")((NewInTy.isInteger() && "Ran out of possibilities!")
? static_cast<void> (0) : __assert_fail ("NewInTy.isInteger() && \"Ran out of possibilities!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2447, __PRETTY_FUNCTION__))
;
2448
2449 // If the target supports SINT_TO_FP of this type, use it.
2450 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2451 OpToUse = ISD::SINT_TO_FP;
2452 break;
2453 }
2454 if (isSigned) continue;
2455
2456 // If the target supports UINT_TO_FP of this type, use it.
2457 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2458 OpToUse = ISD::UINT_TO_FP;
2459 break;
2460 }
2461
2462 // Otherwise, try a larger type.
2463 }
2464
2465 // Okay, we found the operation and type to use. Zero extend our input to the
2466 // desired type then run the operation on it.
2467 return DAG.getNode(OpToUse, dl, DestVT,
2468 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2469 dl, NewInTy, LegalOp));
2470}
2471
2472/// This function is responsible for legalizing a
2473/// FP_TO_*INT operation of the specified operand when the target requests that
2474/// we promote it. At this point, we know that the result and operand types are
2475/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2476/// operation that returns a larger result.
2477SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2478 bool isSigned,
2479 const SDLoc &dl) {
2480 // First step, figure out the appropriate FP_TO*INT operation to use.
2481 EVT NewOutTy = DestVT;
2482
2483 unsigned OpToUse = 0;
2484
2485 // Scan for the appropriate larger type to use.
2486 while (true) {
2487 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2488 assert(NewOutTy.isInteger() && "Ran out of possibilities!")((NewOutTy.isInteger() && "Ran out of possibilities!"
) ? static_cast<void> (0) : __assert_fail ("NewOutTy.isInteger() && \"Ran out of possibilities!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2488, __PRETTY_FUNCTION__))
;
2489
2490 // A larger signed type can hold all unsigned values of the requested type,
2491 // so using FP_TO_SINT is valid
2492 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2493 OpToUse = ISD::FP_TO_SINT;
2494 break;
2495 }
2496
2497 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2498 if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2499 OpToUse = ISD::FP_TO_UINT;
2500 break;
2501 }
2502
2503 // Otherwise, try a larger type.
2504 }
2505
2506 // Okay, we found the operation and type to use.
2507 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2508
2509 // Truncate the result of the extended FP_TO_*INT operation to the desired
2510 // size.
2511 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2512}
2513
2514/// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2515SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2516 EVT VT = Op.getValueType();
2517 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2518 unsigned Sz = VT.getScalarSizeInBits();
2519
2520 SDValue Tmp, Tmp2, Tmp3;
2521
2522 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2523 // and finally the i1 pairs.
2524 // TODO: We can easily support i4/i2 legal types if any target ever does.
2525 if (Sz >= 8 && isPowerOf2_32(Sz)) {
2526 // Create the masks - repeating the pattern every byte.
2527 APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0);
2528 APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0);
2529 for (unsigned J = 0; J != Sz; J += 8) {
2530 MaskHi4 = MaskHi4 | (0xF0ull << J);
2531 MaskLo4 = MaskLo4 | (0x0Full << J);
2532 MaskHi2 = MaskHi2 | (0xCCull << J);
2533 MaskLo2 = MaskLo2 | (0x33ull << J);
2534 MaskHi1 = MaskHi1 | (0xAAull << J);
2535 MaskLo1 = MaskLo1 | (0x55ull << J);
2536 }
2537
2538 // BSWAP if the type is wider than a single byte.
2539 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
2540
2541 // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2542 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
2543 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
2544 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, SHVT));
2545 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT));
2546 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2547
2548 // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2549 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
2550 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
2551 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, SHVT));
2552 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT));
2553 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2554
2555 // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2556 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
2557 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
2558 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, SHVT));
2559 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT));
2560 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2561 return Tmp;
2562 }
2563
2564 Tmp = DAG.getConstant(0, dl, VT);
2565 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
2566 if (I < J)
2567 Tmp2 =
2568 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
2569 else
2570 Tmp2 =
2571 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
2572
2573 APInt Shift(Sz, 1);
2574 Shift <<= J;
2575 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
2576 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
2577 }
2578
2579 return Tmp;
2580}
2581
2582/// Open code the operations for BSWAP of the specified operation.
2583SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2584 EVT VT = Op.getValueType();
2585 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2586 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2587 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
2588 default: llvm_unreachable("Unhandled Expand type in BSWAP!")::llvm::llvm_unreachable_internal("Unhandled Expand type in BSWAP!"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2588)
;
2589 case MVT::i16:
2590 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2591 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2592 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2593 case MVT::i32:
2594 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2595 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2596 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2597 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2598 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2599 DAG.getConstant(0xFF0000, dl, VT));
2600 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2601 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2602 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2603 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2604 case MVT::i64:
2605 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2606 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2607 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2608 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2609 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2610 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2611 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2612 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2613 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2614 DAG.getConstant(255ULL<<48, dl, VT));
2615 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2616 DAG.getConstant(255ULL<<40, dl, VT));
2617 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2618 DAG.getConstant(255ULL<<32, dl, VT));
2619 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2620 DAG.getConstant(255ULL<<24, dl, VT));
2621 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2622 DAG.getConstant(255ULL<<16, dl, VT));
2623 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2624 DAG.getConstant(255ULL<<8 , dl, VT));
2625 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2626 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2627 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2628 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2629 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2630 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2631 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2632 }
2633}
2634
2635bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2636 LLVM_DEBUG(dbgs() << "Trying to expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to expand node\n";
} } while (false)
;
2637 SmallVector<SDValue, 8> Results;
2638 SDLoc dl(Node);
2639 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2640 bool NeedInvert;
2641 switch (Node->getOpcode()) {
2642 case ISD::CTPOP:
2643 if (TLI.expandCTPOP(Node, Tmp1, DAG))
2644 Results.push_back(Tmp1);
2645 break;
2646 case ISD::CTLZ:
2647 case ISD::CTLZ_ZERO_UNDEF:
2648 if (TLI.expandCTLZ(Node, Tmp1, DAG))
2649 Results.push_back(Tmp1);
2650 break;
2651 case ISD::CTTZ:
2652 case ISD::CTTZ_ZERO_UNDEF:
2653 if (TLI.expandCTTZ(Node, Tmp1, DAG))
2654 Results.push_back(Tmp1);
2655 break;
2656 case ISD::BITREVERSE:
2657 Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
2658 break;
2659 case ISD::BSWAP:
2660 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2661 break;
2662 case ISD::FRAMEADDR:
2663 case ISD::RETURNADDR:
2664 case ISD::FRAME_TO_ARGS_OFFSET:
2665 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2666 break;
2667 case ISD::EH_DWARF_CFA: {
2668 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2669 TLI.getPointerTy(DAG.getDataLayout()));
2670 SDValue Offset = DAG.getNode(ISD::ADD, dl,
2671 CfaArg.getValueType(),
2672 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2673 CfaArg.getValueType()),
2674 CfaArg);
2675 SDValue FA = DAG.getNode(
2676 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2677 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2678 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2679 FA, Offset));
2680 break;
2681 }
2682 case ISD::FLT_ROUNDS_:
2683 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2684 break;
2685 case ISD::EH_RETURN:
2686 case ISD::EH_LABEL:
2687 case ISD::PREFETCH:
2688 case ISD::VAEND:
2689 case ISD::EH_SJLJ_LONGJMP:
2690 // If the target didn't expand these, there's nothing to do, so just
2691 // preserve the chain and be done.
2692 Results.push_back(Node->getOperand(0));
2693 break;
2694 case ISD::READCYCLECOUNTER:
2695 // If the target didn't expand this, just return 'zero' and preserve the
2696 // chain.
2697 Results.append(Node->getNumValues() - 1,
2698 DAG.getConstant(0, dl, Node->getValueType(0)));
2699 Results.push_back(Node->getOperand(0));
2700 break;
2701 case ISD::EH_SJLJ_SETJMP:
2702 // If the target didn't expand this, just return 'zero' and preserve the
2703 // chain.
2704 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2705 Results.push_back(Node->getOperand(0));
2706 break;
2707 case ISD::ATOMIC_LOAD: {
2708 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2709 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2710 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2711 SDValue Swap = DAG.getAtomicCmpSwap(
2712 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2713 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2714 cast<AtomicSDNode>(Node)->getMemOperand());
2715 Results.push_back(Swap.getValue(0));
2716 Results.push_back(Swap.getValue(1));
2717 break;
2718 }
2719 case ISD::ATOMIC_STORE: {
2720 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2721 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2722 cast<AtomicSDNode>(Node)->getMemoryVT(),
2723 Node->getOperand(0),
2724 Node->getOperand(1), Node->getOperand(2),
2725 cast<AtomicSDNode>(Node)->getMemOperand());
2726 Results.push_back(Swap.getValue(1));
2727 break;
2728 }
2729 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2730 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2731 // splits out the success value as a comparison. Expanding the resulting
2732 // ATOMIC_CMP_SWAP will produce a libcall.
2733 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2734 SDValue Res = DAG.getAtomicCmpSwap(
2735 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2736 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2737 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2738
2739 SDValue ExtRes = Res;
2740 SDValue LHS = Res;
2741 SDValue RHS = Node->getOperand(1);
2742
2743 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2744 EVT OuterType = Node->getValueType(0);
2745 switch (TLI.getExtendForAtomicOps()) {
2746 case ISD::SIGN_EXTEND:
2747 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2748 DAG.getValueType(AtomicType));
2749 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2750 Node->getOperand(2), DAG.getValueType(AtomicType));
2751 ExtRes = LHS;
2752 break;
2753 case ISD::ZERO_EXTEND:
2754 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2755 DAG.getValueType(AtomicType));
2756 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2757 ExtRes = LHS;
2758 break;
2759 case ISD::ANY_EXTEND:
2760 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2761 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2762 break;
2763 default:
2764 llvm_unreachable("Invalid atomic op extension")::llvm::llvm_unreachable_internal("Invalid atomic op extension"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2764)
;
2765 }
2766
2767 SDValue Success =
2768 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2769
2770 Results.push_back(ExtRes.getValue(0));
2771 Results.push_back(Success);
2772 Results.push_back(Res.getValue(1));
2773 break;
2774 }
2775 case ISD::DYNAMIC_STACKALLOC:
2776 ExpandDYNAMIC_STACKALLOC(Node, Results);
2777 break;
2778 case ISD::MERGE_VALUES:
2779 for (unsigned i = 0; i < Node->getNumValues(); i++)
2780 Results.push_back(Node->getOperand(i));
2781 break;
2782 case ISD::UNDEF: {
2783 EVT VT = Node->getValueType(0);
2784 if (VT.isInteger())
2785 Results.push_back(DAG.getConstant(0, dl, VT));
2786 else {
2787 assert(VT.isFloatingPoint() && "Unknown value type!")((VT.isFloatingPoint() && "Unknown value type!") ? static_cast
<void> (0) : __assert_fail ("VT.isFloatingPoint() && \"Unknown value type!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2787, __PRETTY_FUNCTION__))
;
2788 Results.push_back(DAG.getConstantFP(0, dl, VT));
2789 }
2790 break;
2791 }
2792 case ISD::FP_ROUND:
2793 case ISD::BITCAST:
2794 Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2795 Node->getValueType(0), dl);
2796 Results.push_back(Tmp1);
2797 break;
2798 case ISD::FP_EXTEND:
2799 Tmp1 = EmitStackConvert(Node->getOperand(0),
2800 Node->getOperand(0).getValueType(),
2801 Node->getValueType(0), dl);
2802 Results.push_back(Tmp1);
2803 break;
2804 case ISD::SIGN_EXTEND_INREG: {
2805 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2806 EVT VT = Node->getValueType(0);
2807
2808 // An in-register sign-extend of a boolean is a negation:
2809 // 'true' (1) sign-extended is -1.
2810 // 'false' (0) sign-extended is 0.
2811 // However, we must mask the high bits of the source operand because the
2812 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2813
2814 // TODO: Do this for vectors too?
2815 if (ExtraVT.getSizeInBits() == 1) {
2816 SDValue One = DAG.getConstant(1, dl, VT);
2817 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2818 SDValue Zero = DAG.getConstant(0, dl, VT);
2819 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2820 Results.push_back(Neg);
2821 break;
2822 }
2823
2824 // NOTE: we could fall back on load/store here too for targets without
2825 // SRA. However, it is doubtful that any exist.
2826 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2827 unsigned BitsDiff = VT.getScalarSizeInBits() -
2828 ExtraVT.getScalarSizeInBits();
2829 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2830 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2831 Node->getOperand(0), ShiftCst);
2832 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2833 Results.push_back(Tmp1);
2834 break;
2835 }
2836 case ISD::FP_ROUND_INREG: {
2837 // The only way we can lower this is to turn it into a TRUNCSTORE,
2838 // EXTLOAD pair, targeting a temporary location (a stack slot).
2839
2840 // NOTE: there is a choice here between constantly creating new stack
2841 // slots and always reusing the same one. We currently always create
2842 // new ones, as reuse may inhibit scheduling.
2843 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2844 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
2845 Node->getValueType(0), dl);
2846 Results.push_back(Tmp1);
2847 break;
2848 }
2849 case ISD::UINT_TO_FP:
2850 if (TLI.expandUINT_TO_FP(Node, Tmp1, DAG)) {
2851 Results.push_back(Tmp1);
2852 break;
2853 }
2854 LLVM_FALLTHROUGH[[clang::fallthrough]];
2855 case ISD::SINT_TO_FP:
2856 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
2857 Node->getOperand(0), Node->getValueType(0), dl);
2858 Results.push_back(Tmp1);
2859 break;
2860 case ISD::FP_TO_SINT:
2861 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
2862 Results.push_back(Tmp1);
2863 break;
2864 case ISD::FP_TO_UINT:
2865 if (TLI.expandFP_TO_UINT(Node, Tmp1, DAG))
2866 Results.push_back(Tmp1);
2867 break;
2868 case ISD::VAARG:
2869 Results.push_back(DAG.expandVAArg(Node));
2870 Results.push_back(Results[0].getValue(1));
2871 break;
2872 case ISD::VACOPY:
2873 Results.push_back(DAG.expandVACopy(Node));
2874 break;
2875 case ISD::EXTRACT_VECTOR_ELT:
2876 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
2877 // This must be an access of the only element. Return it.
2878 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
2879 Node->getOperand(0));
2880 else
2881 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
2882 Results.push_back(Tmp1);
2883 break;
2884 case ISD::EXTRACT_SUBVECTOR:
2885 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
2886 break;
2887 case ISD::INSERT_SUBVECTOR:
2888 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
2889 break;
2890 case ISD::CONCAT_VECTORS:
2891 Results.push_back(ExpandVectorBuildThroughStack(Node));
2892 break;
2893 case ISD::SCALAR_TO_VECTOR:
2894 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
2895 break;
2896 case ISD::INSERT_VECTOR_ELT:
2897 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
2898 Node->getOperand(1),
2899 Node->getOperand(2), dl));
2900 break;
2901 case ISD::VECTOR_SHUFFLE: {
2902 SmallVector<int, 32> NewMask;
2903 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
2904
2905 EVT VT = Node->getValueType(0);
2906 EVT EltVT = VT.getVectorElementType();
2907 SDValue Op0 = Node->getOperand(0);
2908 SDValue Op1 = Node->getOperand(1);
2909 if (!TLI.isTypeLegal(EltVT)) {
2910 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
2911
2912 // BUILD_VECTOR operands are allowed to be wider than the element type.
2913 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
2914 // it.
2915 if (NewEltVT.bitsLT(EltVT)) {
2916 // Convert shuffle node.
2917 // If original node was v4i64 and the new EltVT is i32,
2918 // cast operands to v8i32 and re-build the mask.
2919
2920 // Calculate new VT, the size of the new VT should be equal to original.
2921 EVT NewVT =
2922 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
2923 VT.getSizeInBits() / NewEltVT.getSizeInBits());
2924 assert(NewVT.bitsEq(VT))((NewVT.bitsEq(VT)) ? static_cast<void> (0) : __assert_fail
("NewVT.bitsEq(VT)", "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2924, __PRETTY_FUNCTION__))
;
2925
2926 // cast operands to new VT
2927 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
2928 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
2929
2930 // Convert the shuffle mask
2931 unsigned int factor =
2932 NewVT.getVectorNumElements()/VT.getVectorNumElements();
2933
2934 // EltVT gets smaller
2935 assert(factor > 0)((factor > 0) ? static_cast<void> (0) : __assert_fail
("factor > 0", "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2935, __PRETTY_FUNCTION__))
;
2936
2937 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
2938 if (Mask[i] < 0) {
2939 for (unsigned fi = 0; fi < factor; ++fi)
2940 NewMask.push_back(Mask[i]);
2941 }
2942 else {
2943 for (unsigned fi = 0; fi < factor; ++fi)
2944 NewMask.push_back(Mask[i]*factor+fi);
2945 }
2946 }
2947 Mask = NewMask;
2948 VT = NewVT;
2949 }
2950 EltVT = NewEltVT;
2951 }
2952 unsigned NumElems = VT.getVectorNumElements();
2953 SmallVector<SDValue, 16> Ops;
2954 for (unsigned i = 0; i != NumElems; ++i) {
2955 if (Mask[i] < 0) {
2956 Ops.push_back(DAG.getUNDEF(EltVT));
2957 continue;
2958 }
2959 unsigned Idx = Mask[i];
2960 if (Idx < NumElems)
2961 Ops.push_back(DAG.getNode(
2962 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
2963 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2964 else
2965 Ops.push_back(DAG.getNode(
2966 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
2967 DAG.getConstant(Idx - NumElems, dl,
2968 TLI.getVectorIdxTy(DAG.getDataLayout()))));
2969 }
2970
2971 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
2972 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
2973 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
2974 Results.push_back(Tmp1);
2975 break;
2976 }
2977 case ISD::EXTRACT_ELEMENT: {
2978 EVT OpTy = Node->getOperand(0).getValueType();
2979 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
2980 // 1 -> Hi
2981 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
2982 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
2983 TLI.getShiftAmountTy(
2984 Node->getOperand(0).getValueType(),
2985 DAG.getDataLayout())));
2986 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
2987 } else {
2988 // 0 -> Lo
2989 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
2990 Node->getOperand(0));
2991 }
2992 Results.push_back(Tmp1);
2993 break;
2994 }
2995 case ISD::STACKSAVE:
2996 // Expand to CopyFromReg if the target set
2997 // StackPointerRegisterToSaveRestore.
2998 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
2999 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3000 Node->getValueType(0)));
3001 Results.push_back(Results[0].getValue(1));
3002 } else {
3003 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3004 Results.push_back(Node->getOperand(0));
3005 }
3006 break;
3007 case ISD::STACKRESTORE:
3008 // Expand to CopyToReg if the target set
3009 // StackPointerRegisterToSaveRestore.
3010 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3011 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3012 Node->getOperand(1)));
3013 } else {
3014 Results.push_back(Node->getOperand(0));
3015 }
3016 break;
3017 case ISD::GET_DYNAMIC_AREA_OFFSET:
3018 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3019 Results.push_back(Results[0].getValue(0));
3020 break;
3021 case ISD::FCOPYSIGN:
3022 Results.push_back(ExpandFCOPYSIGN(Node));
3023 break;
3024 case ISD::FNEG:
3025 // Expand Y = FNEG(X) -> Y = SUB -0.0, X
3026 Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3027 // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3028 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3029 Node->getOperand(0));
3030 Results.push_back(Tmp1);
3031 break;
3032 case ISD::FABS:
3033 Results.push_back(ExpandFABS(Node));
3034 break;
3035 case ISD::SMIN:
3036 case ISD::SMAX:
3037 case ISD::UMIN:
3038 case ISD::UMAX: {
3039 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3040 ISD::CondCode Pred;
3041 switch (Node->getOpcode()) {
3042 default: llvm_unreachable("How did we get here?")::llvm::llvm_unreachable_internal("How did we get here?", "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3042)
;
3043 case ISD::SMAX: Pred = ISD::SETGT; break;
3044 case ISD::SMIN: Pred = ISD::SETLT; break;
3045 case ISD::UMAX: Pred = ISD::SETUGT; break;
3046 case ISD::UMIN: Pred = ISD::SETULT; break;
3047 }
3048 Tmp1 = Node->getOperand(0);
3049 Tmp2 = Node->getOperand(1);
3050 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3051 Results.push_back(Tmp1);
3052 break;
3053 }
3054 case ISD::FMINNUM:
3055 case ISD::FMAXNUM: {
3056 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3057 Results.push_back(Expanded);
3058 break;
3059 }
3060 case ISD::FSIN:
3061 case ISD::FCOS: {
3062 EVT VT = Node->getValueType(0);
3063 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3064 // fcos which share the same operand and both are used.
3065 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3066 isSinCosLibcallAvailable(Node, TLI))
3067 && useSinCos(Node)) {
3068 SDVTList VTs = DAG.getVTList(VT, VT);
3069 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3070 if (Node->getOpcode() == ISD::FCOS)
3071 Tmp1 = Tmp1.getValue(1);
3072 Results.push_back(Tmp1);
3073 }
3074 break;
3075 }
3076 case ISD::FMAD:
3077 llvm_unreachable("Illegal fmad should never be formed")::llvm::llvm_unreachable_internal("Illegal fmad should never be formed"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3077)
;
3078
3079 case ISD::FP16_TO_FP:
3080 if (Node->getValueType(0) != MVT::f32) {
3081 // We can extend to types bigger than f32 in two steps without changing
3082 // the result. Since "f16 -> f32" is much more commonly available, give
3083 // CodeGen the option of emitting that before resorting to a libcall.
3084 SDValue Res =
3085 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3086 Results.push_back(
3087 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3088 }
3089 break;
3090 case ISD::FP_TO_FP16:
3091 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing FP_TO_FP16\n";
} } while (false)
;
3092 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3093 SDValue Op = Node->getOperand(0);
3094 MVT SVT = Op.getSimpleValueType();
3095 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3096 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3097 // Under fastmath, we can expand this node into a fround followed by
3098 // a float-half conversion.
3099 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3100 DAG.getIntPtrConstant(0, dl));
3101 Results.push_back(
3102 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3103 }
3104 }
3105 break;
3106 case ISD::ConstantFP: {
3107 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3108 // Check to see if this FP immediate is already legal.
3109 // If this is a legal constant, turn it into a TargetConstantFP node.
3110 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
3111 Results.push_back(ExpandConstantFP(CFP, true));
3112 break;
3113 }
3114 case ISD::Constant: {
3115 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3116 Results.push_back(ExpandConstant(CP));
3117 break;
3118 }
3119 case ISD::FSUB: {
3120 EVT VT = Node->getValueType(0);
3121 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3122 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3123 const SDNodeFlags Flags = Node->getFlags();
3124 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3125 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3126 Results.push_back(Tmp1);
3127 }
3128 break;
3129 }
3130 case ISD::SUB: {
3131 EVT VT = Node->getValueType(0);
3132 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3134, __PRETTY_FUNCTION__))
3133 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3134, __PRETTY_FUNCTION__))
3134 "Don't know how to expand this subtraction!")((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3134, __PRETTY_FUNCTION__))
;
3135 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3136 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3137 VT));
3138 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3139 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3140 break;
3141 }
3142 case ISD::UREM:
3143 case ISD::SREM: {
3144 EVT VT = Node->getValueType(0);
3145 bool isSigned = Node->getOpcode() == ISD::SREM;
3146 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3147 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3148 Tmp2 = Node->getOperand(0);
3149 Tmp3 = Node->getOperand(1);
3150 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3151 SDVTList VTs = DAG.getVTList(VT, VT);
3152 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3153 Results.push_back(Tmp1);
3154 } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3155 // X % Y -> X-X/Y*Y
3156 Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3157 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3158 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3159 Results.push_back(Tmp1);
3160 }
3161 break;
3162 }
3163 case ISD::UDIV:
3164 case ISD::SDIV: {
3165 bool isSigned = Node->getOpcode() == ISD::SDIV;
3166 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3167 EVT VT = Node->getValueType(0);
3168 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3169 SDVTList VTs = DAG.getVTList(VT, VT);
3170 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3171 Node->getOperand(1));
3172 Results.push_back(Tmp1);
3173 }
3174 break;
3175 }
3176 case ISD::MULHU:
3177 case ISD::MULHS: {
3178 unsigned ExpandOpcode =
3179 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3180 EVT VT = Node->getValueType(0);
3181 SDVTList VTs = DAG.getVTList(VT, VT);
3182
3183 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3184 Node->getOperand(1));
3185 Results.push_back(Tmp1.getValue(1));
3186 break;
3187 }
3188 case ISD::UMUL_LOHI:
3189 case ISD::SMUL_LOHI: {
3190 SDValue LHS = Node->getOperand(0);
3191 SDValue RHS = Node->getOperand(1);
3192 MVT VT = LHS.getSimpleValueType();
3193 unsigned MULHOpcode =
3194 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3195
3196 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3197 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3198 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3199 break;
3200 }
3201
3202 SmallVector<SDValue, 4> Halves;
3203 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3204 assert(TLI.isTypeLegal(HalfType))((TLI.isTypeLegal(HalfType)) ? static_cast<void> (0) : __assert_fail
("TLI.isTypeLegal(HalfType)", "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3204, __PRETTY_FUNCTION__))
;
3205 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
3206 HalfType, DAG,
3207 TargetLowering::MulExpansionKind::Always)) {
3208 for (unsigned i = 0; i < 2; ++i) {
3209 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3210 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3211 SDValue Shift = DAG.getConstant(
3212 HalfType.getScalarSizeInBits(), dl,
3213 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3214 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3215 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3216 }
3217 break;
3218 }
3219 break;
3220 }
3221 case ISD::MUL: {
3222 EVT VT = Node->getValueType(0);
3223 SDVTList VTs = DAG.getVTList(VT, VT);
3224 // See if multiply or divide can be lowered using two-result operations.
3225 // We just need the low half of the multiply; try both the signed
3226 // and unsigned forms. If the target supports both SMUL_LOHI and
3227 // UMUL_LOHI, form a preference by checking which forms of plain
3228 // MULH it supports.
3229 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3230 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3231 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3232 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3233 unsigned OpToUse = 0;
3234 if (HasSMUL_LOHI && !HasMULHS) {
3235 OpToUse = ISD::SMUL_LOHI;
3236 } else if (HasUMUL_LOHI && !HasMULHU) {
3237 OpToUse = ISD::UMUL_LOHI;
3238 } else if (HasSMUL_LOHI) {
3239 OpToUse = ISD::SMUL_LOHI;
3240 } else if (HasUMUL_LOHI) {
3241 OpToUse = ISD::UMUL_LOHI;
3242 }
3243 if (OpToUse) {
3244 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3245 Node->getOperand(1)));
3246 break;
3247 }
3248
3249 SDValue Lo, Hi;
3250 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3251 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3252 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3253 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3254 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3255 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3256 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3257 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3258 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3259 SDValue Shift =
3260 DAG.getConstant(HalfType.getSizeInBits(), dl,
3261 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3262 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3263 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3264 }
3265 break;
3266 }
3267 case ISD::FSHL:
3268 case ISD::FSHR:
3269 if (TLI.expandFunnelShift(Node, Tmp1, DAG))
3270 Results.push_back(Tmp1);
3271 break;
3272 case ISD::SADDSAT:
3273 case ISD::UADDSAT:
3274 case ISD::SSUBSAT:
3275 case ISD::USUBSAT: {
3276 Results.push_back(TLI.getExpandedSaturationAdditionSubtraction(Node, DAG));
3277 break;
3278 }
3279 case ISD::SADDO:
3280 case ISD::SSUBO: {
3281 SDValue LHS = Node->getOperand(0);
3282 SDValue RHS = Node->getOperand(1);
3283 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3284 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3285 LHS, RHS);
3286 Results.push_back(Sum);
3287 EVT ResultType = Node->getValueType(1);
3288 EVT OType = getSetCCResultType(Node->getValueType(0));
3289
3290 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3291
3292 // LHSSign -> LHS >= 0
3293 // RHSSign -> RHS >= 0
3294 // SumSign -> Sum >= 0
3295 //
3296 // Add:
3297 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3298 // Sub:
3299 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3300 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3301 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3302 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3303 Node->getOpcode() == ISD::SADDO ?
3304 ISD::SETEQ : ISD::SETNE);
3305
3306 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3307 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3308
3309 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3310 Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
3311 break;
3312 }
3313 case ISD::UADDO:
3314 case ISD::USUBO: {
3315 SDValue LHS = Node->getOperand(0);
3316 SDValue RHS = Node->getOperand(1);
3317 bool IsAdd = Node->getOpcode() == ISD::UADDO;
3318 // If ADD/SUBCARRY is legal, use that instead.
3319 unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;
3320 if (TLI.isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
3321 SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
3322 SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
3323 { LHS, RHS, CarryIn });
3324 Results.push_back(SDValue(NodeCarry.getNode(), 0));
3325 Results.push_back(SDValue(NodeCarry.getNode(), 1));
3326 break;
3327 }
3328
3329 SDValue Sum = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
3330 LHS.getValueType(), LHS, RHS);
3331 Results.push_back(Sum);
3332
3333 EVT ResultType = Node->getValueType(1);
3334 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3335 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3336 SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3337
3338 Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
3339 break;
3340 }
3341 case ISD::UMULO:
3342 case ISD::SMULO: {
3343 EVT VT = Node->getValueType(0);
3344 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
3345 SDValue LHS = Node->getOperand(0);
3346 SDValue RHS = Node->getOperand(1);
3347 SDValue BottomHalf;
3348 SDValue TopHalf;
3349 static const unsigned Ops[2][3] =
3350 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
3351 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
3352 bool isSigned = Node->getOpcode() == ISD::SMULO;
3353 if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
3354 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3355 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
3356 } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
3357 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
3358 RHS);
3359 TopHalf = BottomHalf.getValue(1);
3360 } else if (TLI.isTypeLegal(WideVT)) {
3361 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
3362 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
3363 Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3364 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3365 DAG.getIntPtrConstant(0, dl));
3366 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3367 DAG.getIntPtrConstant(1, dl));
3368 } else {
3369 // We can fall back to a libcall with an illegal type for the MUL if we
3370 // have a libcall big enough.
3371 // Also, we can fall back to a division in some cases, but that's a big
3372 // performance hit in the general case.
3373 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3374 if (WideVT == MVT::i16)
3375 LC = RTLIB::MUL_I16;
3376 else if (WideVT == MVT::i32)
3377 LC = RTLIB::MUL_I32;
3378 else if (WideVT == MVT::i64)
3379 LC = RTLIB::MUL_I64;
3380 else if (WideVT == MVT::i128)
3381 LC = RTLIB::MUL_I128;
3382 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!")((LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!"
) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Cannot expand this operation!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3382, __PRETTY_FUNCTION__))
;
3383
3384 SDValue HiLHS;
3385 SDValue HiRHS;
3386 if (isSigned) {
3387 // The high part is obtained by SRA'ing all but one of the bits of low
3388 // part.
3389 unsigned LoSize = VT.getSizeInBits();
3390 HiLHS =
3391 DAG.getNode(ISD::SRA, dl, VT, LHS,
3392 DAG.getConstant(LoSize - 1, dl,
3393 TLI.getPointerTy(DAG.getDataLayout())));
3394 HiRHS =
3395 DAG.getNode(ISD::SRA, dl, VT, RHS,
3396 DAG.getConstant(LoSize - 1, dl,
3397 TLI.getPointerTy(DAG.getDataLayout())));
3398 } else {
3399 HiLHS = DAG.getConstant(0, dl, VT);
3400 HiRHS = DAG.getConstant(0, dl, VT);
3401 }
3402
3403 // Here we're passing the 2 arguments explicitly as 4 arguments that are
3404 // pre-lowered to the correct types. This all depends upon WideVT not
3405 // being a legal type for the architecture and thus has to be split to
3406 // two arguments.
3407 SDValue Ret;
3408 if(DAG.getDataLayout().isLittleEndian()) {
3409 // Halves of WideVT are packed into registers in different order
3410 // depending on platform endianness. This is usually handled by
3411 // the C calling convention, but we can't defer to it in
3412 // the legalizer.
3413 SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
3414 Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3415 } else {
3416 SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
3417 Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3418 }
3419 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&((Ret.getOpcode() == ISD::MERGE_VALUES && "Ret value is a collection of constituent nodes holding result."
) ? static_cast<void> (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3420, __PRETTY_FUNCTION__))
3420 "Ret value is a collection of constituent nodes holding result.")((Ret.getOpcode() == ISD::MERGE_VALUES && "Ret value is a collection of constituent nodes holding result."
) ? static_cast<void> (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3420, __PRETTY_FUNCTION__))
;
3421 BottomHalf = Ret.getOperand(0);
3422 TopHalf = Ret.getOperand(1);
3423 }
3424
3425 if (isSigned) {
3426 Tmp1 = DAG.getConstant(
3427 VT.getSizeInBits() - 1, dl,
3428 TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
3429 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3430 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
3431 ISD::SETNE);
3432 } else {
3433 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
3434 DAG.getConstant(0, dl, VT), ISD::SETNE);
3435 }
3436
3437 // Truncate the result if SetCC returns a larger type than needed.
3438 EVT RType = Node->getValueType(1);
3439 if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
3440 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
3441
3442 assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&((RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
"Unexpected result type for S/UMULO legalization") ? static_cast
<void> (0) : __assert_fail ("RType.getSizeInBits() == TopHalf.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3443, __PRETTY_FUNCTION__))
3443 "Unexpected result type for S/UMULO legalization")((RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
"Unexpected result type for S/UMULO legalization") ? static_cast
<void> (0) : __assert_fail ("RType.getSizeInBits() == TopHalf.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3443, __PRETTY_FUNCTION__))
;
3444
3445 Results.push_back(BottomHalf);
3446 Results.push_back(TopHalf);
3447 break;
3448 }
3449 case ISD::BUILD_PAIR: {
3450 EVT PairTy = Node->getValueType(0);
3451 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3452 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3453 Tmp2 = DAG.getNode(
3454 ISD::SHL, dl, PairTy, Tmp2,
3455 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3456 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3457 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3458 break;
3459 }
3460 case ISD::SELECT:
3461 Tmp1 = Node->getOperand(0);
3462 Tmp2 = Node->getOperand(1);
3463 Tmp3 = Node->getOperand(2);
3464 if (Tmp1.getOpcode() == ISD::SETCC) {
3465 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3466 Tmp2, Tmp3,
3467 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3468 } else {
3469 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3470 DAG.getConstant(0, dl, Tmp1.getValueType()),
3471 Tmp2, Tmp3, ISD::SETNE);
3472 }
3473 Results.push_back(Tmp1);
3474 break;
3475 case ISD::BR_JT: {
3476 SDValue Chain = Node->getOperand(0);
3477 SDValue Table = Node->getOperand(1);
3478 SDValue Index = Node->getOperand(2);
3479
3480 const DataLayout &TD = DAG.getDataLayout();
3481 EVT PTy = TLI.getPointerTy(TD);
3482
3483 unsigned EntrySize =
3484 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3485
3486 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3487 // This transformation needs to be done here since otherwise the MIPS
3488 // backend will end up emitting a three instruction multiply sequence
3489 // instead of a single shift and MSP430 will call a runtime function.
3490 if (llvm::isPowerOf2_32(EntrySize))
3491 Index = DAG.getNode(
3492 ISD::SHL, dl, Index.getValueType(), Index,
3493 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3494 else
3495 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3496 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3497 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3498 Index, Table);
3499
3500 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3501 SDValue LD = DAG.getExtLoad(
3502 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3503 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3504 Addr = LD;
3505 if (TLI.isJumpTableRelative()) {
3506 // For PIC, the sequence is:
3507 // BRIND(load(Jumptable + index) + RelocBase)
3508 // RelocBase can be JumpTable, GOT or some sort of global base.
3509 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3510 TLI.getPICJumpTableRelocBase(Table, DAG));
3511 }
3512
3513 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG);
3514 Results.push_back(Tmp1);
3515 break;
3516 }
3517 case ISD::BRCOND:
3518 // Expand brcond's setcc into its constituent parts and create a BR_CC
3519 // Node.
3520 Tmp1 = Node->getOperand(0);
3521 Tmp2 = Node->getOperand(1);
3522 if (Tmp2.getOpcode() == ISD::SETCC) {
3523 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3524 Tmp1, Tmp2.getOperand(2),
3525 Tmp2.getOperand(0), Tmp2.getOperand(1),
3526 Node->getOperand(2));
3527 } else {
3528 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3529 if (Tmp2.isUndef() ||
3530 (Tmp2.getOpcode() == ISD::AND &&
3531 isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3532 cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3533 Tmp3 = Tmp2;
3534 else
3535 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3536 DAG.getConstant(1, dl, Tmp2.getValueType()));
3537 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3538 DAG.getCondCode(ISD::SETNE), Tmp3,
3539 DAG.getConstant(0, dl, Tmp3.getValueType()),
3540 Node->getOperand(2));
3541 }
3542 Results.push_back(Tmp1);
3543 break;
3544 case ISD::SETCC: {
3545 Tmp1 = Node->getOperand(0);
3546 Tmp2 = Node->getOperand(1);
3547 Tmp3 = Node->getOperand(2);
3548 bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3549 Tmp3, NeedInvert, dl);
3550
3551 if (Legalized) {
3552 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3553 // condition code, create a new SETCC node.
3554 if (Tmp3.getNode())
3555 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3556 Tmp1, Tmp2, Tmp3);
3557
3558 // If we expanded the SETCC by inverting the condition code, then wrap
3559 // the existing SETCC in a NOT to restore the intended condition.
3560 if (NeedInvert)
3561 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3562
3563 Results.push_back(Tmp1);
3564 break;
3565 }
3566
3567 // Otherwise, SETCC for the given comparison type must be completely
3568 // illegal; expand it into a SELECT_CC.
3569 EVT VT = Node->getValueType(0);
3570 int TrueValue;
3571 switch (TLI.getBooleanContents(Tmp1.getValueType())) {
3572 case TargetLowering::ZeroOrOneBooleanContent:
3573 case TargetLowering::UndefinedBooleanContent:
3574 TrueValue = 1;
3575 break;
3576 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3577 TrueValue = -1;
3578 break;
3579 }
3580 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3581 DAG.getConstant(TrueValue, dl, VT),
3582 DAG.getConstant(0, dl, VT),
3583 Tmp3);
3584 Results.push_back(Tmp1);
3585 break;
3586 }
3587 case ISD::SELECT_CC: {
3588 Tmp1 = Node->getOperand(0); // LHS
3589 Tmp2 = Node->getOperand(1); // RHS
3590 Tmp3 = Node->getOperand(2); // True
3591 Tmp4 = Node->getOperand(3); // False
3592 EVT VT = Node->getValueType(0);
3593 SDValue CC = Node->getOperand(4);
3594 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3595
3596 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
3597 // If the condition code is legal, then we need to expand this
3598 // node using SETCC and SELECT.
3599 EVT CmpVT = Tmp1.getValueType();
3600 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3602, __PRETTY_FUNCTION__))
3601 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3602, __PRETTY_FUNCTION__))
3602 "expanded.")((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3602, __PRETTY_FUNCTION__))
;
3603 EVT CCVT =
3604 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
3605 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
3606 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3607 break;
3608 }
3609
3610 // SELECT_CC is legal, so the condition code must not be.
3611 bool Legalized = false;
3612 // Try to legalize by inverting the condition. This is for targets that
3613 // might support an ordered version of a condition, but not the unordered
3614 // version (or vice versa).
3615 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3616 Tmp1.getValueType().isInteger());
3617 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
3618 // Use the new condition code and swap true and false
3619 Legalized = true;
3620 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3621 } else {
3622 // If The inverse is not legal, then try to swap the arguments using
3623 // the inverse condition code.
3624 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3625 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
3626 // The swapped inverse condition is legal, so swap true and false,
3627 // lhs and rhs.
3628 Legalized = true;
3629 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3630 }
3631 }
3632
3633 if (!Legalized) {
3634 Legalized = LegalizeSetCCCondCode(
3635 getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3636 dl);
3637
3638 assert(Legalized && "Can't legalize SELECT_CC with legal condition!")((Legalized && "Can't legalize SELECT_CC with legal condition!"
) ? static_cast<void> (0) : __assert_fail ("Legalized && \"Can't legalize SELECT_CC with legal condition!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3638, __PRETTY_FUNCTION__))
;
3639
3640 // If we expanded the SETCC by inverting the condition code, then swap
3641 // the True/False operands to match.
3642 if (NeedInvert)
3643 std::swap(Tmp3, Tmp4);
3644
3645 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3646 // condition code, create a new SELECT_CC node.
3647 if (CC.getNode()) {
3648 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3649 Tmp1, Tmp2, Tmp3, Tmp4, CC);
3650 } else {
3651 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3652 CC = DAG.getCondCode(ISD::SETNE);
3653 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3654 Tmp2, Tmp3, Tmp4, CC);
3655 }
3656 }
3657 Results.push_back(Tmp1);
3658 break;
3659 }
3660 case ISD::BR_CC: {
3661 Tmp1 = Node->getOperand(0); // Chain
3662 Tmp2 = Node->getOperand(2); // LHS
3663 Tmp3 = Node->getOperand(3); // RHS
3664 Tmp4 = Node->getOperand(1); // CC
3665
3666 bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
3667 Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
3668 (void)Legalized;
3669 assert(Legalized && "Can't legalize BR_CC with legal condition!")((Legalized && "Can't legalize BR_CC with legal condition!"
) ? static_cast<void> (0) : __assert_fail ("Legalized && \"Can't legalize BR_CC with legal condition!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3669, __PRETTY_FUNCTION__))
;
3670
3671 // If we expanded the SETCC by inverting the condition code, then wrap
3672 // the existing SETCC in a NOT to restore the intended condition.
3673 if (NeedInvert)
3674 Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
3675
3676 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3677 // node.
3678 if (Tmp4.getNode()) {
3679 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3680 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3681 } else {
3682 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3683 Tmp4 = DAG.getCondCode(ISD::SETNE);
3684 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3685 Tmp2, Tmp3, Node->getOperand(4));
3686 }
3687 Results.push_back(Tmp1);
3688 break;
3689 }
3690 case ISD::BUILD_VECTOR:
3691 Results.push_back(ExpandBUILD_VECTOR(Node));
3692 break;
3693 case ISD::SRA:
3694 case ISD::SRL:
3695 case ISD::SHL: {
3696 // Scalarize vector SRA/SRL/SHL.
3697 EVT VT = Node->getValueType(0);
3698 assert(VT.isVector() && "Unable to legalize non-vector shift")((VT.isVector() && "Unable to legalize non-vector shift"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Unable to legalize non-vector shift\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3698, __PRETTY_FUNCTION__))
;
3699 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal")((TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"
) ? static_cast<void> (0) : __assert_fail ("TLI.isTypeLegal(VT.getScalarType())&& \"Element type must be legal\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3699, __PRETTY_FUNCTION__))
;
3700 unsigned NumElem = VT.getVectorNumElements();
3701
3702 SmallVector<SDValue, 8> Scalars;
3703 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3704 SDValue Ex = DAG.getNode(
3705 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3706 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3707 SDValue Sh = DAG.getNode(
3708 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3709 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3710 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3711 VT.getScalarType(), Ex, Sh));
3712 }
3713
3714 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3715 ReplaceNode(SDValue(Node, 0), Result);
3716 break;
3717 }
3718 case ISD::ROTL:
3719 case ISD::ROTR: {
3720 bool IsLeft = Node->getOpcode() == ISD::ROTL;
3721 SDValue Op0 = Node->getOperand(0), Op1 = Node->getOperand(1);
3722 EVT ResVT = Node->getValueType(0);
3723 EVT OpVT = Op0.getValueType();
3724 assert(OpVT == ResVT &&((OpVT == ResVT && "The result and the operand types of rotate should match"
) ? static_cast<void> (0) : __assert_fail ("OpVT == ResVT && \"The result and the operand types of rotate should match\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3725, __PRETTY_FUNCTION__))
3725 "The result and the operand types of rotate should match")((OpVT == ResVT && "The result and the operand types of rotate should match"
) ? static_cast<void> (0) : __assert_fail ("OpVT == ResVT && \"The result and the operand types of rotate should match\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3725, __PRETTY_FUNCTION__))
;
3726 EVT ShVT = Op1.getValueType();
3727 SDValue Width = DAG.getConstant(OpVT.getScalarSizeInBits(), dl, ShVT);
3728
3729 // If a rotate in the other direction is legal, use it.
3730 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
3731 if (TLI.isOperationLegal(RevRot, ResVT)) {
3732 SDValue Sub = DAG.getNode(ISD::SUB, dl, ShVT, Width, Op1);
3733 Results.push_back(DAG.getNode(RevRot, dl, ResVT, Op0, Sub));
3734 break;
3735 }
3736
3737 // Otherwise,
3738 // (rotl x, c) -> (or (shl x, (and c, w-1)), (srl x, (and w-c, w-1)))
3739 // (rotr x, c) -> (or (srl x, (and c, w-1)), (shl x, (and w-c, w-1)))
3740 //
3741 assert(isPowerOf2_32(OpVT.getScalarSizeInBits()) &&((isPowerOf2_32(OpVT.getScalarSizeInBits()) && "Expecting the type bitwidth to be a power of 2"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(OpVT.getScalarSizeInBits()) && \"Expecting the type bitwidth to be a power of 2\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3742, __PRETTY_FUNCTION__))
3742 "Expecting the type bitwidth to be a power of 2")((isPowerOf2_32(OpVT.getScalarSizeInBits()) && "Expecting the type bitwidth to be a power of 2"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(OpVT.getScalarSizeInBits()) && \"Expecting the type bitwidth to be a power of 2\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3742, __PRETTY_FUNCTION__))
;
3743 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
3744 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
3745 SDValue Width1 = DAG.getNode(ISD::SUB, dl, ShVT,
3746 Width, DAG.getConstant(1, dl, ShVT));
3747 SDValue NegOp1 = DAG.getNode(ISD::SUB, dl, ShVT, Width, Op1);
3748 SDValue And0 = DAG.getNode(ISD::AND, dl, ShVT, Op1, Width1);
3749 SDValue And1 = DAG.getNode(ISD::AND, dl, ShVT, NegOp1, Width1);
3750
3751 SDValue Or = DAG.getNode(ISD::OR, dl, ResVT,
3752 DAG.getNode(ShOpc, dl, ResVT, Op0, And0),
3753 DAG.getNode(HsOpc, dl, ResVT, Op0, And1));
3754 Results.push_back(Or);
3755 break;
3756 }
3757
3758 case ISD::GLOBAL_OFFSET_TABLE:
3759 case ISD::GlobalAddress:
3760 case ISD::GlobalTLSAddress:
3761 case ISD::ExternalSymbol:
3762 case ISD::ConstantPool:
3763 case ISD::JumpTable:
3764 case ISD::INTRINSIC_W_CHAIN:
3765 case ISD::INTRINSIC_WO_CHAIN:
3766 case ISD::INTRINSIC_VOID:
3767 // FIXME: Custom lowering for these operations shouldn't return null!
3768 break;
3769 }
3770
3771 // Replace the original node with the legalized result.
3772 if (Results.empty()) {
3773 LLVM_DEBUG(dbgs() << "Cannot expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Cannot expand node\n"; } }
while (false)
;
3774 return false;
3775 }
3776
3777 LLVM_DEBUG(dbgs() << "Successfully expanded node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully expanded node\n"
; } } while (false)
;
3778 ReplaceNode(Node, Results.data());
3779 return true;
3780}
3781
3782void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3783 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to convert node to libcall\n"
; } } while (false)
;
3784 SmallVector<SDValue, 8> Results;
3785 SDLoc dl(Node);
3786 // FIXME: Check flags on the node to see if we can use a finite call.
3787 bool CanUseFiniteLibCall = TM.Options.NoInfsFPMath && TM.Options.NoNaNsFPMath;
3788 unsigned Opc = Node->getOpcode();
3789 switch (Opc) {
3790 case ISD::ATOMIC_FENCE: {
3791 // If the target didn't lower this, lower it to '__sync_synchronize()' call
3792 // FIXME: handle "fence singlethread" more efficiently.
3793 TargetLowering::ArgListTy Args;
3794
3795 TargetLowering::CallLoweringInfo CLI(DAG);
3796 CLI.setDebugLoc(dl)
3797 .setChain(Node->getOperand(0))
3798 .setLibCallee(
3799 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3800 DAG.getExternalSymbol("__sync_synchronize",
3801 TLI.getPointerTy(DAG.getDataLayout())),
3802 std::move(Args));
3803
3804 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3805
3806 Results.push_back(CallResult.second);
3807 break;
3808 }
3809 // By default, atomic intrinsics are marked Legal and lowered. Targets
3810 // which don't support them directly, however, may want libcalls, in which
3811 // case they mark them Expand, and we get here.
3812 case ISD::ATOMIC_SWAP:
3813 case ISD::ATOMIC_LOAD_ADD:
3814 case ISD::ATOMIC_LOAD_SUB:
3815 case ISD::ATOMIC_LOAD_AND:
3816 case ISD::ATOMIC_LOAD_CLR:
3817 case ISD::ATOMIC_LOAD_OR:
3818 case ISD::ATOMIC_LOAD_XOR:
3819 case ISD::ATOMIC_LOAD_NAND:
3820 case ISD::ATOMIC_LOAD_MIN:
3821 case ISD::ATOMIC_LOAD_MAX:
3822 case ISD::ATOMIC_LOAD_UMIN:
3823 case ISD::ATOMIC_LOAD_UMAX:
3824 case ISD::ATOMIC_CMP_SWAP: {
3825 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3826 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
3827 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!")((LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"
) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected atomic op or value type!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3827, __PRETTY_FUNCTION__))
;
3828
3829 std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
3830 Results.push_back(Tmp.first);
3831 Results.push_back(Tmp.second);
3832 break;
3833 }
3834 case ISD::TRAP: {
3835 // If this operation is not supported, lower it to 'abort()' call
3836 TargetLowering::ArgListTy Args;
3837 TargetLowering::CallLoweringInfo CLI(DAG);
3838 CLI.setDebugLoc(dl)
3839 .setChain(Node->getOperand(0))
3840 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3841 DAG.getExternalSymbol(
3842 "abort", TLI.getPointerTy(DAG.getDataLayout())),
3843 std::move(Args));
3844 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3845
3846 Results.push_back(CallResult.second);
3847 break;
3848 }
3849 case ISD::FMINNUM:
3850 case ISD::STRICT_FMINNUM:
3851 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3852 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3853 RTLIB::FMIN_PPCF128));
3854 break;
3855 case ISD::FMAXNUM:
3856 case ISD::STRICT_FMAXNUM:
3857 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3858 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3859 RTLIB::FMAX_PPCF128));
3860 break;
3861 case ISD::FSQRT:
3862 case ISD::STRICT_FSQRT:
3863 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3864 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3865 RTLIB::SQRT_PPCF128));
3866 break;
3867 case ISD::FCBRT:
3868 Results.push_back(ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
3869 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
3870 RTLIB::CBRT_PPCF128));
3871 break;
3872 case ISD::FSIN:
3873 case ISD::STRICT_FSIN:
3874 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
3875 RTLIB::SIN_F80, RTLIB::SIN_F128,
3876 RTLIB::SIN_PPCF128));
3877 break;
3878 case ISD::FCOS:
3879 case ISD::STRICT_FCOS:
3880 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
3881 RTLIB::COS_F80, RTLIB::COS_F128,
3882 RTLIB::COS_PPCF128));
3883 break;
3884 case ISD::FSINCOS:
3885 // Expand into sincos libcall.
3886 ExpandSinCosLibCall(Node, Results);
3887 break;
3888 case ISD::FLOG:
3889 case ISD::STRICT_FLOG:
3890 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log_finite))
3891 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_FINITE_F32,
3892 RTLIB::LOG_FINITE_F64,
3893 RTLIB::LOG_FINITE_F80,
3894 RTLIB::LOG_FINITE_F128,
3895 RTLIB::LOG_FINITE_PPCF128));
3896 else
3897 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
3898 RTLIB::LOG_F80, RTLIB::LOG_F128,
3899 RTLIB::LOG_PPCF128));
3900 break;
3901 case ISD::FLOG2:
3902 case ISD::STRICT_FLOG2:
3903 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log2_finite))
3904 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_FINITE_F32,
3905 RTLIB::LOG2_FINITE_F64,
3906 RTLIB::LOG2_FINITE_F80,
3907 RTLIB::LOG2_FINITE_F128,
3908 RTLIB::LOG2_FINITE_PPCF128));
3909 else
3910 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
3911 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
3912 RTLIB::LOG2_PPCF128));
3913 break;
3914 case ISD::FLOG10:
3915 case ISD::STRICT_FLOG10:
3916 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log10_finite))
3917 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_FINITE_F32,
3918 RTLIB::LOG10_FINITE_F64,
3919 RTLIB::LOG10_FINITE_F80,
3920 RTLIB::LOG10_FINITE_F128,
3921 RTLIB::LOG10_FINITE_PPCF128));
3922 else
3923 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
3924 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
3925 RTLIB::LOG10_PPCF128));
3926 break;
3927 case ISD::FEXP:
3928 case ISD::STRICT_FEXP:
3929 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp_finite))
3930 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_FINITE_F32,
3931 RTLIB::EXP_FINITE_F64,
3932 RTLIB::EXP_FINITE_F80,
3933 RTLIB::EXP_FINITE_F128,
3934 RTLIB::EXP_FINITE_PPCF128));
3935 else
3936 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
3937 RTLIB::EXP_F80, RTLIB::EXP_F128,
3938 RTLIB::EXP_PPCF128));
3939 break;
3940 case ISD::FEXP2:
3941 case ISD::STRICT_FEXP2:
3942 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp2_finite))
3943 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_FINITE_F32,
3944 RTLIB::EXP2_FINITE_F64,
3945 RTLIB::EXP2_FINITE_F80,
3946 RTLIB::EXP2_FINITE_F128,
3947 RTLIB::EXP2_FINITE_PPCF128));
3948 else
3949 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
3950 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
3951 RTLIB::EXP2_PPCF128));
3952 break;
3953 case ISD::FTRUNC:
3954 case ISD::STRICT_FTRUNC:
3955 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
3956 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
3957 RTLIB::TRUNC_PPCF128));
3958 break;
3959 case ISD::FFLOOR:
3960 case ISD::STRICT_FFLOOR:
3961 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
3962 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
3963 RTLIB::FLOOR_PPCF128));
3964 break;
3965 case ISD::FCEIL:
3966 case ISD::STRICT_FCEIL:
3967 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
3968 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
3969 RTLIB::CEIL_PPCF128));
3970 break;
3971 case ISD::FRINT:
3972 case ISD::STRICT_FRINT:
3973 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
3974 RTLIB::RINT_F80, RTLIB::RINT_F128,
3975 RTLIB::RINT_PPCF128));
3976 break;
3977 case ISD::FNEARBYINT:
3978 case ISD::STRICT_FNEARBYINT:
3979 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
3980 RTLIB::NEARBYINT_F64,
3981 RTLIB::NEARBYINT_F80,
3982 RTLIB::NEARBYINT_F128,
3983 RTLIB::NEARBYINT_PPCF128));
3984 break;
3985 case ISD::FROUND:
3986 case ISD::STRICT_FROUND:
3987 Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
3988 RTLIB::ROUND_F64,
3989 RTLIB::ROUND_F80,
3990 RTLIB::ROUND_F128,
3991 RTLIB::ROUND_PPCF128));
3992 break;
3993 case ISD::FPOWI:
3994 case ISD::STRICT_FPOWI:
3995 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
3996 RTLIB::POWI_F80, RTLIB::POWI_F128,
3997 RTLIB::POWI_PPCF128));
3998 break;
3999 case ISD::FPOW:
4000 case ISD::STRICT_FPOW:
4001 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_pow_finite))
4002 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_FINITE_F32,
4003 RTLIB::POW_FINITE_F64,
4004 RTLIB::POW_FINITE_F80,
4005 RTLIB::POW_FINITE_F128,
4006 RTLIB::POW_FINITE_PPCF128));
4007 else
4008 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
4009 RTLIB::POW_F80, RTLIB::POW_F128,
4010 RTLIB::POW_PPCF128));
4011 break;
4012 case ISD::FDIV:
4013 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4014 RTLIB::DIV_F80, RTLIB::DIV_F128,
4015 RTLIB::DIV_PPCF128));
4016 break;
4017 case ISD::FREM:
4018 case ISD::STRICT_FREM:
4019 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4020 RTLIB::REM_F80, RTLIB::REM_F128,
4021 RTLIB::REM_PPCF128));
4022 break;
4023 case ISD::FMA:
4024 case ISD::STRICT_FMA:
4025 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4026 RTLIB::FMA_F80, RTLIB::FMA_F128,
4027 RTLIB::FMA_PPCF128));
4028 break;
4029 case ISD::FADD:
4030 Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4031 RTLIB::ADD_F80, RTLIB::ADD_F128,
4032 RTLIB::ADD_PPCF128));
4033 break;
4034 case ISD::FMUL:
4035 Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4036 RTLIB::MUL_F80, RTLIB::MUL_F128,
4037 RTLIB::MUL_PPCF128));
4038 break;
4039 case ISD::FP16_TO_FP:
4040 if (Node->getValueType(0) == MVT::f32) {
4041 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4042 }
4043 break;
4044 case ISD::FP_TO_FP16: {
4045 RTLIB::Libcall LC =
4046 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4047 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16")((LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16"
) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unable to expand fp_to_fp16\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4047, __PRETTY_FUNCTION__))
;
4048 Results.push_back(ExpandLibCall(LC, Node, false));
4049 break;
4050 }
4051 case ISD::FSUB:
4052 Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4053 RTLIB::SUB_F80, RTLIB::SUB_F128,
4054 RTLIB::SUB_PPCF128));
4055 break;
4056 case ISD::SREM:
4057 Results.push_back(ExpandIntLibCall(Node, true,
4058 RTLIB::SREM_I8,
4059 RTLIB::SREM_I16, RTLIB::SREM_I32,
4060 RTLIB::SREM_I64, RTLIB::SREM_I128));
4061 break;
4062 case ISD::UREM:
4063 Results.push_back(ExpandIntLibCall(Node, false,
4064 RTLIB::UREM_I8,
4065 RTLIB::UREM_I16, RTLIB::UREM_I32,
4066 RTLIB::UREM_I64, RTLIB::UREM_I128));
4067 break;
4068 case ISD::SDIV:
4069 Results.push_back(ExpandIntLibCall(Node, true,
4070 RTLIB::SDIV_I8,
4071 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4072 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4073 break;
4074 case ISD::UDIV:
4075 Results.push_back(ExpandIntLibCall(Node, false,
4076 RTLIB::UDIV_I8,
4077 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4078 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4079 break;
4080 case ISD::SDIVREM:
4081 case ISD::UDIVREM:
4082 // Expand into divrem libcall
4083 ExpandDivRemLibCall(Node, Results);
4084 break;
4085 case ISD::MUL:
4086 Results.push_back(ExpandIntLibCall(Node, false,
4087 RTLIB::MUL_I8,
4088 RTLIB::MUL_I16, RTLIB::MUL_I32,
4089 RTLIB::MUL_I64, RTLIB::MUL_I128));
4090 break;
4091 case ISD::CTLZ_ZERO_UNDEF:
4092 switch (Node->getSimpleValueType(0).SimpleTy) {
4093 default:
4094 llvm_unreachable("LibCall explicitly requested, but not available")::llvm::llvm_unreachable_internal("LibCall explicitly requested, but not available"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4094)
;
4095 case MVT::i32:
4096 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false));
4097 break;
4098 case MVT::i64:
4099 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false));
4100 break;
4101 case MVT::i128:
4102 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false));
4103 break;
4104 }
4105 break;
4106 }
4107
4108 // Replace the original node with the legalized result.
4109 if (!Results.empty()) {
4110 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully converted node to libcall\n"
; } } while (false)
;
4111 ReplaceNode(Node, Results.data());
4112 } else
4113 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not convert node to libcall\n"
; } } while (false)
;
4114}
4115
4116// Determine the vector type to use in place of an original scalar element when
4117// promoting equally sized vectors.
4118static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4119 MVT EltVT, MVT NewEltVT) {
4120 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4121 MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4122 assert(TLI.isTypeLegal(MidVT) && "unexpected")((TLI.isTypeLegal(MidVT) && "unexpected") ? static_cast
<void> (0) : __assert_fail ("TLI.isTypeLegal(MidVT) && \"unexpected\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4122, __PRETTY_FUNCTION__))
;
4123 return MidVT;
4124}
4125
4126void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4127 LLVM_DEBUG(dbgs() << "Trying to promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to promote node\n"
; } } while (false)
;
4128 SmallVector<SDValue, 8> Results;
4129 MVT OVT = Node->getSimpleValueType(0);
4130 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4131 Node->getOpcode() == ISD::SINT_TO_FP ||
4132 Node->getOpcode() == ISD::SETCC ||
4133 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4134 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4135 OVT = Node->getOperand(0).getSimpleValueType();
4136 }
4137 if (Node->getOpcode() == ISD::BR_CC)
4138 OVT = Node->getOperand(2).getSimpleValueType();
4139 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4140 SDLoc dl(Node);
4141 SDValue Tmp1, Tmp2, Tmp3;
4142 switch (Node->getOpcode()) {
4143 case ISD::CTTZ:
4144 case ISD::CTTZ_ZERO_UNDEF:
4145 case ISD::CTLZ:
4146 case ISD::CTLZ_ZERO_UNDEF:
4147 case ISD::CTPOP:
4148 // Zero extend the argument.
4149 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4150 if (Node->getOpcode() == ISD::CTTZ) {
4151 // The count is the same in the promoted type except if the original
4152 // value was zero. This can be handled by setting the bit just off
4153 // the top of the original type.
4154 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4155 OVT.getSizeInBits());
4156 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4157 DAG.getConstant(TopBit, dl, NVT));
4158 }
4159 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4160 // already the correct result.
4161 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4162 if (Node->getOpcode() == ISD::CTLZ ||
4163 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4164 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4165 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4166 DAG.getConstant(NVT.getSizeInBits() -
4167 OVT.getSizeInBits(), dl, NVT));
4168 }
4169 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4170 break;
4171 case ISD::BITREVERSE:
4172 case ISD::BSWAP: {
4173 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4174 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4175 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4176 Tmp1 = DAG.getNode(
4177 ISD::SRL, dl, NVT, Tmp1,
4178 DAG.getConstant(DiffBits, dl,
4179 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4180
4181 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4182 break;
4183 }
4184 case ISD::FP_TO_UINT:
4185 case ISD::FP_TO_SINT:
4186 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4187 Node->getOpcode() == ISD::FP_TO_SINT, dl);
4188 Results.push_back(Tmp1);
4189 break;
4190 case ISD::UINT_TO_FP:
4191 case ISD::SINT_TO_FP:
4192 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4193 Node->getOpcode() == ISD::SINT_TO_FP, dl);
4194 Results.push_back(Tmp1);
4195 break;
4196 case ISD::VAARG: {
4197 SDValue Chain = Node->getOperand(0); // Get the chain.
4198 SDValue Ptr = Node->getOperand(1); // Get the pointer.
4199
4200 unsigned TruncOp;
4201 if (OVT.isVector()) {
4202 TruncOp = ISD::BITCAST;
4203 } else {
4204 assert(OVT.isInteger()((OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4205, __PRETTY_FUNCTION__))
4205 && "VAARG promotion is supported only for vectors or integer types")((OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4205, __PRETTY_FUNCTION__))
;
4206 TruncOp = ISD::TRUNCATE;
4207 }
4208
4209 // Perform the larger operation, then convert back
4210 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4211 Node->getConstantOperandVal(3));
4212 Chain = Tmp1.getValue(1);
4213
4214 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4215
4216 // Modified the chain result - switch anything that used the old chain to
4217 // use the new one.
4218 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4219 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4220 if (UpdatedNodes) {
4221 UpdatedNodes->insert(Tmp2.getNode());
4222 UpdatedNodes->insert(Chain.getNode());
4223 }
4224 ReplacedNode(Node);
4225 break;
4226 }
4227 case ISD::MUL:
4228 case ISD::SDIV:
4229 case ISD::SREM:
4230 case ISD::UDIV:
4231 case ISD::UREM:
4232 case ISD::AND:
4233 case ISD::OR:
4234 case ISD::XOR: {
4235 unsigned ExtOp, TruncOp;
4236 if (OVT.isVector()) {
4237 ExtOp = ISD::BITCAST;
4238 TruncOp = ISD::BITCAST;
4239 } else {
4240 assert(OVT.isInteger() && "Cannot promote logic operation")((OVT.isInteger() && "Cannot promote logic operation"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"Cannot promote logic operation\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4240, __PRETTY_FUNCTION__))
;
4241
4242 switch (Node->getOpcode()) {
4243 default:
4244 ExtOp = ISD::ANY_EXTEND;
4245 break;
4246 case ISD::SDIV:
4247 case ISD::SREM:
4248 ExtOp = ISD::SIGN_EXTEND;
4249 break;
4250 case ISD::UDIV:
4251 case ISD::UREM:
4252 ExtOp = ISD::ZERO_EXTEND;
4253 break;
4254 }
4255 TruncOp = ISD::TRUNCATE;
4256 }
4257 // Promote each of the values to the new type.
4258 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4259 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4260 // Perform the larger operation, then convert back
4261 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4262 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4263 break;
4264 }
4265 case ISD::UMUL_LOHI:
4266 case ISD::SMUL_LOHI: {
4267 // Promote to a multiply in a wider integer type.
4268 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4269 : ISD::SIGN_EXTEND;
4270 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4271 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4272 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4273
4274 auto &DL = DAG.getDataLayout();
4275 unsigned OriginalSize = OVT.getScalarSizeInBits();
4276 Tmp2 = DAG.getNode(
4277 ISD::SRL, dl, NVT, Tmp1,
4278 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4279 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4280 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4281 break;
4282 }
4283 case ISD::SELECT: {
4284 unsigned ExtOp, TruncOp;
4285 if (Node->getValueType(0).isVector() ||
4286 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4287 ExtOp = ISD::BITCAST;
4288 TruncOp = ISD::BITCAST;
4289 } else if (Node->getValueType(0).isInteger()) {
4290 ExtOp = ISD::ANY_EXTEND;
4291 TruncOp = ISD::TRUNCATE;
4292 } else {
4293 ExtOp = ISD::FP_EXTEND;
4294 TruncOp = ISD::FP_ROUND;
4295 }
4296 Tmp1 = Node->getOperand(0);
4297 // Promote each of the values to the new type.
4298 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4299 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4300 // Perform the larger operation, then round down.
4301 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4302 if (TruncOp != ISD::FP_ROUND)
4303 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4304 else
4305 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4306 DAG.getIntPtrConstant(0, dl));
4307 Results.push_back(Tmp1);
4308 break;
4309 }
4310 case ISD::VECTOR_SHUFFLE: {
4311 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4312
4313 // Cast the two input vectors.
4314 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4315 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4316
4317 // Convert the shuffle mask to the right # elements.
4318 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4319 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4320 Results.push_back(Tmp1);
4321 break;
4322 }
4323 case ISD::SETCC: {
4324 unsigned ExtOp = ISD::FP_EXTEND;
4325 if (NVT.isInteger()) {
4326 ISD::CondCode CCCode =
4327 cast<CondCodeSDNode>(Node->getOperand(2))->get();
4328 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4329 }
4330 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4331 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4332 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
4333 Tmp1, Tmp2, Node->getOperand(2)));
4334 break;
4335 }
4336 case ISD::BR_CC: {
4337 unsigned ExtOp = ISD::FP_EXTEND;
4338 if (NVT.isInteger()) {
4339 ISD::CondCode CCCode =
4340 cast<CondCodeSDNode>(Node->getOperand(1))->get();
4341 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4342 }
4343 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4344 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4345 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4346 Node->getOperand(0), Node->getOperand(1),
4347 Tmp1, Tmp2, Node->getOperand(4)));
4348 break;
4349 }
4350 case ISD::FADD:
4351 case ISD::FSUB:
4352 case ISD::FMUL:
4353 case ISD::FDIV:
4354 case ISD::FREM:
4355 case ISD::FMINNUM:
4356 case ISD::FMAXNUM:
4357 case ISD::FPOW:
4358 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4359 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4360 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4361 Node->getFlags());
4362 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4363 Tmp3, DAG.getIntPtrConstant(0, dl)));
4364 break;
4365 case ISD::FMA:
4366 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4367 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4368 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4369 Results.push_back(
4370 DAG.getNode(ISD::FP_ROUND, dl, OVT,
4371 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4372 DAG.getIntPtrConstant(0, dl)));
4373 break;
4374 case ISD::FCOPYSIGN:
4375 case ISD::FPOWI: {
4376 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4377 Tmp2 = Node->getOperand(1);
4378 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4379
4380 // fcopysign doesn't change anything but the sign bit, so
4381 // (fp_round (fcopysign (fpext a), b))
4382 // is as precise as
4383 // (fp_round (fpext a))
4384 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4385 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4386 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4387 Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4388 break;
4389 }
4390 case ISD::FFLOOR:
4391 case ISD::FCEIL:
4392 case ISD::FRINT:
4393 case ISD::FNEARBYINT:
4394 case ISD::FROUND:
4395 case ISD::FTRUNC:
4396 case ISD::FNEG:
4397 case ISD::FSQRT:
4398 case ISD::FSIN:
4399 case ISD::FCOS:
4400 case ISD::FLOG:
4401 case ISD::FLOG2:
4402 case ISD::FLOG10:
4403 case ISD::FABS:
4404 case ISD::FEXP:
4405 case ISD::FEXP2:
4406 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4407 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4408 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4409 Tmp2, DAG.getIntPtrConstant(0, dl)));
4410 break;
4411 case ISD::BUILD_VECTOR: {
4412 MVT EltVT = OVT.getVectorElementType();
4413 MVT NewEltVT = NVT.getVectorElementType();
4414
4415 // Handle bitcasts to a different vector type with the same total bit size
4416 //
4417 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4418 // =>
4419 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4420
4421 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for build_vector") ? static_cast
<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4422, __PRETTY_FUNCTION__))
4422 "Invalid promote type for build_vector")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for build_vector") ? static_cast
<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4422, __PRETTY_FUNCTION__))
;
4423 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4423, __PRETTY_FUNCTION__))
;
4424
4425 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4426
4427 SmallVector<SDValue, 8> NewOps;
4428 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4429 SDValue Op = Node->getOperand(I);
4430 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4431 }
4432
4433 SDLoc SL(Node);
4434 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4435 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4436 Results.push_back(CvtVec);
4437 break;
4438 }
4439 case ISD::EXTRACT_VECTOR_ELT: {
4440 MVT EltVT = OVT.getVectorElementType();
4441 MVT NewEltVT = NVT.getVectorElementType();
4442
4443 // Handle bitcasts to a different vector type with the same total bit size.
4444 //
4445 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4446 // =>
4447 // v4i32:castx = bitcast x:v2i64
4448 //
4449 // i64 = bitcast
4450 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4451 // (i32 (extract_vector_elt castx, (2 * y + 1)))
4452 //
4453
4454 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for extract_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4455, __PRETTY_FUNCTION__))
4455 "Invalid promote type for extract_vector_elt")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for extract_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4455, __PRETTY_FUNCTION__))
;
4456 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4456, __PRETTY_FUNCTION__))
;
4457
4458 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4459 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4460
4461 SDValue Idx = Node->getOperand(1);
4462 EVT IdxVT = Idx.getValueType();
4463 SDLoc SL(Node);
4464 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4465 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4466
4467 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4468
4469 SmallVector<SDValue, 8> NewOps;
4470 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4471 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4472 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4473
4474 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4475 CastVec, TmpIdx);
4476 NewOps.push_back(Elt);
4477 }
4478
4479 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4480 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4481 break;
4482 }
4483 case ISD::INSERT_VECTOR_ELT: {
4484 MVT EltVT = OVT.getVectorElementType();
4485 MVT NewEltVT = NVT.getVectorElementType();
4486
4487 // Handle bitcasts to a different vector type with the same total bit size
4488 //
4489 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4490 // =>
4491 // v4i32:castx = bitcast x:v2i64
4492 // v2i32:casty = bitcast y:i64
4493 //
4494 // v2i64 = bitcast
4495 // (v4i32 insert_vector_elt
4496 // (v4i32 insert_vector_elt v4i32:castx,
4497 // (extract_vector_elt casty, 0), 2 * z),
4498 // (extract_vector_elt casty, 1), (2 * z + 1))
4499
4500 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for insert_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4501, __PRETTY_FUNCTION__))
4501 "Invalid promote type for insert_vector_elt")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for insert_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4501, __PRETTY_FUNCTION__))
;
4502 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4502, __PRETTY_FUNCTION__))
;
4503
4504 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4505 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4506
4507 SDValue Val = Node->getOperand(1);
4508 SDValue Idx = Node->getOperand(2);
4509 EVT IdxVT = Idx.getValueType();
4510 SDLoc SL(Node);
4511
4512 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4513 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4514
4515 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4516 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4517
4518 SDValue NewVec = CastVec;
4519 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4520 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4521 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4522
4523 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4524 CastVal, IdxOffset);
4525
4526 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4527 NewVec, Elt, InEltIdx);
4528 }
4529
4530 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4531 break;
4532 }
4533 case ISD::SCALAR_TO_VECTOR: {
4534 MVT EltVT = OVT.getVectorElementType();
4535 MVT NewEltVT = NVT.getVectorElementType();
4536
4537 // Handle bitcasts to different vector type with the same total bit size.
4538 //
4539 // e.g. v2i64 = scalar_to_vector x:i64
4540 // =>
4541 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4542 //
4543
4544 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4545 SDValue Val = Node->getOperand(0);
4546 SDLoc SL(Node);
4547
4548 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4549 SDValue Undef = DAG.getUNDEF(MidVT);
4550
4551 SmallVector<SDValue, 8> NewElts;
4552 NewElts.push_back(CastVal);
4553 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4554 NewElts.push_back(Undef);
4555
4556 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4557 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4558 Results.push_back(CvtVec);
4559 break;
4560 }
4561 }
4562
4563 // Replace the original node with the legalized result.
4564 if (!Results.empty()) {
4565 LLVM_DEBUG(dbgs() << "Successfully promoted node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully promoted node\n"
; } } while (false)
;
4566 ReplaceNode(Node, Results.data());
4567 } else
4568 LLVM_DEBUG(dbgs() << "Could not promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not promote node\n"
; } } while (false)
;
4569}
4570
4571/// This is the entry point for the file.
4572void SelectionDAG::Legalize() {
4573 AssignTopologicalOrder();
4574
4575 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4576 // Use a delete listener to remove nodes which were deleted during
4577 // legalization from LegalizeNodes. This is needed to handle the situation
4578 // where a new node is allocated by the object pool to the same address of a
4579 // previously deleted node.
4580 DAGNodeDeletedListener DeleteListener(
4581 *this,
4582 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4583
4584 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4585
4586 // Visit all the nodes. We start in topological order, so that we see
4587 // nodes with their original operands intact. Legalization can produce
4588 // new nodes which may themselves need to be legalized. Iterate until all
4589 // nodes have been legalized.
4590 while (true) {
4591 bool AnyLegalized = false;
4592 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4593 --NI;
4594
4595 SDNode *N = &*NI;
4596 if (N->use_empty() && N != getRoot().getNode()) {
4597 ++NI;
4598 DeleteNode(N);
4599 continue;
4600 }
4601
4602 if (LegalizedNodes.insert(N).second) {
4603 AnyLegalized = true;
4604 Legalizer.LegalizeOp(N);
4605
4606 if (N->use_empty() && N != getRoot().getNode()) {
4607 ++NI;
4608 DeleteNode(N);
4609 }
4610 }
4611 }
4612 if (!AnyLegalized)
4613 break;
4614
4615 }
4616
4617 // Remove dead nodes now.
4618 RemoveDeadNodes();
4619}
4620
4621bool SelectionDAG::LegalizeOp(SDNode *N,
4622 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4623 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4624 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4625
4626 // Directly insert the node in question, and legalize it. This will recurse
4627 // as needed through operands.
4628 LegalizedNodes.insert(N);
4629 Legalizer.LegalizeOp(N);
4630
4631 return LegalizedNodes.count(N);
4632}

/build/llvm-toolchain-snapshot-8~svn348900/include/llvm/Support/MathExtras.h

1//===-- llvm/Support/MathExtras.h - Useful math functions -------*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file contains some functions that are useful for math stuff.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_SUPPORT_MATHEXTRAS_H
15#define LLVM_SUPPORT_MATHEXTRAS_H
16
17#include "llvm/Support/Compiler.h"
18#include "llvm/Support/SwapByteOrder.h"
19#include <algorithm>
20#include <cassert>
21#include <climits>
22#include <cstring>
23#include <limits>
24#include <type_traits>
25
26#ifdef __ANDROID_NDK__
27#include <android/api-level.h>
28#endif
29
30#ifdef _MSC_VER
31// Declare these intrinsics manually rather including intrin.h. It's very
32// expensive, and MathExtras.h is popular.
33// #include <intrin.h>
34extern "C" {
35unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask);
36unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask);
37unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask);
38unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask);
39}
40#endif
41
42namespace llvm {
43/// The behavior an operation has on an input of 0.
44enum ZeroBehavior {
45 /// The returned value is undefined.
46 ZB_Undefined,
47 /// The returned value is numeric_limits<T>::max()
48 ZB_Max,
49 /// The returned value is numeric_limits<T>::digits
50 ZB_Width
51};
52
53namespace detail {
54template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter {
55 static std::size_t count(T Val, ZeroBehavior) {
56 if (!Val)
57 return std::numeric_limits<T>::digits;
58 if (Val & 0x1)
59 return 0;
60
61 // Bisection method.
62 std::size_t ZeroBits = 0;
63 T Shift = std::numeric_limits<T>::digits >> 1;
64 T Mask = std::numeric_limits<T>::max() >> Shift;
65 while (Shift) {
66 if ((Val & Mask) == 0) {
67 Val >>= Shift;
68 ZeroBits |= Shift;
69 }
70 Shift >>= 1;
71 Mask >>= Shift;
72 }
73 return ZeroBits;
74 }
75};
76
77#if __GNUC__4 >= 4 || defined(_MSC_VER)
78template <typename T> struct TrailingZerosCounter<T, 4> {
79 static std::size_t count(T Val, ZeroBehavior ZB) {
80 if (ZB != ZB_Undefined && Val == 0)
81 return 32;
82
83#if __has_builtin(__builtin_ctz)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
84 return __builtin_ctz(Val);
85#elif defined(_MSC_VER)
86 unsigned long Index;
87 _BitScanForward(&Index, Val);
88 return Index;
89#endif
90 }
91};
92
93#if !defined(_MSC_VER) || defined(_M_X64)
94template <typename T> struct TrailingZerosCounter<T, 8> {
95 static std::size_t count(T Val, ZeroBehavior ZB) {
96 if (ZB != ZB_Undefined && Val == 0)
97 return 64;
98
99#if __has_builtin(__builtin_ctzll)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
100 return __builtin_ctzll(Val);
101#elif defined(_MSC_VER)
102 unsigned long Index;
103 _BitScanForward64(&Index, Val);
104 return Index;
105#endif
106 }
107};
108#endif
109#endif
110} // namespace detail
111
112/// Count number of 0's from the least significant bit to the most
113/// stopping at the first 1.
114///
115/// Only unsigned integral types are allowed.
116///
117/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
118/// valid arguments.
119template <typename T>
120std::size_t countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
121 static_assert(std::numeric_limits<T>::is_integer &&
122 !std::numeric_limits<T>::is_signed,
123 "Only unsigned integral types are allowed.");
124 return llvm::detail::TrailingZerosCounter<T, sizeof(T)>::count(Val, ZB);
125}
126
127namespace detail {
128template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter {
129 static std::size_t count(T Val, ZeroBehavior) {
130 if (!Val)
131 return std::numeric_limits<T>::digits;
132
133 // Bisection method.
134 std::size_t ZeroBits = 0;
135 for (T Shift = std::numeric_limits<T>::digits >> 1; Shift; Shift >>= 1) {
136 T Tmp = Val >> Shift;
137 if (Tmp)
138 Val = Tmp;
139 else
140 ZeroBits |= Shift;
141 }
142 return ZeroBits;
143 }
144};
145
146#if __GNUC__4 >= 4 || defined(_MSC_VER)
147template <typename T> struct LeadingZerosCounter<T, 4> {
148 static std::size_t count(T Val, ZeroBehavior ZB) {
149 if (ZB != ZB_Undefined && Val == 0)
150 return 32;
151
152#if __has_builtin(__builtin_clz)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
153 return __builtin_clz(Val);
154#elif defined(_MSC_VER)
155 unsigned long Index;
156 _BitScanReverse(&Index, Val);
157 return Index ^ 31;
158#endif
159 }
160};
161
162#if !defined(_MSC_VER) || defined(_M_X64)
163template <typename T> struct LeadingZerosCounter<T, 8> {
164 static std::size_t count(T Val, ZeroBehavior ZB) {
165 if (ZB != ZB_Undefined && Val == 0)
166 return 64;
167
168#if __has_builtin(__builtin_clzll)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
169 return __builtin_clzll(Val);
170#elif defined(_MSC_VER)
171 unsigned long Index;
172 _BitScanReverse64(&Index, Val);
173 return Index ^ 63;
174#endif
175 }
176};
177#endif
178#endif
179} // namespace detail
180
181/// Count number of 0's from the most significant bit to the least
182/// stopping at the first 1.
183///
184/// Only unsigned integral types are allowed.
185///
186/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
187/// valid arguments.
188template <typename T>
189std::size_t countLeadingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
190 static_assert(std::numeric_limits<T>::is_integer &&
191 !std::numeric_limits<T>::is_signed,
192 "Only unsigned integral types are allowed.");
193 return llvm::detail::LeadingZerosCounter<T, sizeof(T)>::count(Val, ZB);
194}
195
196/// Get the index of the first set bit starting from the least
197/// significant bit.
198///
199/// Only unsigned integral types are allowed.
200///
201/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
202/// valid arguments.
203template <typename T> T findFirstSet(T Val, ZeroBehavior ZB = ZB_Max) {
204 if (ZB == ZB_Max && Val == 0)
205 return std::numeric_limits<T>::max();
206
207 return countTrailingZeros(Val, ZB_Undefined);
208}
209
210/// Create a bitmask with the N right-most bits set to 1, and all other
211/// bits set to 0. Only unsigned types are allowed.
212template <typename T> T maskTrailingOnes(unsigned N) {
213 static_assert(std::is_unsigned<T>::value, "Invalid type!");
214 const unsigned Bits = CHAR_BIT8 * sizeof(T);
215 assert(N <= Bits && "Invalid bit index")((N <= Bits && "Invalid bit index") ? static_cast<
void> (0) : __assert_fail ("N <= Bits && \"Invalid bit index\""
, "/build/llvm-toolchain-snapshot-8~svn348900/include/llvm/Support/MathExtras.h"
, 215, __PRETTY_FUNCTION__))
;
216 return N == 0 ? 0 : (T(-1) >> (Bits - N));
217}
218
219/// Create a bitmask with the N left-most bits set to 1, and all other
220/// bits set to 0. Only unsigned types are allowed.
221template <typename T> T maskLeadingOnes(unsigned N) {
222 return ~maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
223}
224
225/// Create a bitmask with the N right-most bits set to 0, and all other
226/// bits set to 1. Only unsigned types are allowed.
227template <typename T> T maskTrailingZeros(unsigned N) {
228 return maskLeadingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
229}
230
231/// Create a bitmask with the N left-most bits set to 0, and all other
232/// bits set to 1. Only unsigned types are allowed.
233template <typename T> T maskLeadingZeros(unsigned N) {
234 return maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
235}
236
237/// Get the index of the last set bit starting from the least
238/// significant bit.
239///
240/// Only unsigned integral types are allowed.
241///
242/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
243/// valid arguments.
244template <typename T> T findLastSet(T Val, ZeroBehavior ZB = ZB_Max) {
245 if (ZB == ZB_Max && Val == 0)
246 return std::numeric_limits<T>::max();
247
248 // Use ^ instead of - because both gcc and llvm can remove the associated ^
249 // in the __builtin_clz intrinsic on x86.
250 return countLeadingZeros(Val, ZB_Undefined) ^
251 (std::numeric_limits<T>::digits - 1);
252}
253
254/// Macro compressed bit reversal table for 256 bits.
255///
256/// http://graphics.stanford.edu/~seander/bithacks.html#BitReverseTable
257static const unsigned char BitReverseTable256[256] = {
258#define R2(n) n, n + 2 * 64, n + 1 * 64, n + 3 * 64
259#define R4(n) R2(n), R2(n + 2 * 16), R2(n + 1 * 16), R2(n + 3 * 16)
260#define R6(n) R4(n), R4(n + 2 * 4), R4(n + 1 * 4), R4(n + 3 * 4)
261 R6(0), R6(2), R6(1), R6(3)
262#undef R2
263#undef R4
264#undef R6
265};
266
267/// Reverse the bits in \p Val.
268template <typename T>
269T reverseBits(T Val) {
270 unsigned char in[sizeof(Val)];
271 unsigned char out[sizeof(Val)];
272 std::memcpy(in, &Val, sizeof(Val));
273 for (unsigned i = 0; i < sizeof(Val); ++i)
274 out[(sizeof(Val) - i) - 1] = BitReverseTable256[in[i]];
275 std::memcpy(&Val, out, sizeof(Val));
276 return Val;
277}
278
279// NOTE: The following support functions use the _32/_64 extensions instead of
280// type overloading so that signed and unsigned integers can be used without
281// ambiguity.
282
283/// Return the high 32 bits of a 64 bit value.
284constexpr inline uint32_t Hi_32(uint64_t Value) {
285 return static_cast<uint32_t>(Value >> 32);
286}
287
288/// Return the low 32 bits of a 64 bit value.
289constexpr inline uint32_t Lo_32(uint64_t Value) {
290 return static_cast<uint32_t>(Value);
291}
292
293/// Make a 64-bit integer from a high / low pair of 32-bit integers.
294constexpr inline uint64_t Make_64(uint32_t High, uint32_t Low) {
295 return ((uint64_t)High << 32) | (uint64_t)Low;
296}
297
298/// Checks if an integer fits into the given bit width.
299template <unsigned N> constexpr inline bool isInt(int64_t x) {
300 return N >= 64 || (-(INT64_C(1)1L<<(N-1)) <= x && x < (INT64_C(1)1L<<(N-1)));
301}
302// Template specializations to get better code for common cases.
303template <> constexpr inline bool isInt<8>(int64_t x) {
304 return static_cast<int8_t>(x) == x;
305}
306template <> constexpr inline bool isInt<16>(int64_t x) {
307 return static_cast<int16_t>(x) == x;
308}
309template <> constexpr inline bool isInt<32>(int64_t x) {
310 return static_cast<int32_t>(x) == x;
311}
312
313/// Checks if a signed integer is an N bit number shifted left by S.
314template <unsigned N, unsigned S>
315constexpr inline bool isShiftedInt(int64_t x) {
316 static_assert(
317 N > 0, "isShiftedInt<0> doesn't make sense (refers to a 0-bit number.");
318 static_assert(N + S <= 64, "isShiftedInt<N, S> with N + S > 64 is too wide.");
319 return isInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
320}
321
322/// Checks if an unsigned integer fits into the given bit width.
323///
324/// This is written as two functions rather than as simply
325///
326/// return N >= 64 || X < (UINT64_C(1) << N);
327///
328/// to keep MSVC from (incorrectly) warning on isUInt<64> that we're shifting
329/// left too many places.
330template <unsigned N>
331constexpr inline typename std::enable_if<(N < 64), bool>::type
332isUInt(uint64_t X) {
333 static_assert(N > 0, "isUInt<0> doesn't make sense");
334 return X < (UINT64_C(1)1UL << (N));
335}
336template <unsigned N>
337constexpr inline typename std::enable_if<N >= 64, bool>::type
338isUInt(uint64_t X) {
339 return true;
340}
341
342// Template specializations to get better code for common cases.
343template <> constexpr inline bool isUInt<8>(uint64_t x) {
344 return static_cast<uint8_t>(x) == x;
345}
346template <> constexpr inline bool isUInt<16>(uint64_t x) {
347 return static_cast<uint16_t>(x) == x;
348}
349template <> constexpr inline bool isUInt<32>(uint64_t x) {
350 return static_cast<uint32_t>(x) == x;
351}
352
353/// Checks if a unsigned integer is an N bit number shifted left by S.
354template <unsigned N, unsigned S>
355constexpr inline bool isShiftedUInt(uint64_t x) {
356 static_assert(
357 N > 0, "isShiftedUInt<0> doesn't make sense (refers to a 0-bit number)");
358 static_assert(N + S <= 64,
359 "isShiftedUInt<N, S> with N + S > 64 is too wide.");
360 // Per the two static_asserts above, S must be strictly less than 64. So
361 // 1 << S is not undefined behavior.
362 return isUInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
363}
364
365/// Gets the maximum value for a N-bit unsigned integer.
366inline uint64_t maxUIntN(uint64_t N) {
367 assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-8~svn348900/include/llvm/Support/MathExtras.h"
, 367, __PRETTY_FUNCTION__))
;
368
369 // uint64_t(1) << 64 is undefined behavior, so we can't do
370 // (uint64_t(1) << N) - 1
371 // without checking first that N != 64. But this works and doesn't have a
372 // branch.
373 return UINT64_MAX(18446744073709551615UL) >> (64 - N);
374}
375
376/// Gets the minimum value for a N-bit signed integer.
377inline int64_t minIntN(int64_t N) {
378 assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-8~svn348900/include/llvm/Support/MathExtras.h"
, 378, __PRETTY_FUNCTION__))
;
379
380 return -(UINT64_C(1)1UL<<(N-1));
381}
382
383/// Gets the maximum value for a N-bit signed integer.
384inline int64_t maxIntN(int64_t N) {
385 assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-8~svn348900/include/llvm/Support/MathExtras.h"
, 385, __PRETTY_FUNCTION__))
;
386
387 // This relies on two's complement wraparound when N == 64, so we convert to
388 // int64_t only at the very end to avoid UB.
389 return (UINT64_C(1)1UL << (N - 1)) - 1;
390}
391
392/// Checks if an unsigned integer fits into the given (dynamic) bit width.
393inline bool isUIntN(unsigned N, uint64_t x) {
394 return N >= 64 || x <= maxUIntN(N);
395}
396
397/// Checks if an signed integer fits into the given (dynamic) bit width.
398inline bool isIntN(unsigned N, int64_t x) {
399 return N >= 64 || (minIntN(N) <= x && x <= maxIntN(N));
400}
401
402/// Return true if the argument is a non-empty sequence of ones starting at the
403/// least significant bit with the remainder zero (32 bit version).
404/// Ex. isMask_32(0x0000FFFFU) == true.
405constexpr inline bool isMask_32(uint32_t Value) {
406 return Value && ((Value + 1) & Value) == 0;
407}
408
409/// Return true if the argument is a non-empty sequence of ones starting at the
410/// least significant bit with the remainder zero (64 bit version).
411constexpr inline bool isMask_64(uint64_t Value) {
412 return Value && ((Value + 1) & Value) == 0;
413}
414
415/// Return true if the argument contains a non-empty sequence of ones with the
416/// remainder zero (32 bit version.) Ex. isShiftedMask_32(0x0000FF00U) == true.
417constexpr inline bool isShiftedMask_32(uint32_t Value) {
418 return Value && isMask_32((Value - 1) | Value);
419}
420
421/// Return true if the argument contains a non-empty sequence of ones with the
422/// remainder zero (64 bit version.)
423constexpr inline bool isShiftedMask_64(uint64_t Value) {
424 return Value && isMask_64((Value - 1) | Value);
425}
426
427/// Return true if the argument is a power of two > 0.
428/// Ex. isPowerOf2_32(0x00100000U) == true (32 bit edition.)
429constexpr inline bool isPowerOf2_32(uint32_t Value) {
430 return Value && !(Value & (Value - 1));
431}
432
433/// Return true if the argument is a power of two > 0 (64 bit edition.)
434constexpr inline bool isPowerOf2_64(uint64_t Value) {
435 return Value && !(Value & (Value - 1));
436}
437
438/// Return a byte-swapped representation of the 16-bit argument.
439inline uint16_t ByteSwap_16(uint16_t Value) {
440 return sys::SwapByteOrder_16(Value);
441}
442
443/// Return a byte-swapped representation of the 32-bit argument.
444inline uint32_t ByteSwap_32(uint32_t Value) {
445 return sys::SwapByteOrder_32(Value);
446}
447
448/// Return a byte-swapped representation of the 64-bit argument.
449inline uint64_t ByteSwap_64(uint64_t Value) {
450 return sys::SwapByteOrder_64(Value);
451}
452
453/// Count the number of ones from the most significant bit to the first
454/// zero bit.
455///
456/// Ex. countLeadingOnes(0xFF0FFF00) == 8.
457/// Only unsigned integral types are allowed.
458///
459/// \param ZB the behavior on an input of all ones. Only ZB_Width and
460/// ZB_Undefined are valid arguments.
461template <typename T>
462std::size_t countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
463 static_assert(std::numeric_limits<T>::is_integer &&
464 !std::numeric_limits<T>::is_signed,
465 "Only unsigned integral types are allowed.");
466 return countLeadingZeros<T>(~Value, ZB);
467}
468
469/// Count the number of ones from the least significant bit to the first
470/// zero bit.
471///
472/// Ex. countTrailingOnes(0x00FF00FF) == 8.
473/// Only unsigned integral types are allowed.
474///
475/// \param ZB the behavior on an input of all ones. Only ZB_Width and
476/// ZB_Undefined are valid arguments.
477template <typename T>
478std::size_t countTrailingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
479 static_assert(std::numeric_limits<T>::is_integer &&
480 !std::numeric_limits<T>::is_signed,
481 "Only unsigned integral types are allowed.");
482 return countTrailingZeros<T>(~Value, ZB);
483}
484
485namespace detail {
486template <typename T, std::size_t SizeOfT> struct PopulationCounter {
487 static unsigned count(T Value) {
488 // Generic version, forward to 32 bits.
489 static_assert(SizeOfT <= 4, "Not implemented!");
490#if __GNUC__4 >= 4
491 return __builtin_popcount(Value);
492#else
493 uint32_t v = Value;
494 v = v - ((v >> 1) & 0x55555555);
495 v = (v & 0x33333333) + ((v >> 2) & 0x33333333);
496 return ((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24;
497#endif
498 }
499};
500
501template <typename T> struct PopulationCounter<T, 8> {
502 static unsigned count(T Value) {
503#if __GNUC__4 >= 4
504 return __builtin_popcountll(Value);
505#else
506 uint64_t v = Value;
507 v = v - ((v >> 1) & 0x5555555555555555ULL);
508 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
509 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
510 return unsigned((uint64_t)(v * 0x0101010101010101ULL) >> 56);
511#endif
512 }
513};
514} // namespace detail
515
516/// Count the number of set bits in a value.
517/// Ex. countPopulation(0xF000F000) = 8
518/// Returns 0 if the word is zero.
519template <typename T>
520inline unsigned countPopulation(T Value) {
521 static_assert(std::numeric_limits<T>::is_integer &&
522 !std::numeric_limits<T>::is_signed,
523 "Only unsigned integral types are allowed.");
524 return detail::PopulationCounter<T, sizeof(T)>::count(Value);
525}
526
527/// Return the log base 2 of the specified value.
528inline double Log2(double Value) {
529#if defined(__ANDROID_API__) && __ANDROID_API__ < 18
530 return __builtin_log(Value) / __builtin_log(2.0);
531#else
532 return log2(Value);
533#endif
534}
535
536/// Return the floor log base 2 of the specified value, -1 if the value is zero.
537/// (32 bit edition.)
538/// Ex. Log2_32(32) == 5, Log2_32(1) == 0, Log2_32(0) == -1, Log2_32(6) == 2
539inline unsigned Log2_32(uint32_t Value) {
540 return 31 - countLeadingZeros(Value);
24
Returning the value 4294967295
541}
542
543/// Return the floor log base 2 of the specified value, -1 if the value is zero.
544/// (64 bit edition.)
545inline unsigned Log2_64(uint64_t Value) {
546 return 63 - countLeadingZeros(Value);
547}
548
549/// Return the ceil log base 2 of the specified value, 32 if the value is zero.
550/// (32 bit edition).
551/// Ex. Log2_32_Ceil(32) == 5, Log2_32_Ceil(1) == 0, Log2_32_Ceil(6) == 3
552inline unsigned Log2_32_Ceil(uint32_t Value) {
553 return 32 - countLeadingZeros(Value - 1);
554}
555
556/// Return the ceil log base 2 of the specified value, 64 if the value is zero.
557/// (64 bit edition.)
558inline unsigned Log2_64_Ceil(uint64_t Value) {
559 return 64 - countLeadingZeros(Value - 1);
560}
561
562/// Return the greatest common divisor of the values using Euclid's algorithm.
563inline uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B) {
564 while (B) {
565 uint64_t T = B;
566 B = A % B;
567 A = T;
568 }
569 return A;
570}
571
572/// This function takes a 64-bit integer and returns the bit equivalent double.
573inline double BitsToDouble(uint64_t Bits) {
574 double D;
575 static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
576 memcpy(&D, &Bits, sizeof(Bits));
577 return D;
578}
579
580/// This function takes a 32-bit integer and returns the bit equivalent float.
581inline float BitsToFloat(uint32_t Bits) {
582 float F;
583 static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
584 memcpy(&F, &Bits, sizeof(Bits));
585 return F;
586}
587
588/// This function takes a double and returns the bit equivalent 64-bit integer.
589/// Note that copying doubles around changes the bits of NaNs on some hosts,
590/// notably x86, so this routine cannot be used if these bits are needed.
591inline uint64_t DoubleToBits(double Double) {
592 uint64_t Bits;
593 static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
594 memcpy(&Bits, &Double, sizeof(Double));
595 return Bits;
596}
597
598/// This function takes a float and returns the bit equivalent 32-bit integer.
599/// Note that copying floats around changes the bits of NaNs on some hosts,
600/// notably x86, so this routine cannot be used if these bits are needed.
601inline uint32_t FloatToBits(float Float) {
602 uint32_t Bits;
603 static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
604 memcpy(&Bits, &Float, sizeof(Float));
605 return Bits;
606}
607
608/// A and B are either alignments or offsets. Return the minimum alignment that
609/// may be assumed after adding the two together.
610constexpr inline uint64_t MinAlign(uint64_t A, uint64_t B) {
611 // The largest power of 2 that divides both A and B.
612 //
613 // Replace "-Value" by "1+~Value" in the following commented code to avoid
614 // MSVC warning C4146
615 // return (A | B) & -(A | B);
616 return (A | B) & (1 + ~(A | B));
617}
618
619/// Aligns \c Addr to \c Alignment bytes, rounding up.
620///
621/// Alignment should be a power of two. This method rounds up, so
622/// alignAddr(7, 4) == 8 and alignAddr(8, 4) == 8.
623inline uintptr_t alignAddr(const void *Addr, size_t Alignment) {
624 assert(Alignment && isPowerOf2_64((uint64_t)Alignment) &&((Alignment && isPowerOf2_64((uint64_t)Alignment) &&
"Alignment is not a power of two!") ? static_cast<void>
(0) : __assert_fail ("Alignment && isPowerOf2_64((uint64_t)Alignment) && \"Alignment is not a power of two!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/include/llvm/Support/MathExtras.h"
, 625, __PRETTY_FUNCTION__))
625 "Alignment is not a power of two!")((Alignment && isPowerOf2_64((uint64_t)Alignment) &&
"Alignment is not a power of two!") ? static_cast<void>
(0) : __assert_fail ("Alignment && isPowerOf2_64((uint64_t)Alignment) && \"Alignment is not a power of two!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/include/llvm/Support/MathExtras.h"
, 625, __PRETTY_FUNCTION__))
;
626
627 assert((uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr)(((uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr) ? static_cast
<void> (0) : __assert_fail ("(uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr"
, "/build/llvm-toolchain-snapshot-8~svn348900/include/llvm/Support/MathExtras.h"
, 627, __PRETTY_FUNCTION__))
;
628
629 return (((uintptr_t)Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1));
630}
631
632/// Returns the necessary adjustment for aligning \c Ptr to \c Alignment
633/// bytes, rounding up.
634inline size_t alignmentAdjustment(const void *Ptr, size_t Alignment) {
635 return alignAddr(Ptr, Alignment) - (uintptr_t)Ptr;
636}
637
638/// Returns the next power of two (in 64-bits) that is strictly greater than A.
639/// Returns zero on overflow.
640inline uint64_t NextPowerOf2(uint64_t A) {
641 A |= (A >> 1);
642 A |= (A >> 2);
643 A |= (A >> 4);
644 A |= (A >> 8);
645 A |= (A >> 16);
646 A |= (A >> 32);
647 return A + 1;
648}
649
650/// Returns the power of two which is less than or equal to the given value.
651/// Essentially, it is a floor operation across the domain of powers of two.
652inline uint64_t PowerOf2Floor(uint64_t A) {
653 if (!A) return 0;
654 return 1ull << (63 - countLeadingZeros(A, ZB_Undefined));
655}
656
657/// Returns the power of two which is greater than or equal to the given value.
658/// Essentially, it is a ceil operation across the domain of powers of two.
659inline uint64_t PowerOf2Ceil(uint64_t A) {
660 if (!A)
661 return 0;
662 return NextPowerOf2(A - 1);
663}
664
665/// Returns the next integer (mod 2**64) that is greater than or equal to
666/// \p Value and is a multiple of \p Align. \p Align must be non-zero.
667///
668/// If non-zero \p Skew is specified, the return value will be a minimal
669/// integer that is greater than or equal to \p Value and equal to
670/// \p Align * N + \p Skew for some integer N. If \p Skew is larger than
671/// \p Align, its value is adjusted to '\p Skew mod \p Align'.
672///
673/// Examples:
674/// \code
675/// alignTo(5, 8) = 8
676/// alignTo(17, 8) = 24
677/// alignTo(~0LL, 8) = 0
678/// alignTo(321, 255) = 510
679///
680/// alignTo(5, 8, 7) = 7
681/// alignTo(17, 8, 1) = 17
682/// alignTo(~0LL, 8, 3) = 3
683/// alignTo(321, 255, 42) = 552
684/// \endcode
685inline uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
686 assert(Align != 0u && "Align can't be 0.")((Align != 0u && "Align can't be 0.") ? static_cast<
void> (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/include/llvm/Support/MathExtras.h"
, 686, __PRETTY_FUNCTION__))
;
687 Skew %= Align;
688 return (Value + Align - 1 - Skew) / Align * Align + Skew;
689}
690
691/// Returns the next integer (mod 2**64) that is greater than or equal to
692/// \p Value and is a multiple of \c Align. \c Align must be non-zero.
693template <uint64_t Align> constexpr inline uint64_t alignTo(uint64_t Value) {
694 static_assert(Align != 0u, "Align must be non-zero");
695 return (Value + Align - 1) / Align * Align;
696}
697
698/// Returns the integer ceil(Numerator / Denominator).
699inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) {
700 return alignTo(Numerator, Denominator) / Denominator;
701}
702
703/// \c alignTo for contexts where a constant expression is required.
704/// \sa alignTo
705///
706/// \todo FIXME: remove when \c constexpr becomes really \c constexpr
707template <uint64_t Align>
708struct AlignTo {
709 static_assert(Align != 0u, "Align must be non-zero");
710 template <uint64_t Value>
711 struct from_value {
712 static const uint64_t value = (Value + Align - 1) / Align * Align;
713 };
714};
715
716/// Returns the largest uint64_t less than or equal to \p Value and is
717/// \p Skew mod \p Align. \p Align must be non-zero
718inline uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
719 assert(Align != 0u && "Align can't be 0.")((Align != 0u && "Align can't be 0.") ? static_cast<
void> (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/include/llvm/Support/MathExtras.h"
, 719, __PRETTY_FUNCTION__))
;
720 Skew %= Align;
721 return (Value - Skew) / Align * Align + Skew;
722}
723
724/// Returns the offset to the next integer (mod 2**64) that is greater than
725/// or equal to \p Value and is a multiple of \p Align. \p Align must be
726/// non-zero.
727inline uint64_t OffsetToAlignment(uint64_t Value, uint64_t Align) {
728 return alignTo(Value, Align) - Value;
729}
730
731/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
732/// Requires 0 < B <= 32.
733template <unsigned B> constexpr inline int32_t SignExtend32(uint32_t X) {
734 static_assert(B > 0, "Bit width can't be 0.");
735 static_assert(B <= 32, "Bit width out of range.");
736 return int32_t(X << (32 - B)) >> (32 - B);
737}
738
739/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
740/// Requires 0 < B < 32.
741inline int32_t SignExtend32(uint32_t X, unsigned B) {
742 assert(B > 0 && "Bit width can't be 0.")((B > 0 && "Bit width can't be 0.") ? static_cast<
void> (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/include/llvm/Support/MathExtras.h"
, 742, __PRETTY_FUNCTION__))
;
743 assert(B <= 32 && "Bit width out of range.")((B <= 32 && "Bit width out of range.") ? static_cast
<void> (0) : __assert_fail ("B <= 32 && \"Bit width out of range.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/include/llvm/Support/MathExtras.h"
, 743, __PRETTY_FUNCTION__))
;
744 return int32_t(X << (32 - B)) >> (32 - B);
745}
746
747/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
748/// Requires 0 < B < 64.
749template <unsigned B> constexpr inline int64_t SignExtend64(uint64_t x) {
750 static_assert(B > 0, "Bit width can't be 0.");
751 static_assert(B <= 64, "Bit width out of range.");
752 return int64_t(x << (64 - B)) >> (64 - B);
753}
754
755/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
756/// Requires 0 < B < 64.
757inline int64_t SignExtend64(uint64_t X, unsigned B) {
758 assert(B > 0 && "Bit width can't be 0.")((B > 0 && "Bit width can't be 0.") ? static_cast<
void> (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/include/llvm/Support/MathExtras.h"
, 758, __PRETTY_FUNCTION__))
;
759 assert(B <= 64 && "Bit width out of range.")((B <= 64 && "Bit width out of range.") ? static_cast
<void> (0) : __assert_fail ("B <= 64 && \"Bit width out of range.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/include/llvm/Support/MathExtras.h"
, 759, __PRETTY_FUNCTION__))
;
760 return int64_t(X << (64 - B)) >> (64 - B);
761}
762
763/// Subtract two unsigned integers, X and Y, of type T and return the absolute
764/// value of the result.
765template <typename T>
766typename std::enable_if<std::is_unsigned<T>::value, T>::type
767AbsoluteDifference(T X, T Y) {
768 return std::max(X, Y) - std::min(X, Y);
769}
770
771/// Add two unsigned integers, X and Y, of type T. Clamp the result to the
772/// maximum representable value of T on overflow. ResultOverflowed indicates if
773/// the result is larger than the maximum representable value of type T.
774template <typename T>
775typename std::enable_if<std::is_unsigned<T>::value, T>::type
776SaturatingAdd(T X, T Y, bool *ResultOverflowed = nullptr) {
777 bool Dummy;
778 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
779 // Hacker's Delight, p. 29
780 T Z = X + Y;
781 Overflowed = (Z < X || Z < Y);
782 if (Overflowed)
783 return std::numeric_limits<T>::max();
784 else
785 return Z;
786}
787
788/// Multiply two unsigned integers, X and Y, of type T. Clamp the result to the
789/// maximum representable value of T on overflow. ResultOverflowed indicates if
790/// the result is larger than the maximum representable value of type T.
791template <typename T>
792typename std::enable_if<std::is_unsigned<T>::value, T>::type
793SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) {
794 bool Dummy;
795 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
796
797 // Hacker's Delight, p. 30 has a different algorithm, but we don't use that
798 // because it fails for uint16_t (where multiplication can have undefined
799 // behavior due to promotion to int), and requires a division in addition
800 // to the multiplication.
801
802 Overflowed = false;
803
804 // Log2(Z) would be either Log2Z or Log2Z + 1.
805 // Special case: if X or Y is 0, Log2_64 gives -1, and Log2Z
806 // will necessarily be less than Log2Max as desired.
807 int Log2Z = Log2_64(X) + Log2_64(Y);
808 const T Max = std::numeric_limits<T>::max();
809 int Log2Max = Log2_64(Max);
810 if (Log2Z < Log2Max) {
811 return X * Y;
812 }
813 if (Log2Z > Log2Max) {
814 Overflowed = true;
815 return Max;
816 }
817
818 // We're going to use the top bit, and maybe overflow one
819 // bit past it. Multiply all but the bottom bit then add
820 // that on at the end.
821 T Z = (X >> 1) * Y;
822 if (Z & ~(Max >> 1)) {
823 Overflowed = true;
824 return Max;
825 }
826 Z <<= 1;
827 if (X & 1)
828 return SaturatingAdd(Z, Y, ResultOverflowed);
829
830 return Z;
831}
832
833/// Multiply two unsigned integers, X and Y, and add the unsigned integer, A to
834/// the product. Clamp the result to the maximum representable value of T on
835/// overflow. ResultOverflowed indicates if the result is larger than the
836/// maximum representable value of type T.
837template <typename T>
838typename std::enable_if<std::is_unsigned<T>::value, T>::type
839SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed = nullptr) {
840 bool Dummy;
841 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
842
843 T Product = SaturatingMultiply(X, Y, &Overflowed);
844 if (Overflowed)
845 return Product;
846
847 return SaturatingAdd(A, Product, &Overflowed);
848}
849
850/// Use this rather than HUGE_VALF; the latter causes warnings on MSVC.
851extern const float huge_valf;
852} // End llvm namespace
853
854#endif