Bug Summary

File:lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
Warning:line 545, column 29
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizeDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-8/lib/clang/8.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/include -I /build/llvm-toolchain-snapshot-8~svn350071/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/8.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-8/lib/clang/8.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/lib/CodeGen/SelectionDAG -fdebug-prefix-map=/build/llvm-toolchain-snapshot-8~svn350071=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-12-27-042839-1215-1 -x c++ /build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp -faddrsig

/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the SelectionDAG::Legalize method.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ADT/APFloat.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/CodeGen/ISDOpcodes.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineMemOperand.h"
25#include "llvm/CodeGen/RuntimeLibcalls.h"
26#include "llvm/CodeGen/SelectionDAG.h"
27#include "llvm/CodeGen/SelectionDAGNodes.h"
28#include "llvm/CodeGen/TargetFrameLowering.h"
29#include "llvm/CodeGen/TargetLowering.h"
30#include "llvm/CodeGen/TargetSubtargetInfo.h"
31#include "llvm/CodeGen/ValueTypes.h"
32#include "llvm/IR/CallingConv.h"
33#include "llvm/IR/Constants.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/DerivedTypes.h"
36#include "llvm/IR/Function.h"
37#include "llvm/IR/Metadata.h"
38#include "llvm/IR/Type.h"
39#include "llvm/Support/Casting.h"
40#include "llvm/Support/Compiler.h"
41#include "llvm/Support/Debug.h"
42#include "llvm/Support/ErrorHandling.h"
43#include "llvm/Support/MachineValueType.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/raw_ostream.h"
46#include "llvm/Target/TargetMachine.h"
47#include "llvm/Target/TargetOptions.h"
48#include <algorithm>
49#include <cassert>
50#include <cstdint>
51#include <tuple>
52#include <utility>
53
54using namespace llvm;
55
56#define DEBUG_TYPE"legalizedag" "legalizedag"
57
58namespace {
59
60/// Keeps track of state when getting the sign of a floating-point value as an
61/// integer.
62struct FloatSignAsInt {
63 EVT FloatVT;
64 SDValue Chain;
65 SDValue FloatPtr;
66 SDValue IntPtr;
67 MachinePointerInfo IntPointerInfo;
68 MachinePointerInfo FloatPointerInfo;
69 SDValue IntValue;
70 APInt SignMask;
71 uint8_t SignBit;
72};
73
74//===----------------------------------------------------------------------===//
75/// This takes an arbitrary SelectionDAG as input and
76/// hacks on it until the target machine can handle it. This involves
77/// eliminating value sizes the machine cannot handle (promoting small sizes to
78/// large sizes or splitting up large values into small values) as well as
79/// eliminating operations the machine cannot handle.
80///
81/// This code also does a small amount of optimization and recognition of idioms
82/// as part of its processing. For example, if a target does not support a
83/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
84/// will attempt merge setcc and brc instructions into brcc's.
85class SelectionDAGLegalize {
86 const TargetMachine &TM;
87 const TargetLowering &TLI;
88 SelectionDAG &DAG;
89
90 /// The set of nodes which have already been legalized. We hold a
91 /// reference to it in order to update as necessary on node deletion.
92 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93
94 /// A set of all the nodes updated during legalization.
95 SmallSetVector<SDNode *, 16> *UpdatedNodes;
96
97 EVT getSetCCResultType(EVT VT) const {
98 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
99 }
100
101 // Libcall insertion helpers.
102
103public:
104 SelectionDAGLegalize(SelectionDAG &DAG,
105 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
106 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
107 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
108 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109
110 /// Legalizes the given operation.
111 void LegalizeOp(SDNode *Node);
112
113private:
114 SDValue OptimizeFloatStore(StoreSDNode *ST);
115
116 void LegalizeLoadOps(SDNode *Node);
117 void LegalizeStoreOps(SDNode *Node);
118
119 /// Some targets cannot handle a variable
120 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
121 /// is necessary to spill the vector being inserted into to memory, perform
122 /// the insert there, and then read the result back.
123 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
124 const SDLoc &dl);
125 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
126 const SDLoc &dl);
127
128 /// Return a vector shuffle operation which
129 /// performs the same shuffe in terms of order or result bytes, but on a type
130 /// whose vector element type is narrower than the original shuffle type.
131 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
132 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133 SDValue N1, SDValue N2,
134 ArrayRef<int> Mask) const;
135
136 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
137 bool &NeedInvert, const SDLoc &dl);
138
139 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
140 SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
141 unsigned NumOps, bool isSigned, const SDLoc &dl);
142
143 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
144 SDNode *Node, bool isSigned);
145 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
146 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
147 RTLIB::Libcall Call_F128,
148 RTLIB::Libcall Call_PPCF128);
149 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
150 RTLIB::Libcall Call_I8,
151 RTLIB::Libcall Call_I16,
152 RTLIB::Libcall Call_I32,
153 RTLIB::Libcall Call_I64,
154 RTLIB::Libcall Call_I128);
155 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157
158 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
159 const SDLoc &dl);
160 SDValue ExpandBUILD_VECTOR(SDNode *Node);
161 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
162 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
163 SmallVectorImpl<SDValue> &Results);
164 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
165 SDValue Value) const;
166 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
167 SDValue NewIntValue) const;
168 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
169 SDValue ExpandFABS(SDNode *Node) const;
170 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, EVT DestVT,
171 const SDLoc &dl);
172 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
173 const SDLoc &dl);
174 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
175 const SDLoc &dl);
176
177 SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
178 SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
179
180 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
181 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
182 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
183
184 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
185 SDValue ExpandConstant(ConstantSDNode *CP);
186
187 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
188 bool ExpandNode(SDNode *Node);
189 void ConvertNodeToLibcall(SDNode *Node);
190 void PromoteNode(SDNode *Node);
191
192public:
193 // Node replacement helpers
194
195 void ReplacedNode(SDNode *N) {
196 LegalizedNodes.erase(N);
197 if (UpdatedNodes)
198 UpdatedNodes->insert(N);
199 }
200
201 void ReplaceNode(SDNode *Old, SDNode *New) {
202 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
203 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
204
205 assert(Old->getNumValues() == New->getNumValues() &&((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 207, __PRETTY_FUNCTION__))
206 "Replacing one node with another that produces a different number "((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 207, __PRETTY_FUNCTION__))
207 "of values!")((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 207, __PRETTY_FUNCTION__))
;
208 DAG.ReplaceAllUsesWith(Old, New);
209 if (UpdatedNodes)
210 UpdatedNodes->insert(New);
211 ReplacedNode(Old);
212 }
213
214 void ReplaceNode(SDValue Old, SDValue New) {
215 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
216 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
217
218 DAG.ReplaceAllUsesWith(Old, New);
219 if (UpdatedNodes)
220 UpdatedNodes->insert(New.getNode());
221 ReplacedNode(Old.getNode());
222 }
223
224 void ReplaceNode(SDNode *Old, const SDValue *New) {
225 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); } } while (false)
;
226
227 DAG.ReplaceAllUsesWith(Old, New);
228 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
229 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
230 New[i]->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
;
231 if (UpdatedNodes)
232 UpdatedNodes->insert(New[i].getNode());
233 }
234 ReplacedNode(Old);
235 }
236};
237
238} // end anonymous namespace
239
240/// Return a vector shuffle operation which
241/// performs the same shuffle in terms of order or result bytes, but on a type
242/// whose vector element type is narrower than the original shuffle type.
243/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
244SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
245 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
246 ArrayRef<int> Mask) const {
247 unsigned NumMaskElts = VT.getVectorNumElements();
248 unsigned NumDestElts = NVT.getVectorNumElements();
249 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
250
251 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!")((NumEltsGrowth && "Cannot promote to vector type with fewer elts!"
) ? static_cast<void> (0) : __assert_fail ("NumEltsGrowth && \"Cannot promote to vector type with fewer elts!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 251, __PRETTY_FUNCTION__))
;
252
253 if (NumEltsGrowth == 1)
254 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
255
256 SmallVector<int, 8> NewMask;
257 for (unsigned i = 0; i != NumMaskElts; ++i) {
258 int Idx = Mask[i];
259 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
260 if (Idx < 0)
261 NewMask.push_back(-1);
262 else
263 NewMask.push_back(Idx * NumEltsGrowth + j);
264 }
265 }
266 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?")((NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"
) ? static_cast<void> (0) : __assert_fail ("NewMask.size() == NumDestElts && \"Non-integer NumEltsGrowth?\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 266, __PRETTY_FUNCTION__))
;
267 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?")((TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"
) ? static_cast<void> (0) : __assert_fail ("TLI.isShuffleMaskLegal(NewMask, NVT) && \"Shuffle not legal?\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 267, __PRETTY_FUNCTION__))
;
268 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
269}
270
271/// Expands the ConstantFP node to an integer constant or
272/// a load from the constant pool.
273SDValue
274SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
275 bool Extend = false;
276 SDLoc dl(CFP);
277
278 // If a FP immediate is precise when represented as a float and if the
279 // target can do an extending load from float to double, we put it into
280 // the constant pool as a float, even if it's is statically typed as a
281 // double. This shrinks FP constants and canonicalizes them for targets where
282 // an FP extending load is the same cost as a normal load (such as on the x87
283 // fp stack or PPC FP unit).
284 EVT VT = CFP->getValueType(0);
285 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
286 if (!UseCP) {
287 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion")(((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::f64 || VT == MVT::f32) && \"Invalid type expansion\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 287, __PRETTY_FUNCTION__))
;
288 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
289 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
290 }
291
292 APFloat APF = CFP->getValueAPF();
293 EVT OrigVT = VT;
294 EVT SVT = VT;
295
296 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
297 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
298 if (!APF.isSignaling()) {
299 while (SVT != MVT::f32 && SVT != MVT::f16) {
300 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
301 if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
302 // Only do this if the target has a native EXTLOAD instruction from
303 // smaller type.
304 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
305 TLI.ShouldShrinkFPConstant(OrigVT)) {
306 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
307 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
308 VT = SVT;
309 Extend = true;
310 }
311 }
312 }
313
314 SDValue CPIdx =
315 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
316 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
317 if (Extend) {
318 SDValue Result = DAG.getExtLoad(
319 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
320 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
321 Alignment);
322 return Result;
323 }
324 SDValue Result = DAG.getLoad(
325 OrigVT, dl, DAG.getEntryNode(), CPIdx,
326 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
327 return Result;
328}
329
330/// Expands the Constant node to a load from the constant pool.
331SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
332 SDLoc dl(CP);
333 EVT VT = CP->getValueType(0);
334 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
335 TLI.getPointerTy(DAG.getDataLayout()));
336 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
337 SDValue Result = DAG.getLoad(
338 VT, dl, DAG.getEntryNode(), CPIdx,
339 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
340 return Result;
341}
342
343/// Some target cannot handle a variable insertion index for the
344/// INSERT_VECTOR_ELT instruction. In this case, it
345/// is necessary to spill the vector being inserted into to memory, perform
346/// the insert there, and then read the result back.
347SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
348 SDValue Val,
349 SDValue Idx,
350 const SDLoc &dl) {
351 SDValue Tmp1 = Vec;
352 SDValue Tmp2 = Val;
353 SDValue Tmp3 = Idx;
354
355 // If the target doesn't support this, we have to spill the input vector
356 // to a temporary stack slot, update the element, then reload it. This is
357 // badness. We could also load the value into a vector register (either
358 // with a "move to register" or "extload into register" instruction, then
359 // permute it into place, if the idx is a constant and if the idx is
360 // supported by the target.
361 EVT VT = Tmp1.getValueType();
362 EVT EltVT = VT.getVectorElementType();
363 SDValue StackPtr = DAG.CreateStackTemporary(VT);
364
365 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
366
367 // Store the vector.
368 SDValue Ch = DAG.getStore(
369 DAG.getEntryNode(), dl, Tmp1, StackPtr,
370 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
371
372 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
373
374 // Store the scalar value.
375 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
376 // Load the updated vector.
377 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
378 DAG.getMachineFunction(), SPFI));
379}
380
381SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
382 SDValue Idx,
383 const SDLoc &dl) {
384 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
385 // SCALAR_TO_VECTOR requires that the type of the value being inserted
386 // match the element type of the vector being created, except for
387 // integers in which case the inserted value can be over width.
388 EVT EltVT = Vec.getValueType().getVectorElementType();
389 if (Val.getValueType() == EltVT ||
390 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
391 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
392 Vec.getValueType(), Val);
393
394 unsigned NumElts = Vec.getValueType().getVectorNumElements();
395 // We generate a shuffle of InVec and ScVec, so the shuffle mask
396 // should be 0,1,2,3,4,5... with the appropriate element replaced with
397 // elt 0 of the RHS.
398 SmallVector<int, 8> ShufOps;
399 for (unsigned i = 0; i != NumElts; ++i)
400 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
401
402 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
403 }
404 }
405 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
406}
407
408SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
409 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Optimizing float store operations\n"
; } } while (false)
;
410 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
411 // FIXME: We shouldn't do this for TargetConstantFP's.
412 // FIXME: move this to the DAG Combiner! Note that we can't regress due
413 // to phase ordering between legalized code and the dag combiner. This
414 // probably means that we need to integrate dag combiner and legalizer
415 // together.
416 // We generally can't do this one for long doubles.
417 SDValue Chain = ST->getChain();
418 SDValue Ptr = ST->getBasePtr();
419 unsigned Alignment = ST->getAlignment();
420 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
421 AAMDNodes AAInfo = ST->getAAInfo();
422 SDLoc dl(ST);
423 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
424 if (CFP->getValueType(0) == MVT::f32 &&
425 TLI.isTypeLegal(MVT::i32)) {
426 SDValue Con = DAG.getConstant(CFP->getValueAPF().
427 bitcastToAPInt().zextOrTrunc(32),
428 SDLoc(CFP), MVT::i32);
429 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
430 MMOFlags, AAInfo);
431 }
432
433 if (CFP->getValueType(0) == MVT::f64) {
434 // If this target supports 64-bit registers, do a single 64-bit store.
435 if (TLI.isTypeLegal(MVT::i64)) {
436 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
437 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
438 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
439 Alignment, MMOFlags, AAInfo);
440 }
441
442 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
443 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
444 // stores. If the target supports neither 32- nor 64-bits, this
445 // xform is certainly not worth it.
446 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
447 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
448 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
449 if (DAG.getDataLayout().isBigEndian())
450 std::swap(Lo, Hi);
451
452 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
453 MMOFlags, AAInfo);
454 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
455 DAG.getConstant(4, dl, Ptr.getValueType()));
456 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
457 ST->getPointerInfo().getWithOffset(4),
458 MinAlign(Alignment, 4U), MMOFlags, AAInfo);
459
460 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
461 }
462 }
463 }
464 return SDValue(nullptr, 0);
465}
466
467void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
468 StoreSDNode *ST = cast<StoreSDNode>(Node);
469 SDValue Chain = ST->getChain();
470 SDValue Ptr = ST->getBasePtr();
471 SDLoc dl(Node);
472
473 unsigned Alignment = ST->getAlignment();
474 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
475 AAMDNodes AAInfo = ST->getAAInfo();
476
477 if (!ST->isTruncatingStore()) {
1
Assuming the condition is false
2
Taking false branch
478 LLVM_DEBUG(dbgs() << "Legalizing store operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing store operation\n"
; } } while (false)
;
479 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
480 ReplaceNode(ST, OptStore);
481 return;
482 }
483
484 SDValue Value = ST->getValue();
485 MVT VT = Value.getSimpleValueType();
486 switch (TLI.getOperationAction(ISD::STORE, VT)) {
487 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 487)
;
488 case TargetLowering::Legal: {
489 // If this is an unaligned store and the target doesn't support it,
490 // expand it.
491 EVT MemVT = ST->getMemoryVT();
492 unsigned AS = ST->getAddressSpace();
493 unsigned Align = ST->getAlignment();
494 const DataLayout &DL = DAG.getDataLayout();
495 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
496 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Expanding unsupported unaligned store\n"
; } } while (false)
;
497 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
498 ReplaceNode(SDValue(ST, 0), Result);
499 } else
500 LLVM_DEBUG(dbgs() << "Legal store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legal store\n"; } } while
(false)
;
501 break;
502 }
503 case TargetLowering::Custom: {
504 LLVM_DEBUG(dbgs() << "Trying custom lowering\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying custom lowering\n"
; } } while (false)
;
505 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
506 if (Res && Res != SDValue(Node, 0))
507 ReplaceNode(SDValue(Node, 0), Res);
508 return;
509 }
510 case TargetLowering::Promote: {
511 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
512 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote stores to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 513, __PRETTY_FUNCTION__))
513 "Can only promote stores to same size type")((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote stores to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 513, __PRETTY_FUNCTION__))
;
514 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
515 SDValue Result =
516 DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
517 Alignment, MMOFlags, AAInfo);
518 ReplaceNode(SDValue(Node, 0), Result);
519 break;
520 }
521 }
522 return;
523 }
524
525 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing truncating store operations\n"
; } } while (false)
;
3
Assuming 'DebugFlag' is 0
4
Loop condition is false. Exiting loop
526 SDValue Value = ST->getValue();
527 EVT StVT = ST->getMemoryVT();
528 unsigned StWidth = StVT.getSizeInBits();
529 auto &DL = DAG.getDataLayout();
530
531 if (StWidth != StVT.getStoreSizeInBits()) {
5
Assuming the condition is false
6
Taking false branch
532 // Promote to a byte-sized store with upper bits zero if not
533 // storing an integral number of bytes. For example, promote
534 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
535 EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
536 StVT.getStoreSizeInBits());
537 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
538 SDValue Result =
539 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
540 Alignment, MMOFlags, AAInfo);
541 ReplaceNode(SDValue(Node, 0), Result);
542 } else if (StWidth & (StWidth - 1)) {
7
Assuming the condition is true
8
Taking true branch
543 // If not storing a power-of-2 number of bits, expand as two stores.
544 assert(!StVT.isVector() && "Unsupported truncstore!")((!StVT.isVector() && "Unsupported truncstore!") ? static_cast
<void> (0) : __assert_fail ("!StVT.isVector() && \"Unsupported truncstore!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 544, __PRETTY_FUNCTION__))
;
9
Assuming the condition is true
10
'?' condition is true
545 unsigned RoundWidth = 1 << Log2_32(StWidth);
11
Calling 'Log2_32'
13
Returning from 'Log2_32'
14
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'
546 assert(RoundWidth < StWidth)((RoundWidth < StWidth) ? static_cast<void> (0) : __assert_fail
("RoundWidth < StWidth", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 546, __PRETTY_FUNCTION__))
;
547 unsigned ExtraWidth = StWidth - RoundWidth;
548 assert(ExtraWidth < RoundWidth)((ExtraWidth < RoundWidth) ? static_cast<void> (0) :
__assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 548, __PRETTY_FUNCTION__))
;
549 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Store size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 550, __PRETTY_FUNCTION__))
550 "Store size not an integral number of bytes!")((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Store size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 550, __PRETTY_FUNCTION__))
;
551 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
552 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
553 SDValue Lo, Hi;
554 unsigned IncrementSize;
555
556 if (DL.isLittleEndian()) {
557 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
558 // Store the bottom RoundWidth bits.
559 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
560 RoundVT, Alignment, MMOFlags, AAInfo);
561
562 // Store the remaining ExtraWidth bits.
563 IncrementSize = RoundWidth / 8;
564 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
565 DAG.getConstant(IncrementSize, dl,
566 Ptr.getValueType()));
567 Hi = DAG.getNode(
568 ISD::SRL, dl, Value.getValueType(), Value,
569 DAG.getConstant(RoundWidth, dl,
570 TLI.getShiftAmountTy(Value.getValueType(), DL)));
571 Hi = DAG.getTruncStore(
572 Chain, dl, Hi, Ptr,
573 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
574 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
575 } else {
576 // Big endian - avoid unaligned stores.
577 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
578 // Store the top RoundWidth bits.
579 Hi = DAG.getNode(
580 ISD::SRL, dl, Value.getValueType(), Value,
581 DAG.getConstant(ExtraWidth, dl,
582 TLI.getShiftAmountTy(Value.getValueType(), DL)));
583 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
584 RoundVT, Alignment, MMOFlags, AAInfo);
585
586 // Store the remaining ExtraWidth bits.
587 IncrementSize = RoundWidth / 8;
588 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
589 DAG.getConstant(IncrementSize, dl,
590 Ptr.getValueType()));
591 Lo = DAG.getTruncStore(
592 Chain, dl, Value, Ptr,
593 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
594 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
595 }
596
597 // The order of the stores doesn't matter.
598 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
599 ReplaceNode(SDValue(Node, 0), Result);
600 } else {
601 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
602 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 602)
;
603 case TargetLowering::Legal: {
604 EVT MemVT = ST->getMemoryVT();
605 unsigned AS = ST->getAddressSpace();
606 unsigned Align = ST->getAlignment();
607 // If this is an unaligned store and the target doesn't support it,
608 // expand it.
609 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
610 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
611 ReplaceNode(SDValue(ST, 0), Result);
612 }
613 break;
614 }
615 case TargetLowering::Custom: {
616 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
617 if (Res && Res != SDValue(Node, 0))
618 ReplaceNode(SDValue(Node, 0), Res);
619 return;
620 }
621 case TargetLowering::Expand:
622 assert(!StVT.isVector() &&((!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 623, __PRETTY_FUNCTION__))
623 "Vector Stores are handled in LegalizeVectorOps")((!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 623, __PRETTY_FUNCTION__))
;
624
625 SDValue Result;
626
627 // TRUNCSTORE:i16 i32 -> STORE i16
628 if (TLI.isTypeLegal(StVT)) {
629 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
630 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
631 Alignment, MMOFlags, AAInfo);
632 } else {
633 // The in-memory type isn't legal. Truncate to the type it would promote
634 // to, and then do a truncstore.
635 Value = DAG.getNode(ISD::TRUNCATE, dl,
636 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
637 Value);
638 Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
639 StVT, Alignment, MMOFlags, AAInfo);
640 }
641
642 ReplaceNode(SDValue(Node, 0), Result);
643 break;
644 }
645 }
646}
647
648void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
649 LoadSDNode *LD = cast<LoadSDNode>(Node);
650 SDValue Chain = LD->getChain(); // The chain.
651 SDValue Ptr = LD->getBasePtr(); // The base pointer.
652 SDValue Value; // The value returned by the load op.
653 SDLoc dl(Node);
654
655 ISD::LoadExtType ExtType = LD->getExtensionType();
656 if (ExtType == ISD::NON_EXTLOAD) {
657 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing non-extending load operation\n"
; } } while (false)
;
658 MVT VT = Node->getSimpleValueType(0);
659 SDValue RVal = SDValue(Node, 0);
660 SDValue RChain = SDValue(Node, 1);
661
662 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
663 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 663)
;
664 case TargetLowering::Legal: {
665 EVT MemVT = LD->getMemoryVT();
666 unsigned AS = LD->getAddressSpace();
667 unsigned Align = LD->getAlignment();
668 const DataLayout &DL = DAG.getDataLayout();
669 // If this is an unaligned load and the target doesn't support it,
670 // expand it.
671 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
672 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
673 }
674 break;
675 }
676 case TargetLowering::Custom:
677 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
678 RVal = Res;
679 RChain = Res.getValue(1);
680 }
681 break;
682
683 case TargetLowering::Promote: {
684 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
685 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote loads to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 686, __PRETTY_FUNCTION__))
686 "Can only promote loads to same size type")((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote loads to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 686, __PRETTY_FUNCTION__))
;
687
688 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
689 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
690 RChain = Res.getValue(1);
691 break;
692 }
693 }
694 if (RChain.getNode() != Node) {
695 assert(RVal.getNode() != Node && "Load must be completely replaced")((RVal.getNode() != Node && "Load must be completely replaced"
) ? static_cast<void> (0) : __assert_fail ("RVal.getNode() != Node && \"Load must be completely replaced\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 695, __PRETTY_FUNCTION__))
;
696 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
697 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
698 if (UpdatedNodes) {
699 UpdatedNodes->insert(RVal.getNode());
700 UpdatedNodes->insert(RChain.getNode());
701 }
702 ReplacedNode(Node);
703 }
704 return;
705 }
706
707 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing extending load operation\n"
; } } while (false)
;
708 EVT SrcVT = LD->getMemoryVT();
709 unsigned SrcWidth = SrcVT.getSizeInBits();
710 unsigned Alignment = LD->getAlignment();
711 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
712 AAMDNodes AAInfo = LD->getAAInfo();
713
714 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
715 // Some targets pretend to have an i1 loading operation, and actually
716 // load an i8. This trick is correct for ZEXTLOAD because the top 7
717 // bits are guaranteed to be zero; it helps the optimizers understand
718 // that these bits are zero. It is also useful for EXTLOAD, since it
719 // tells the optimizers that those bits are undefined. It would be
720 // nice to have an effective generic way of getting these benefits...
721 // Until such a way is found, don't insist on promoting i1 here.
722 (SrcVT != MVT::i1 ||
723 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
724 TargetLowering::Promote)) {
725 // Promote to a byte-sized load if not loading an integral number of
726 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
727 unsigned NewWidth = SrcVT.getStoreSizeInBits();
728 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
729 SDValue Ch;
730
731 // The extra bits are guaranteed to be zero, since we stored them that
732 // way. A zext load from NVT thus automatically gives zext from SrcVT.
733
734 ISD::LoadExtType NewExtType =
735 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
736
737 SDValue Result =
738 DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
739 LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
740
741 Ch = Result.getValue(1); // The chain.
742
743 if (ExtType == ISD::SEXTLOAD)
744 // Having the top bits zero doesn't help when sign extending.
745 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
746 Result.getValueType(),
747 Result, DAG.getValueType(SrcVT));
748 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
749 // All the top bits are guaranteed to be zero - inform the optimizers.
750 Result = DAG.getNode(ISD::AssertZext, dl,
751 Result.getValueType(), Result,
752 DAG.getValueType(SrcVT));
753
754 Value = Result;
755 Chain = Ch;
756 } else if (SrcWidth & (SrcWidth - 1)) {
757 // If not loading a power-of-2 number of bits, expand as two loads.
758 assert(!SrcVT.isVector() && "Unsupported extload!")((!SrcVT.isVector() && "Unsupported extload!") ? static_cast
<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Unsupported extload!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 758, __PRETTY_FUNCTION__))
;
759 unsigned RoundWidth = 1 << Log2_32(SrcWidth);
760 assert(RoundWidth < SrcWidth)((RoundWidth < SrcWidth) ? static_cast<void> (0) : __assert_fail
("RoundWidth < SrcWidth", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 760, __PRETTY_FUNCTION__))
;
761 unsigned ExtraWidth = SrcWidth - RoundWidth;
762 assert(ExtraWidth < RoundWidth)((ExtraWidth < RoundWidth) ? static_cast<void> (0) :
__assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 762, __PRETTY_FUNCTION__))
;
763 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Load size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 764, __PRETTY_FUNCTION__))
764 "Load size not an integral number of bytes!")((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Load size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 764, __PRETTY_FUNCTION__))
;
765 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
766 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
767 SDValue Lo, Hi, Ch;
768 unsigned IncrementSize;
769 auto &DL = DAG.getDataLayout();
770
771 if (DL.isLittleEndian()) {
772 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
773 // Load the bottom RoundWidth bits.
774 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
775 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
776 AAInfo);
777
778 // Load the remaining ExtraWidth bits.
779 IncrementSize = RoundWidth / 8;
780 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
781 DAG.getConstant(IncrementSize, dl,
782 Ptr.getValueType()));
783 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
784 LD->getPointerInfo().getWithOffset(IncrementSize),
785 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
786 AAInfo);
787
788 // Build a factor node to remember that this load is independent of
789 // the other one.
790 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
791 Hi.getValue(1));
792
793 // Move the top bits to the right place.
794 Hi = DAG.getNode(
795 ISD::SHL, dl, Hi.getValueType(), Hi,
796 DAG.getConstant(RoundWidth, dl,
797 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
798
799 // Join the hi and lo parts.
800 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
801 } else {
802 // Big endian - avoid unaligned loads.
803 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
804 // Load the top RoundWidth bits.
805 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
806 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
807 AAInfo);
808
809 // Load the remaining ExtraWidth bits.
810 IncrementSize = RoundWidth / 8;
811 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
812 DAG.getConstant(IncrementSize, dl,
813 Ptr.getValueType()));
814 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
815 LD->getPointerInfo().getWithOffset(IncrementSize),
816 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
817 AAInfo);
818
819 // Build a factor node to remember that this load is independent of
820 // the other one.
821 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
822 Hi.getValue(1));
823
824 // Move the top bits to the right place.
825 Hi = DAG.getNode(
826 ISD::SHL, dl, Hi.getValueType(), Hi,
827 DAG.getConstant(ExtraWidth, dl,
828 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
829
830 // Join the hi and lo parts.
831 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
832 }
833
834 Chain = Ch;
835 } else {
836 bool isCustom = false;
837 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
838 SrcVT.getSimpleVT())) {
839 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 839)
;
840 case TargetLowering::Custom:
841 isCustom = true;
842 LLVM_FALLTHROUGH[[clang::fallthrough]];
843 case TargetLowering::Legal:
844 Value = SDValue(Node, 0);
845 Chain = SDValue(Node, 1);
846
847 if (isCustom) {
848 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
849 Value = Res;
850 Chain = Res.getValue(1);
851 }
852 } else {
853 // If this is an unaligned load and the target doesn't support it,
854 // expand it.
855 EVT MemVT = LD->getMemoryVT();
856 unsigned AS = LD->getAddressSpace();
857 unsigned Align = LD->getAlignment();
858 const DataLayout &DL = DAG.getDataLayout();
859 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
860 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
861 }
862 }
863 break;
864
865 case TargetLowering::Expand: {
866 EVT DestVT = Node->getValueType(0);
867 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
868 // If the source type is not legal, see if there is a legal extload to
869 // an intermediate type that we can then extend further.
870 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
871 if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
872 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
873 // If we are loading a legal type, this is a non-extload followed by a
874 // full extend.
875 ISD::LoadExtType MidExtType =
876 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
877
878 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
879 SrcVT, LD->getMemOperand());
880 unsigned ExtendOp =
881 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
882 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
883 Chain = Load.getValue(1);
884 break;
885 }
886
887 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
888 // normal undefined upper bits behavior to allow using an in-reg extend
889 // with the illegal FP type, so load as an integer and do the
890 // from-integer conversion.
891 if (SrcVT.getScalarType() == MVT::f16) {
892 EVT ISrcVT = SrcVT.changeTypeToInteger();
893 EVT IDestVT = DestVT.changeTypeToInteger();
894 EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
895
896 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
897 Chain, Ptr, ISrcVT,
898 LD->getMemOperand());
899 Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
900 Chain = Result.getValue(1);
901 break;
902 }
903 }
904
905 assert(!SrcVT.isVector() &&((!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 906, __PRETTY_FUNCTION__))
906 "Vector Loads are handled in LegalizeVectorOps")((!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 906, __PRETTY_FUNCTION__))
;
907
908 // FIXME: This does not work for vectors on most targets. Sign-
909 // and zero-extend operations are currently folded into extending
910 // loads, whether they are legal or not, and then we end up here
911 // without any support for legalizing them.
912 assert(ExtType != ISD::EXTLOAD &&((ExtType != ISD::EXTLOAD && "EXTLOAD should always be supported!"
) ? static_cast<void> (0) : __assert_fail ("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 913, __PRETTY_FUNCTION__))
913 "EXTLOAD should always be supported!")((ExtType != ISD::EXTLOAD && "EXTLOAD should always be supported!"
) ? static_cast<void> (0) : __assert_fail ("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 913, __PRETTY_FUNCTION__))
;
914 // Turn the unsupported load into an EXTLOAD followed by an
915 // explicit zero/sign extend inreg.
916 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
917 Node->getValueType(0),
918 Chain, Ptr, SrcVT,
919 LD->getMemOperand());
920 SDValue ValRes;
921 if (ExtType == ISD::SEXTLOAD)
922 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
923 Result.getValueType(),
924 Result, DAG.getValueType(SrcVT));
925 else
926 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
927 Value = ValRes;
928 Chain = Result.getValue(1);
929 break;
930 }
931 }
932 }
933
934 // Since loads produce two values, make sure to remember that we legalized
935 // both of them.
936 if (Chain.getNode() != Node) {
937 assert(Value.getNode() != Node && "Load must be completely replaced")((Value.getNode() != Node && "Load must be completely replaced"
) ? static_cast<void> (0) : __assert_fail ("Value.getNode() != Node && \"Load must be completely replaced\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 937, __PRETTY_FUNCTION__))
;
938 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
939 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
940 if (UpdatedNodes) {
941 UpdatedNodes->insert(Value.getNode());
942 UpdatedNodes->insert(Chain.getNode());
943 }
944 ReplacedNode(Node);
945 }
946}
947
948/// Return a legal replacement for the given operation, with all legal operands.
949void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
950 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "\nLegalizing: "; Node->
dump(&DAG); } } while (false)
;
951
952 // Allow illegal target nodes and illegal registers.
953 if (Node->getOpcode() == ISD::TargetConstant ||
954 Node->getOpcode() == ISD::Register)
955 return;
956
957#ifndef NDEBUG
958 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
959 assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 962, __PRETTY_FUNCTION__))
960 TargetLowering::TypeLegal ||(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 962, __PRETTY_FUNCTION__))
961 TLI.isTypeLegal(Node->getValueType(i))) &&(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 962, __PRETTY_FUNCTION__))
962 "Unexpected illegal type!")(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 962, __PRETTY_FUNCTION__))
;
963
964 for (const SDValue &Op : Node->op_values())
965 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
966 TargetLowering::TypeLegal ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
967 TLI.isTypeLegal(Op.getValueType()) ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
968 Op.getOpcode() == ISD::TargetConstant ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
969 Op.getOpcode() == ISD::Register) &&(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
970 "Unexpected illegal type!")(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
;
971#endif
972
973 // Figure out the correct action; the way to query this varies by opcode
974 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
975 bool SimpleFinishLegalizing = true;
976 switch (Node->getOpcode()) {
977 case ISD::INTRINSIC_W_CHAIN:
978 case ISD::INTRINSIC_WO_CHAIN:
979 case ISD::INTRINSIC_VOID:
980 case ISD::STACKSAVE:
981 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
982 break;
983 case ISD::GET_DYNAMIC_AREA_OFFSET:
984 Action = TLI.getOperationAction(Node->getOpcode(),
985 Node->getValueType(0));
986 break;
987 case ISD::VAARG:
988 Action = TLI.getOperationAction(Node->getOpcode(),
989 Node->getValueType(0));
990 if (Action != TargetLowering::Promote)
991 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
992 break;
993 case ISD::FP_TO_FP16:
994 case ISD::SINT_TO_FP:
995 case ISD::UINT_TO_FP:
996 case ISD::EXTRACT_VECTOR_ELT:
997 Action = TLI.getOperationAction(Node->getOpcode(),
998 Node->getOperand(0).getValueType());
999 break;
1000 case ISD::FP_ROUND_INREG:
1001 case ISD::SIGN_EXTEND_INREG: {
1002 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1003 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1004 break;
1005 }
1006 case ISD::ATOMIC_STORE:
1007 Action = TLI.getOperationAction(Node->getOpcode(),
1008 Node->getOperand(2).getValueType());
1009 break;
1010 case ISD::SELECT_CC:
1011 case ISD::SETCC:
1012 case ISD::BR_CC: {
1013 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1014 Node->getOpcode() == ISD::SETCC ? 2 : 1;
1015 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
1016 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1017 ISD::CondCode CCCode =
1018 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1019 Action = TLI.getCondCodeAction(CCCode, OpVT);
1020 if (Action == TargetLowering::Legal) {
1021 if (Node->getOpcode() == ISD::SELECT_CC)
1022 Action = TLI.getOperationAction(Node->getOpcode(),
1023 Node->getValueType(0));
1024 else
1025 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1026 }
1027 break;
1028 }
1029 case ISD::LOAD:
1030 case ISD::STORE:
1031 // FIXME: Model these properly. LOAD and STORE are complicated, and
1032 // STORE expects the unlegalized operand in some cases.
1033 SimpleFinishLegalizing = false;
1034 break;
1035 case ISD::CALLSEQ_START:
1036 case ISD::CALLSEQ_END:
1037 // FIXME: This shouldn't be necessary. These nodes have special properties
1038 // dealing with the recursive nature of legalization. Removing this
1039 // special case should be done as part of making LegalizeDAG non-recursive.
1040 SimpleFinishLegalizing = false;
1041 break;
1042 case ISD::EXTRACT_ELEMENT:
1043 case ISD::FLT_ROUNDS_:
1044 case ISD::MERGE_VALUES:
1045 case ISD::EH_RETURN:
1046 case ISD::FRAME_TO_ARGS_OFFSET:
1047 case ISD::EH_DWARF_CFA:
1048 case ISD::EH_SJLJ_SETJMP:
1049 case ISD::EH_SJLJ_LONGJMP:
1050 case ISD::EH_SJLJ_SETUP_DISPATCH:
1051 // These operations lie about being legal: when they claim to be legal,
1052 // they should actually be expanded.
1053 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1054 if (Action == TargetLowering::Legal)
1055 Action = TargetLowering::Expand;
1056 break;
1057 case ISD::INIT_TRAMPOLINE:
1058 case ISD::ADJUST_TRAMPOLINE:
1059 case ISD::FRAMEADDR:
1060 case ISD::RETURNADDR:
1061 case ISD::ADDROFRETURNADDR:
1062 case ISD::SPONENTRY:
1063 // These operations lie about being legal: when they claim to be legal,
1064 // they should actually be custom-lowered.
1065 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1066 if (Action == TargetLowering::Legal)
1067 Action = TargetLowering::Custom;
1068 break;
1069 case ISD::READCYCLECOUNTER:
1070 // READCYCLECOUNTER returns an i64, even if type legalization might have
1071 // expanded that to several smaller types.
1072 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1073 break;
1074 case ISD::READ_REGISTER:
1075 case ISD::WRITE_REGISTER:
1076 // Named register is legal in the DAG, but blocked by register name
1077 // selection if not implemented by target (to chose the correct register)
1078 // They'll be converted to Copy(To/From)Reg.
1079 Action = TargetLowering::Legal;
1080 break;
1081 case ISD::DEBUGTRAP:
1082 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1083 if (Action == TargetLowering::Expand) {
1084 // replace ISD::DEBUGTRAP with ISD::TRAP
1085 SDValue NewVal;
1086 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1087 Node->getOperand(0));
1088 ReplaceNode(Node, NewVal.getNode());
1089 LegalizeOp(NewVal.getNode());
1090 return;
1091 }
1092 break;
1093 case ISD::STRICT_FADD:
1094 case ISD::STRICT_FSUB:
1095 case ISD::STRICT_FMUL:
1096 case ISD::STRICT_FDIV:
1097 case ISD::STRICT_FREM:
1098 case ISD::STRICT_FSQRT:
1099 case ISD::STRICT_FMA:
1100 case ISD::STRICT_FPOW:
1101 case ISD::STRICT_FPOWI:
1102 case ISD::STRICT_FSIN:
1103 case ISD::STRICT_FCOS:
1104 case ISD::STRICT_FEXP:
1105 case ISD::STRICT_FEXP2:
1106 case ISD::STRICT_FLOG:
1107 case ISD::STRICT_FLOG10:
1108 case ISD::STRICT_FLOG2:
1109 case ISD::STRICT_FRINT:
1110 case ISD::STRICT_FNEARBYINT:
1111 case ISD::STRICT_FMAXNUM:
1112 case ISD::STRICT_FMINNUM:
1113 case ISD::STRICT_FCEIL:
1114 case ISD::STRICT_FFLOOR:
1115 case ISD::STRICT_FROUND:
1116 case ISD::STRICT_FTRUNC:
1117 // These pseudo-ops get legalized as if they were their non-strict
1118 // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
1119 // is also legal, but if ISD::FSQRT requires expansion then so does
1120 // ISD::STRICT_FSQRT.
1121 Action = TLI.getStrictFPOperationAction(Node->getOpcode(),
1122 Node->getValueType(0));
1123 break;
1124 case ISD::SADDSAT:
1125 case ISD::UADDSAT:
1126 case ISD::SSUBSAT:
1127 case ISD::USUBSAT: {
1128 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1129 break;
1130 }
1131 case ISD::SMULFIX: {
1132 unsigned Scale = Node->getConstantOperandVal(2);
1133 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1134 Node->getValueType(0), Scale);
1135 break;
1136 }
1137 case ISD::MSCATTER:
1138 Action = TLI.getOperationAction(Node->getOpcode(),
1139 cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1140 break;
1141 case ISD::MSTORE:
1142 Action = TLI.getOperationAction(Node->getOpcode(),
1143 cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1144 break;
1145 default:
1146 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1147 Action = TargetLowering::Legal;
1148 } else {
1149 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1150 }
1151 break;
1152 }
1153
1154 if (SimpleFinishLegalizing) {
1155 SDNode *NewNode = Node;
1156 switch (Node->getOpcode()) {
1157 default: break;
1158 case ISD::SHL:
1159 case ISD::SRL:
1160 case ISD::SRA:
1161 case ISD::ROTL:
1162 case ISD::ROTR: {
1163 // Legalizing shifts/rotates requires adjusting the shift amount
1164 // to the appropriate width.
1165 SDValue Op0 = Node->getOperand(0);
1166 SDValue Op1 = Node->getOperand(1);
1167 if (!Op1.getValueType().isVector()) {
1168 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1169 // The getShiftAmountOperand() may create a new operand node or
1170 // return the existing one. If new operand is created we need
1171 // to update the parent node.
1172 // Do not try to legalize SAO here! It will be automatically legalized
1173 // in the next round.
1174 if (SAO != Op1)
1175 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1176 }
1177 }
1178 break;
1179 case ISD::FSHL:
1180 case ISD::FSHR:
1181 case ISD::SRL_PARTS:
1182 case ISD::SRA_PARTS:
1183 case ISD::SHL_PARTS: {
1184 // Legalizing shifts/rotates requires adjusting the shift amount
1185 // to the appropriate width.
1186 SDValue Op0 = Node->getOperand(0);
1187 SDValue Op1 = Node->getOperand(1);
1188 SDValue Op2 = Node->getOperand(2);
1189 if (!Op2.getValueType().isVector()) {
1190 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1191 // The getShiftAmountOperand() may create a new operand node or
1192 // return the existing one. If new operand is created we need
1193 // to update the parent node.
1194 if (SAO != Op2)
1195 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1196 }
1197 break;
1198 }
1199 }
1200
1201 if (NewNode != Node) {
1202 ReplaceNode(Node, NewNode);
1203 Node = NewNode;
1204 }
1205 switch (Action) {
1206 case TargetLowering::Legal:
1207 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legal node: nothing to do\n"
; } } while (false)
;
1208 return;
1209 case TargetLowering::Custom:
1210 LLVM_DEBUG(dbgs() << "Trying custom legalization\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying custom legalization\n"
; } } while (false)
;
1211 // FIXME: The handling for custom lowering with multiple results is
1212 // a complete mess.
1213 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1214 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1215 return;
1216
1217 if (Node->getNumValues() == 1) {
1218 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully custom legalized node\n"
; } } while (false)
;
1219 // We can just directly replace this node with the lowered value.
1220 ReplaceNode(SDValue(Node, 0), Res);
1221 return;
1222 }
1223
1224 SmallVector<SDValue, 8> ResultVals;
1225 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1226 ResultVals.push_back(Res.getValue(i));
1227 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully custom legalized node\n"
; } } while (false)
;
1228 ReplaceNode(Node, ResultVals.data());
1229 return;
1230 }
1231 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not custom legalize node\n"
; } } while (false)
;
1232 LLVM_FALLTHROUGH[[clang::fallthrough]];
1233 case TargetLowering::Expand:
1234 if (ExpandNode(Node))
1235 return;
1236 LLVM_FALLTHROUGH[[clang::fallthrough]];
1237 case TargetLowering::LibCall:
1238 ConvertNodeToLibcall(Node);
1239 return;
1240 case TargetLowering::Promote:
1241 PromoteNode(Node);
1242 return;
1243 }
1244 }
1245
1246 switch (Node->getOpcode()) {
1247 default:
1248#ifndef NDEBUG
1249 dbgs() << "NODE: ";
1250 Node->dump( &DAG);
1251 dbgs() << "\n";
1252#endif
1253 llvm_unreachable("Do not know how to legalize this operator!")::llvm::llvm_unreachable_internal("Do not know how to legalize this operator!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1253)
;
1254
1255 case ISD::CALLSEQ_START:
1256 case ISD::CALLSEQ_END:
1257 break;
1258 case ISD::LOAD:
1259 return LegalizeLoadOps(Node);
1260 case ISD::STORE:
1261 return LegalizeStoreOps(Node);
1262 }
1263}
1264
1265SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1266 SDValue Vec = Op.getOperand(0);
1267 SDValue Idx = Op.getOperand(1);
1268 SDLoc dl(Op);
1269
1270 // Before we generate a new store to a temporary stack slot, see if there is
1271 // already one that we can use. There often is because when we scalarize
1272 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1273 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1274 // the vector. If all are expanded here, we don't want one store per vector
1275 // element.
1276
1277 // Caches for hasPredecessorHelper
1278 SmallPtrSet<const SDNode *, 32> Visited;
1279 SmallVector<const SDNode *, 16> Worklist;
1280 Visited.insert(Op.getNode());
1281 Worklist.push_back(Idx.getNode());
1282 SDValue StackPtr, Ch;
1283 for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1284 UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1285 SDNode *User = *UI;
1286 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1287 if (ST->isIndexed() || ST->isTruncatingStore() ||
1288 ST->getValue() != Vec)
1289 continue;
1290
1291 // Make sure that nothing else could have stored into the destination of
1292 // this store.
1293 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1294 continue;
1295
1296 // If the index is dependent on the store we will introduce a cycle when
1297 // creating the load (the load uses the index, and by replacing the chain
1298 // we will make the index dependent on the load). Also, the store might be
1299 // dependent on the extractelement and introduce a cycle when creating
1300 // the load.
1301 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1302 ST->hasPredecessor(Op.getNode()))
1303 continue;
1304
1305 StackPtr = ST->getBasePtr();
1306 Ch = SDValue(ST, 0);
1307 break;
1308 }
1309 }
1310
1311 EVT VecVT = Vec.getValueType();
1312
1313 if (!Ch.getNode()) {
1314 // Store the value to a temporary stack slot, then LOAD the returned part.
1315 StackPtr = DAG.CreateStackTemporary(VecVT);
1316 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1317 MachinePointerInfo());
1318 }
1319
1320 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1321
1322 SDValue NewLoad;
1323
1324 if (Op.getValueType().isVector())
1325 NewLoad =
1326 DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1327 else
1328 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1329 MachinePointerInfo(),
1330 VecVT.getVectorElementType());
1331
1332 // Replace the chain going out of the store, by the one out of the load.
1333 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1334
1335 // We introduced a cycle though, so update the loads operands, making sure
1336 // to use the original store's chain as an incoming chain.
1337 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1338 NewLoad->op_end());
1339 NewLoadOperands[0] = Ch;
1340 NewLoad =
1341 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1342 return NewLoad;
1343}
1344
1345SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1346 assert(Op.getValueType().isVector() && "Non-vector insert subvector!")((Op.getValueType().isVector() && "Non-vector insert subvector!"
) ? static_cast<void> (0) : __assert_fail ("Op.getValueType().isVector() && \"Non-vector insert subvector!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1346, __PRETTY_FUNCTION__))
;
1347
1348 SDValue Vec = Op.getOperand(0);
1349 SDValue Part = Op.getOperand(1);
1350 SDValue Idx = Op.getOperand(2);
1351 SDLoc dl(Op);
1352
1353 // Store the value to a temporary stack slot, then LOAD the returned part.
1354 EVT VecVT = Vec.getValueType();
1355 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1356 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1357 MachinePointerInfo PtrInfo =
1358 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1359
1360 // First store the whole vector.
1361 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1362
1363 // Then store the inserted part.
1364 SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1365
1366 // Store the subvector.
1367 Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
1368
1369 // Finally, load the updated vector.
1370 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1371}
1372
1373SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1374 // We can't handle this case efficiently. Allocate a sufficiently
1375 // aligned object on the stack, store each element into it, then load
1376 // the result as a vector.
1377 // Create the stack frame object.
1378 EVT VT = Node->getValueType(0);
1379 EVT EltVT = VT.getVectorElementType();
1380 SDLoc dl(Node);
1381 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1382 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1383 MachinePointerInfo PtrInfo =
1384 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1385
1386 // Emit a store of each element to the stack slot.
1387 SmallVector<SDValue, 8> Stores;
1388 unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1389 // Store (in the right endianness) the elements to memory.
1390 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1391 // Ignore undef elements.
1392 if (Node->getOperand(i).isUndef()) continue;
1393
1394 unsigned Offset = TypeByteSize*i;
1395
1396 SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1397 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1398
1399 // If the destination vector element type is narrower than the source
1400 // element type, only store the bits necessary.
1401 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1402 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1403 Node->getOperand(i), Idx,
1404 PtrInfo.getWithOffset(Offset), EltVT));
1405 } else
1406 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1407 Idx, PtrInfo.getWithOffset(Offset)));
1408 }
1409
1410 SDValue StoreChain;
1411 if (!Stores.empty()) // Not all undef elements?
1412 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1413 else
1414 StoreChain = DAG.getEntryNode();
1415
1416 // Result is a load from the stack slot.
1417 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1418}
1419
1420/// Bitcast a floating-point value to an integer value. Only bitcast the part
1421/// containing the sign bit if the target has no integer value capable of
1422/// holding all bits of the floating-point value.
1423void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1424 const SDLoc &DL,
1425 SDValue Value) const {
1426 EVT FloatVT = Value.getValueType();
1427 unsigned NumBits = FloatVT.getSizeInBits();
1428 State.FloatVT = FloatVT;
1429 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1430 // Convert to an integer of the same size.
1431 if (TLI.isTypeLegal(IVT)) {
1432 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1433 State.SignMask = APInt::getSignMask(NumBits);
1434 State.SignBit = NumBits - 1;
1435 return;
1436 }
1437
1438 auto &DataLayout = DAG.getDataLayout();
1439 // Store the float to memory, then load the sign part out as an integer.
1440 MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1441 // First create a temporary that is aligned for both the load and store.
1442 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1443 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1444 // Then store the float to it.
1445 State.FloatPtr = StackPtr;
1446 MachineFunction &MF = DAG.getMachineFunction();
1447 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1448 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1449 State.FloatPointerInfo);
1450
1451 SDValue IntPtr;
1452 if (DataLayout.isBigEndian()) {
1453 assert(FloatVT.isByteSized() && "Unsupported floating point type!")((FloatVT.isByteSized() && "Unsupported floating point type!"
) ? static_cast<void> (0) : __assert_fail ("FloatVT.isByteSized() && \"Unsupported floating point type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1453, __PRETTY_FUNCTION__))
;
1454 // Load out a legal integer with the same sign bit as the float.
1455 IntPtr = StackPtr;
1456 State.IntPointerInfo = State.FloatPointerInfo;
1457 } else {
1458 // Advance the pointer so that the loaded byte will contain the sign bit.
1459 unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
1460 IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
1461 DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
1462 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1463 ByteOffset);
1464 }
1465
1466 State.IntPtr = IntPtr;
1467 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1468 State.IntPointerInfo, MVT::i8);
1469 State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
1470 State.SignBit = 7;
1471}
1472
1473/// Replace the integer value produced by getSignAsIntValue() with a new value
1474/// and cast the result back to a floating-point type.
1475SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1476 const SDLoc &DL,
1477 SDValue NewIntValue) const {
1478 if (!State.Chain)
1479 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1480
1481 // Override the part containing the sign bit in the value stored on the stack.
1482 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1483 State.IntPointerInfo, MVT::i8);
1484 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1485 State.FloatPointerInfo);
1486}
1487
1488SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1489 SDLoc DL(Node);
1490 SDValue Mag = Node->getOperand(0);
1491 SDValue Sign = Node->getOperand(1);
1492
1493 // Get sign bit into an integer value.
1494 FloatSignAsInt SignAsInt;
1495 getSignAsIntValue(SignAsInt, DL, Sign);
1496
1497 EVT IntVT = SignAsInt.IntValue.getValueType();
1498 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1499 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1500 SignMask);
1501
1502 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1503 EVT FloatVT = Mag.getValueType();
1504 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1505 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1506 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1507 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1508 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1509 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1510 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1511 }
1512
1513 // Transform Mag value to integer, and clear the sign bit.
1514 FloatSignAsInt MagAsInt;
1515 getSignAsIntValue(MagAsInt, DL, Mag);
1516 EVT MagVT = MagAsInt.IntValue.getValueType();
1517 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1518 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1519 ClearSignMask);
1520
1521 // Get the signbit at the right position for MagAsInt.
1522 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1523 EVT ShiftVT = IntVT;
1524 if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
1525 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1526 ShiftVT = MagVT;
1527 }
1528 if (ShiftAmount > 0) {
1529 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1530 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1531 } else if (ShiftAmount < 0) {
1532 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1533 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1534 }
1535 if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
1536 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1537 }
1538
1539 // Store the part with the modified sign and convert back to float.
1540 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1541 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1542}
1543
1544SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1545 SDLoc DL(Node);
1546 SDValue Value = Node->getOperand(0);
1547
1548 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1549 EVT FloatVT = Value.getValueType();
1550 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1551 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1552 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1553 }
1554
1555 // Transform value to integer, clear the sign bit and transform back.
1556 FloatSignAsInt ValueAsInt;
1557 getSignAsIntValue(ValueAsInt, DL, Value);
1558 EVT IntVT = ValueAsInt.IntValue.getValueType();
1559 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1560 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1561 ClearSignMask);
1562 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1563}
1564
1565void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1566 SmallVectorImpl<SDValue> &Results) {
1567 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1568 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"((SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? static_cast
<void> (0) : __assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1569, __PRETTY_FUNCTION__))
1569 " not tell us which reg is the stack pointer!")((SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? static_cast
<void> (0) : __assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1569, __PRETTY_FUNCTION__))
;
1570 SDLoc dl(Node);
1571 EVT VT = Node->getValueType(0);
1572 SDValue Tmp1 = SDValue(Node, 0);
1573 SDValue Tmp2 = SDValue(Node, 1);
1574 SDValue Tmp3 = Node->getOperand(2);
1575 SDValue Chain = Tmp1.getOperand(0);
1576
1577 // Chain the dynamic stack allocation so that it doesn't modify the stack
1578 // pointer when other instructions are using the stack.
1579 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1580
1581 SDValue Size = Tmp2.getOperand(1);
1582 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1583 Chain = SP.getValue(1);
1584 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1585 unsigned StackAlign =
1586 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1587 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1588 if (Align > StackAlign)
1589 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1590 DAG.getConstant(-(uint64_t)Align, dl, VT));
1591 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1592
1593 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1594 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1595
1596 Results.push_back(Tmp1);
1597 Results.push_back(Tmp2);
1598}
1599
1600/// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1601/// target.
1602///
1603/// If the SETCC has been legalized using AND / OR, then the legalized node
1604/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1605/// will be set to false.
1606///
1607/// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1608/// then the values of LHS and RHS will be swapped, CC will be set to the
1609/// new condition, and NeedInvert will be set to false.
1610///
1611/// If the SETCC has been legalized using the inverse condcode, then LHS and
1612/// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1613/// will be set to true. The caller must invert the result of the SETCC with
1614/// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1615/// of a true/false result.
1616///
1617/// \returns true if the SetCC has been legalized, false if it hasn't.
1618bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1619 SDValue &RHS, SDValue &CC,
1620 bool &NeedInvert,
1621 const SDLoc &dl) {
1622 MVT OpVT = LHS.getSimpleValueType();
1623 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1624 NeedInvert = false;
1625 bool NeedSwap = false;
1626 switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1627 default: llvm_unreachable("Unknown condition code action!")::llvm::llvm_unreachable_internal("Unknown condition code action!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1627)
;
1628 case TargetLowering::Legal:
1629 // Nothing to do.
1630 break;
1631 case TargetLowering::Expand: {
1632 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
1633 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1634 std::swap(LHS, RHS);
1635 CC = DAG.getCondCode(InvCC);
1636 return true;
1637 }
1638 // Swapping operands didn't work. Try inverting the condition.
1639 InvCC = getSetCCInverse(CCCode, OpVT.isInteger());
1640 if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1641 // If inverting the condition is not enough, try swapping operands
1642 // on top of it.
1643 InvCC = ISD::getSetCCSwappedOperands(InvCC);
1644 NeedSwap = true;
1645 }
1646 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1647 CC = DAG.getCondCode(InvCC);
1648 NeedInvert = true;
1649 if (NeedSwap)
1650 std::swap(LHS, RHS);
1651 return true;
1652 }
1653
1654 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1655 unsigned Opc = 0;
1656 switch (CCCode) {
1657 default: llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1657)
;
1658 case ISD::SETO:
1659 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT)((TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && "If SETO is expanded, SETOEQ must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1660, __PRETTY_FUNCTION__))
1660 && "If SETO is expanded, SETOEQ must be legal!")((TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && "If SETO is expanded, SETOEQ must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1660, __PRETTY_FUNCTION__))
;
1661 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1662 case ISD::SETUO:
1663 assert(TLI.isCondCodeLegal(ISD::SETUNE, OpVT)((TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && "If SETUO is expanded, SETUNE must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1664, __PRETTY_FUNCTION__))
1664 && "If SETUO is expanded, SETUNE must be legal!")((TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && "If SETUO is expanded, SETUNE must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1664, __PRETTY_FUNCTION__))
;
1665 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break;
1666 case ISD::SETOEQ:
1667 case ISD::SETOGT:
1668 case ISD::SETOGE:
1669 case ISD::SETOLT:
1670 case ISD::SETOLE:
1671 case ISD::SETONE:
1672 case ISD::SETUEQ:
1673 case ISD::SETUNE:
1674 case ISD::SETUGT:
1675 case ISD::SETUGE:
1676 case ISD::SETULT:
1677 case ISD::SETULE:
1678 // If we are floating point, assign and break, otherwise fall through.
1679 if (!OpVT.isInteger()) {
1680 // We can use the 4th bit to tell if we are the unordered
1681 // or ordered version of the opcode.
1682 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
1683 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
1684 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1685 break;
1686 }
1687 // Fallthrough if we are unsigned integer.
1688 LLVM_FALLTHROUGH[[clang::fallthrough]];
1689 case ISD::SETLE:
1690 case ISD::SETGT:
1691 case ISD::SETGE:
1692 case ISD::SETLT:
1693 case ISD::SETNE:
1694 case ISD::SETEQ:
1695 // If all combinations of inverting the condition and swapping operands
1696 // didn't work then we have no means to expand the condition.
1697 llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1697)
;
1698 }
1699
1700 SDValue SetCC1, SetCC2;
1701 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
1702 // If we aren't the ordered or unorder operation,
1703 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1704 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1705 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1706 } else {
1707 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1708 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1709 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1710 }
1711 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1712 RHS = SDValue();
1713 CC = SDValue();
1714 return true;
1715 }
1716 }
1717 return false;
1718}
1719
1720/// Emit a store/load combination to the stack. This stores
1721/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1722/// a load from the stack slot to DestVT, extending it if needed.
1723/// The resultant code need not be legal.
1724SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1725 EVT DestVT, const SDLoc &dl) {
1726 // Create the stack frame object.
1727 unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1728 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1729 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1730
1731 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1732 int SPFI = StackPtrFI->getIndex();
1733 MachinePointerInfo PtrInfo =
1734 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1735
1736 unsigned SrcSize = SrcOp.getValueSizeInBits();
1737 unsigned SlotSize = SlotVT.getSizeInBits();
1738 unsigned DestSize = DestVT.getSizeInBits();
1739 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1740 unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1741
1742 // Emit a store to the stack slot. Use a truncstore if the input value is
1743 // later than DestVT.
1744 SDValue Store;
1745
1746 if (SrcSize > SlotSize)
1747 Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
1748 SlotVT, SrcAlign);
1749 else {
1750 assert(SrcSize == SlotSize && "Invalid store")((SrcSize == SlotSize && "Invalid store") ? static_cast
<void> (0) : __assert_fail ("SrcSize == SlotSize && \"Invalid store\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1750, __PRETTY_FUNCTION__))
;
1751 Store =
1752 DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1753 }
1754
1755 // Result is a load from the stack slot.
1756 if (SlotSize == DestSize)
1757 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1758
1759 assert(SlotSize < DestSize && "Unknown extension!")((SlotSize < DestSize && "Unknown extension!") ? static_cast
<void> (0) : __assert_fail ("SlotSize < DestSize && \"Unknown extension!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1759, __PRETTY_FUNCTION__))
;
1760 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1761 DestAlign);
1762}
1763
1764SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1765 SDLoc dl(Node);
1766 // Create a vector sized/aligned stack slot, store the value to element #0,
1767 // then load the whole vector back out.
1768 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1769
1770 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1771 int SPFI = StackPtrFI->getIndex();
1772
1773 SDValue Ch = DAG.getTruncStore(
1774 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1775 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1776 Node->getValueType(0).getVectorElementType());
1777 return DAG.getLoad(
1778 Node->getValueType(0), dl, Ch, StackPtr,
1779 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1780}
1781
1782static bool
1783ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1784 const TargetLowering &TLI, SDValue &Res) {
1785 unsigned NumElems = Node->getNumOperands();
1786 SDLoc dl(Node);
1787 EVT VT = Node->getValueType(0);
1788
1789 // Try to group the scalars into pairs, shuffle the pairs together, then
1790 // shuffle the pairs of pairs together, etc. until the vector has
1791 // been built. This will work only if all of the necessary shuffle masks
1792 // are legal.
1793
1794 // We do this in two phases; first to check the legality of the shuffles,
1795 // and next, assuming that all shuffles are legal, to create the new nodes.
1796 for (int Phase = 0; Phase < 2; ++Phase) {
1797 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1798 NewIntermedVals;
1799 for (unsigned i = 0; i < NumElems; ++i) {
1800 SDValue V = Node->getOperand(i);
1801 if (V.isUndef())
1802 continue;
1803
1804 SDValue Vec;
1805 if (Phase)
1806 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1807 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1808 }
1809
1810 while (IntermedVals.size() > 2) {
1811 NewIntermedVals.clear();
1812 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1813 // This vector and the next vector are shuffled together (simply to
1814 // append the one to the other).
1815 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1816
1817 SmallVector<int, 16> FinalIndices;
1818 FinalIndices.reserve(IntermedVals[i].second.size() +
1819 IntermedVals[i+1].second.size());
1820
1821 int k = 0;
1822 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1823 ++j, ++k) {
1824 ShuffleVec[k] = j;
1825 FinalIndices.push_back(IntermedVals[i].second[j]);
1826 }
1827 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1828 ++j, ++k) {
1829 ShuffleVec[k] = NumElems + j;
1830 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1831 }
1832
1833 SDValue Shuffle;
1834 if (Phase)
1835 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1836 IntermedVals[i+1].first,
1837 ShuffleVec);
1838 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1839 return false;
1840 NewIntermedVals.push_back(
1841 std::make_pair(Shuffle, std::move(FinalIndices)));
1842 }
1843
1844 // If we had an odd number of defined values, then append the last
1845 // element to the array of new vectors.
1846 if ((IntermedVals.size() & 1) != 0)
1847 NewIntermedVals.push_back(IntermedVals.back());
1848
1849 IntermedVals.swap(NewIntermedVals);
1850 }
1851
1852 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&((IntermedVals.size() <= 2 && IntermedVals.size() >
0 && "Invalid number of intermediate vectors") ? static_cast
<void> (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1853, __PRETTY_FUNCTION__))
1853 "Invalid number of intermediate vectors")((IntermedVals.size() <= 2 && IntermedVals.size() >
0 && "Invalid number of intermediate vectors") ? static_cast
<void> (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1853, __PRETTY_FUNCTION__))
;
1854 SDValue Vec1 = IntermedVals[0].first;
1855 SDValue Vec2;
1856 if (IntermedVals.size() > 1)
1857 Vec2 = IntermedVals[1].first;
1858 else if (Phase)
1859 Vec2 = DAG.getUNDEF(VT);
1860
1861 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1862 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1863 ShuffleVec[IntermedVals[0].second[i]] = i;
1864 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1865 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1866
1867 if (Phase)
1868 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1869 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1870 return false;
1871 }
1872
1873 return true;
1874}
1875
1876/// Expand a BUILD_VECTOR node on targets that don't
1877/// support the operation, but do support the resultant vector type.
1878SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1879 unsigned NumElems = Node->getNumOperands();
1880 SDValue Value1, Value2;
1881 SDLoc dl(Node);
1882 EVT VT = Node->getValueType(0);
1883 EVT OpVT = Node->getOperand(0).getValueType();
1884 EVT EltVT = VT.getVectorElementType();
1885
1886 // If the only non-undef value is the low element, turn this into a
1887 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1888 bool isOnlyLowElement = true;
1889 bool MoreThanTwoValues = false;
1890 bool isConstant = true;
1891 for (unsigned i = 0; i < NumElems; ++i) {
1892 SDValue V = Node->getOperand(i);
1893 if (V.isUndef())
1894 continue;
1895 if (i > 0)
1896 isOnlyLowElement = false;
1897 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1898 isConstant = false;
1899
1900 if (!Value1.getNode()) {
1901 Value1 = V;
1902 } else if (!Value2.getNode()) {
1903 if (V != Value1)
1904 Value2 = V;
1905 } else if (V != Value1 && V != Value2) {
1906 MoreThanTwoValues = true;
1907 }
1908 }
1909
1910 if (!Value1.getNode())
1911 return DAG.getUNDEF(VT);
1912
1913 if (isOnlyLowElement)
1914 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1915
1916 // If all elements are constants, create a load from the constant pool.
1917 if (isConstant) {
1918 SmallVector<Constant*, 16> CV;
1919 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1920 if (ConstantFPSDNode *V =
1921 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1922 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1923 } else if (ConstantSDNode *V =
1924 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1925 if (OpVT==EltVT)
1926 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1927 else {
1928 // If OpVT and EltVT don't match, EltVT is not legal and the
1929 // element values have been promoted/truncated earlier. Undo this;
1930 // we don't want a v16i8 to become a v16i32 for example.
1931 const ConstantInt *CI = V->getConstantIntValue();
1932 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1933 CI->getZExtValue()));
1934 }
1935 } else {
1936 assert(Node->getOperand(i).isUndef())((Node->getOperand(i).isUndef()) ? static_cast<void>
(0) : __assert_fail ("Node->getOperand(i).isUndef()", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1936, __PRETTY_FUNCTION__))
;
1937 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1938 CV.push_back(UndefValue::get(OpNTy));
1939 }
1940 }
1941 Constant *CP = ConstantVector::get(CV);
1942 SDValue CPIdx =
1943 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1944 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1945 return DAG.getLoad(
1946 VT, dl, DAG.getEntryNode(), CPIdx,
1947 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1948 Alignment);
1949 }
1950
1951 SmallSet<SDValue, 16> DefinedValues;
1952 for (unsigned i = 0; i < NumElems; ++i) {
1953 if (Node->getOperand(i).isUndef())
1954 continue;
1955 DefinedValues.insert(Node->getOperand(i));
1956 }
1957
1958 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1959 if (!MoreThanTwoValues) {
1960 SmallVector<int, 8> ShuffleVec(NumElems, -1);
1961 for (unsigned i = 0; i < NumElems; ++i) {
1962 SDValue V = Node->getOperand(i);
1963 if (V.isUndef())
1964 continue;
1965 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1966 }
1967 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1968 // Get the splatted value into the low element of a vector register.
1969 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1970 SDValue Vec2;
1971 if (Value2.getNode())
1972 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1973 else
1974 Vec2 = DAG.getUNDEF(VT);
1975
1976 // Return shuffle(LowValVec, undef, <0,0,0,0>)
1977 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1978 }
1979 } else {
1980 SDValue Res;
1981 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1982 return Res;
1983 }
1984 }
1985
1986 // Otherwise, we can't handle this case efficiently.
1987 return ExpandVectorBuildThroughStack(Node);
1988}
1989
1990// Expand a node into a call to a libcall. If the result value
1991// does not fit into a register, return the lo part and set the hi part to the
1992// by-reg argument. If it does fit into a single register, return the result
1993// and leave the Hi part unset.
1994SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
1995 bool isSigned) {
1996 TargetLowering::ArgListTy Args;
1997 TargetLowering::ArgListEntry Entry;
1998 for (const SDValue &Op : Node->op_values()) {
1999 EVT ArgVT = Op.getValueType();
2000 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2001 Entry.Node = Op;
2002 Entry.Ty = ArgTy;
2003 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2004 Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2005 Args.push_back(Entry);
2006 }
2007 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2008 TLI.getPointerTy(DAG.getDataLayout()));
2009
2010 EVT RetVT = Node->getValueType(0);
2011 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2012
2013 // By default, the input chain to this libcall is the entry node of the
2014 // function. If the libcall is going to be emitted as a tail call then
2015 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2016 // node which is being folded has a non-entry input chain.
2017 SDValue InChain = DAG.getEntryNode();
2018
2019 // isTailCall may be true since the callee does not reference caller stack
2020 // frame. Check if it's in the right position and that the return types match.
2021 SDValue TCChain = InChain;
2022 const Function &F = DAG.getMachineFunction().getFunction();
2023 bool isTailCall =
2024 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2025 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2026 if (isTailCall)
2027 InChain = TCChain;
2028
2029 TargetLowering::CallLoweringInfo CLI(DAG);
2030 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2031 CLI.setDebugLoc(SDLoc(Node))
2032 .setChain(InChain)
2033 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2034 std::move(Args))
2035 .setTailCall(isTailCall)
2036 .setSExtResult(signExtend)
2037 .setZExtResult(!signExtend)
2038 .setIsPostTypeLegalization(true);
2039
2040 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2041
2042 if (!CallInfo.second.getNode()) {
2043 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Created tailcall: "; DAG.
getRoot().dump(); } } while (false)
;
2044 // It's a tailcall, return the chain (which is the DAG root).
2045 return DAG.getRoot();
2046 }
2047
2048 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Created libcall: "; CallInfo
.first.dump(); } } while (false)
;
2049 return CallInfo.first;
2050}
2051
2052/// Generate a libcall taking the given operands as arguments
2053/// and returning a result of type RetVT.
2054SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
2055 const SDValue *Ops, unsigned NumOps,
2056 bool isSigned, const SDLoc &dl) {
2057 TargetLowering::ArgListTy Args;
2058 Args.reserve(NumOps);
2059
2060 TargetLowering::ArgListEntry Entry;
2061 for (unsigned i = 0; i != NumOps; ++i) {
2062 Entry.Node = Ops[i];
2063 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
2064 Entry.IsSExt = isSigned;
2065 Entry.IsZExt = !isSigned;
2066 Args.push_back(Entry);
2067 }
2068 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2069 TLI.getPointerTy(DAG.getDataLayout()));
2070
2071 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2072
2073 TargetLowering::CallLoweringInfo CLI(DAG);
2074 CLI.setDebugLoc(dl)
2075 .setChain(DAG.getEntryNode())
2076 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2077 std::move(Args))
2078 .setSExtResult(isSigned)
2079 .setZExtResult(!isSigned)
2080 .setIsPostTypeLegalization(true);
2081
2082 std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
2083
2084 return CallInfo.first;
2085}
2086
2087// Expand a node into a call to a libcall. Similar to
2088// ExpandLibCall except that the first operand is the in-chain.
2089std::pair<SDValue, SDValue>
2090SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2091 SDNode *Node,
2092 bool isSigned) {
2093 SDValue InChain = Node->getOperand(0);
2094
2095 TargetLowering::ArgListTy Args;
2096 TargetLowering::ArgListEntry Entry;
2097 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
2098 EVT ArgVT = Node->getOperand(i).getValueType();
2099 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2100 Entry.Node = Node->getOperand(i);
2101 Entry.Ty = ArgTy;
2102 Entry.IsSExt = isSigned;
2103 Entry.IsZExt = !isSigned;
2104 Args.push_back(Entry);
2105 }
2106 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2107 TLI.getPointerTy(DAG.getDataLayout()));
2108
2109 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2110
2111 TargetLowering::CallLoweringInfo CLI(DAG);
2112 CLI.setDebugLoc(SDLoc(Node))
2113 .setChain(InChain)
2114 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2115 std::move(Args))
2116 .setSExtResult(isSigned)
2117 .setZExtResult(!isSigned);
2118
2119 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2120
2121 return CallInfo;
2122}
2123
2124SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2125 RTLIB::Libcall Call_F32,
2126 RTLIB::Libcall Call_F64,
2127 RTLIB::Libcall Call_F80,
2128 RTLIB::Libcall Call_F128,
2129 RTLIB::Libcall Call_PPCF128) {
2130 if (Node->isStrictFPOpcode())
2131 Node = DAG.mutateStrictFPToFP(Node);
2132
2133 RTLIB::Libcall LC;
2134 switch (Node->getSimpleValueType(0).SimpleTy) {
2135 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2135)
;
2136 case MVT::f32: LC = Call_F32; break;
2137 case MVT::f64: LC = Call_F64; break;
2138 case MVT::f80: LC = Call_F80; break;
2139 case MVT::f128: LC = Call_F128; break;
2140 case MVT::ppcf128: LC = Call_PPCF128; break;
2141 }
2142 return ExpandLibCall(LC, Node, false);
2143}
2144
2145SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2146 RTLIB::Libcall Call_I8,
2147 RTLIB::Libcall Call_I16,
2148 RTLIB::Libcall Call_I32,
2149 RTLIB::Libcall Call_I64,
2150 RTLIB::Libcall Call_I128) {
2151 RTLIB::Libcall LC;
2152 switch (Node->getSimpleValueType(0).SimpleTy) {
2153 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2153)
;
2154 case MVT::i8: LC = Call_I8; break;
2155 case MVT::i16: LC = Call_I16; break;
2156 case MVT::i32: LC = Call_I32; break;
2157 case MVT::i64: LC = Call_I64; break;
2158 case MVT::i128: LC = Call_I128; break;
2159 }
2160 return ExpandLibCall(LC, Node, isSigned);
2161}
2162
2163/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2164void
2165SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2166 SmallVectorImpl<SDValue> &Results) {
2167 unsigned Opcode = Node->getOpcode();
2168 bool isSigned = Opcode == ISD::SDIVREM;
2169
2170 RTLIB::Libcall LC;
2171 switch (Node->getSimpleValueType(0).SimpleTy) {
2172 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2172)
;
2173 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2174 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2175 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2176 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2177 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2178 }
2179
2180 // The input chain to this libcall is the entry node of the function.
2181 // Legalizing the call will automatically add the previous call to the
2182 // dependence.
2183 SDValue InChain = DAG.getEntryNode();
2184
2185 EVT RetVT = Node->getValueType(0);
2186 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2187
2188 TargetLowering::ArgListTy Args;
2189 TargetLowering::ArgListEntry Entry;
2190 for (const SDValue &Op : Node->op_values()) {
2191 EVT ArgVT = Op.getValueType();
2192 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2193 Entry.Node = Op;
2194 Entry.Ty = ArgTy;
2195 Entry.IsSExt = isSigned;
2196 Entry.IsZExt = !isSigned;
2197 Args.push_back(Entry);
2198 }
2199
2200 // Also pass the return address of the remainder.
2201 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2202 Entry.Node = FIPtr;
2203 Entry.Ty = RetTy->getPointerTo();
2204 Entry.IsSExt = isSigned;
2205 Entry.IsZExt = !isSigned;
2206 Args.push_back(Entry);
2207
2208 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2209 TLI.getPointerTy(DAG.getDataLayout()));
2210
2211 SDLoc dl(Node);
2212 TargetLowering::CallLoweringInfo CLI(DAG);
2213 CLI.setDebugLoc(dl)
2214 .setChain(InChain)
2215 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2216 std::move(Args))
2217 .setSExtResult(isSigned)
2218 .setZExtResult(!isSigned);
2219
2220 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2221
2222 // Remainder is loaded back from the stack frame.
2223 SDValue Rem =
2224 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2225 Results.push_back(CallInfo.first);
2226 Results.push_back(Rem);
2227}
2228
2229/// Return true if sincos libcall is available.
2230static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2231 RTLIB::Libcall LC;
2232 switch (Node->getSimpleValueType(0).SimpleTy) {
2233 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2233)
;
2234 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2235 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2236 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2237 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2238 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2239 }
2240 return TLI.getLibcallName(LC) != nullptr;
2241}
2242
2243/// Only issue sincos libcall if both sin and cos are needed.
2244static bool useSinCos(SDNode *Node) {
2245 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2246 ? ISD::FCOS : ISD::FSIN;
2247
2248 SDValue Op0 = Node->getOperand(0);
2249 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2250 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2251 SDNode *User = *UI;
2252 if (User == Node)
2253 continue;
2254 // The other user might have been turned into sincos already.
2255 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2256 return true;
2257 }
2258 return false;
2259}
2260
2261/// Issue libcalls to sincos to compute sin / cos pairs.
2262void
2263SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2264 SmallVectorImpl<SDValue> &Results) {
2265 RTLIB::Libcall LC;
2266 switch (Node->getSimpleValueType(0).SimpleTy) {
2267 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2267)
;
2268 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2269 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2270 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2271 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2272 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2273 }
2274
2275 // The input chain to this libcall is the entry node of the function.
2276 // Legalizing the call will automatically add the previous call to the
2277 // dependence.
2278 SDValue InChain = DAG.getEntryNode();
2279
2280 EVT RetVT = Node->getValueType(0);
2281 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2282
2283 TargetLowering::ArgListTy Args;
2284 TargetLowering::ArgListEntry Entry;
2285
2286 // Pass the argument.
2287 Entry.Node = Node->getOperand(0);
2288 Entry.Ty = RetTy;
2289 Entry.IsSExt = false;
2290 Entry.IsZExt = false;
2291 Args.push_back(Entry);
2292
2293 // Pass the return address of sin.
2294 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2295 Entry.Node = SinPtr;
2296 Entry.Ty = RetTy->getPointerTo();
2297 Entry.IsSExt = false;
2298 Entry.IsZExt = false;
2299 Args.push_back(Entry);
2300
2301 // Also pass the return address of the cos.
2302 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2303 Entry.Node = CosPtr;
2304 Entry.Ty = RetTy->getPointerTo();
2305 Entry.IsSExt = false;
2306 Entry.IsZExt = false;
2307 Args.push_back(Entry);
2308
2309 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2310 TLI.getPointerTy(DAG.getDataLayout()));
2311
2312 SDLoc dl(Node);
2313 TargetLowering::CallLoweringInfo CLI(DAG);
2314 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2315 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2316 std::move(Args));
2317
2318 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2319
2320 Results.push_back(
2321 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2322 Results.push_back(
2323 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2324}
2325
2326/// This function is responsible for legalizing a
2327/// INT_TO_FP operation of the specified operand when the target requests that
2328/// we expand it. At this point, we know that the result and operand types are
2329/// legal for the target.
2330SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2331 EVT DestVT,
2332 const SDLoc &dl) {
2333 EVT SrcVT = Op0.getValueType();
2334
2335 // TODO: Should any fast-math-flags be set for the created nodes?
2336 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing INT_TO_FP\n"; }
} while (false)
;
2337 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
2338 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double "
"expansion\n"; } } while (false)
2339 "expansion\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double "
"expansion\n"; } } while (false)
;
2340
2341 // Get the stack frame index of a 8 byte buffer.
2342 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2343
2344 // word offset constant for Hi/Lo address computation
2345 SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2346 StackSlot.getValueType());
2347 // set up Hi and Lo (into buffer) address based on endian
2348 SDValue Hi = StackSlot;
2349 SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2350 StackSlot, WordOff);
2351 if (DAG.getDataLayout().isLittleEndian())
2352 std::swap(Hi, Lo);
2353
2354 // if signed map to unsigned space
2355 SDValue Op0Mapped;
2356 if (isSigned) {
2357 // constant used to invert sign bit (signed to unsigned mapping)
2358 SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2359 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2360 } else {
2361 Op0Mapped = Op0;
2362 }
2363 // store the lo of the constructed double - based on integer input
2364 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2365 MachinePointerInfo());
2366 // initial hi portion of constructed double
2367 SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2368 // store the hi of the constructed double - biased exponent
2369 SDValue Store2 =
2370 DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
2371 // load the constructed double
2372 SDValue Load =
2373 DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
2374 // FP constant to bias correct the final result
2375 SDValue Bias = DAG.getConstantFP(isSigned ?
2376 BitsToDouble(0x4330000080000000ULL) :
2377 BitsToDouble(0x4330000000000000ULL),
2378 dl, MVT::f64);
2379 // subtract the bias
2380 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2381 // final result
2382 SDValue Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2383 return Result;
2384 }
2385 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet")((!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"
) ? static_cast<void> (0) : __assert_fail ("!isSigned && \"Legalize cannot Expand SINT_TO_FP for i64 yet\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2385, __PRETTY_FUNCTION__))
;
2386 // Code below here assumes !isSigned without checking again.
2387
2388 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2389
2390 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2391 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2392 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2393 Four = DAG.getIntPtrConstant(4, dl);
2394 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2395 SignSet, Four, Zero);
2396
2397 // If the sign bit of the integer is set, the large number will be treated
2398 // as a negative number. To counteract this, the dynamic code adds an
2399 // offset depending on the data type.
2400 uint64_t FF;
2401 switch (SrcVT.getSimpleVT().SimpleTy) {
2402 default: llvm_unreachable("Unsupported integer type!")::llvm::llvm_unreachable_internal("Unsupported integer type!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2402)
;
2403 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2404 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2405 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2406 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2407 }
2408 if (DAG.getDataLayout().isLittleEndian())
2409 FF <<= 32;
2410 Constant *FudgeFactor = ConstantInt::get(
2411 Type::getInt64Ty(*DAG.getContext()), FF);
2412
2413 SDValue CPIdx =
2414 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2415 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2416 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2417 Alignment = std::min(Alignment, 4u);
2418 SDValue FudgeInReg;
2419 if (DestVT == MVT::f32)
2420 FudgeInReg = DAG.getLoad(
2421 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2422 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2423 Alignment);
2424 else {
2425 SDValue Load = DAG.getExtLoad(
2426 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2427 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2428 Alignment);
2429 HandleSDNode Handle(Load);
2430 LegalizeOp(Load.getNode());
2431 FudgeInReg = Handle.getValue();
2432 }
2433
2434 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2435}
2436
2437/// This function is responsible for legalizing a
2438/// *INT_TO_FP operation of the specified operand when the target requests that
2439/// we promote it. At this point, we know that the result and operand types are
2440/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2441/// operation that takes a larger input.
2442SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2443 bool isSigned,
2444 const SDLoc &dl) {
2445 // First step, figure out the appropriate *INT_TO_FP operation to use.
2446 EVT NewInTy = LegalOp.getValueType();
2447
2448 unsigned OpToUse = 0;
2449
2450 // Scan for the appropriate larger type to use.
2451 while (true) {
2452 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2453 assert(NewInTy.isInteger() && "Ran out of possibilities!")((NewInTy.isInteger() && "Ran out of possibilities!")
? static_cast<void> (0) : __assert_fail ("NewInTy.isInteger() && \"Ran out of possibilities!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2453, __PRETTY_FUNCTION__))
;
2454
2455 // If the target supports SINT_TO_FP of this type, use it.
2456 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2457 OpToUse = ISD::SINT_TO_FP;
2458 break;
2459 }
2460 if (isSigned) continue;
2461
2462 // If the target supports UINT_TO_FP of this type, use it.
2463 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2464 OpToUse = ISD::UINT_TO_FP;
2465 break;
2466 }
2467
2468 // Otherwise, try a larger type.
2469 }
2470
2471 // Okay, we found the operation and type to use. Zero extend our input to the
2472 // desired type then run the operation on it.
2473 return DAG.getNode(OpToUse, dl, DestVT,
2474 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2475 dl, NewInTy, LegalOp));
2476}
2477
2478/// This function is responsible for legalizing a
2479/// FP_TO_*INT operation of the specified operand when the target requests that
2480/// we promote it. At this point, we know that the result and operand types are
2481/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2482/// operation that returns a larger result.
2483SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2484 bool isSigned,
2485 const SDLoc &dl) {
2486 // First step, figure out the appropriate FP_TO*INT operation to use.
2487 EVT NewOutTy = DestVT;
2488
2489 unsigned OpToUse = 0;
2490
2491 // Scan for the appropriate larger type to use.
2492 while (true) {
2493 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2494 assert(NewOutTy.isInteger() && "Ran out of possibilities!")((NewOutTy.isInteger() && "Ran out of possibilities!"
) ? static_cast<void> (0) : __assert_fail ("NewOutTy.isInteger() && \"Ran out of possibilities!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2494, __PRETTY_FUNCTION__))
;
2495
2496 // A larger signed type can hold all unsigned values of the requested type,
2497 // so using FP_TO_SINT is valid
2498 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2499 OpToUse = ISD::FP_TO_SINT;
2500 break;
2501 }
2502
2503 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2504 if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2505 OpToUse = ISD::FP_TO_UINT;
2506 break;
2507 }
2508
2509 // Otherwise, try a larger type.
2510 }
2511
2512 // Okay, we found the operation and type to use.
2513 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2514
2515 // Truncate the result of the extended FP_TO_*INT operation to the desired
2516 // size.
2517 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2518}
2519
2520/// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2521SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2522 EVT VT = Op.getValueType();
2523 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2524 unsigned Sz = VT.getScalarSizeInBits();
2525
2526 SDValue Tmp, Tmp2, Tmp3;
2527
2528 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2529 // and finally the i1 pairs.
2530 // TODO: We can easily support i4/i2 legal types if any target ever does.
2531 if (Sz >= 8 && isPowerOf2_32(Sz)) {
2532 // Create the masks - repeating the pattern every byte.
2533 APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0);
2534 APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0);
2535 for (unsigned J = 0; J != Sz; J += 8) {
2536 MaskHi4 = MaskHi4 | (0xF0ull << J);
2537 MaskLo4 = MaskLo4 | (0x0Full << J);
2538 MaskHi2 = MaskHi2 | (0xCCull << J);
2539 MaskLo2 = MaskLo2 | (0x33ull << J);
2540 MaskHi1 = MaskHi1 | (0xAAull << J);
2541 MaskLo1 = MaskLo1 | (0x55ull << J);
2542 }
2543
2544 // BSWAP if the type is wider than a single byte.
2545 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
2546
2547 // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2548 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
2549 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
2550 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, SHVT));
2551 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT));
2552 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2553
2554 // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2555 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
2556 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
2557 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, SHVT));
2558 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT));
2559 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2560
2561 // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2562 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
2563 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
2564 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, SHVT));
2565 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT));
2566 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2567 return Tmp;
2568 }
2569
2570 Tmp = DAG.getConstant(0, dl, VT);
2571 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
2572 if (I < J)
2573 Tmp2 =
2574 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
2575 else
2576 Tmp2 =
2577 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
2578
2579 APInt Shift(Sz, 1);
2580 Shift <<= J;
2581 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
2582 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
2583 }
2584
2585 return Tmp;
2586}
2587
2588/// Open code the operations for BSWAP of the specified operation.
2589SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2590 EVT VT = Op.getValueType();
2591 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2592 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2593 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
2594 default: llvm_unreachable("Unhandled Expand type in BSWAP!")::llvm::llvm_unreachable_internal("Unhandled Expand type in BSWAP!"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2594)
;
2595 case MVT::i16:
2596 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2597 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2598 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2599 case MVT::i32:
2600 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2601 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2602 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2603 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2604 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2605 DAG.getConstant(0xFF0000, dl, VT));
2606 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2607 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2608 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2609 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2610 case MVT::i64:
2611 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2612 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2613 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2614 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2615 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2616 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2617 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2618 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2619 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2620 DAG.getConstant(255ULL<<48, dl, VT));
2621 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2622 DAG.getConstant(255ULL<<40, dl, VT));
2623 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2624 DAG.getConstant(255ULL<<32, dl, VT));
2625 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2626 DAG.getConstant(255ULL<<24, dl, VT));
2627 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2628 DAG.getConstant(255ULL<<16, dl, VT));
2629 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2630 DAG.getConstant(255ULL<<8 , dl, VT));
2631 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2632 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2633 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2634 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2635 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2636 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2637 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2638 }
2639}
2640
2641bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2642 LLVM_DEBUG(dbgs() << "Trying to expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to expand node\n";
} } while (false)
;
2643 SmallVector<SDValue, 8> Results;
2644 SDLoc dl(Node);
2645 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2646 bool NeedInvert;
2647 switch (Node->getOpcode()) {
2648 case ISD::CTPOP:
2649 if (TLI.expandCTPOP(Node, Tmp1, DAG))
2650 Results.push_back(Tmp1);
2651 break;
2652 case ISD::CTLZ:
2653 case ISD::CTLZ_ZERO_UNDEF:
2654 if (TLI.expandCTLZ(Node, Tmp1, DAG))
2655 Results.push_back(Tmp1);
2656 break;
2657 case ISD::CTTZ:
2658 case ISD::CTTZ_ZERO_UNDEF:
2659 if (TLI.expandCTTZ(Node, Tmp1, DAG))
2660 Results.push_back(Tmp1);
2661 break;
2662 case ISD::BITREVERSE:
2663 Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
2664 break;
2665 case ISD::BSWAP:
2666 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2667 break;
2668 case ISD::FRAMEADDR:
2669 case ISD::RETURNADDR:
2670 case ISD::FRAME_TO_ARGS_OFFSET:
2671 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2672 break;
2673 case ISD::EH_DWARF_CFA: {
2674 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2675 TLI.getPointerTy(DAG.getDataLayout()));
2676 SDValue Offset = DAG.getNode(ISD::ADD, dl,
2677 CfaArg.getValueType(),
2678 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2679 CfaArg.getValueType()),
2680 CfaArg);
2681 SDValue FA = DAG.getNode(
2682 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2683 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2684 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2685 FA, Offset));
2686 break;
2687 }
2688 case ISD::FLT_ROUNDS_:
2689 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2690 break;
2691 case ISD::EH_RETURN:
2692 case ISD::EH_LABEL:
2693 case ISD::PREFETCH:
2694 case ISD::VAEND:
2695 case ISD::EH_SJLJ_LONGJMP:
2696 // If the target didn't expand these, there's nothing to do, so just
2697 // preserve the chain and be done.
2698 Results.push_back(Node->getOperand(0));
2699 break;
2700 case ISD::READCYCLECOUNTER:
2701 // If the target didn't expand this, just return 'zero' and preserve the
2702 // chain.
2703 Results.append(Node->getNumValues() - 1,
2704 DAG.getConstant(0, dl, Node->getValueType(0)));
2705 Results.push_back(Node->getOperand(0));
2706 break;
2707 case ISD::EH_SJLJ_SETJMP:
2708 // If the target didn't expand this, just return 'zero' and preserve the
2709 // chain.
2710 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2711 Results.push_back(Node->getOperand(0));
2712 break;
2713 case ISD::ATOMIC_LOAD: {
2714 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2715 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2716 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2717 SDValue Swap = DAG.getAtomicCmpSwap(
2718 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2719 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2720 cast<AtomicSDNode>(Node)->getMemOperand());
2721 Results.push_back(Swap.getValue(0));
2722 Results.push_back(Swap.getValue(1));
2723 break;
2724 }
2725 case ISD::ATOMIC_STORE: {
2726 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2727 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2728 cast<AtomicSDNode>(Node)->getMemoryVT(),
2729 Node->getOperand(0),
2730 Node->getOperand(1), Node->getOperand(2),
2731 cast<AtomicSDNode>(Node)->getMemOperand());
2732 Results.push_back(Swap.getValue(1));
2733 break;
2734 }
2735 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2736 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2737 // splits out the success value as a comparison. Expanding the resulting
2738 // ATOMIC_CMP_SWAP will produce a libcall.
2739 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2740 SDValue Res = DAG.getAtomicCmpSwap(
2741 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2742 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2743 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2744
2745 SDValue ExtRes = Res;
2746 SDValue LHS = Res;
2747 SDValue RHS = Node->getOperand(1);
2748
2749 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2750 EVT OuterType = Node->getValueType(0);
2751 switch (TLI.getExtendForAtomicOps()) {
2752 case ISD::SIGN_EXTEND:
2753 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2754 DAG.getValueType(AtomicType));
2755 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2756 Node->getOperand(2), DAG.getValueType(AtomicType));
2757 ExtRes = LHS;
2758 break;
2759 case ISD::ZERO_EXTEND:
2760 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2761 DAG.getValueType(AtomicType));
2762 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2763 ExtRes = LHS;
2764 break;
2765 case ISD::ANY_EXTEND:
2766 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2767 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2768 break;
2769 default:
2770 llvm_unreachable("Invalid atomic op extension")::llvm::llvm_unreachable_internal("Invalid atomic op extension"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2770)
;
2771 }
2772
2773 SDValue Success =
2774 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2775
2776 Results.push_back(ExtRes.getValue(0));
2777 Results.push_back(Success);
2778 Results.push_back(Res.getValue(1));
2779 break;
2780 }
2781 case ISD::DYNAMIC_STACKALLOC:
2782 ExpandDYNAMIC_STACKALLOC(Node, Results);
2783 break;
2784 case ISD::MERGE_VALUES:
2785 for (unsigned i = 0; i < Node->getNumValues(); i++)
2786 Results.push_back(Node->getOperand(i));
2787 break;
2788 case ISD::UNDEF: {
2789 EVT VT = Node->getValueType(0);
2790 if (VT.isInteger())
2791 Results.push_back(DAG.getConstant(0, dl, VT));
2792 else {
2793 assert(VT.isFloatingPoint() && "Unknown value type!")((VT.isFloatingPoint() && "Unknown value type!") ? static_cast
<void> (0) : __assert_fail ("VT.isFloatingPoint() && \"Unknown value type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2793, __PRETTY_FUNCTION__))
;
2794 Results.push_back(DAG.getConstantFP(0, dl, VT));
2795 }
2796 break;
2797 }
2798 case ISD::FP_ROUND:
2799 case ISD::BITCAST:
2800 Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2801 Node->getValueType(0), dl);
2802 Results.push_back(Tmp1);
2803 break;
2804 case ISD::FP_EXTEND:
2805 Tmp1 = EmitStackConvert(Node->getOperand(0),
2806 Node->getOperand(0).getValueType(),
2807 Node->getValueType(0), dl);
2808 Results.push_back(Tmp1);
2809 break;
2810 case ISD::SIGN_EXTEND_INREG: {
2811 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2812 EVT VT = Node->getValueType(0);
2813
2814 // An in-register sign-extend of a boolean is a negation:
2815 // 'true' (1) sign-extended is -1.
2816 // 'false' (0) sign-extended is 0.
2817 // However, we must mask the high bits of the source operand because the
2818 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2819
2820 // TODO: Do this for vectors too?
2821 if (ExtraVT.getSizeInBits() == 1) {
2822 SDValue One = DAG.getConstant(1, dl, VT);
2823 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2824 SDValue Zero = DAG.getConstant(0, dl, VT);
2825 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2826 Results.push_back(Neg);
2827 break;
2828 }
2829
2830 // NOTE: we could fall back on load/store here too for targets without
2831 // SRA. However, it is doubtful that any exist.
2832 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2833 unsigned BitsDiff = VT.getScalarSizeInBits() -
2834 ExtraVT.getScalarSizeInBits();
2835 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2836 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2837 Node->getOperand(0), ShiftCst);
2838 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2839 Results.push_back(Tmp1);
2840 break;
2841 }
2842 case ISD::FP_ROUND_INREG: {
2843 // The only way we can lower this is to turn it into a TRUNCSTORE,
2844 // EXTLOAD pair, targeting a temporary location (a stack slot).
2845
2846 // NOTE: there is a choice here between constantly creating new stack
2847 // slots and always reusing the same one. We currently always create
2848 // new ones, as reuse may inhibit scheduling.
2849 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2850 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
2851 Node->getValueType(0), dl);
2852 Results.push_back(Tmp1);
2853 break;
2854 }
2855 case ISD::UINT_TO_FP:
2856 if (TLI.expandUINT_TO_FP(Node, Tmp1, DAG)) {
2857 Results.push_back(Tmp1);
2858 break;
2859 }
2860 LLVM_FALLTHROUGH[[clang::fallthrough]];
2861 case ISD::SINT_TO_FP:
2862 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
2863 Node->getOperand(0), Node->getValueType(0), dl);
2864 Results.push_back(Tmp1);
2865 break;
2866 case ISD::FP_TO_SINT:
2867 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
2868 Results.push_back(Tmp1);
2869 break;
2870 case ISD::FP_TO_UINT:
2871 if (TLI.expandFP_TO_UINT(Node, Tmp1, DAG))
2872 Results.push_back(Tmp1);
2873 break;
2874 case ISD::VAARG:
2875 Results.push_back(DAG.expandVAArg(Node));
2876 Results.push_back(Results[0].getValue(1));
2877 break;
2878 case ISD::VACOPY:
2879 Results.push_back(DAG.expandVACopy(Node));
2880 break;
2881 case ISD::EXTRACT_VECTOR_ELT:
2882 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
2883 // This must be an access of the only element. Return it.
2884 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
2885 Node->getOperand(0));
2886 else
2887 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
2888 Results.push_back(Tmp1);
2889 break;
2890 case ISD::EXTRACT_SUBVECTOR:
2891 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
2892 break;
2893 case ISD::INSERT_SUBVECTOR:
2894 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
2895 break;
2896 case ISD::CONCAT_VECTORS:
2897 Results.push_back(ExpandVectorBuildThroughStack(Node));
2898 break;
2899 case ISD::SCALAR_TO_VECTOR:
2900 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
2901 break;
2902 case ISD::INSERT_VECTOR_ELT:
2903 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
2904 Node->getOperand(1),
2905 Node->getOperand(2), dl));
2906 break;
2907 case ISD::VECTOR_SHUFFLE: {
2908 SmallVector<int, 32> NewMask;
2909 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
2910
2911 EVT VT = Node->getValueType(0);
2912 EVT EltVT = VT.getVectorElementType();
2913 SDValue Op0 = Node->getOperand(0);
2914 SDValue Op1 = Node->getOperand(1);
2915 if (!TLI.isTypeLegal(EltVT)) {
2916 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
2917
2918 // BUILD_VECTOR operands are allowed to be wider than the element type.
2919 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
2920 // it.
2921 if (NewEltVT.bitsLT(EltVT)) {
2922 // Convert shuffle node.
2923 // If original node was v4i64 and the new EltVT is i32,
2924 // cast operands to v8i32 and re-build the mask.
2925
2926 // Calculate new VT, the size of the new VT should be equal to original.
2927 EVT NewVT =
2928 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
2929 VT.getSizeInBits() / NewEltVT.getSizeInBits());
2930 assert(NewVT.bitsEq(VT))((NewVT.bitsEq(VT)) ? static_cast<void> (0) : __assert_fail
("NewVT.bitsEq(VT)", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2930, __PRETTY_FUNCTION__))
;
2931
2932 // cast operands to new VT
2933 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
2934 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
2935
2936 // Convert the shuffle mask
2937 unsigned int factor =
2938 NewVT.getVectorNumElements()/VT.getVectorNumElements();
2939
2940 // EltVT gets smaller
2941 assert(factor > 0)((factor > 0) ? static_cast<void> (0) : __assert_fail
("factor > 0", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2941, __PRETTY_FUNCTION__))
;
2942
2943 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
2944 if (Mask[i] < 0) {
2945 for (unsigned fi = 0; fi < factor; ++fi)
2946 NewMask.push_back(Mask[i]);
2947 }
2948 else {
2949 for (unsigned fi = 0; fi < factor; ++fi)
2950 NewMask.push_back(Mask[i]*factor+fi);
2951 }
2952 }
2953 Mask = NewMask;
2954 VT = NewVT;
2955 }
2956 EltVT = NewEltVT;
2957 }
2958 unsigned NumElems = VT.getVectorNumElements();
2959 SmallVector<SDValue, 16> Ops;
2960 for (unsigned i = 0; i != NumElems; ++i) {
2961 if (Mask[i] < 0) {
2962 Ops.push_back(DAG.getUNDEF(EltVT));
2963 continue;
2964 }
2965 unsigned Idx = Mask[i];
2966 if (Idx < NumElems)
2967 Ops.push_back(DAG.getNode(
2968 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
2969 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2970 else
2971 Ops.push_back(DAG.getNode(
2972 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
2973 DAG.getConstant(Idx - NumElems, dl,
2974 TLI.getVectorIdxTy(DAG.getDataLayout()))));
2975 }
2976
2977 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
2978 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
2979 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
2980 Results.push_back(Tmp1);
2981 break;
2982 }
2983 case ISD::EXTRACT_ELEMENT: {
2984 EVT OpTy = Node->getOperand(0).getValueType();
2985 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
2986 // 1 -> Hi
2987 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
2988 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
2989 TLI.getShiftAmountTy(
2990 Node->getOperand(0).getValueType(),
2991 DAG.getDataLayout())));
2992 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
2993 } else {
2994 // 0 -> Lo
2995 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
2996 Node->getOperand(0));
2997 }
2998 Results.push_back(Tmp1);
2999 break;
3000 }
3001 case ISD::STACKSAVE:
3002 // Expand to CopyFromReg if the target set
3003 // StackPointerRegisterToSaveRestore.
3004 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3005 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3006 Node->getValueType(0)));
3007 Results.push_back(Results[0].getValue(1));
3008 } else {
3009 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3010 Results.push_back(Node->getOperand(0));
3011 }
3012 break;
3013 case ISD::STACKRESTORE:
3014 // Expand to CopyToReg if the target set
3015 // StackPointerRegisterToSaveRestore.
3016 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3017 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3018 Node->getOperand(1)));
3019 } else {
3020 Results.push_back(Node->getOperand(0));
3021 }
3022 break;
3023 case ISD::GET_DYNAMIC_AREA_OFFSET:
3024 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3025 Results.push_back(Results[0].getValue(0));
3026 break;
3027 case ISD::FCOPYSIGN:
3028 Results.push_back(ExpandFCOPYSIGN(Node));
3029 break;
3030 case ISD::FNEG:
3031 // Expand Y = FNEG(X) -> Y = SUB -0.0, X
3032 Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3033 // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3034 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3035 Node->getOperand(0));
3036 Results.push_back(Tmp1);
3037 break;
3038 case ISD::FABS:
3039 Results.push_back(ExpandFABS(Node));
3040 break;
3041 case ISD::SMIN:
3042 case ISD::SMAX:
3043 case ISD::UMIN:
3044 case ISD::UMAX: {
3045 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3046 ISD::CondCode Pred;
3047 switch (Node->getOpcode()) {
3048 default: llvm_unreachable("How did we get here?")::llvm::llvm_unreachable_internal("How did we get here?", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3048)
;
3049 case ISD::SMAX: Pred = ISD::SETGT; break;
3050 case ISD::SMIN: Pred = ISD::SETLT; break;
3051 case ISD::UMAX: Pred = ISD::SETUGT; break;
3052 case ISD::UMIN: Pred = ISD::SETULT; break;
3053 }
3054 Tmp1 = Node->getOperand(0);
3055 Tmp2 = Node->getOperand(1);
3056 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3057 Results.push_back(Tmp1);
3058 break;
3059 }
3060 case ISD::FMINNUM:
3061 case ISD::FMAXNUM: {
3062 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3063 Results.push_back(Expanded);
3064 break;
3065 }
3066 case ISD::FSIN:
3067 case ISD::FCOS: {
3068 EVT VT = Node->getValueType(0);
3069 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3070 // fcos which share the same operand and both are used.
3071 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3072 isSinCosLibcallAvailable(Node, TLI))
3073 && useSinCos(Node)) {
3074 SDVTList VTs = DAG.getVTList(VT, VT);
3075 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3076 if (Node->getOpcode() == ISD::FCOS)
3077 Tmp1 = Tmp1.getValue(1);
3078 Results.push_back(Tmp1);
3079 }
3080 break;
3081 }
3082 case ISD::FMAD:
3083 llvm_unreachable("Illegal fmad should never be formed")::llvm::llvm_unreachable_internal("Illegal fmad should never be formed"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3083)
;
3084
3085 case ISD::FP16_TO_FP:
3086 if (Node->getValueType(0) != MVT::f32) {
3087 // We can extend to types bigger than f32 in two steps without changing
3088 // the result. Since "f16 -> f32" is much more commonly available, give
3089 // CodeGen the option of emitting that before resorting to a libcall.
3090 SDValue Res =
3091 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3092 Results.push_back(
3093 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3094 }
3095 break;
3096 case ISD::FP_TO_FP16:
3097 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing FP_TO_FP16\n";
} } while (false)
;
3098 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3099 SDValue Op = Node->getOperand(0);
3100 MVT SVT = Op.getSimpleValueType();
3101 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3102 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3103 // Under fastmath, we can expand this node into a fround followed by
3104 // a float-half conversion.
3105 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3106 DAG.getIntPtrConstant(0, dl));
3107 Results.push_back(
3108 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3109 }
3110 }
3111 break;
3112 case ISD::ConstantFP: {
3113 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3114 // Check to see if this FP immediate is already legal.
3115 // If this is a legal constant, turn it into a TargetConstantFP node.
3116 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
3117 Results.push_back(ExpandConstantFP(CFP, true));
3118 break;
3119 }
3120 case ISD::Constant: {
3121 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3122 Results.push_back(ExpandConstant(CP));
3123 break;
3124 }
3125 case ISD::FSUB: {
3126 EVT VT = Node->getValueType(0);
3127 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3128 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3129 const SDNodeFlags Flags = Node->getFlags();
3130 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3131 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3132 Results.push_back(Tmp1);
3133 }
3134 break;
3135 }
3136 case ISD::SUB: {
3137 EVT VT = Node->getValueType(0);
3138 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3140, __PRETTY_FUNCTION__))
3139 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3140, __PRETTY_FUNCTION__))
3140 "Don't know how to expand this subtraction!")((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3140, __PRETTY_FUNCTION__))
;
3141 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3142 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3143 VT));
3144 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3145 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3146 break;
3147 }
3148 case ISD::UREM:
3149 case ISD::SREM: {
3150 EVT VT = Node->getValueType(0);
3151 bool isSigned = Node->getOpcode() == ISD::SREM;
3152 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3153 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3154 Tmp2 = Node->getOperand(0);
3155 Tmp3 = Node->getOperand(1);
3156 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3157 SDVTList VTs = DAG.getVTList(VT, VT);
3158 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3159 Results.push_back(Tmp1);
3160 } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3161 // X % Y -> X-X/Y*Y
3162 Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3163 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3164 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3165 Results.push_back(Tmp1);
3166 }
3167 break;
3168 }
3169 case ISD::UDIV:
3170 case ISD::SDIV: {
3171 bool isSigned = Node->getOpcode() == ISD::SDIV;
3172 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3173 EVT VT = Node->getValueType(0);
3174 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3175 SDVTList VTs = DAG.getVTList(VT, VT);
3176 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3177 Node->getOperand(1));
3178 Results.push_back(Tmp1);
3179 }
3180 break;
3181 }
3182 case ISD::MULHU:
3183 case ISD::MULHS: {
3184 unsigned ExpandOpcode =
3185 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3186 EVT VT = Node->getValueType(0);
3187 SDVTList VTs = DAG.getVTList(VT, VT);
3188
3189 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3190 Node->getOperand(1));
3191 Results.push_back(Tmp1.getValue(1));
3192 break;
3193 }
3194 case ISD::UMUL_LOHI:
3195 case ISD::SMUL_LOHI: {
3196 SDValue LHS = Node->getOperand(0);
3197 SDValue RHS = Node->getOperand(1);
3198 MVT VT = LHS.getSimpleValueType();
3199 unsigned MULHOpcode =
3200 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3201
3202 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3203 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3204 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3205 break;
3206 }
3207
3208 SmallVector<SDValue, 4> Halves;
3209 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3210 assert(TLI.isTypeLegal(HalfType))((TLI.isTypeLegal(HalfType)) ? static_cast<void> (0) : __assert_fail
("TLI.isTypeLegal(HalfType)", "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3210, __PRETTY_FUNCTION__))
;
3211 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
3212 HalfType, DAG,
3213 TargetLowering::MulExpansionKind::Always)) {
3214 for (unsigned i = 0; i < 2; ++i) {
3215 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3216 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3217 SDValue Shift = DAG.getConstant(
3218 HalfType.getScalarSizeInBits(), dl,
3219 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3220 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3221 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3222 }
3223 break;
3224 }
3225 break;
3226 }
3227 case ISD::MUL: {
3228 EVT VT = Node->getValueType(0);
3229 SDVTList VTs = DAG.getVTList(VT, VT);
3230 // See if multiply or divide can be lowered using two-result operations.
3231 // We just need the low half of the multiply; try both the signed
3232 // and unsigned forms. If the target supports both SMUL_LOHI and
3233 // UMUL_LOHI, form a preference by checking which forms of plain
3234 // MULH it supports.
3235 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3236 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3237 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3238 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3239 unsigned OpToUse = 0;
3240 if (HasSMUL_LOHI && !HasMULHS) {
3241 OpToUse = ISD::SMUL_LOHI;
3242 } else if (HasUMUL_LOHI && !HasMULHU) {
3243 OpToUse = ISD::UMUL_LOHI;
3244 } else if (HasSMUL_LOHI) {
3245 OpToUse = ISD::SMUL_LOHI;
3246 } else if (HasUMUL_LOHI) {
3247 OpToUse = ISD::UMUL_LOHI;
3248 }
3249 if (OpToUse) {
3250 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3251 Node->getOperand(1)));
3252 break;
3253 }
3254
3255 SDValue Lo, Hi;
3256 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3257 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3258 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3259 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3260 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3261 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3262 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3263 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3264 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3265 SDValue Shift =
3266 DAG.getConstant(HalfType.getSizeInBits(), dl,
3267 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3268 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3269 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3270 }
3271 break;
3272 }
3273 case ISD::FSHL:
3274 case ISD::FSHR:
3275 if (TLI.expandFunnelShift(Node, Tmp1, DAG))
3276 Results.push_back(Tmp1);
3277 break;
3278 case ISD::ROTL:
3279 case ISD::ROTR:
3280 if (TLI.expandROT(Node, Tmp1, DAG))
3281 Results.push_back(Tmp1);
3282 break;
3283 case ISD::SADDSAT:
3284 case ISD::UADDSAT:
3285 case ISD::SSUBSAT:
3286 case ISD::USUBSAT: {
3287 Results.push_back(TLI.getExpandedSaturationAdditionSubtraction(Node, DAG));
3288 break;
3289 }
3290 case ISD::SMULFIX: {
3291 Results.push_back(TLI.getExpandedFixedPointMultiplication(Node, DAG));
3292 break;
3293 }
3294 case ISD::SADDO:
3295 case ISD::SSUBO: {
3296 SDValue LHS = Node->getOperand(0);
3297 SDValue RHS = Node->getOperand(1);
3298 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3299 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3300 LHS, RHS);
3301 Results.push_back(Sum);
3302 EVT ResultType = Node->getValueType(1);
3303 EVT OType = getSetCCResultType(Node->getValueType(0));
3304
3305 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3306
3307 // LHSSign -> LHS >= 0
3308 // RHSSign -> RHS >= 0
3309 // SumSign -> Sum >= 0
3310 //
3311 // Add:
3312 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3313 // Sub:
3314 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3315 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3316 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3317 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3318 Node->getOpcode() == ISD::SADDO ?
3319 ISD::SETEQ : ISD::SETNE);
3320
3321 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3322 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3323
3324 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3325 Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
3326 break;
3327 }
3328 case ISD::UADDO:
3329 case ISD::USUBO: {
3330 SDValue LHS = Node->getOperand(0);
3331 SDValue RHS = Node->getOperand(1);
3332 bool IsAdd = Node->getOpcode() == ISD::UADDO;
3333 // If ADD/SUBCARRY is legal, use that instead.
3334 unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;
3335 if (TLI.isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
3336 SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
3337 SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
3338 { LHS, RHS, CarryIn });
3339 Results.push_back(SDValue(NodeCarry.getNode(), 0));
3340 Results.push_back(SDValue(NodeCarry.getNode(), 1));
3341 break;
3342 }
3343
3344 SDValue Sum = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
3345 LHS.getValueType(), LHS, RHS);
3346 Results.push_back(Sum);
3347
3348 EVT ResultType = Node->getValueType(1);
3349 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3350 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3351 SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3352
3353 Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
3354 break;
3355 }
3356 case ISD::UMULO:
3357 case ISD::SMULO: {
3358 EVT VT = Node->getValueType(0);
3359 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
3360 SDValue LHS = Node->getOperand(0);
3361 SDValue RHS = Node->getOperand(1);
3362 SDValue BottomHalf;
3363 SDValue TopHalf;
3364 static const unsigned Ops[2][3] =
3365 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
3366 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
3367 bool isSigned = Node->getOpcode() == ISD::SMULO;
3368 if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
3369 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3370 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
3371 } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
3372 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
3373 RHS);
3374 TopHalf = BottomHalf.getValue(1);
3375 } else if (TLI.isTypeLegal(WideVT)) {
3376 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
3377 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
3378 Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3379 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3380 DAG.getIntPtrConstant(0, dl));
3381 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3382 DAG.getIntPtrConstant(1, dl));
3383 } else {
3384 // We can fall back to a libcall with an illegal type for the MUL if we
3385 // have a libcall big enough.
3386 // Also, we can fall back to a division in some cases, but that's a big
3387 // performance hit in the general case.
3388 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3389 if (WideVT == MVT::i16)
3390 LC = RTLIB::MUL_I16;
3391 else if (WideVT == MVT::i32)
3392 LC = RTLIB::MUL_I32;
3393 else if (WideVT == MVT::i64)
3394 LC = RTLIB::MUL_I64;
3395 else if (WideVT == MVT::i128)
3396 LC = RTLIB::MUL_I128;
3397 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!")((LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!"
) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Cannot expand this operation!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3397, __PRETTY_FUNCTION__))
;
3398
3399 SDValue HiLHS;
3400 SDValue HiRHS;
3401 if (isSigned) {
3402 // The high part is obtained by SRA'ing all but one of the bits of low
3403 // part.
3404 unsigned LoSize = VT.getSizeInBits();
3405 HiLHS =
3406 DAG.getNode(ISD::SRA, dl, VT, LHS,
3407 DAG.getConstant(LoSize - 1, dl,
3408 TLI.getPointerTy(DAG.getDataLayout())));
3409 HiRHS =
3410 DAG.getNode(ISD::SRA, dl, VT, RHS,
3411 DAG.getConstant(LoSize - 1, dl,
3412 TLI.getPointerTy(DAG.getDataLayout())));
3413 } else {
3414 HiLHS = DAG.getConstant(0, dl, VT);
3415 HiRHS = DAG.getConstant(0, dl, VT);
3416 }
3417
3418 // Here we're passing the 2 arguments explicitly as 4 arguments that are
3419 // pre-lowered to the correct types. This all depends upon WideVT not
3420 // being a legal type for the architecture and thus has to be split to
3421 // two arguments.
3422 SDValue Ret;
3423 if(DAG.getDataLayout().isLittleEndian()) {
3424 // Halves of WideVT are packed into registers in different order
3425 // depending on platform endianness. This is usually handled by
3426 // the C calling convention, but we can't defer to it in
3427 // the legalizer.
3428 SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
3429 Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3430 } else {
3431 SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
3432 Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3433 }
3434 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&((Ret.getOpcode() == ISD::MERGE_VALUES && "Ret value is a collection of constituent nodes holding result."
) ? static_cast<void> (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3435, __PRETTY_FUNCTION__))
3435 "Ret value is a collection of constituent nodes holding result.")((Ret.getOpcode() == ISD::MERGE_VALUES && "Ret value is a collection of constituent nodes holding result."
) ? static_cast<void> (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3435, __PRETTY_FUNCTION__))
;
3436 BottomHalf = Ret.getOperand(0);
3437 TopHalf = Ret.getOperand(1);
3438 }
3439
3440 if (isSigned) {
3441 Tmp1 = DAG.getConstant(
3442 VT.getSizeInBits() - 1, dl,
3443 TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
3444 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3445 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
3446 ISD::SETNE);
3447 } else {
3448 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
3449 DAG.getConstant(0, dl, VT), ISD::SETNE);
3450 }
3451
3452 // Truncate the result if SetCC returns a larger type than needed.
3453 EVT RType = Node->getValueType(1);
3454 if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
3455 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
3456
3457 assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&((RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
"Unexpected result type for S/UMULO legalization") ? static_cast
<void> (0) : __assert_fail ("RType.getSizeInBits() == TopHalf.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3458, __PRETTY_FUNCTION__))
3458 "Unexpected result type for S/UMULO legalization")((RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
"Unexpected result type for S/UMULO legalization") ? static_cast
<void> (0) : __assert_fail ("RType.getSizeInBits() == TopHalf.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3458, __PRETTY_FUNCTION__))
;
3459
3460 Results.push_back(BottomHalf);
3461 Results.push_back(TopHalf);
3462 break;
3463 }
3464 case ISD::BUILD_PAIR: {
3465 EVT PairTy = Node->getValueType(0);
3466 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3467 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3468 Tmp2 = DAG.getNode(
3469 ISD::SHL, dl, PairTy, Tmp2,
3470 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3471 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3472 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3473 break;
3474 }
3475 case ISD::SELECT:
3476 Tmp1 = Node->getOperand(0);
3477 Tmp2 = Node->getOperand(1);
3478 Tmp3 = Node->getOperand(2);
3479 if (Tmp1.getOpcode() == ISD::SETCC) {
3480 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3481 Tmp2, Tmp3,
3482 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3483 } else {
3484 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3485 DAG.getConstant(0, dl, Tmp1.getValueType()),
3486 Tmp2, Tmp3, ISD::SETNE);
3487 }
3488 Results.push_back(Tmp1);
3489 break;
3490 case ISD::BR_JT: {
3491 SDValue Chain = Node->getOperand(0);
3492 SDValue Table = Node->getOperand(1);
3493 SDValue Index = Node->getOperand(2);
3494
3495 const DataLayout &TD = DAG.getDataLayout();
3496 EVT PTy = TLI.getPointerTy(TD);
3497
3498 unsigned EntrySize =
3499 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3500
3501 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3502 // This transformation needs to be done here since otherwise the MIPS
3503 // backend will end up emitting a three instruction multiply sequence
3504 // instead of a single shift and MSP430 will call a runtime function.
3505 if (llvm::isPowerOf2_32(EntrySize))
3506 Index = DAG.getNode(
3507 ISD::SHL, dl, Index.getValueType(), Index,
3508 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3509 else
3510 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3511 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3512 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3513 Index, Table);
3514
3515 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3516 SDValue LD = DAG.getExtLoad(
3517 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3518 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3519 Addr = LD;
3520 if (TLI.isJumpTableRelative()) {
3521 // For PIC, the sequence is:
3522 // BRIND(load(Jumptable + index) + RelocBase)
3523 // RelocBase can be JumpTable, GOT or some sort of global base.
3524 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3525 TLI.getPICJumpTableRelocBase(Table, DAG));
3526 }
3527
3528 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG);
3529 Results.push_back(Tmp1);
3530 break;
3531 }
3532 case ISD::BRCOND:
3533 // Expand brcond's setcc into its constituent parts and create a BR_CC
3534 // Node.
3535 Tmp1 = Node->getOperand(0);
3536 Tmp2 = Node->getOperand(1);
3537 if (Tmp2.getOpcode() == ISD::SETCC) {
3538 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3539 Tmp1, Tmp2.getOperand(2),
3540 Tmp2.getOperand(0), Tmp2.getOperand(1),
3541 Node->getOperand(2));
3542 } else {
3543 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3544 if (Tmp2.isUndef() ||
3545 (Tmp2.getOpcode() == ISD::AND &&
3546 isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3547 cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3548 Tmp3 = Tmp2;
3549 else
3550 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3551 DAG.getConstant(1, dl, Tmp2.getValueType()));
3552 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3553 DAG.getCondCode(ISD::SETNE), Tmp3,
3554 DAG.getConstant(0, dl, Tmp3.getValueType()),
3555 Node->getOperand(2));
3556 }
3557 Results.push_back(Tmp1);
3558 break;
3559 case ISD::SETCC: {
3560 Tmp1 = Node->getOperand(0);
3561 Tmp2 = Node->getOperand(1);
3562 Tmp3 = Node->getOperand(2);
3563 bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3564 Tmp3, NeedInvert, dl);
3565
3566 if (Legalized) {
3567 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3568 // condition code, create a new SETCC node.
3569 if (Tmp3.getNode())
3570 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3571 Tmp1, Tmp2, Tmp3);
3572
3573 // If we expanded the SETCC by inverting the condition code, then wrap
3574 // the existing SETCC in a NOT to restore the intended condition.
3575 if (NeedInvert)
3576 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3577
3578 Results.push_back(Tmp1);
3579 break;
3580 }
3581
3582 // Otherwise, SETCC for the given comparison type must be completely
3583 // illegal; expand it into a SELECT_CC.
3584 EVT VT = Node->getValueType(0);
3585 int TrueValue;
3586 switch (TLI.getBooleanContents(Tmp1.getValueType())) {
3587 case TargetLowering::ZeroOrOneBooleanContent:
3588 case TargetLowering::UndefinedBooleanContent:
3589 TrueValue = 1;
3590 break;
3591 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3592 TrueValue = -1;
3593 break;
3594 }
3595 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3596 DAG.getConstant(TrueValue, dl, VT),
3597 DAG.getConstant(0, dl, VT),
3598 Tmp3);
3599 Results.push_back(Tmp1);
3600 break;
3601 }
3602 case ISD::SELECT_CC: {
3603 Tmp1 = Node->getOperand(0); // LHS
3604 Tmp2 = Node->getOperand(1); // RHS
3605 Tmp3 = Node->getOperand(2); // True
3606 Tmp4 = Node->getOperand(3); // False
3607 EVT VT = Node->getValueType(0);
3608 SDValue CC = Node->getOperand(4);
3609 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3610
3611 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
3612 // If the condition code is legal, then we need to expand this
3613 // node using SETCC and SELECT.
3614 EVT CmpVT = Tmp1.getValueType();
3615 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3617, __PRETTY_FUNCTION__))
3616 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3617, __PRETTY_FUNCTION__))
3617 "expanded.")((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3617, __PRETTY_FUNCTION__))
;
3618 EVT CCVT =
3619 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
3620 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
3621 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3622 break;
3623 }
3624
3625 // SELECT_CC is legal, so the condition code must not be.
3626 bool Legalized = false;
3627 // Try to legalize by inverting the condition. This is for targets that
3628 // might support an ordered version of a condition, but not the unordered
3629 // version (or vice versa).
3630 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3631 Tmp1.getValueType().isInteger());
3632 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
3633 // Use the new condition code and swap true and false
3634 Legalized = true;
3635 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3636 } else {
3637 // If The inverse is not legal, then try to swap the arguments using
3638 // the inverse condition code.
3639 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3640 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
3641 // The swapped inverse condition is legal, so swap true and false,
3642 // lhs and rhs.
3643 Legalized = true;
3644 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3645 }
3646 }
3647
3648 if (!Legalized) {
3649 Legalized = LegalizeSetCCCondCode(
3650 getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3651 dl);
3652
3653 assert(Legalized && "Can't legalize SELECT_CC with legal condition!")((Legalized && "Can't legalize SELECT_CC with legal condition!"
) ? static_cast<void> (0) : __assert_fail ("Legalized && \"Can't legalize SELECT_CC with legal condition!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3653, __PRETTY_FUNCTION__))
;
3654
3655 // If we expanded the SETCC by inverting the condition code, then swap
3656 // the True/False operands to match.
3657 if (NeedInvert)
3658 std::swap(Tmp3, Tmp4);
3659
3660 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3661 // condition code, create a new SELECT_CC node.
3662 if (CC.getNode()) {
3663 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3664 Tmp1, Tmp2, Tmp3, Tmp4, CC);
3665 } else {
3666 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3667 CC = DAG.getCondCode(ISD::SETNE);
3668 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3669 Tmp2, Tmp3, Tmp4, CC);
3670 }
3671 }
3672 Results.push_back(Tmp1);
3673 break;
3674 }
3675 case ISD::BR_CC: {
3676 Tmp1 = Node->getOperand(0); // Chain
3677 Tmp2 = Node->getOperand(2); // LHS
3678 Tmp3 = Node->getOperand(3); // RHS
3679 Tmp4 = Node->getOperand(1); // CC
3680
3681 bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
3682 Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
3683 (void)Legalized;
3684 assert(Legalized && "Can't legalize BR_CC with legal condition!")((Legalized && "Can't legalize BR_CC with legal condition!"
) ? static_cast<void> (0) : __assert_fail ("Legalized && \"Can't legalize BR_CC with legal condition!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3684, __PRETTY_FUNCTION__))
;
3685
3686 // If we expanded the SETCC by inverting the condition code, then wrap
3687 // the existing SETCC in a NOT to restore the intended condition.
3688 if (NeedInvert)
3689 Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
3690
3691 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3692 // node.
3693 if (Tmp4.getNode()) {
3694 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3695 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3696 } else {
3697 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3698 Tmp4 = DAG.getCondCode(ISD::SETNE);
3699 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3700 Tmp2, Tmp3, Node->getOperand(4));
3701 }
3702 Results.push_back(Tmp1);
3703 break;
3704 }
3705 case ISD::BUILD_VECTOR:
3706 Results.push_back(ExpandBUILD_VECTOR(Node));
3707 break;
3708 case ISD::SRA:
3709 case ISD::SRL:
3710 case ISD::SHL: {
3711 // Scalarize vector SRA/SRL/SHL.
3712 EVT VT = Node->getValueType(0);
3713 assert(VT.isVector() && "Unable to legalize non-vector shift")((VT.isVector() && "Unable to legalize non-vector shift"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Unable to legalize non-vector shift\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3713, __PRETTY_FUNCTION__))
;
3714 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal")((TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"
) ? static_cast<void> (0) : __assert_fail ("TLI.isTypeLegal(VT.getScalarType())&& \"Element type must be legal\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3714, __PRETTY_FUNCTION__))
;
3715 unsigned NumElem = VT.getVectorNumElements();
3716
3717 SmallVector<SDValue, 8> Scalars;
3718 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3719 SDValue Ex = DAG.getNode(
3720 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3721 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3722 SDValue Sh = DAG.getNode(
3723 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3724 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3725 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3726 VT.getScalarType(), Ex, Sh));
3727 }
3728
3729 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3730 ReplaceNode(SDValue(Node, 0), Result);
3731 break;
3732 }
3733 case ISD::GLOBAL_OFFSET_TABLE:
3734 case ISD::GlobalAddress:
3735 case ISD::GlobalTLSAddress:
3736 case ISD::ExternalSymbol:
3737 case ISD::ConstantPool:
3738 case ISD::JumpTable:
3739 case ISD::INTRINSIC_W_CHAIN:
3740 case ISD::INTRINSIC_WO_CHAIN:
3741 case ISD::INTRINSIC_VOID:
3742 // FIXME: Custom lowering for these operations shouldn't return null!
3743 break;
3744 }
3745
3746 // Replace the original node with the legalized result.
3747 if (Results.empty()) {
3748 LLVM_DEBUG(dbgs() << "Cannot expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Cannot expand node\n"; } }
while (false)
;
3749 return false;
3750 }
3751
3752 LLVM_DEBUG(dbgs() << "Successfully expanded node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully expanded node\n"
; } } while (false)
;
3753 ReplaceNode(Node, Results.data());
3754 return true;
3755}
3756
3757void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3758 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to convert node to libcall\n"
; } } while (false)
;
3759 SmallVector<SDValue, 8> Results;
3760 SDLoc dl(Node);
3761 // FIXME: Check flags on the node to see if we can use a finite call.
3762 bool CanUseFiniteLibCall = TM.Options.NoInfsFPMath && TM.Options.NoNaNsFPMath;
3763 unsigned Opc = Node->getOpcode();
3764 switch (Opc) {
3765 case ISD::ATOMIC_FENCE: {
3766 // If the target didn't lower this, lower it to '__sync_synchronize()' call
3767 // FIXME: handle "fence singlethread" more efficiently.
3768 TargetLowering::ArgListTy Args;
3769
3770 TargetLowering::CallLoweringInfo CLI(DAG);
3771 CLI.setDebugLoc(dl)
3772 .setChain(Node->getOperand(0))
3773 .setLibCallee(
3774 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3775 DAG.getExternalSymbol("__sync_synchronize",
3776 TLI.getPointerTy(DAG.getDataLayout())),
3777 std::move(Args));
3778
3779 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3780
3781 Results.push_back(CallResult.second);
3782 break;
3783 }
3784 // By default, atomic intrinsics are marked Legal and lowered. Targets
3785 // which don't support them directly, however, may want libcalls, in which
3786 // case they mark them Expand, and we get here.
3787 case ISD::ATOMIC_SWAP:
3788 case ISD::ATOMIC_LOAD_ADD:
3789 case ISD::ATOMIC_LOAD_SUB:
3790 case ISD::ATOMIC_LOAD_AND:
3791 case ISD::ATOMIC_LOAD_CLR:
3792 case ISD::ATOMIC_LOAD_OR:
3793 case ISD::ATOMIC_LOAD_XOR:
3794 case ISD::ATOMIC_LOAD_NAND:
3795 case ISD::ATOMIC_LOAD_MIN:
3796 case ISD::ATOMIC_LOAD_MAX:
3797 case ISD::ATOMIC_LOAD_UMIN:
3798 case ISD::ATOMIC_LOAD_UMAX:
3799 case ISD::ATOMIC_CMP_SWAP: {
3800 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3801 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
3802 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!")((LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"
) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected atomic op or value type!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3802, __PRETTY_FUNCTION__))
;
3803
3804 std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
3805 Results.push_back(Tmp.first);
3806 Results.push_back(Tmp.second);
3807 break;
3808 }
3809 case ISD::TRAP: {
3810 // If this operation is not supported, lower it to 'abort()' call
3811 TargetLowering::ArgListTy Args;
3812 TargetLowering::CallLoweringInfo CLI(DAG);
3813 CLI.setDebugLoc(dl)
3814 .setChain(Node->getOperand(0))
3815 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3816 DAG.getExternalSymbol(
3817 "abort", TLI.getPointerTy(DAG.getDataLayout())),
3818 std::move(Args));
3819 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3820
3821 Results.push_back(CallResult.second);
3822 break;
3823 }
3824 case ISD::FMINNUM:
3825 case ISD::STRICT_FMINNUM:
3826 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3827 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3828 RTLIB::FMIN_PPCF128));
3829 break;
3830 case ISD::FMAXNUM:
3831 case ISD::STRICT_FMAXNUM:
3832 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3833 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3834 RTLIB::FMAX_PPCF128));
3835 break;
3836 case ISD::FSQRT:
3837 case ISD::STRICT_FSQRT:
3838 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3839 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3840 RTLIB::SQRT_PPCF128));
3841 break;
3842 case ISD::FCBRT:
3843 Results.push_back(ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
3844 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
3845 RTLIB::CBRT_PPCF128));
3846 break;
3847 case ISD::FSIN:
3848 case ISD::STRICT_FSIN:
3849 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
3850 RTLIB::SIN_F80, RTLIB::SIN_F128,
3851 RTLIB::SIN_PPCF128));
3852 break;
3853 case ISD::FCOS:
3854 case ISD::STRICT_FCOS:
3855 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
3856 RTLIB::COS_F80, RTLIB::COS_F128,
3857 RTLIB::COS_PPCF128));
3858 break;
3859 case ISD::FSINCOS:
3860 // Expand into sincos libcall.
3861 ExpandSinCosLibCall(Node, Results);
3862 break;
3863 case ISD::FLOG:
3864 case ISD::STRICT_FLOG:
3865 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log_finite))
3866 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_FINITE_F32,
3867 RTLIB::LOG_FINITE_F64,
3868 RTLIB::LOG_FINITE_F80,
3869 RTLIB::LOG_FINITE_F128,
3870 RTLIB::LOG_FINITE_PPCF128));
3871 else
3872 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
3873 RTLIB::LOG_F80, RTLIB::LOG_F128,
3874 RTLIB::LOG_PPCF128));
3875 break;
3876 case ISD::FLOG2:
3877 case ISD::STRICT_FLOG2:
3878 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log2_finite))
3879 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_FINITE_F32,
3880 RTLIB::LOG2_FINITE_F64,
3881 RTLIB::LOG2_FINITE_F80,
3882 RTLIB::LOG2_FINITE_F128,
3883 RTLIB::LOG2_FINITE_PPCF128));
3884 else
3885 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
3886 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
3887 RTLIB::LOG2_PPCF128));
3888 break;
3889 case ISD::FLOG10:
3890 case ISD::STRICT_FLOG10:
3891 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log10_finite))
3892 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_FINITE_F32,
3893 RTLIB::LOG10_FINITE_F64,
3894 RTLIB::LOG10_FINITE_F80,
3895 RTLIB::LOG10_FINITE_F128,
3896 RTLIB::LOG10_FINITE_PPCF128));
3897 else
3898 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
3899 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
3900 RTLIB::LOG10_PPCF128));
3901 break;
3902 case ISD::FEXP:
3903 case ISD::STRICT_FEXP:
3904 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp_finite))
3905 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_FINITE_F32,
3906 RTLIB::EXP_FINITE_F64,
3907 RTLIB::EXP_FINITE_F80,
3908 RTLIB::EXP_FINITE_F128,
3909 RTLIB::EXP_FINITE_PPCF128));
3910 else
3911 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
3912 RTLIB::EXP_F80, RTLIB::EXP_F128,
3913 RTLIB::EXP_PPCF128));
3914 break;
3915 case ISD::FEXP2:
3916 case ISD::STRICT_FEXP2:
3917 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp2_finite))
3918 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_FINITE_F32,
3919 RTLIB::EXP2_FINITE_F64,
3920 RTLIB::EXP2_FINITE_F80,
3921 RTLIB::EXP2_FINITE_F128,
3922 RTLIB::EXP2_FINITE_PPCF128));
3923 else
3924 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
3925 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
3926 RTLIB::EXP2_PPCF128));
3927 break;
3928 case ISD::FTRUNC:
3929 case ISD::STRICT_FTRUNC:
3930 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
3931 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
3932 RTLIB::TRUNC_PPCF128));
3933 break;
3934 case ISD::FFLOOR:
3935 case ISD::STRICT_FFLOOR:
3936 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
3937 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
3938 RTLIB::FLOOR_PPCF128));
3939 break;
3940 case ISD::FCEIL:
3941 case ISD::STRICT_FCEIL:
3942 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
3943 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
3944 RTLIB::CEIL_PPCF128));
3945 break;
3946 case ISD::FRINT:
3947 case ISD::STRICT_FRINT:
3948 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
3949 RTLIB::RINT_F80, RTLIB::RINT_F128,
3950 RTLIB::RINT_PPCF128));
3951 break;
3952 case ISD::FNEARBYINT:
3953 case ISD::STRICT_FNEARBYINT:
3954 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
3955 RTLIB::NEARBYINT_F64,
3956 RTLIB::NEARBYINT_F80,
3957 RTLIB::NEARBYINT_F128,
3958 RTLIB::NEARBYINT_PPCF128));
3959 break;
3960 case ISD::FROUND:
3961 case ISD::STRICT_FROUND:
3962 Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
3963 RTLIB::ROUND_F64,
3964 RTLIB::ROUND_F80,
3965 RTLIB::ROUND_F128,
3966 RTLIB::ROUND_PPCF128));
3967 break;
3968 case ISD::FPOWI:
3969 case ISD::STRICT_FPOWI:
3970 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
3971 RTLIB::POWI_F80, RTLIB::POWI_F128,
3972 RTLIB::POWI_PPCF128));
3973 break;
3974 case ISD::FPOW:
3975 case ISD::STRICT_FPOW:
3976 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_pow_finite))
3977 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_FINITE_F32,
3978 RTLIB::POW_FINITE_F64,
3979 RTLIB::POW_FINITE_F80,
3980 RTLIB::POW_FINITE_F128,
3981 RTLIB::POW_FINITE_PPCF128));
3982 else
3983 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
3984 RTLIB::POW_F80, RTLIB::POW_F128,
3985 RTLIB::POW_PPCF128));
3986 break;
3987 case ISD::FDIV:
3988 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
3989 RTLIB::DIV_F80, RTLIB::DIV_F128,
3990 RTLIB::DIV_PPCF128));
3991 break;
3992 case ISD::FREM:
3993 case ISD::STRICT_FREM:
3994 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
3995 RTLIB::REM_F80, RTLIB::REM_F128,
3996 RTLIB::REM_PPCF128));
3997 break;
3998 case ISD::FMA:
3999 case ISD::STRICT_FMA:
4000 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4001 RTLIB::FMA_F80, RTLIB::FMA_F128,
4002 RTLIB::FMA_PPCF128));
4003 break;
4004 case ISD::FADD:
4005 Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4006 RTLIB::ADD_F80, RTLIB::ADD_F128,
4007 RTLIB::ADD_PPCF128));
4008 break;
4009 case ISD::FMUL:
4010 Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4011 RTLIB::MUL_F80, RTLIB::MUL_F128,
4012 RTLIB::MUL_PPCF128));
4013 break;
4014 case ISD::FP16_TO_FP:
4015 if (Node->getValueType(0) == MVT::f32) {
4016 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4017 }
4018 break;
4019 case ISD::FP_TO_FP16: {
4020 RTLIB::Libcall LC =
4021 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4022 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16")((LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16"
) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unable to expand fp_to_fp16\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4022, __PRETTY_FUNCTION__))
;
4023 Results.push_back(ExpandLibCall(LC, Node, false));
4024 break;
4025 }
4026 case ISD::FSUB:
4027 Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4028 RTLIB::SUB_F80, RTLIB::SUB_F128,
4029 RTLIB::SUB_PPCF128));
4030 break;
4031 case ISD::SREM:
4032 Results.push_back(ExpandIntLibCall(Node, true,
4033 RTLIB::SREM_I8,
4034 RTLIB::SREM_I16, RTLIB::SREM_I32,
4035 RTLIB::SREM_I64, RTLIB::SREM_I128));
4036 break;
4037 case ISD::UREM:
4038 Results.push_back(ExpandIntLibCall(Node, false,
4039 RTLIB::UREM_I8,
4040 RTLIB::UREM_I16, RTLIB::UREM_I32,
4041 RTLIB::UREM_I64, RTLIB::UREM_I128));
4042 break;
4043 case ISD::SDIV:
4044 Results.push_back(ExpandIntLibCall(Node, true,
4045 RTLIB::SDIV_I8,
4046 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4047 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4048 break;
4049 case ISD::UDIV:
4050 Results.push_back(ExpandIntLibCall(Node, false,
4051 RTLIB::UDIV_I8,
4052 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4053 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4054 break;
4055 case ISD::SDIVREM:
4056 case ISD::UDIVREM:
4057 // Expand into divrem libcall
4058 ExpandDivRemLibCall(Node, Results);
4059 break;
4060 case ISD::MUL:
4061 Results.push_back(ExpandIntLibCall(Node, false,
4062 RTLIB::MUL_I8,
4063 RTLIB::MUL_I16, RTLIB::MUL_I32,
4064 RTLIB::MUL_I64, RTLIB::MUL_I128));
4065 break;
4066 case ISD::CTLZ_ZERO_UNDEF:
4067 switch (Node->getSimpleValueType(0).SimpleTy) {
4068 default:
4069 llvm_unreachable("LibCall explicitly requested, but not available")::llvm::llvm_unreachable_internal("LibCall explicitly requested, but not available"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4069)
;
4070 case MVT::i32:
4071 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false));
4072 break;
4073 case MVT::i64:
4074 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false));
4075 break;
4076 case MVT::i128:
4077 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false));
4078 break;
4079 }
4080 break;
4081 }
4082
4083 // Replace the original node with the legalized result.
4084 if (!Results.empty()) {
4085 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully converted node to libcall\n"
; } } while (false)
;
4086 ReplaceNode(Node, Results.data());
4087 } else
4088 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not convert node to libcall\n"
; } } while (false)
;
4089}
4090
4091// Determine the vector type to use in place of an original scalar element when
4092// promoting equally sized vectors.
4093static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4094 MVT EltVT, MVT NewEltVT) {
4095 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4096 MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4097 assert(TLI.isTypeLegal(MidVT) && "unexpected")((TLI.isTypeLegal(MidVT) && "unexpected") ? static_cast
<void> (0) : __assert_fail ("TLI.isTypeLegal(MidVT) && \"unexpected\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4097, __PRETTY_FUNCTION__))
;
4098 return MidVT;
4099}
4100
4101void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4102 LLVM_DEBUG(dbgs() << "Trying to promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to promote node\n"
; } } while (false)
;
4103 SmallVector<SDValue, 8> Results;
4104 MVT OVT = Node->getSimpleValueType(0);
4105 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4106 Node->getOpcode() == ISD::SINT_TO_FP ||
4107 Node->getOpcode() == ISD::SETCC ||
4108 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4109 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4110 OVT = Node->getOperand(0).getSimpleValueType();
4111 }
4112 if (Node->getOpcode() == ISD::BR_CC)
4113 OVT = Node->getOperand(2).getSimpleValueType();
4114 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4115 SDLoc dl(Node);
4116 SDValue Tmp1, Tmp2, Tmp3;
4117 switch (Node->getOpcode()) {
4118 case ISD::CTTZ:
4119 case ISD::CTTZ_ZERO_UNDEF:
4120 case ISD::CTLZ:
4121 case ISD::CTLZ_ZERO_UNDEF:
4122 case ISD::CTPOP:
4123 // Zero extend the argument.
4124 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4125 if (Node->getOpcode() == ISD::CTTZ) {
4126 // The count is the same in the promoted type except if the original
4127 // value was zero. This can be handled by setting the bit just off
4128 // the top of the original type.
4129 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4130 OVT.getSizeInBits());
4131 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4132 DAG.getConstant(TopBit, dl, NVT));
4133 }
4134 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4135 // already the correct result.
4136 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4137 if (Node->getOpcode() == ISD::CTLZ ||
4138 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4139 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4140 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4141 DAG.getConstant(NVT.getSizeInBits() -
4142 OVT.getSizeInBits(), dl, NVT));
4143 }
4144 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4145 break;
4146 case ISD::BITREVERSE:
4147 case ISD::BSWAP: {
4148 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4149 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4150 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4151 Tmp1 = DAG.getNode(
4152 ISD::SRL, dl, NVT, Tmp1,
4153 DAG.getConstant(DiffBits, dl,
4154 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4155
4156 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4157 break;
4158 }
4159 case ISD::FP_TO_UINT:
4160 case ISD::FP_TO_SINT:
4161 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4162 Node->getOpcode() == ISD::FP_TO_SINT, dl);
4163 Results.push_back(Tmp1);
4164 break;
4165 case ISD::UINT_TO_FP:
4166 case ISD::SINT_TO_FP:
4167 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4168 Node->getOpcode() == ISD::SINT_TO_FP, dl);
4169 Results.push_back(Tmp1);
4170 break;
4171 case ISD::VAARG: {
4172 SDValue Chain = Node->getOperand(0); // Get the chain.
4173 SDValue Ptr = Node->getOperand(1); // Get the pointer.
4174
4175 unsigned TruncOp;
4176 if (OVT.isVector()) {
4177 TruncOp = ISD::BITCAST;
4178 } else {
4179 assert(OVT.isInteger()((OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4180, __PRETTY_FUNCTION__))
4180 && "VAARG promotion is supported only for vectors or integer types")((OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4180, __PRETTY_FUNCTION__))
;
4181 TruncOp = ISD::TRUNCATE;
4182 }
4183
4184 // Perform the larger operation, then convert back
4185 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4186 Node->getConstantOperandVal(3));
4187 Chain = Tmp1.getValue(1);
4188
4189 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4190
4191 // Modified the chain result - switch anything that used the old chain to
4192 // use the new one.
4193 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4194 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4195 if (UpdatedNodes) {
4196 UpdatedNodes->insert(Tmp2.getNode());
4197 UpdatedNodes->insert(Chain.getNode());
4198 }
4199 ReplacedNode(Node);
4200 break;
4201 }
4202 case ISD::MUL:
4203 case ISD::SDIV:
4204 case ISD::SREM:
4205 case ISD::UDIV:
4206 case ISD::UREM:
4207 case ISD::AND:
4208 case ISD::OR:
4209 case ISD::XOR: {
4210 unsigned ExtOp, TruncOp;
4211 if (OVT.isVector()) {
4212 ExtOp = ISD::BITCAST;
4213 TruncOp = ISD::BITCAST;
4214 } else {
4215 assert(OVT.isInteger() && "Cannot promote logic operation")((OVT.isInteger() && "Cannot promote logic operation"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"Cannot promote logic operation\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4215, __PRETTY_FUNCTION__))
;
4216
4217 switch (Node->getOpcode()) {
4218 default:
4219 ExtOp = ISD::ANY_EXTEND;
4220 break;
4221 case ISD::SDIV:
4222 case ISD::SREM:
4223 ExtOp = ISD::SIGN_EXTEND;
4224 break;
4225 case ISD::UDIV:
4226 case ISD::UREM:
4227 ExtOp = ISD::ZERO_EXTEND;
4228 break;
4229 }
4230 TruncOp = ISD::TRUNCATE;
4231 }
4232 // Promote each of the values to the new type.
4233 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4234 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4235 // Perform the larger operation, then convert back
4236 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4237 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4238 break;
4239 }
4240 case ISD::UMUL_LOHI:
4241 case ISD::SMUL_LOHI: {
4242 // Promote to a multiply in a wider integer type.
4243 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4244 : ISD::SIGN_EXTEND;
4245 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4246 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4247 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4248
4249 auto &DL = DAG.getDataLayout();
4250 unsigned OriginalSize = OVT.getScalarSizeInBits();
4251 Tmp2 = DAG.getNode(
4252 ISD::SRL, dl, NVT, Tmp1,
4253 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4254 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4255 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4256 break;
4257 }
4258 case ISD::SELECT: {
4259 unsigned ExtOp, TruncOp;
4260 if (Node->getValueType(0).isVector() ||
4261 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4262 ExtOp = ISD::BITCAST;
4263 TruncOp = ISD::BITCAST;
4264 } else if (Node->getValueType(0).isInteger()) {
4265 ExtOp = ISD::ANY_EXTEND;
4266 TruncOp = ISD::TRUNCATE;
4267 } else {
4268 ExtOp = ISD::FP_EXTEND;
4269 TruncOp = ISD::FP_ROUND;
4270 }
4271 Tmp1 = Node->getOperand(0);
4272 // Promote each of the values to the new type.
4273 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4274 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4275 // Perform the larger operation, then round down.
4276 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4277 if (TruncOp != ISD::FP_ROUND)
4278 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4279 else
4280 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4281 DAG.getIntPtrConstant(0, dl));
4282 Results.push_back(Tmp1);
4283 break;
4284 }
4285 case ISD::VECTOR_SHUFFLE: {
4286 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4287
4288 // Cast the two input vectors.
4289 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4290 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4291
4292 // Convert the shuffle mask to the right # elements.
4293 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4294 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4295 Results.push_back(Tmp1);
4296 break;
4297 }
4298 case ISD::SETCC: {
4299 unsigned ExtOp = ISD::FP_EXTEND;
4300 if (NVT.isInteger()) {
4301 ISD::CondCode CCCode =
4302 cast<CondCodeSDNode>(Node->getOperand(2))->get();
4303 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4304 }
4305 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4306 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4307 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
4308 Tmp1, Tmp2, Node->getOperand(2)));
4309 break;
4310 }
4311 case ISD::BR_CC: {
4312 unsigned ExtOp = ISD::FP_EXTEND;
4313 if (NVT.isInteger()) {
4314 ISD::CondCode CCCode =
4315 cast<CondCodeSDNode>(Node->getOperand(1))->get();
4316 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4317 }
4318 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4319 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4320 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4321 Node->getOperand(0), Node->getOperand(1),
4322 Tmp1, Tmp2, Node->getOperand(4)));
4323 break;
4324 }
4325 case ISD::FADD:
4326 case ISD::FSUB:
4327 case ISD::FMUL:
4328 case ISD::FDIV:
4329 case ISD::FREM:
4330 case ISD::FMINNUM:
4331 case ISD::FMAXNUM:
4332 case ISD::FPOW:
4333 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4334 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4335 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4336 Node->getFlags());
4337 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4338 Tmp3, DAG.getIntPtrConstant(0, dl)));
4339 break;
4340 case ISD::FMA:
4341 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4342 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4343 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4344 Results.push_back(
4345 DAG.getNode(ISD::FP_ROUND, dl, OVT,
4346 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4347 DAG.getIntPtrConstant(0, dl)));
4348 break;
4349 case ISD::FCOPYSIGN:
4350 case ISD::FPOWI: {
4351 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4352 Tmp2 = Node->getOperand(1);
4353 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4354
4355 // fcopysign doesn't change anything but the sign bit, so
4356 // (fp_round (fcopysign (fpext a), b))
4357 // is as precise as
4358 // (fp_round (fpext a))
4359 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4360 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4361 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4362 Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4363 break;
4364 }
4365 case ISD::FFLOOR:
4366 case ISD::FCEIL:
4367 case ISD::FRINT:
4368 case ISD::FNEARBYINT:
4369 case ISD::FROUND:
4370 case ISD::FTRUNC:
4371 case ISD::FNEG:
4372 case ISD::FSQRT:
4373 case ISD::FSIN:
4374 case ISD::FCOS:
4375 case ISD::FLOG:
4376 case ISD::FLOG2:
4377 case ISD::FLOG10:
4378 case ISD::FABS:
4379 case ISD::FEXP:
4380 case ISD::FEXP2:
4381 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4382 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4383 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4384 Tmp2, DAG.getIntPtrConstant(0, dl)));
4385 break;
4386 case ISD::BUILD_VECTOR: {
4387 MVT EltVT = OVT.getVectorElementType();
4388 MVT NewEltVT = NVT.getVectorElementType();
4389
4390 // Handle bitcasts to a different vector type with the same total bit size
4391 //
4392 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4393 // =>
4394 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4395
4396 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for build_vector") ? static_cast
<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4397, __PRETTY_FUNCTION__))
4397 "Invalid promote type for build_vector")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for build_vector") ? static_cast
<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4397, __PRETTY_FUNCTION__))
;
4398 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4398, __PRETTY_FUNCTION__))
;
4399
4400 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4401
4402 SmallVector<SDValue, 8> NewOps;
4403 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4404 SDValue Op = Node->getOperand(I);
4405 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4406 }
4407
4408 SDLoc SL(Node);
4409 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4410 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4411 Results.push_back(CvtVec);
4412 break;
4413 }
4414 case ISD::EXTRACT_VECTOR_ELT: {
4415 MVT EltVT = OVT.getVectorElementType();
4416 MVT NewEltVT = NVT.getVectorElementType();
4417
4418 // Handle bitcasts to a different vector type with the same total bit size.
4419 //
4420 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4421 // =>
4422 // v4i32:castx = bitcast x:v2i64
4423 //
4424 // i64 = bitcast
4425 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4426 // (i32 (extract_vector_elt castx, (2 * y + 1)))
4427 //
4428
4429 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for extract_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4430, __PRETTY_FUNCTION__))
4430 "Invalid promote type for extract_vector_elt")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for extract_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4430, __PRETTY_FUNCTION__))
;
4431 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4431, __PRETTY_FUNCTION__))
;
4432
4433 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4434 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4435
4436 SDValue Idx = Node->getOperand(1);
4437 EVT IdxVT = Idx.getValueType();
4438 SDLoc SL(Node);
4439 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4440 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4441
4442 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4443
4444 SmallVector<SDValue, 8> NewOps;
4445 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4446 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4447 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4448
4449 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4450 CastVec, TmpIdx);
4451 NewOps.push_back(Elt);
4452 }
4453
4454 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4455 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4456 break;
4457 }
4458 case ISD::INSERT_VECTOR_ELT: {
4459 MVT EltVT = OVT.getVectorElementType();
4460 MVT NewEltVT = NVT.getVectorElementType();
4461
4462 // Handle bitcasts to a different vector type with the same total bit size
4463 //
4464 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4465 // =>
4466 // v4i32:castx = bitcast x:v2i64
4467 // v2i32:casty = bitcast y:i64
4468 //
4469 // v2i64 = bitcast
4470 // (v4i32 insert_vector_elt
4471 // (v4i32 insert_vector_elt v4i32:castx,
4472 // (extract_vector_elt casty, 0), 2 * z),
4473 // (extract_vector_elt casty, 1), (2 * z + 1))
4474
4475 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for insert_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4476, __PRETTY_FUNCTION__))
4476 "Invalid promote type for insert_vector_elt")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for insert_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4476, __PRETTY_FUNCTION__))
;
4477 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4477, __PRETTY_FUNCTION__))
;
4478
4479 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4480 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4481
4482 SDValue Val = Node->getOperand(1);
4483 SDValue Idx = Node->getOperand(2);
4484 EVT IdxVT = Idx.getValueType();
4485 SDLoc SL(Node);
4486
4487 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4488 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4489
4490 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4491 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4492
4493 SDValue NewVec = CastVec;
4494 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4495 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4496 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4497
4498 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4499 CastVal, IdxOffset);
4500
4501 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4502 NewVec, Elt, InEltIdx);
4503 }
4504
4505 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4506 break;
4507 }
4508 case ISD::SCALAR_TO_VECTOR: {
4509 MVT EltVT = OVT.getVectorElementType();
4510 MVT NewEltVT = NVT.getVectorElementType();
4511
4512 // Handle bitcasts to different vector type with the same total bit size.
4513 //
4514 // e.g. v2i64 = scalar_to_vector x:i64
4515 // =>
4516 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4517 //
4518
4519 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4520 SDValue Val = Node->getOperand(0);
4521 SDLoc SL(Node);
4522
4523 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4524 SDValue Undef = DAG.getUNDEF(MidVT);
4525
4526 SmallVector<SDValue, 8> NewElts;
4527 NewElts.push_back(CastVal);
4528 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4529 NewElts.push_back(Undef);
4530
4531 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4532 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4533 Results.push_back(CvtVec);
4534 break;
4535 }
4536 }
4537
4538 // Replace the original node with the legalized result.
4539 if (!Results.empty()) {
4540 LLVM_DEBUG(dbgs() << "Successfully promoted node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully promoted node\n"
; } } while (false)
;
4541 ReplaceNode(Node, Results.data());
4542 } else
4543 LLVM_DEBUG(dbgs() << "Could not promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not promote node\n"
; } } while (false)
;
4544}
4545
4546/// This is the entry point for the file.
4547void SelectionDAG::Legalize() {
4548 AssignTopologicalOrder();
4549
4550 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4551 // Use a delete listener to remove nodes which were deleted during
4552 // legalization from LegalizeNodes. This is needed to handle the situation
4553 // where a new node is allocated by the object pool to the same address of a
4554 // previously deleted node.
4555 DAGNodeDeletedListener DeleteListener(
4556 *this,
4557 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4558
4559 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4560
4561 // Visit all the nodes. We start in topological order, so that we see
4562 // nodes with their original operands intact. Legalization can produce
4563 // new nodes which may themselves need to be legalized. Iterate until all
4564 // nodes have been legalized.
4565 while (true) {
4566 bool AnyLegalized = false;
4567 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4568 --NI;
4569
4570 SDNode *N = &*NI;
4571 if (N->use_empty() && N != getRoot().getNode()) {
4572 ++NI;
4573 DeleteNode(N);
4574 continue;
4575 }
4576
4577 if (LegalizedNodes.insert(N).second) {
4578 AnyLegalized = true;
4579 Legalizer.LegalizeOp(N);
4580
4581 if (N->use_empty() && N != getRoot().getNode()) {
4582 ++NI;
4583 DeleteNode(N);
4584 }
4585 }
4586 }
4587 if (!AnyLegalized)
4588 break;
4589
4590 }
4591
4592 // Remove dead nodes now.
4593 RemoveDeadNodes();
4594}
4595
4596bool SelectionDAG::LegalizeOp(SDNode *N,
4597 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4598 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4599 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4600
4601 // Directly insert the node in question, and legalize it. This will recurse
4602 // as needed through operands.
4603 LegalizedNodes.insert(N);
4604 Legalizer.LegalizeOp(N);
4605
4606 return LegalizedNodes.count(N);
4607}

/build/llvm-toolchain-snapshot-8~svn350071/include/llvm/Support/MathExtras.h

1//===-- llvm/Support/MathExtras.h - Useful math functions -------*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file contains some functions that are useful for math stuff.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_SUPPORT_MATHEXTRAS_H
15#define LLVM_SUPPORT_MATHEXTRAS_H
16
17#include "llvm/Support/Compiler.h"
18#include "llvm/Support/SwapByteOrder.h"
19#include <algorithm>
20#include <cassert>
21#include <climits>
22#include <cstring>
23#include <limits>
24#include <type_traits>
25
26#ifdef __ANDROID_NDK__
27#include <android/api-level.h>
28#endif
29
30#ifdef _MSC_VER
31// Declare these intrinsics manually rather including intrin.h. It's very
32// expensive, and MathExtras.h is popular.
33// #include <intrin.h>
34extern "C" {
35unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask);
36unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask);
37unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask);
38unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask);
39}
40#endif
41
42namespace llvm {
43/// The behavior an operation has on an input of 0.
44enum ZeroBehavior {
45 /// The returned value is undefined.
46 ZB_Undefined,
47 /// The returned value is numeric_limits<T>::max()
48 ZB_Max,
49 /// The returned value is numeric_limits<T>::digits
50 ZB_Width
51};
52
53namespace detail {
54template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter {
55 static std::size_t count(T Val, ZeroBehavior) {
56 if (!Val)
57 return std::numeric_limits<T>::digits;
58 if (Val & 0x1)
59 return 0;
60
61 // Bisection method.
62 std::size_t ZeroBits = 0;
63 T Shift = std::numeric_limits<T>::digits >> 1;
64 T Mask = std::numeric_limits<T>::max() >> Shift;
65 while (Shift) {
66 if ((Val & Mask) == 0) {
67 Val >>= Shift;
68 ZeroBits |= Shift;
69 }
70 Shift >>= 1;
71 Mask >>= Shift;
72 }
73 return ZeroBits;
74 }
75};
76
77#if __GNUC__4 >= 4 || defined(_MSC_VER)
78template <typename T> struct TrailingZerosCounter<T, 4> {
79 static std::size_t count(T Val, ZeroBehavior ZB) {
80 if (ZB != ZB_Undefined && Val == 0)
81 return 32;
82
83#if __has_builtin(__builtin_ctz)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
84 return __builtin_ctz(Val);
85#elif defined(_MSC_VER)
86 unsigned long Index;
87 _BitScanForward(&Index, Val);
88 return Index;
89#endif
90 }
91};
92
93#if !defined(_MSC_VER) || defined(_M_X64)
94template <typename T> struct TrailingZerosCounter<T, 8> {
95 static std::size_t count(T Val, ZeroBehavior ZB) {
96 if (ZB != ZB_Undefined && Val == 0)
97 return 64;
98
99#if __has_builtin(__builtin_ctzll)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
100 return __builtin_ctzll(Val);
101#elif defined(_MSC_VER)
102 unsigned long Index;
103 _BitScanForward64(&Index, Val);
104 return Index;
105#endif
106 }
107};
108#endif
109#endif
110} // namespace detail
111
112/// Count number of 0's from the least significant bit to the most
113/// stopping at the first 1.
114///
115/// Only unsigned integral types are allowed.
116///
117/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
118/// valid arguments.
119template <typename T>
120std::size_t countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
121 static_assert(std::numeric_limits<T>::is_integer &&
122 !std::numeric_limits<T>::is_signed,
123 "Only unsigned integral types are allowed.");
124 return llvm::detail::TrailingZerosCounter<T, sizeof(T)>::count(Val, ZB);
125}
126
127namespace detail {
128template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter {
129 static std::size_t count(T Val, ZeroBehavior) {
130 if (!Val)
131 return std::numeric_limits<T>::digits;
132
133 // Bisection method.
134 std::size_t ZeroBits = 0;
135 for (T Shift = std::numeric_limits<T>::digits >> 1; Shift; Shift >>= 1) {
136 T Tmp = Val >> Shift;
137 if (Tmp)
138 Val = Tmp;
139 else
140 ZeroBits |= Shift;
141 }
142 return ZeroBits;
143 }
144};
145
146#if __GNUC__4 >= 4 || defined(_MSC_VER)
147template <typename T> struct LeadingZerosCounter<T, 4> {
148 static std::size_t count(T Val, ZeroBehavior ZB) {
149 if (ZB != ZB_Undefined && Val == 0)
150 return 32;
151
152#if __has_builtin(__builtin_clz)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
153 return __builtin_clz(Val);
154#elif defined(_MSC_VER)
155 unsigned long Index;
156 _BitScanReverse(&Index, Val);
157 return Index ^ 31;
158#endif
159 }
160};
161
162#if !defined(_MSC_VER) || defined(_M_X64)
163template <typename T> struct LeadingZerosCounter<T, 8> {
164 static std::size_t count(T Val, ZeroBehavior ZB) {
165 if (ZB != ZB_Undefined && Val == 0)
166 return 64;
167
168#if __has_builtin(__builtin_clzll)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
169 return __builtin_clzll(Val);
170#elif defined(_MSC_VER)
171 unsigned long Index;
172 _BitScanReverse64(&Index, Val);
173 return Index ^ 63;
174#endif
175 }
176};
177#endif
178#endif
179} // namespace detail
180
181/// Count number of 0's from the most significant bit to the least
182/// stopping at the first 1.
183///
184/// Only unsigned integral types are allowed.
185///
186/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
187/// valid arguments.
188template <typename T>
189std::size_t countLeadingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
190 static_assert(std::numeric_limits<T>::is_integer &&
191 !std::numeric_limits<T>::is_signed,
192 "Only unsigned integral types are allowed.");
193 return llvm::detail::LeadingZerosCounter<T, sizeof(T)>::count(Val, ZB);
194}
195
196/// Get the index of the first set bit starting from the least
197/// significant bit.
198///
199/// Only unsigned integral types are allowed.
200///
201/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
202/// valid arguments.
203template <typename T> T findFirstSet(T Val, ZeroBehavior ZB = ZB_Max) {
204 if (ZB == ZB_Max && Val == 0)
205 return std::numeric_limits<T>::max();
206
207 return countTrailingZeros(Val, ZB_Undefined);
208}
209
210/// Create a bitmask with the N right-most bits set to 1, and all other
211/// bits set to 0. Only unsigned types are allowed.
212template <typename T> T maskTrailingOnes(unsigned N) {
213 static_assert(std::is_unsigned<T>::value, "Invalid type!");
214 const unsigned Bits = CHAR_BIT8 * sizeof(T);
215 assert(N <= Bits && "Invalid bit index")((N <= Bits && "Invalid bit index") ? static_cast<
void> (0) : __assert_fail ("N <= Bits && \"Invalid bit index\""
, "/build/llvm-toolchain-snapshot-8~svn350071/include/llvm/Support/MathExtras.h"
, 215, __PRETTY_FUNCTION__))
;
216 return N == 0 ? 0 : (T(-1) >> (Bits - N));
217}
218
219/// Create a bitmask with the N left-most bits set to 1, and all other
220/// bits set to 0. Only unsigned types are allowed.
221template <typename T> T maskLeadingOnes(unsigned N) {
222 return ~maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
223}
224
225/// Create a bitmask with the N right-most bits set to 0, and all other
226/// bits set to 1. Only unsigned types are allowed.
227template <typename T> T maskTrailingZeros(unsigned N) {
228 return maskLeadingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
229}
230
231/// Create a bitmask with the N left-most bits set to 0, and all other
232/// bits set to 1. Only unsigned types are allowed.
233template <typename T> T maskLeadingZeros(unsigned N) {
234 return maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
235}
236
237/// Get the index of the last set bit starting from the least
238/// significant bit.
239///
240/// Only unsigned integral types are allowed.
241///
242/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
243/// valid arguments.
244template <typename T> T findLastSet(T Val, ZeroBehavior ZB = ZB_Max) {
245 if (ZB == ZB_Max && Val == 0)
246 return std::numeric_limits<T>::max();
247
248 // Use ^ instead of - because both gcc and llvm can remove the associated ^
249 // in the __builtin_clz intrinsic on x86.
250 return countLeadingZeros(Val, ZB_Undefined) ^
251 (std::numeric_limits<T>::digits - 1);
252}
253
254/// Macro compressed bit reversal table for 256 bits.
255///
256/// http://graphics.stanford.edu/~seander/bithacks.html#BitReverseTable
257static const unsigned char BitReverseTable256[256] = {
258#define R2(n) n, n + 2 * 64, n + 1 * 64, n + 3 * 64
259#define R4(n) R2(n), R2(n + 2 * 16), R2(n + 1 * 16), R2(n + 3 * 16)
260#define R6(n) R4(n), R4(n + 2 * 4), R4(n + 1 * 4), R4(n + 3 * 4)
261 R6(0), R6(2), R6(1), R6(3)
262#undef R2
263#undef R4
264#undef R6
265};
266
267/// Reverse the bits in \p Val.
268template <typename T>
269T reverseBits(T Val) {
270 unsigned char in[sizeof(Val)];
271 unsigned char out[sizeof(Val)];
272 std::memcpy(in, &Val, sizeof(Val));
273 for (unsigned i = 0; i < sizeof(Val); ++i)
274 out[(sizeof(Val) - i) - 1] = BitReverseTable256[in[i]];
275 std::memcpy(&Val, out, sizeof(Val));
276 return Val;
277}
278
279// NOTE: The following support functions use the _32/_64 extensions instead of
280// type overloading so that signed and unsigned integers can be used without
281// ambiguity.
282
283/// Return the high 32 bits of a 64 bit value.
284constexpr inline uint32_t Hi_32(uint64_t Value) {
285 return static_cast<uint32_t>(Value >> 32);
286}
287
288/// Return the low 32 bits of a 64 bit value.
289constexpr inline uint32_t Lo_32(uint64_t Value) {
290 return static_cast<uint32_t>(Value);
291}
292
293/// Make a 64-bit integer from a high / low pair of 32-bit integers.
294constexpr inline uint64_t Make_64(uint32_t High, uint32_t Low) {
295 return ((uint64_t)High << 32) | (uint64_t)Low;
296}
297
298/// Checks if an integer fits into the given bit width.
299template <unsigned N> constexpr inline bool isInt(int64_t x) {
300 return N >= 64 || (-(INT64_C(1)1L<<(N-1)) <= x && x < (INT64_C(1)1L<<(N-1)));
301}
302// Template specializations to get better code for common cases.
303template <> constexpr inline bool isInt<8>(int64_t x) {
304 return static_cast<int8_t>(x) == x;
305}
306template <> constexpr inline bool isInt<16>(int64_t x) {
307 return static_cast<int16_t>(x) == x;
308}
309template <> constexpr inline bool isInt<32>(int64_t x) {
310 return static_cast<int32_t>(x) == x;
311}
312
313/// Checks if a signed integer is an N bit number shifted left by S.
314template <unsigned N, unsigned S>
315constexpr inline bool isShiftedInt(int64_t x) {
316 static_assert(
317 N > 0, "isShiftedInt<0> doesn't make sense (refers to a 0-bit number.");
318 static_assert(N + S <= 64, "isShiftedInt<N, S> with N + S > 64 is too wide.");
319 return isInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
320}
321
322/// Checks if an unsigned integer fits into the given bit width.
323///
324/// This is written as two functions rather than as simply
325///
326/// return N >= 64 || X < (UINT64_C(1) << N);
327///
328/// to keep MSVC from (incorrectly) warning on isUInt<64> that we're shifting
329/// left too many places.
330template <unsigned N>
331constexpr inline typename std::enable_if<(N < 64), bool>::type
332isUInt(uint64_t X) {
333 static_assert(N > 0, "isUInt<0> doesn't make sense");
334 return X < (UINT64_C(1)1UL << (N));
335}
336template <unsigned N>
337constexpr inline typename std::enable_if<N >= 64, bool>::type
338isUInt(uint64_t X) {
339 return true;
340}
341
342// Template specializations to get better code for common cases.
343template <> constexpr inline bool isUInt<8>(uint64_t x) {
344 return static_cast<uint8_t>(x) == x;
345}
346template <> constexpr inline bool isUInt<16>(uint64_t x) {
347 return static_cast<uint16_t>(x) == x;
348}
349template <> constexpr inline bool isUInt<32>(uint64_t x) {
350 return static_cast<uint32_t>(x) == x;
351}
352
353/// Checks if a unsigned integer is an N bit number shifted left by S.
354template <unsigned N, unsigned S>
355constexpr inline bool isShiftedUInt(uint64_t x) {
356 static_assert(
357 N > 0, "isShiftedUInt<0> doesn't make sense (refers to a 0-bit number)");
358 static_assert(N + S <= 64,
359 "isShiftedUInt<N, S> with N + S > 64 is too wide.");
360 // Per the two static_asserts above, S must be strictly less than 64. So
361 // 1 << S is not undefined behavior.
362 return isUInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
363}
364
365/// Gets the maximum value for a N-bit unsigned integer.
366inline uint64_t maxUIntN(uint64_t N) {
367 assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-8~svn350071/include/llvm/Support/MathExtras.h"
, 367, __PRETTY_FUNCTION__))
;
368
369 // uint64_t(1) << 64 is undefined behavior, so we can't do
370 // (uint64_t(1) << N) - 1
371 // without checking first that N != 64. But this works and doesn't have a
372 // branch.
373 return UINT64_MAX(18446744073709551615UL) >> (64 - N);
374}
375
376/// Gets the minimum value for a N-bit signed integer.
377inline int64_t minIntN(int64_t N) {
378 assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-8~svn350071/include/llvm/Support/MathExtras.h"
, 378, __PRETTY_FUNCTION__))
;
379
380 return -(UINT64_C(1)1UL<<(N-1));
381}
382
383/// Gets the maximum value for a N-bit signed integer.
384inline int64_t maxIntN(int64_t N) {
385 assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-8~svn350071/include/llvm/Support/MathExtras.h"
, 385, __PRETTY_FUNCTION__))
;
386
387 // This relies on two's complement wraparound when N == 64, so we convert to
388 // int64_t only at the very end to avoid UB.
389 return (UINT64_C(1)1UL << (N - 1)) - 1;
390}
391
392/// Checks if an unsigned integer fits into the given (dynamic) bit width.
393inline bool isUIntN(unsigned N, uint64_t x) {
394 return N >= 64 || x <= maxUIntN(N);
395}
396
397/// Checks if an signed integer fits into the given (dynamic) bit width.
398inline bool isIntN(unsigned N, int64_t x) {
399 return N >= 64 || (minIntN(N) <= x && x <= maxIntN(N));
400}
401
402/// Return true if the argument is a non-empty sequence of ones starting at the
403/// least significant bit with the remainder zero (32 bit version).
404/// Ex. isMask_32(0x0000FFFFU) == true.
405constexpr inline bool isMask_32(uint32_t Value) {
406 return Value && ((Value + 1) & Value) == 0;
407}
408
409/// Return true if the argument is a non-empty sequence of ones starting at the
410/// least significant bit with the remainder zero (64 bit version).
411constexpr inline bool isMask_64(uint64_t Value) {
412 return Value && ((Value + 1) & Value) == 0;
413}
414
415/// Return true if the argument contains a non-empty sequence of ones with the
416/// remainder zero (32 bit version.) Ex. isShiftedMask_32(0x0000FF00U) == true.
417constexpr inline bool isShiftedMask_32(uint32_t Value) {
418 return Value && isMask_32((Value - 1) | Value);
419}
420
421/// Return true if the argument contains a non-empty sequence of ones with the
422/// remainder zero (64 bit version.)
423constexpr inline bool isShiftedMask_64(uint64_t Value) {
424 return Value && isMask_64((Value - 1) | Value);
425}
426
427/// Return true if the argument is a power of two > 0.
428/// Ex. isPowerOf2_32(0x00100000U) == true (32 bit edition.)
429constexpr inline bool isPowerOf2_32(uint32_t Value) {
430 return Value && !(Value & (Value - 1));
431}
432
433/// Return true if the argument is a power of two > 0 (64 bit edition.)
434constexpr inline bool isPowerOf2_64(uint64_t Value) {
435 return Value && !(Value & (Value - 1));
436}
437
438/// Return a byte-swapped representation of the 16-bit argument.
439inline uint16_t ByteSwap_16(uint16_t Value) {
440 return sys::SwapByteOrder_16(Value);
441}
442
443/// Return a byte-swapped representation of the 32-bit argument.
444inline uint32_t ByteSwap_32(uint32_t Value) {
445 return sys::SwapByteOrder_32(Value);
446}
447
448/// Return a byte-swapped representation of the 64-bit argument.
449inline uint64_t ByteSwap_64(uint64_t Value) {
450 return sys::SwapByteOrder_64(Value);
451}
452
453/// Count the number of ones from the most significant bit to the first
454/// zero bit.
455///
456/// Ex. countLeadingOnes(0xFF0FFF00) == 8.
457/// Only unsigned integral types are allowed.
458///
459/// \param ZB the behavior on an input of all ones. Only ZB_Width and
460/// ZB_Undefined are valid arguments.
461template <typename T>
462std::size_t countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
463 static_assert(std::numeric_limits<T>::is_integer &&
464 !std::numeric_limits<T>::is_signed,
465 "Only unsigned integral types are allowed.");
466 return countLeadingZeros<T>(~Value, ZB);
467}
468
469/// Count the number of ones from the least significant bit to the first
470/// zero bit.
471///
472/// Ex. countTrailingOnes(0x00FF00FF) == 8.
473/// Only unsigned integral types are allowed.
474///
475/// \param ZB the behavior on an input of all ones. Only ZB_Width and
476/// ZB_Undefined are valid arguments.
477template <typename T>
478std::size_t countTrailingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
479 static_assert(std::numeric_limits<T>::is_integer &&
480 !std::numeric_limits<T>::is_signed,
481 "Only unsigned integral types are allowed.");
482 return countTrailingZeros<T>(~Value, ZB);
483}
484
485namespace detail {
486template <typename T, std::size_t SizeOfT> struct PopulationCounter {
487 static unsigned count(T Value) {
488 // Generic version, forward to 32 bits.
489 static_assert(SizeOfT <= 4, "Not implemented!");
490#if __GNUC__4 >= 4
491 return __builtin_popcount(Value);
492#else
493 uint32_t v = Value;
494 v = v - ((v >> 1) & 0x55555555);
495 v = (v & 0x33333333) + ((v >> 2) & 0x33333333);
496 return ((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24;
497#endif
498 }
499};
500
501template <typename T> struct PopulationCounter<T, 8> {
502 static unsigned count(T Value) {
503#if __GNUC__4 >= 4
504 return __builtin_popcountll(Value);
505#else
506 uint64_t v = Value;
507 v = v - ((v >> 1) & 0x5555555555555555ULL);
508 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
509 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
510 return unsigned((uint64_t)(v * 0x0101010101010101ULL) >> 56);
511#endif
512 }
513};
514} // namespace detail
515
516/// Count the number of set bits in a value.
517/// Ex. countPopulation(0xF000F000) = 8
518/// Returns 0 if the word is zero.
519template <typename T>
520inline unsigned countPopulation(T Value) {
521 static_assert(std::numeric_limits<T>::is_integer &&
522 !std::numeric_limits<T>::is_signed,
523 "Only unsigned integral types are allowed.");
524 return detail::PopulationCounter<T, sizeof(T)>::count(Value);
525}
526
527/// Return the log base 2 of the specified value.
528inline double Log2(double Value) {
529#if defined(__ANDROID_API__) && __ANDROID_API__ < 18
530 return __builtin_log(Value) / __builtin_log(2.0);
531#else
532 return log2(Value);
533#endif
534}
535
536/// Return the floor log base 2 of the specified value, -1 if the value is zero.
537/// (32 bit edition.)
538/// Ex. Log2_32(32) == 5, Log2_32(1) == 0, Log2_32(0) == -1, Log2_32(6) == 2
539inline unsigned Log2_32(uint32_t Value) {
540 return 31 - countLeadingZeros(Value);
12
Returning the value 4294967295
541}
542
543/// Return the floor log base 2 of the specified value, -1 if the value is zero.
544/// (64 bit edition.)
545inline unsigned Log2_64(uint64_t Value) {
546 return 63 - countLeadingZeros(Value);
547}
548
549/// Return the ceil log base 2 of the specified value, 32 if the value is zero.
550/// (32 bit edition).
551/// Ex. Log2_32_Ceil(32) == 5, Log2_32_Ceil(1) == 0, Log2_32_Ceil(6) == 3
552inline unsigned Log2_32_Ceil(uint32_t Value) {
553 return 32 - countLeadingZeros(Value - 1);
554}
555
556/// Return the ceil log base 2 of the specified value, 64 if the value is zero.
557/// (64 bit edition.)
558inline unsigned Log2_64_Ceil(uint64_t Value) {
559 return 64 - countLeadingZeros(Value - 1);
560}
561
562/// Return the greatest common divisor of the values using Euclid's algorithm.
563inline uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B) {
564 while (B) {
565 uint64_t T = B;
566 B = A % B;
567 A = T;
568 }
569 return A;
570}
571
572/// This function takes a 64-bit integer and returns the bit equivalent double.
573inline double BitsToDouble(uint64_t Bits) {
574 double D;
575 static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
576 memcpy(&D, &Bits, sizeof(Bits));
577 return D;
578}
579
580/// This function takes a 32-bit integer and returns the bit equivalent float.
581inline float BitsToFloat(uint32_t Bits) {
582 float F;
583 static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
584 memcpy(&F, &Bits, sizeof(Bits));
585 return F;
586}
587
588/// This function takes a double and returns the bit equivalent 64-bit integer.
589/// Note that copying doubles around changes the bits of NaNs on some hosts,
590/// notably x86, so this routine cannot be used if these bits are needed.
591inline uint64_t DoubleToBits(double Double) {
592 uint64_t Bits;
593 static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
594 memcpy(&Bits, &Double, sizeof(Double));
595 return Bits;
596}
597
598/// This function takes a float and returns the bit equivalent 32-bit integer.
599/// Note that copying floats around changes the bits of NaNs on some hosts,
600/// notably x86, so this routine cannot be used if these bits are needed.
601inline uint32_t FloatToBits(float Float) {
602 uint32_t Bits;
603 static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
604 memcpy(&Bits, &Float, sizeof(Float));
605 return Bits;
606}
607
608/// A and B are either alignments or offsets. Return the minimum alignment that
609/// may be assumed after adding the two together.
610constexpr inline uint64_t MinAlign(uint64_t A, uint64_t B) {
611 // The largest power of 2 that divides both A and B.
612 //
613 // Replace "-Value" by "1+~Value" in the following commented code to avoid
614 // MSVC warning C4146
615 // return (A | B) & -(A | B);
616 return (A | B) & (1 + ~(A | B));
617}
618
619/// Aligns \c Addr to \c Alignment bytes, rounding up.
620///
621/// Alignment should be a power of two. This method rounds up, so
622/// alignAddr(7, 4) == 8 and alignAddr(8, 4) == 8.
623inline uintptr_t alignAddr(const void *Addr, size_t Alignment) {
624 assert(Alignment && isPowerOf2_64((uint64_t)Alignment) &&((Alignment && isPowerOf2_64((uint64_t)Alignment) &&
"Alignment is not a power of two!") ? static_cast<void>
(0) : __assert_fail ("Alignment && isPowerOf2_64((uint64_t)Alignment) && \"Alignment is not a power of two!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/include/llvm/Support/MathExtras.h"
, 625, __PRETTY_FUNCTION__))
625 "Alignment is not a power of two!")((Alignment && isPowerOf2_64((uint64_t)Alignment) &&
"Alignment is not a power of two!") ? static_cast<void>
(0) : __assert_fail ("Alignment && isPowerOf2_64((uint64_t)Alignment) && \"Alignment is not a power of two!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/include/llvm/Support/MathExtras.h"
, 625, __PRETTY_FUNCTION__))
;
626
627 assert((uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr)(((uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr) ? static_cast
<void> (0) : __assert_fail ("(uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr"
, "/build/llvm-toolchain-snapshot-8~svn350071/include/llvm/Support/MathExtras.h"
, 627, __PRETTY_FUNCTION__))
;
628
629 return (((uintptr_t)Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1));
630}
631
632/// Returns the necessary adjustment for aligning \c Ptr to \c Alignment
633/// bytes, rounding up.
634inline size_t alignmentAdjustment(const void *Ptr, size_t Alignment) {
635 return alignAddr(Ptr, Alignment) - (uintptr_t)Ptr;
636}
637
638/// Returns the next power of two (in 64-bits) that is strictly greater than A.
639/// Returns zero on overflow.
640inline uint64_t NextPowerOf2(uint64_t A) {
641 A |= (A >> 1);
642 A |= (A >> 2);
643 A |= (A >> 4);
644 A |= (A >> 8);
645 A |= (A >> 16);
646 A |= (A >> 32);
647 return A + 1;
648}
649
650/// Returns the power of two which is less than or equal to the given value.
651/// Essentially, it is a floor operation across the domain of powers of two.
652inline uint64_t PowerOf2Floor(uint64_t A) {
653 if (!A) return 0;
654 return 1ull << (63 - countLeadingZeros(A, ZB_Undefined));
655}
656
657/// Returns the power of two which is greater than or equal to the given value.
658/// Essentially, it is a ceil operation across the domain of powers of two.
659inline uint64_t PowerOf2Ceil(uint64_t A) {
660 if (!A)
661 return 0;
662 return NextPowerOf2(A - 1);
663}
664
665/// Returns the next integer (mod 2**64) that is greater than or equal to
666/// \p Value and is a multiple of \p Align. \p Align must be non-zero.
667///
668/// If non-zero \p Skew is specified, the return value will be a minimal
669/// integer that is greater than or equal to \p Value and equal to
670/// \p Align * N + \p Skew for some integer N. If \p Skew is larger than
671/// \p Align, its value is adjusted to '\p Skew mod \p Align'.
672///
673/// Examples:
674/// \code
675/// alignTo(5, 8) = 8
676/// alignTo(17, 8) = 24
677/// alignTo(~0LL, 8) = 0
678/// alignTo(321, 255) = 510
679///
680/// alignTo(5, 8, 7) = 7
681/// alignTo(17, 8, 1) = 17
682/// alignTo(~0LL, 8, 3) = 3
683/// alignTo(321, 255, 42) = 552
684/// \endcode
685inline uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
686 assert(Align != 0u && "Align can't be 0.")((Align != 0u && "Align can't be 0.") ? static_cast<
void> (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/include/llvm/Support/MathExtras.h"
, 686, __PRETTY_FUNCTION__))
;
687 Skew %= Align;
688 return (Value + Align - 1 - Skew) / Align * Align + Skew;
689}
690
691/// Returns the next integer (mod 2**64) that is greater than or equal to
692/// \p Value and is a multiple of \c Align. \c Align must be non-zero.
693template <uint64_t Align> constexpr inline uint64_t alignTo(uint64_t Value) {
694 static_assert(Align != 0u, "Align must be non-zero");
695 return (Value + Align - 1) / Align * Align;
696}
697
698/// Returns the integer ceil(Numerator / Denominator).
699inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) {
700 return alignTo(Numerator, Denominator) / Denominator;
701}
702
703/// \c alignTo for contexts where a constant expression is required.
704/// \sa alignTo
705///
706/// \todo FIXME: remove when \c constexpr becomes really \c constexpr
707template <uint64_t Align>
708struct AlignTo {
709 static_assert(Align != 0u, "Align must be non-zero");
710 template <uint64_t Value>
711 struct from_value {
712 static const uint64_t value = (Value + Align - 1) / Align * Align;
713 };
714};
715
716/// Returns the largest uint64_t less than or equal to \p Value and is
717/// \p Skew mod \p Align. \p Align must be non-zero
718inline uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
719 assert(Align != 0u && "Align can't be 0.")((Align != 0u && "Align can't be 0.") ? static_cast<
void> (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/include/llvm/Support/MathExtras.h"
, 719, __PRETTY_FUNCTION__))
;
720 Skew %= Align;
721 return (Value - Skew) / Align * Align + Skew;
722}
723
724/// Returns the offset to the next integer (mod 2**64) that is greater than
725/// or equal to \p Value and is a multiple of \p Align. \p Align must be
726/// non-zero.
727inline uint64_t OffsetToAlignment(uint64_t Value, uint64_t Align) {
728 return alignTo(Value, Align) - Value;
729}
730
731/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
732/// Requires 0 < B <= 32.
733template <unsigned B> constexpr inline int32_t SignExtend32(uint32_t X) {
734 static_assert(B > 0, "Bit width can't be 0.");
735 static_assert(B <= 32, "Bit width out of range.");
736 return int32_t(X << (32 - B)) >> (32 - B);
737}
738
739/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
740/// Requires 0 < B < 32.
741inline int32_t SignExtend32(uint32_t X, unsigned B) {
742 assert(B > 0 && "Bit width can't be 0.")((B > 0 && "Bit width can't be 0.") ? static_cast<
void> (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/include/llvm/Support/MathExtras.h"
, 742, __PRETTY_FUNCTION__))
;
743 assert(B <= 32 && "Bit width out of range.")((B <= 32 && "Bit width out of range.") ? static_cast
<void> (0) : __assert_fail ("B <= 32 && \"Bit width out of range.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/include/llvm/Support/MathExtras.h"
, 743, __PRETTY_FUNCTION__))
;
744 return int32_t(X << (32 - B)) >> (32 - B);
745}
746
747/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
748/// Requires 0 < B < 64.
749template <unsigned B> constexpr inline int64_t SignExtend64(uint64_t x) {
750 static_assert(B > 0, "Bit width can't be 0.");
751 static_assert(B <= 64, "Bit width out of range.");
752 return int64_t(x << (64 - B)) >> (64 - B);
753}
754
755/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
756/// Requires 0 < B < 64.
757inline int64_t SignExtend64(uint64_t X, unsigned B) {
758 assert(B > 0 && "Bit width can't be 0.")((B > 0 && "Bit width can't be 0.") ? static_cast<
void> (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/include/llvm/Support/MathExtras.h"
, 758, __PRETTY_FUNCTION__))
;
759 assert(B <= 64 && "Bit width out of range.")((B <= 64 && "Bit width out of range.") ? static_cast
<void> (0) : __assert_fail ("B <= 64 && \"Bit width out of range.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/include/llvm/Support/MathExtras.h"
, 759, __PRETTY_FUNCTION__))
;
760 return int64_t(X << (64 - B)) >> (64 - B);
761}
762
763/// Subtract two unsigned integers, X and Y, of type T and return the absolute
764/// value of the result.
765template <typename T>
766typename std::enable_if<std::is_unsigned<T>::value, T>::type
767AbsoluteDifference(T X, T Y) {
768 return std::max(X, Y) - std::min(X, Y);
769}
770
771/// Add two unsigned integers, X and Y, of type T. Clamp the result to the
772/// maximum representable value of T on overflow. ResultOverflowed indicates if
773/// the result is larger than the maximum representable value of type T.
774template <typename T>
775typename std::enable_if<std::is_unsigned<T>::value, T>::type
776SaturatingAdd(T X, T Y, bool *ResultOverflowed = nullptr) {
777 bool Dummy;
778 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
779 // Hacker's Delight, p. 29
780 T Z = X + Y;
781 Overflowed = (Z < X || Z < Y);
782 if (Overflowed)
783 return std::numeric_limits<T>::max();
784 else
785 return Z;
786}
787
788/// Multiply two unsigned integers, X and Y, of type T. Clamp the result to the
789/// maximum representable value of T on overflow. ResultOverflowed indicates if
790/// the result is larger than the maximum representable value of type T.
791template <typename T>
792typename std::enable_if<std::is_unsigned<T>::value, T>::type
793SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) {
794 bool Dummy;
795 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
796
797 // Hacker's Delight, p. 30 has a different algorithm, but we don't use that
798 // because it fails for uint16_t (where multiplication can have undefined
799 // behavior due to promotion to int), and requires a division in addition
800 // to the multiplication.
801
802 Overflowed = false;
803
804 // Log2(Z) would be either Log2Z or Log2Z + 1.
805 // Special case: if X or Y is 0, Log2_64 gives -1, and Log2Z
806 // will necessarily be less than Log2Max as desired.
807 int Log2Z = Log2_64(X) + Log2_64(Y);
808 const T Max = std::numeric_limits<T>::max();
809 int Log2Max = Log2_64(Max);
810 if (Log2Z < Log2Max) {
811 return X * Y;
812 }
813 if (Log2Z > Log2Max) {
814 Overflowed = true;
815 return Max;
816 }
817
818 // We're going to use the top bit, and maybe overflow one
819 // bit past it. Multiply all but the bottom bit then add
820 // that on at the end.
821 T Z = (X >> 1) * Y;
822 if (Z & ~(Max >> 1)) {
823 Overflowed = true;
824 return Max;
825 }
826 Z <<= 1;
827 if (X & 1)
828 return SaturatingAdd(Z, Y, ResultOverflowed);
829
830 return Z;
831}
832
833/// Multiply two unsigned integers, X and Y, and add the unsigned integer, A to
834/// the product. Clamp the result to the maximum representable value of T on
835/// overflow. ResultOverflowed indicates if the result is larger than the
836/// maximum representable value of type T.
837template <typename T>
838typename std::enable_if<std::is_unsigned<T>::value, T>::type
839SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed = nullptr) {
840 bool Dummy;
841 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
842
843 T Product = SaturatingMultiply(X, Y, &Overflowed);
844 if (Overflowed)
845 return Product;
846
847 return SaturatingAdd(A, Product, &Overflowed);
848}
849
850/// Use this rather than HUGE_VALF; the latter causes warnings on MSVC.
851extern const float huge_valf;
852} // End llvm namespace
853
854#endif