Bug Summary

File:lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
Warning:line 545, column 29
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizeDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-8/lib/clang/8.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/include -I /build/llvm-toolchain-snapshot-8~svn345461/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/8.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-8/lib/clang/8.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/lib/CodeGen/SelectionDAG -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-10-27-211344-32123-1 -x c++ /build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp -faddrsig

/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the SelectionDAG::Legalize method.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ADT/APFloat.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/CodeGen/ISDOpcodes.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineMemOperand.h"
25#include "llvm/CodeGen/RuntimeLibcalls.h"
26#include "llvm/CodeGen/SelectionDAG.h"
27#include "llvm/CodeGen/SelectionDAGNodes.h"
28#include "llvm/CodeGen/TargetFrameLowering.h"
29#include "llvm/CodeGen/TargetLowering.h"
30#include "llvm/CodeGen/TargetSubtargetInfo.h"
31#include "llvm/CodeGen/ValueTypes.h"
32#include "llvm/IR/CallingConv.h"
33#include "llvm/IR/Constants.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/DerivedTypes.h"
36#include "llvm/IR/Function.h"
37#include "llvm/IR/Metadata.h"
38#include "llvm/IR/Type.h"
39#include "llvm/Support/Casting.h"
40#include "llvm/Support/Compiler.h"
41#include "llvm/Support/Debug.h"
42#include "llvm/Support/ErrorHandling.h"
43#include "llvm/Support/MachineValueType.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/raw_ostream.h"
46#include "llvm/Target/TargetMachine.h"
47#include "llvm/Target/TargetOptions.h"
48#include <algorithm>
49#include <cassert>
50#include <cstdint>
51#include <tuple>
52#include <utility>
53
54using namespace llvm;
55
56#define DEBUG_TYPE"legalizedag" "legalizedag"
57
58namespace {
59
60/// Keeps track of state when getting the sign of a floating-point value as an
61/// integer.
62struct FloatSignAsInt {
63 EVT FloatVT;
64 SDValue Chain;
65 SDValue FloatPtr;
66 SDValue IntPtr;
67 MachinePointerInfo IntPointerInfo;
68 MachinePointerInfo FloatPointerInfo;
69 SDValue IntValue;
70 APInt SignMask;
71 uint8_t SignBit;
72};
73
74//===----------------------------------------------------------------------===//
75/// This takes an arbitrary SelectionDAG as input and
76/// hacks on it until the target machine can handle it. This involves
77/// eliminating value sizes the machine cannot handle (promoting small sizes to
78/// large sizes or splitting up large values into small values) as well as
79/// eliminating operations the machine cannot handle.
80///
81/// This code also does a small amount of optimization and recognition of idioms
82/// as part of its processing. For example, if a target does not support a
83/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
84/// will attempt merge setcc and brc instructions into brcc's.
85class SelectionDAGLegalize {
86 const TargetMachine &TM;
87 const TargetLowering &TLI;
88 SelectionDAG &DAG;
89
90 /// The set of nodes which have already been legalized. We hold a
91 /// reference to it in order to update as necessary on node deletion.
92 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93
94 /// A set of all the nodes updated during legalization.
95 SmallSetVector<SDNode *, 16> *UpdatedNodes;
96
97 EVT getSetCCResultType(EVT VT) const {
98 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
99 }
100
101 // Libcall insertion helpers.
102
103public:
104 SelectionDAGLegalize(SelectionDAG &DAG,
105 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
106 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
107 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
108 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109
110 /// Legalizes the given operation.
111 void LegalizeOp(SDNode *Node);
112
113private:
114 SDValue OptimizeFloatStore(StoreSDNode *ST);
115
116 void LegalizeLoadOps(SDNode *Node);
117 void LegalizeStoreOps(SDNode *Node);
118
119 /// Some targets cannot handle a variable
120 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
121 /// is necessary to spill the vector being inserted into to memory, perform
122 /// the insert there, and then read the result back.
123 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
124 const SDLoc &dl);
125 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
126 const SDLoc &dl);
127
128 /// Return a vector shuffle operation which
129 /// performs the same shuffe in terms of order or result bytes, but on a type
130 /// whose vector element type is narrower than the original shuffle type.
131 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
132 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133 SDValue N1, SDValue N2,
134 ArrayRef<int> Mask) const;
135
136 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
137 bool &NeedInvert, const SDLoc &dl);
138
139 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
140 SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
141 unsigned NumOps, bool isSigned, const SDLoc &dl);
142
143 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
144 SDNode *Node, bool isSigned);
145 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
146 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
147 RTLIB::Libcall Call_F128,
148 RTLIB::Libcall Call_PPCF128);
149 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
150 RTLIB::Libcall Call_I8,
151 RTLIB::Libcall Call_I16,
152 RTLIB::Libcall Call_I32,
153 RTLIB::Libcall Call_I64,
154 RTLIB::Libcall Call_I128);
155 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157
158 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
159 const SDLoc &dl);
160 SDValue ExpandBUILD_VECTOR(SDNode *Node);
161 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
162 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
163 SmallVectorImpl<SDValue> &Results);
164 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
165 SDValue Value) const;
166 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
167 SDValue NewIntValue) const;
168 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
169 SDValue ExpandFABS(SDNode *Node) const;
170 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, EVT DestVT,
171 const SDLoc &dl);
172 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
173 const SDLoc &dl);
174 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
175 const SDLoc &dl);
176
177 SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
178 SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
179
180 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
181 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
182 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
183
184 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
185 SDValue ExpandConstant(ConstantSDNode *CP);
186
187 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
188 bool ExpandNode(SDNode *Node);
189 void ConvertNodeToLibcall(SDNode *Node);
190 void PromoteNode(SDNode *Node);
191
192public:
193 // Node replacement helpers
194
195 void ReplacedNode(SDNode *N) {
196 LegalizedNodes.erase(N);
197 if (UpdatedNodes)
198 UpdatedNodes->insert(N);
199 }
200
201 void ReplaceNode(SDNode *Old, SDNode *New) {
202 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
203 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
204
205 assert(Old->getNumValues() == New->getNumValues() &&((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 207, __PRETTY_FUNCTION__))
206 "Replacing one node with another that produces a different number "((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 207, __PRETTY_FUNCTION__))
207 "of values!")((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 207, __PRETTY_FUNCTION__))
;
208 DAG.ReplaceAllUsesWith(Old, New);
209 if (UpdatedNodes)
210 UpdatedNodes->insert(New);
211 ReplacedNode(Old);
212 }
213
214 void ReplaceNode(SDValue Old, SDValue New) {
215 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
216 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
217
218 DAG.ReplaceAllUsesWith(Old, New);
219 if (UpdatedNodes)
220 UpdatedNodes->insert(New.getNode());
221 ReplacedNode(Old.getNode());
222 }
223
224 void ReplaceNode(SDNode *Old, const SDValue *New) {
225 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); } } while (false)
;
226
227 DAG.ReplaceAllUsesWith(Old, New);
228 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
229 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
230 New[i]->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
;
231 if (UpdatedNodes)
232 UpdatedNodes->insert(New[i].getNode());
233 }
234 ReplacedNode(Old);
235 }
236};
237
238} // end anonymous namespace
239
240/// Return a vector shuffle operation which
241/// performs the same shuffe in terms of order or result bytes, but on a type
242/// whose vector element type is narrower than the original shuffle type.
243/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
244SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
245 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
246 ArrayRef<int> Mask) const {
247 unsigned NumMaskElts = VT.getVectorNumElements();
248 unsigned NumDestElts = NVT.getVectorNumElements();
249 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
250
251 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!")((NumEltsGrowth && "Cannot promote to vector type with fewer elts!"
) ? static_cast<void> (0) : __assert_fail ("NumEltsGrowth && \"Cannot promote to vector type with fewer elts!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 251, __PRETTY_FUNCTION__))
;
252
253 if (NumEltsGrowth == 1)
254 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
255
256 SmallVector<int, 8> NewMask;
257 for (unsigned i = 0; i != NumMaskElts; ++i) {
258 int Idx = Mask[i];
259 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
260 if (Idx < 0)
261 NewMask.push_back(-1);
262 else
263 NewMask.push_back(Idx * NumEltsGrowth + j);
264 }
265 }
266 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?")((NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"
) ? static_cast<void> (0) : __assert_fail ("NewMask.size() == NumDestElts && \"Non-integer NumEltsGrowth?\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 266, __PRETTY_FUNCTION__))
;
267 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?")((TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"
) ? static_cast<void> (0) : __assert_fail ("TLI.isShuffleMaskLegal(NewMask, NVT) && \"Shuffle not legal?\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 267, __PRETTY_FUNCTION__))
;
268 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
269}
270
271/// Expands the ConstantFP node to an integer constant or
272/// a load from the constant pool.
273SDValue
274SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
275 bool Extend = false;
276 SDLoc dl(CFP);
277
278 // If a FP immediate is precise when represented as a float and if the
279 // target can do an extending load from float to double, we put it into
280 // the constant pool as a float, even if it's is statically typed as a
281 // double. This shrinks FP constants and canonicalizes them for targets where
282 // an FP extending load is the same cost as a normal load (such as on the x87
283 // fp stack or PPC FP unit).
284 EVT VT = CFP->getValueType(0);
285 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
286 if (!UseCP) {
287 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion")(((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::f64 || VT == MVT::f32) && \"Invalid type expansion\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 287, __PRETTY_FUNCTION__))
;
288 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
289 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
290 }
291
292 APFloat APF = CFP->getValueAPF();
293 EVT OrigVT = VT;
294 EVT SVT = VT;
295
296 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
297 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
298 if (!APF.isSignaling()) {
299 while (SVT != MVT::f32 && SVT != MVT::f16) {
300 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
301 if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
302 // Only do this if the target has a native EXTLOAD instruction from
303 // smaller type.
304 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
305 TLI.ShouldShrinkFPConstant(OrigVT)) {
306 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
307 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
308 VT = SVT;
309 Extend = true;
310 }
311 }
312 }
313
314 SDValue CPIdx =
315 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
316 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
317 if (Extend) {
318 SDValue Result = DAG.getExtLoad(
319 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
320 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
321 Alignment);
322 return Result;
323 }
324 SDValue Result = DAG.getLoad(
325 OrigVT, dl, DAG.getEntryNode(), CPIdx,
326 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
327 return Result;
328}
329
330/// Expands the Constant node to a load from the constant pool.
331SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
332 SDLoc dl(CP);
333 EVT VT = CP->getValueType(0);
334 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
335 TLI.getPointerTy(DAG.getDataLayout()));
336 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
337 SDValue Result = DAG.getLoad(
338 VT, dl, DAG.getEntryNode(), CPIdx,
339 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
340 return Result;
341}
342
343/// Some target cannot handle a variable insertion index for the
344/// INSERT_VECTOR_ELT instruction. In this case, it
345/// is necessary to spill the vector being inserted into to memory, perform
346/// the insert there, and then read the result back.
347SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
348 SDValue Val,
349 SDValue Idx,
350 const SDLoc &dl) {
351 SDValue Tmp1 = Vec;
352 SDValue Tmp2 = Val;
353 SDValue Tmp3 = Idx;
354
355 // If the target doesn't support this, we have to spill the input vector
356 // to a temporary stack slot, update the element, then reload it. This is
357 // badness. We could also load the value into a vector register (either
358 // with a "move to register" or "extload into register" instruction, then
359 // permute it into place, if the idx is a constant and if the idx is
360 // supported by the target.
361 EVT VT = Tmp1.getValueType();
362 EVT EltVT = VT.getVectorElementType();
363 SDValue StackPtr = DAG.CreateStackTemporary(VT);
364
365 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
366
367 // Store the vector.
368 SDValue Ch = DAG.getStore(
369 DAG.getEntryNode(), dl, Tmp1, StackPtr,
370 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
371
372 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
373
374 // Store the scalar value.
375 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
376 // Load the updated vector.
377 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
378 DAG.getMachineFunction(), SPFI));
379}
380
381SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
382 SDValue Idx,
383 const SDLoc &dl) {
384 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
385 // SCALAR_TO_VECTOR requires that the type of the value being inserted
386 // match the element type of the vector being created, except for
387 // integers in which case the inserted value can be over width.
388 EVT EltVT = Vec.getValueType().getVectorElementType();
389 if (Val.getValueType() == EltVT ||
390 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
391 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
392 Vec.getValueType(), Val);
393
394 unsigned NumElts = Vec.getValueType().getVectorNumElements();
395 // We generate a shuffle of InVec and ScVec, so the shuffle mask
396 // should be 0,1,2,3,4,5... with the appropriate element replaced with
397 // elt 0 of the RHS.
398 SmallVector<int, 8> ShufOps;
399 for (unsigned i = 0; i != NumElts; ++i)
400 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
401
402 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
403 }
404 }
405 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
406}
407
408SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
409 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Optimizing float store operations\n"
; } } while (false)
;
410 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
411 // FIXME: We shouldn't do this for TargetConstantFP's.
412 // FIXME: move this to the DAG Combiner! Note that we can't regress due
413 // to phase ordering between legalized code and the dag combiner. This
414 // probably means that we need to integrate dag combiner and legalizer
415 // together.
416 // We generally can't do this one for long doubles.
417 SDValue Chain = ST->getChain();
418 SDValue Ptr = ST->getBasePtr();
419 unsigned Alignment = ST->getAlignment();
420 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
421 AAMDNodes AAInfo = ST->getAAInfo();
422 SDLoc dl(ST);
423 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
424 if (CFP->getValueType(0) == MVT::f32 &&
425 TLI.isTypeLegal(MVT::i32)) {
426 SDValue Con = DAG.getConstant(CFP->getValueAPF().
427 bitcastToAPInt().zextOrTrunc(32),
428 SDLoc(CFP), MVT::i32);
429 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
430 MMOFlags, AAInfo);
431 }
432
433 if (CFP->getValueType(0) == MVT::f64) {
434 // If this target supports 64-bit registers, do a single 64-bit store.
435 if (TLI.isTypeLegal(MVT::i64)) {
436 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
437 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
438 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
439 Alignment, MMOFlags, AAInfo);
440 }
441
442 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
443 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
444 // stores. If the target supports neither 32- nor 64-bits, this
445 // xform is certainly not worth it.
446 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
447 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
448 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
449 if (DAG.getDataLayout().isBigEndian())
450 std::swap(Lo, Hi);
451
452 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
453 MMOFlags, AAInfo);
454 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
455 DAG.getConstant(4, dl, Ptr.getValueType()));
456 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
457 ST->getPointerInfo().getWithOffset(4),
458 MinAlign(Alignment, 4U), MMOFlags, AAInfo);
459
460 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
461 }
462 }
463 }
464 return SDValue(nullptr, 0);
465}
466
467void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
468 StoreSDNode *ST = cast<StoreSDNode>(Node);
469 SDValue Chain = ST->getChain();
470 SDValue Ptr = ST->getBasePtr();
471 SDLoc dl(Node);
472
473 unsigned Alignment = ST->getAlignment();
474 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
475 AAMDNodes AAInfo = ST->getAAInfo();
476
477 if (!ST->isTruncatingStore()) {
11
Assuming the condition is false
12
Taking false branch
478 LLVM_DEBUG(dbgs() << "Legalizing store operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing store operation\n"
; } } while (false)
;
479 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
480 ReplaceNode(ST, OptStore);
481 return;
482 }
483
484 SDValue Value = ST->getValue();
485 MVT VT = Value.getSimpleValueType();
486 switch (TLI.getOperationAction(ISD::STORE, VT)) {
487 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 487)
;
488 case TargetLowering::Legal: {
489 // If this is an unaligned store and the target doesn't support it,
490 // expand it.
491 EVT MemVT = ST->getMemoryVT();
492 unsigned AS = ST->getAddressSpace();
493 unsigned Align = ST->getAlignment();
494 const DataLayout &DL = DAG.getDataLayout();
495 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
496 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Expanding unsupported unaligned store\n"
; } } while (false)
;
497 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
498 ReplaceNode(SDValue(ST, 0), Result);
499 } else
500 LLVM_DEBUG(dbgs() << "Legal store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legal store\n"; } } while
(false)
;
501 break;
502 }
503 case TargetLowering::Custom: {
504 LLVM_DEBUG(dbgs() << "Trying custom lowering\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying custom lowering\n"
; } } while (false)
;
505 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
506 if (Res && Res != SDValue(Node, 0))
507 ReplaceNode(SDValue(Node, 0), Res);
508 return;
509 }
510 case TargetLowering::Promote: {
511 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
512 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote stores to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 513, __PRETTY_FUNCTION__))
513 "Can only promote stores to same size type")((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote stores to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 513, __PRETTY_FUNCTION__))
;
514 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
515 SDValue Result =
516 DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
517 Alignment, MMOFlags, AAInfo);
518 ReplaceNode(SDValue(Node, 0), Result);
519 break;
520 }
521 }
522 return;
523 }
524
525 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing truncating store operations\n"
; } } while (false)
;
526 SDValue Value = ST->getValue();
527 EVT StVT = ST->getMemoryVT();
528 unsigned StWidth = StVT.getSizeInBits();
529 auto &DL = DAG.getDataLayout();
530
531 if (StWidth != StVT.getStoreSizeInBits()) {
13
Assuming the condition is false
14
Taking false branch
532 // Promote to a byte-sized store with upper bits zero if not
533 // storing an integral number of bytes. For example, promote
534 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
535 EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
536 StVT.getStoreSizeInBits());
537 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
538 SDValue Result =
539 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
540 Alignment, MMOFlags, AAInfo);
541 ReplaceNode(SDValue(Node, 0), Result);
542 } else if (StWidth & (StWidth - 1)) {
15
Assuming the condition is true
16
Taking true branch
543 // If not storing a power-of-2 number of bits, expand as two stores.
544 assert(!StVT.isVector() && "Unsupported truncstore!")((!StVT.isVector() && "Unsupported truncstore!") ? static_cast
<void> (0) : __assert_fail ("!StVT.isVector() && \"Unsupported truncstore!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 544, __PRETTY_FUNCTION__))
;
545 unsigned RoundWidth = 1 << Log2_32(StWidth);
17
Calling 'Log2_32'
19
Returning from 'Log2_32'
20
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'
546 assert(RoundWidth < StWidth)((RoundWidth < StWidth) ? static_cast<void> (0) : __assert_fail
("RoundWidth < StWidth", "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 546, __PRETTY_FUNCTION__))
;
547 unsigned ExtraWidth = StWidth - RoundWidth;
548 assert(ExtraWidth < RoundWidth)((ExtraWidth < RoundWidth) ? static_cast<void> (0) :
__assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 548, __PRETTY_FUNCTION__))
;
549 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Store size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 550, __PRETTY_FUNCTION__))
550 "Store size not an integral number of bytes!")((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Store size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 550, __PRETTY_FUNCTION__))
;
551 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
552 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
553 SDValue Lo, Hi;
554 unsigned IncrementSize;
555
556 if (DL.isLittleEndian()) {
557 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
558 // Store the bottom RoundWidth bits.
559 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
560 RoundVT, Alignment, MMOFlags, AAInfo);
561
562 // Store the remaining ExtraWidth bits.
563 IncrementSize = RoundWidth / 8;
564 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
565 DAG.getConstant(IncrementSize, dl,
566 Ptr.getValueType()));
567 Hi = DAG.getNode(
568 ISD::SRL, dl, Value.getValueType(), Value,
569 DAG.getConstant(RoundWidth, dl,
570 TLI.getShiftAmountTy(Value.getValueType(), DL)));
571 Hi = DAG.getTruncStore(
572 Chain, dl, Hi, Ptr,
573 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
574 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
575 } else {
576 // Big endian - avoid unaligned stores.
577 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
578 // Store the top RoundWidth bits.
579 Hi = DAG.getNode(
580 ISD::SRL, dl, Value.getValueType(), Value,
581 DAG.getConstant(ExtraWidth, dl,
582 TLI.getShiftAmountTy(Value.getValueType(), DL)));
583 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
584 RoundVT, Alignment, MMOFlags, AAInfo);
585
586 // Store the remaining ExtraWidth bits.
587 IncrementSize = RoundWidth / 8;
588 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
589 DAG.getConstant(IncrementSize, dl,
590 Ptr.getValueType()));
591 Lo = DAG.getTruncStore(
592 Chain, dl, Value, Ptr,
593 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
594 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
595 }
596
597 // The order of the stores doesn't matter.
598 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
599 ReplaceNode(SDValue(Node, 0), Result);
600 } else {
601 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
602 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 602)
;
603 case TargetLowering::Legal: {
604 EVT MemVT = ST->getMemoryVT();
605 unsigned AS = ST->getAddressSpace();
606 unsigned Align = ST->getAlignment();
607 // If this is an unaligned store and the target doesn't support it,
608 // expand it.
609 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
610 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
611 ReplaceNode(SDValue(ST, 0), Result);
612 }
613 break;
614 }
615 case TargetLowering::Custom: {
616 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
617 if (Res && Res != SDValue(Node, 0))
618 ReplaceNode(SDValue(Node, 0), Res);
619 return;
620 }
621 case TargetLowering::Expand:
622 assert(!StVT.isVector() &&((!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 623, __PRETTY_FUNCTION__))
623 "Vector Stores are handled in LegalizeVectorOps")((!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 623, __PRETTY_FUNCTION__))
;
624
625 SDValue Result;
626
627 // TRUNCSTORE:i16 i32 -> STORE i16
628 if (TLI.isTypeLegal(StVT)) {
629 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
630 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
631 Alignment, MMOFlags, AAInfo);
632 } else {
633 // The in-memory type isn't legal. Truncate to the type it would promote
634 // to, and then do a truncstore.
635 Value = DAG.getNode(ISD::TRUNCATE, dl,
636 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
637 Value);
638 Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
639 StVT, Alignment, MMOFlags, AAInfo);
640 }
641
642 ReplaceNode(SDValue(Node, 0), Result);
643 break;
644 }
645 }
646}
647
648void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
649 LoadSDNode *LD = cast<LoadSDNode>(Node);
650 SDValue Chain = LD->getChain(); // The chain.
651 SDValue Ptr = LD->getBasePtr(); // The base pointer.
652 SDValue Value; // The value returned by the load op.
653 SDLoc dl(Node);
654
655 ISD::LoadExtType ExtType = LD->getExtensionType();
656 if (ExtType == ISD::NON_EXTLOAD) {
657 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing non-extending load operation\n"
; } } while (false)
;
658 MVT VT = Node->getSimpleValueType(0);
659 SDValue RVal = SDValue(Node, 0);
660 SDValue RChain = SDValue(Node, 1);
661
662 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
663 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 663)
;
664 case TargetLowering::Legal: {
665 EVT MemVT = LD->getMemoryVT();
666 unsigned AS = LD->getAddressSpace();
667 unsigned Align = LD->getAlignment();
668 const DataLayout &DL = DAG.getDataLayout();
669 // If this is an unaligned load and the target doesn't support it,
670 // expand it.
671 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
672 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
673 }
674 break;
675 }
676 case TargetLowering::Custom:
677 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
678 RVal = Res;
679 RChain = Res.getValue(1);
680 }
681 break;
682
683 case TargetLowering::Promote: {
684 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
685 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote loads to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 686, __PRETTY_FUNCTION__))
686 "Can only promote loads to same size type")((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote loads to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 686, __PRETTY_FUNCTION__))
;
687
688 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
689 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
690 RChain = Res.getValue(1);
691 break;
692 }
693 }
694 if (RChain.getNode() != Node) {
695 assert(RVal.getNode() != Node && "Load must be completely replaced")((RVal.getNode() != Node && "Load must be completely replaced"
) ? static_cast<void> (0) : __assert_fail ("RVal.getNode() != Node && \"Load must be completely replaced\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 695, __PRETTY_FUNCTION__))
;
696 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
697 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
698 if (UpdatedNodes) {
699 UpdatedNodes->insert(RVal.getNode());
700 UpdatedNodes->insert(RChain.getNode());
701 }
702 ReplacedNode(Node);
703 }
704 return;
705 }
706
707 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing extending load operation\n"
; } } while (false)
;
708 EVT SrcVT = LD->getMemoryVT();
709 unsigned SrcWidth = SrcVT.getSizeInBits();
710 unsigned Alignment = LD->getAlignment();
711 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
712 AAMDNodes AAInfo = LD->getAAInfo();
713
714 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
715 // Some targets pretend to have an i1 loading operation, and actually
716 // load an i8. This trick is correct for ZEXTLOAD because the top 7
717 // bits are guaranteed to be zero; it helps the optimizers understand
718 // that these bits are zero. It is also useful for EXTLOAD, since it
719 // tells the optimizers that those bits are undefined. It would be
720 // nice to have an effective generic way of getting these benefits...
721 // Until such a way is found, don't insist on promoting i1 here.
722 (SrcVT != MVT::i1 ||
723 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
724 TargetLowering::Promote)) {
725 // Promote to a byte-sized load if not loading an integral number of
726 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
727 unsigned NewWidth = SrcVT.getStoreSizeInBits();
728 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
729 SDValue Ch;
730
731 // The extra bits are guaranteed to be zero, since we stored them that
732 // way. A zext load from NVT thus automatically gives zext from SrcVT.
733
734 ISD::LoadExtType NewExtType =
735 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
736
737 SDValue Result =
738 DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
739 LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
740
741 Ch = Result.getValue(1); // The chain.
742
743 if (ExtType == ISD::SEXTLOAD)
744 // Having the top bits zero doesn't help when sign extending.
745 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
746 Result.getValueType(),
747 Result, DAG.getValueType(SrcVT));
748 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
749 // All the top bits are guaranteed to be zero - inform the optimizers.
750 Result = DAG.getNode(ISD::AssertZext, dl,
751 Result.getValueType(), Result,
752 DAG.getValueType(SrcVT));
753
754 Value = Result;
755 Chain = Ch;
756 } else if (SrcWidth & (SrcWidth - 1)) {
757 // If not loading a power-of-2 number of bits, expand as two loads.
758 assert(!SrcVT.isVector() && "Unsupported extload!")((!SrcVT.isVector() && "Unsupported extload!") ? static_cast
<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Unsupported extload!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 758, __PRETTY_FUNCTION__))
;
759 unsigned RoundWidth = 1 << Log2_32(SrcWidth);
760 assert(RoundWidth < SrcWidth)((RoundWidth < SrcWidth) ? static_cast<void> (0) : __assert_fail
("RoundWidth < SrcWidth", "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 760, __PRETTY_FUNCTION__))
;
761 unsigned ExtraWidth = SrcWidth - RoundWidth;
762 assert(ExtraWidth < RoundWidth)((ExtraWidth < RoundWidth) ? static_cast<void> (0) :
__assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 762, __PRETTY_FUNCTION__))
;
763 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Load size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 764, __PRETTY_FUNCTION__))
764 "Load size not an integral number of bytes!")((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Load size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 764, __PRETTY_FUNCTION__))
;
765 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
766 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
767 SDValue Lo, Hi, Ch;
768 unsigned IncrementSize;
769 auto &DL = DAG.getDataLayout();
770
771 if (DL.isLittleEndian()) {
772 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
773 // Load the bottom RoundWidth bits.
774 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
775 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
776 AAInfo);
777
778 // Load the remaining ExtraWidth bits.
779 IncrementSize = RoundWidth / 8;
780 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
781 DAG.getConstant(IncrementSize, dl,
782 Ptr.getValueType()));
783 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
784 LD->getPointerInfo().getWithOffset(IncrementSize),
785 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
786 AAInfo);
787
788 // Build a factor node to remember that this load is independent of
789 // the other one.
790 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
791 Hi.getValue(1));
792
793 // Move the top bits to the right place.
794 Hi = DAG.getNode(
795 ISD::SHL, dl, Hi.getValueType(), Hi,
796 DAG.getConstant(RoundWidth, dl,
797 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
798
799 // Join the hi and lo parts.
800 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
801 } else {
802 // Big endian - avoid unaligned loads.
803 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
804 // Load the top RoundWidth bits.
805 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
806 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
807 AAInfo);
808
809 // Load the remaining ExtraWidth bits.
810 IncrementSize = RoundWidth / 8;
811 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
812 DAG.getConstant(IncrementSize, dl,
813 Ptr.getValueType()));
814 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
815 LD->getPointerInfo().getWithOffset(IncrementSize),
816 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
817 AAInfo);
818
819 // Build a factor node to remember that this load is independent of
820 // the other one.
821 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
822 Hi.getValue(1));
823
824 // Move the top bits to the right place.
825 Hi = DAG.getNode(
826 ISD::SHL, dl, Hi.getValueType(), Hi,
827 DAG.getConstant(ExtraWidth, dl,
828 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
829
830 // Join the hi and lo parts.
831 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
832 }
833
834 Chain = Ch;
835 } else {
836 bool isCustom = false;
837 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
838 SrcVT.getSimpleVT())) {
839 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 839)
;
840 case TargetLowering::Custom:
841 isCustom = true;
842 LLVM_FALLTHROUGH[[clang::fallthrough]];
843 case TargetLowering::Legal:
844 Value = SDValue(Node, 0);
845 Chain = SDValue(Node, 1);
846
847 if (isCustom) {
848 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
849 Value = Res;
850 Chain = Res.getValue(1);
851 }
852 } else {
853 // If this is an unaligned load and the target doesn't support it,
854 // expand it.
855 EVT MemVT = LD->getMemoryVT();
856 unsigned AS = LD->getAddressSpace();
857 unsigned Align = LD->getAlignment();
858 const DataLayout &DL = DAG.getDataLayout();
859 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
860 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
861 }
862 }
863 break;
864
865 case TargetLowering::Expand: {
866 EVT DestVT = Node->getValueType(0);
867 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
868 // If the source type is not legal, see if there is a legal extload to
869 // an intermediate type that we can then extend further.
870 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
871 if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
872 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
873 // If we are loading a legal type, this is a non-extload followed by a
874 // full extend.
875 ISD::LoadExtType MidExtType =
876 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
877
878 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
879 SrcVT, LD->getMemOperand());
880 unsigned ExtendOp =
881 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
882 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
883 Chain = Load.getValue(1);
884 break;
885 }
886
887 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
888 // normal undefined upper bits behavior to allow using an in-reg extend
889 // with the illegal FP type, so load as an integer and do the
890 // from-integer conversion.
891 if (SrcVT.getScalarType() == MVT::f16) {
892 EVT ISrcVT = SrcVT.changeTypeToInteger();
893 EVT IDestVT = DestVT.changeTypeToInteger();
894 EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
895
896 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
897 Chain, Ptr, ISrcVT,
898 LD->getMemOperand());
899 Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
900 Chain = Result.getValue(1);
901 break;
902 }
903 }
904
905 assert(!SrcVT.isVector() &&((!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 906, __PRETTY_FUNCTION__))
906 "Vector Loads are handled in LegalizeVectorOps")((!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 906, __PRETTY_FUNCTION__))
;
907
908 // FIXME: This does not work for vectors on most targets. Sign-
909 // and zero-extend operations are currently folded into extending
910 // loads, whether they are legal or not, and then we end up here
911 // without any support for legalizing them.
912 assert(ExtType != ISD::EXTLOAD &&((ExtType != ISD::EXTLOAD && "EXTLOAD should always be supported!"
) ? static_cast<void> (0) : __assert_fail ("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 913, __PRETTY_FUNCTION__))
913 "EXTLOAD should always be supported!")((ExtType != ISD::EXTLOAD && "EXTLOAD should always be supported!"
) ? static_cast<void> (0) : __assert_fail ("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 913, __PRETTY_FUNCTION__))
;
914 // Turn the unsupported load into an EXTLOAD followed by an
915 // explicit zero/sign extend inreg.
916 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
917 Node->getValueType(0),
918 Chain, Ptr, SrcVT,
919 LD->getMemOperand());
920 SDValue ValRes;
921 if (ExtType == ISD::SEXTLOAD)
922 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
923 Result.getValueType(),
924 Result, DAG.getValueType(SrcVT));
925 else
926 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
927 Value = ValRes;
928 Chain = Result.getValue(1);
929 break;
930 }
931 }
932 }
933
934 // Since loads produce two values, make sure to remember that we legalized
935 // both of them.
936 if (Chain.getNode() != Node) {
937 assert(Value.getNode() != Node && "Load must be completely replaced")((Value.getNode() != Node && "Load must be completely replaced"
) ? static_cast<void> (0) : __assert_fail ("Value.getNode() != Node && \"Load must be completely replaced\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 937, __PRETTY_FUNCTION__))
;
938 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
939 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
940 if (UpdatedNodes) {
941 UpdatedNodes->insert(Value.getNode());
942 UpdatedNodes->insert(Chain.getNode());
943 }
944 ReplacedNode(Node);
945 }
946}
947
948/// Return a legal replacement for the given operation, with all legal operands.
949void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
950 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "\nLegalizing: "; Node->
dump(&DAG); } } while (false)
;
951
952 // Allow illegal target nodes and illegal registers.
953 if (Node->getOpcode() == ISD::TargetConstant ||
1
Assuming the condition is false
3
Taking false branch
954 Node->getOpcode() == ISD::Register)
2
Assuming the condition is false
955 return;
956
957#ifndef NDEBUG
958 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
4
Assuming 'i' is equal to 'e'
5
Loop condition is false. Execution continues on line 964
959 assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 962, __PRETTY_FUNCTION__))
960 TargetLowering::TypeLegal ||(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 962, __PRETTY_FUNCTION__))
961 TLI.isTypeLegal(Node->getValueType(i))) &&(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 962, __PRETTY_FUNCTION__))
962 "Unexpected illegal type!")(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 962, __PRETTY_FUNCTION__))
;
963
964 for (const SDValue &Op : Node->op_values())
965 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
966 TargetLowering::TypeLegal ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
967 TLI.isTypeLegal(Op.getValueType()) ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
968 Op.getOpcode() == ISD::TargetConstant ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
969 Op.getOpcode() == ISD::Register) &&(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
970 "Unexpected illegal type!")(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 970, __PRETTY_FUNCTION__))
;
971#endif
972
973 // Figure out the correct action; the way to query this varies by opcode
974 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
975 bool SimpleFinishLegalizing = true;
976 switch (Node->getOpcode()) {
6
Control jumps to 'case STORE:' at line 1030
977 case ISD::INTRINSIC_W_CHAIN:
978 case ISD::INTRINSIC_WO_CHAIN:
979 case ISD::INTRINSIC_VOID:
980 case ISD::STACKSAVE:
981 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
982 break;
983 case ISD::GET_DYNAMIC_AREA_OFFSET:
984 Action = TLI.getOperationAction(Node->getOpcode(),
985 Node->getValueType(0));
986 break;
987 case ISD::VAARG:
988 Action = TLI.getOperationAction(Node->getOpcode(),
989 Node->getValueType(0));
990 if (Action != TargetLowering::Promote)
991 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
992 break;
993 case ISD::FP_TO_FP16:
994 case ISD::SINT_TO_FP:
995 case ISD::UINT_TO_FP:
996 case ISD::EXTRACT_VECTOR_ELT:
997 Action = TLI.getOperationAction(Node->getOpcode(),
998 Node->getOperand(0).getValueType());
999 break;
1000 case ISD::FP_ROUND_INREG:
1001 case ISD::SIGN_EXTEND_INREG: {
1002 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1003 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1004 break;
1005 }
1006 case ISD::ATOMIC_STORE:
1007 Action = TLI.getOperationAction(Node->getOpcode(),
1008 Node->getOperand(2).getValueType());
1009 break;
1010 case ISD::SELECT_CC:
1011 case ISD::SETCC:
1012 case ISD::BR_CC: {
1013 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1014 Node->getOpcode() == ISD::SETCC ? 2 : 1;
1015 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
1016 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1017 ISD::CondCode CCCode =
1018 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1019 Action = TLI.getCondCodeAction(CCCode, OpVT);
1020 if (Action == TargetLowering::Legal) {
1021 if (Node->getOpcode() == ISD::SELECT_CC)
1022 Action = TLI.getOperationAction(Node->getOpcode(),
1023 Node->getValueType(0));
1024 else
1025 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1026 }
1027 break;
1028 }
1029 case ISD::LOAD:
1030 case ISD::STORE:
1031 // FIXME: Model these properly. LOAD and STORE are complicated, and
1032 // STORE expects the unlegalized operand in some cases.
1033 SimpleFinishLegalizing = false;
1034 break;
7
Execution continues on line 1139
1035 case ISD::CALLSEQ_START:
1036 case ISD::CALLSEQ_END:
1037 // FIXME: This shouldn't be necessary. These nodes have special properties
1038 // dealing with the recursive nature of legalization. Removing this
1039 // special case should be done as part of making LegalizeDAG non-recursive.
1040 SimpleFinishLegalizing = false;
1041 break;
1042 case ISD::EXTRACT_ELEMENT:
1043 case ISD::FLT_ROUNDS_:
1044 case ISD::MERGE_VALUES:
1045 case ISD::EH_RETURN:
1046 case ISD::FRAME_TO_ARGS_OFFSET:
1047 case ISD::EH_DWARF_CFA:
1048 case ISD::EH_SJLJ_SETJMP:
1049 case ISD::EH_SJLJ_LONGJMP:
1050 case ISD::EH_SJLJ_SETUP_DISPATCH:
1051 // These operations lie about being legal: when they claim to be legal,
1052 // they should actually be expanded.
1053 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1054 if (Action == TargetLowering::Legal)
1055 Action = TargetLowering::Expand;
1056 break;
1057 case ISD::INIT_TRAMPOLINE:
1058 case ISD::ADJUST_TRAMPOLINE:
1059 case ISD::FRAMEADDR:
1060 case ISD::RETURNADDR:
1061 case ISD::ADDROFRETURNADDR:
1062 // These operations lie about being legal: when they claim to be legal,
1063 // they should actually be custom-lowered.
1064 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1065 if (Action == TargetLowering::Legal)
1066 Action = TargetLowering::Custom;
1067 break;
1068 case ISD::READCYCLECOUNTER:
1069 // READCYCLECOUNTER returns an i64, even if type legalization might have
1070 // expanded that to several smaller types.
1071 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1072 break;
1073 case ISD::READ_REGISTER:
1074 case ISD::WRITE_REGISTER:
1075 // Named register is legal in the DAG, but blocked by register name
1076 // selection if not implemented by target (to chose the correct register)
1077 // They'll be converted to Copy(To/From)Reg.
1078 Action = TargetLowering::Legal;
1079 break;
1080 case ISD::DEBUGTRAP:
1081 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1082 if (Action == TargetLowering::Expand) {
1083 // replace ISD::DEBUGTRAP with ISD::TRAP
1084 SDValue NewVal;
1085 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1086 Node->getOperand(0));
1087 ReplaceNode(Node, NewVal.getNode());
1088 LegalizeOp(NewVal.getNode());
1089 return;
1090 }
1091 break;
1092 case ISD::STRICT_FADD:
1093 case ISD::STRICT_FSUB:
1094 case ISD::STRICT_FMUL:
1095 case ISD::STRICT_FDIV:
1096 case ISD::STRICT_FREM:
1097 case ISD::STRICT_FSQRT:
1098 case ISD::STRICT_FMA:
1099 case ISD::STRICT_FPOW:
1100 case ISD::STRICT_FPOWI:
1101 case ISD::STRICT_FSIN:
1102 case ISD::STRICT_FCOS:
1103 case ISD::STRICT_FEXP:
1104 case ISD::STRICT_FEXP2:
1105 case ISD::STRICT_FLOG:
1106 case ISD::STRICT_FLOG10:
1107 case ISD::STRICT_FLOG2:
1108 case ISD::STRICT_FRINT:
1109 case ISD::STRICT_FNEARBYINT:
1110 // These pseudo-ops get legalized as if they were their non-strict
1111 // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
1112 // is also legal, but if ISD::FSQRT requires expansion then so does
1113 // ISD::STRICT_FSQRT.
1114 Action = TLI.getStrictFPOperationAction(Node->getOpcode(),
1115 Node->getValueType(0));
1116 break;
1117 case ISD::SADDSAT:
1118 case ISD::UADDSAT: {
1119 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1120 break;
1121 }
1122 case ISD::MSCATTER:
1123 Action = TLI.getOperationAction(Node->getOpcode(),
1124 cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1125 break;
1126 case ISD::MSTORE:
1127 Action = TLI.getOperationAction(Node->getOpcode(),
1128 cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1129 break;
1130 default:
1131 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1132 Action = TargetLowering::Legal;
1133 } else {
1134 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1135 }
1136 break;
1137 }
1138
1139 if (SimpleFinishLegalizing) {
8
Taking false branch
1140 SDNode *NewNode = Node;
1141 switch (Node->getOpcode()) {
1142 default: break;
1143 case ISD::SHL:
1144 case ISD::SRL:
1145 case ISD::SRA:
1146 case ISD::ROTL:
1147 case ISD::ROTR: {
1148 // Legalizing shifts/rotates requires adjusting the shift amount
1149 // to the appropriate width.
1150 SDValue Op0 = Node->getOperand(0);
1151 SDValue Op1 = Node->getOperand(1);
1152 if (!Op1.getValueType().isVector()) {
1153 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1154 // The getShiftAmountOperand() may create a new operand node or
1155 // return the existing one. If new operand is created we need
1156 // to update the parent node.
1157 // Do not try to legalize SAO here! It will be automatically legalized
1158 // in the next round.
1159 if (SAO != Op1)
1160 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1161 }
1162 }
1163 break;
1164 case ISD::SRL_PARTS:
1165 case ISD::SRA_PARTS:
1166 case ISD::SHL_PARTS: {
1167 // Legalizing shifts/rotates requires adjusting the shift amount
1168 // to the appropriate width.
1169 SDValue Op0 = Node->getOperand(0);
1170 SDValue Op1 = Node->getOperand(1);
1171 SDValue Op2 = Node->getOperand(2);
1172 if (!Op2.getValueType().isVector()) {
1173 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1174 // The getShiftAmountOperand() may create a new operand node or
1175 // return the existing one. If new operand is created we need
1176 // to update the parent node.
1177 if (SAO != Op2)
1178 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1179 }
1180 break;
1181 }
1182 }
1183
1184 if (NewNode != Node) {
1185 ReplaceNode(Node, NewNode);
1186 Node = NewNode;
1187 }
1188 switch (Action) {
1189 case TargetLowering::Legal:
1190 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legal node: nothing to do\n"
; } } while (false)
;
1191 return;
1192 case TargetLowering::Custom:
1193 LLVM_DEBUG(dbgs() << "Trying custom legalization\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying custom legalization\n"
; } } while (false)
;
1194 // FIXME: The handling for custom lowering with multiple results is
1195 // a complete mess.
1196 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1197 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1198 return;
1199
1200 if (Node->getNumValues() == 1) {
1201 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully custom legalized node\n"
; } } while (false)
;
1202 // We can just directly replace this node with the lowered value.
1203 ReplaceNode(SDValue(Node, 0), Res);
1204 return;
1205 }
1206
1207 SmallVector<SDValue, 8> ResultVals;
1208 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1209 ResultVals.push_back(Res.getValue(i));
1210 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully custom legalized node\n"
; } } while (false)
;
1211 ReplaceNode(Node, ResultVals.data());
1212 return;
1213 }
1214 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not custom legalize node\n"
; } } while (false)
;
1215 LLVM_FALLTHROUGH[[clang::fallthrough]];
1216 case TargetLowering::Expand:
1217 if (ExpandNode(Node))
1218 return;
1219 LLVM_FALLTHROUGH[[clang::fallthrough]];
1220 case TargetLowering::LibCall:
1221 ConvertNodeToLibcall(Node);
1222 return;
1223 case TargetLowering::Promote:
1224 PromoteNode(Node);
1225 return;
1226 }
1227 }
1228
1229 switch (Node->getOpcode()) {
9
Control jumps to 'case STORE:' at line 1243
1230 default:
1231#ifndef NDEBUG
1232 dbgs() << "NODE: ";
1233 Node->dump( &DAG);
1234 dbgs() << "\n";
1235#endif
1236 llvm_unreachable("Do not know how to legalize this operator!")::llvm::llvm_unreachable_internal("Do not know how to legalize this operator!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1236)
;
1237
1238 case ISD::CALLSEQ_START:
1239 case ISD::CALLSEQ_END:
1240 break;
1241 case ISD::LOAD:
1242 return LegalizeLoadOps(Node);
1243 case ISD::STORE:
1244 return LegalizeStoreOps(Node);
10
Calling 'SelectionDAGLegalize::LegalizeStoreOps'
1245 }
1246}
1247
1248SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1249 SDValue Vec = Op.getOperand(0);
1250 SDValue Idx = Op.getOperand(1);
1251 SDLoc dl(Op);
1252
1253 // Before we generate a new store to a temporary stack slot, see if there is
1254 // already one that we can use. There often is because when we scalarize
1255 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1256 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1257 // the vector. If all are expanded here, we don't want one store per vector
1258 // element.
1259
1260 // Caches for hasPredecessorHelper
1261 SmallPtrSet<const SDNode *, 32> Visited;
1262 SmallVector<const SDNode *, 16> Worklist;
1263 Visited.insert(Op.getNode());
1264 Worklist.push_back(Idx.getNode());
1265 SDValue StackPtr, Ch;
1266 for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1267 UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1268 SDNode *User = *UI;
1269 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1270 if (ST->isIndexed() || ST->isTruncatingStore() ||
1271 ST->getValue() != Vec)
1272 continue;
1273
1274 // Make sure that nothing else could have stored into the destination of
1275 // this store.
1276 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1277 continue;
1278
1279 // If the index is dependent on the store we will introduce a cycle when
1280 // creating the load (the load uses the index, and by replacing the chain
1281 // we will make the index dependent on the load). Also, the store might be
1282 // dependent on the extractelement and introduce a cycle when creating
1283 // the load.
1284 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1285 ST->hasPredecessor(Op.getNode()))
1286 continue;
1287
1288 StackPtr = ST->getBasePtr();
1289 Ch = SDValue(ST, 0);
1290 break;
1291 }
1292 }
1293
1294 EVT VecVT = Vec.getValueType();
1295
1296 if (!Ch.getNode()) {
1297 // Store the value to a temporary stack slot, then LOAD the returned part.
1298 StackPtr = DAG.CreateStackTemporary(VecVT);
1299 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1300 MachinePointerInfo());
1301 }
1302
1303 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1304
1305 SDValue NewLoad;
1306
1307 if (Op.getValueType().isVector())
1308 NewLoad =
1309 DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1310 else
1311 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1312 MachinePointerInfo(),
1313 VecVT.getVectorElementType());
1314
1315 // Replace the chain going out of the store, by the one out of the load.
1316 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1317
1318 // We introduced a cycle though, so update the loads operands, making sure
1319 // to use the original store's chain as an incoming chain.
1320 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1321 NewLoad->op_end());
1322 NewLoadOperands[0] = Ch;
1323 NewLoad =
1324 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1325 return NewLoad;
1326}
1327
1328SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1329 assert(Op.getValueType().isVector() && "Non-vector insert subvector!")((Op.getValueType().isVector() && "Non-vector insert subvector!"
) ? static_cast<void> (0) : __assert_fail ("Op.getValueType().isVector() && \"Non-vector insert subvector!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1329, __PRETTY_FUNCTION__))
;
1330
1331 SDValue Vec = Op.getOperand(0);
1332 SDValue Part = Op.getOperand(1);
1333 SDValue Idx = Op.getOperand(2);
1334 SDLoc dl(Op);
1335
1336 // Store the value to a temporary stack slot, then LOAD the returned part.
1337 EVT VecVT = Vec.getValueType();
1338 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1339 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1340 MachinePointerInfo PtrInfo =
1341 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1342
1343 // First store the whole vector.
1344 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1345
1346 // Then store the inserted part.
1347 SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1348
1349 // Store the subvector.
1350 Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
1351
1352 // Finally, load the updated vector.
1353 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1354}
1355
1356SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1357 // We can't handle this case efficiently. Allocate a sufficiently
1358 // aligned object on the stack, store each element into it, then load
1359 // the result as a vector.
1360 // Create the stack frame object.
1361 EVT VT = Node->getValueType(0);
1362 EVT EltVT = VT.getVectorElementType();
1363 SDLoc dl(Node);
1364 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1365 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1366 MachinePointerInfo PtrInfo =
1367 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1368
1369 // Emit a store of each element to the stack slot.
1370 SmallVector<SDValue, 8> Stores;
1371 unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1372 // Store (in the right endianness) the elements to memory.
1373 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1374 // Ignore undef elements.
1375 if (Node->getOperand(i).isUndef()) continue;
1376
1377 unsigned Offset = TypeByteSize*i;
1378
1379 SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1380 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1381
1382 // If the destination vector element type is narrower than the source
1383 // element type, only store the bits necessary.
1384 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1385 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1386 Node->getOperand(i), Idx,
1387 PtrInfo.getWithOffset(Offset), EltVT));
1388 } else
1389 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1390 Idx, PtrInfo.getWithOffset(Offset)));
1391 }
1392
1393 SDValue StoreChain;
1394 if (!Stores.empty()) // Not all undef elements?
1395 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1396 else
1397 StoreChain = DAG.getEntryNode();
1398
1399 // Result is a load from the stack slot.
1400 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1401}
1402
1403/// Bitcast a floating-point value to an integer value. Only bitcast the part
1404/// containing the sign bit if the target has no integer value capable of
1405/// holding all bits of the floating-point value.
1406void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1407 const SDLoc &DL,
1408 SDValue Value) const {
1409 EVT FloatVT = Value.getValueType();
1410 unsigned NumBits = FloatVT.getSizeInBits();
1411 State.FloatVT = FloatVT;
1412 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1413 // Convert to an integer of the same size.
1414 if (TLI.isTypeLegal(IVT)) {
1415 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1416 State.SignMask = APInt::getSignMask(NumBits);
1417 State.SignBit = NumBits - 1;
1418 return;
1419 }
1420
1421 auto &DataLayout = DAG.getDataLayout();
1422 // Store the float to memory, then load the sign part out as an integer.
1423 MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1424 // First create a temporary that is aligned for both the load and store.
1425 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1426 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1427 // Then store the float to it.
1428 State.FloatPtr = StackPtr;
1429 MachineFunction &MF = DAG.getMachineFunction();
1430 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1431 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1432 State.FloatPointerInfo);
1433
1434 SDValue IntPtr;
1435 if (DataLayout.isBigEndian()) {
1436 assert(FloatVT.isByteSized() && "Unsupported floating point type!")((FloatVT.isByteSized() && "Unsupported floating point type!"
) ? static_cast<void> (0) : __assert_fail ("FloatVT.isByteSized() && \"Unsupported floating point type!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1436, __PRETTY_FUNCTION__))
;
1437 // Load out a legal integer with the same sign bit as the float.
1438 IntPtr = StackPtr;
1439 State.IntPointerInfo = State.FloatPointerInfo;
1440 } else {
1441 // Advance the pointer so that the loaded byte will contain the sign bit.
1442 unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
1443 IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
1444 DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
1445 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1446 ByteOffset);
1447 }
1448
1449 State.IntPtr = IntPtr;
1450 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1451 State.IntPointerInfo, MVT::i8);
1452 State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
1453 State.SignBit = 7;
1454}
1455
1456/// Replace the integer value produced by getSignAsIntValue() with a new value
1457/// and cast the result back to a floating-point type.
1458SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1459 const SDLoc &DL,
1460 SDValue NewIntValue) const {
1461 if (!State.Chain)
1462 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1463
1464 // Override the part containing the sign bit in the value stored on the stack.
1465 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1466 State.IntPointerInfo, MVT::i8);
1467 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1468 State.FloatPointerInfo);
1469}
1470
1471SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1472 SDLoc DL(Node);
1473 SDValue Mag = Node->getOperand(0);
1474 SDValue Sign = Node->getOperand(1);
1475
1476 // Get sign bit into an integer value.
1477 FloatSignAsInt SignAsInt;
1478 getSignAsIntValue(SignAsInt, DL, Sign);
1479
1480 EVT IntVT = SignAsInt.IntValue.getValueType();
1481 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1482 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1483 SignMask);
1484
1485 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1486 EVT FloatVT = Mag.getValueType();
1487 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1488 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1489 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1490 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1491 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1492 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1493 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1494 }
1495
1496 // Transform Mag value to integer, and clear the sign bit.
1497 FloatSignAsInt MagAsInt;
1498 getSignAsIntValue(MagAsInt, DL, Mag);
1499 EVT MagVT = MagAsInt.IntValue.getValueType();
1500 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1501 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1502 ClearSignMask);
1503
1504 // Get the signbit at the right position for MagAsInt.
1505 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1506 EVT ShiftVT = IntVT;
1507 if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
1508 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1509 ShiftVT = MagVT;
1510 }
1511 if (ShiftAmount > 0) {
1512 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1513 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1514 } else if (ShiftAmount < 0) {
1515 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1516 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1517 }
1518 if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
1519 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1520 }
1521
1522 // Store the part with the modified sign and convert back to float.
1523 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1524 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1525}
1526
1527SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1528 SDLoc DL(Node);
1529 SDValue Value = Node->getOperand(0);
1530
1531 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1532 EVT FloatVT = Value.getValueType();
1533 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1534 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1535 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1536 }
1537
1538 // Transform value to integer, clear the sign bit and transform back.
1539 FloatSignAsInt ValueAsInt;
1540 getSignAsIntValue(ValueAsInt, DL, Value);
1541 EVT IntVT = ValueAsInt.IntValue.getValueType();
1542 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1543 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1544 ClearSignMask);
1545 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1546}
1547
1548void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1549 SmallVectorImpl<SDValue> &Results) {
1550 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1551 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"((SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? static_cast
<void> (0) : __assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1552, __PRETTY_FUNCTION__))
1552 " not tell us which reg is the stack pointer!")((SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? static_cast
<void> (0) : __assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1552, __PRETTY_FUNCTION__))
;
1553 SDLoc dl(Node);
1554 EVT VT = Node->getValueType(0);
1555 SDValue Tmp1 = SDValue(Node, 0);
1556 SDValue Tmp2 = SDValue(Node, 1);
1557 SDValue Tmp3 = Node->getOperand(2);
1558 SDValue Chain = Tmp1.getOperand(0);
1559
1560 // Chain the dynamic stack allocation so that it doesn't modify the stack
1561 // pointer when other instructions are using the stack.
1562 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1563
1564 SDValue Size = Tmp2.getOperand(1);
1565 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1566 Chain = SP.getValue(1);
1567 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1568 unsigned StackAlign =
1569 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1570 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1571 if (Align > StackAlign)
1572 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1573 DAG.getConstant(-(uint64_t)Align, dl, VT));
1574 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1575
1576 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1577 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1578
1579 Results.push_back(Tmp1);
1580 Results.push_back(Tmp2);
1581}
1582
1583/// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1584/// target.
1585///
1586/// If the SETCC has been legalized using AND / OR, then the legalized node
1587/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1588/// will be set to false.
1589///
1590/// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1591/// then the values of LHS and RHS will be swapped, CC will be set to the
1592/// new condition, and NeedInvert will be set to false.
1593///
1594/// If the SETCC has been legalized using the inverse condcode, then LHS and
1595/// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1596/// will be set to true. The caller must invert the result of the SETCC with
1597/// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1598/// of a true/false result.
1599///
1600/// \returns true if the SetCC has been legalized, false if it hasn't.
1601bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1602 SDValue &RHS, SDValue &CC,
1603 bool &NeedInvert,
1604 const SDLoc &dl) {
1605 MVT OpVT = LHS.getSimpleValueType();
1606 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1607 NeedInvert = false;
1608 bool NeedSwap = false;
1609 switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1610 default: llvm_unreachable("Unknown condition code action!")::llvm::llvm_unreachable_internal("Unknown condition code action!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1610)
;
1611 case TargetLowering::Legal:
1612 // Nothing to do.
1613 break;
1614 case TargetLowering::Expand: {
1615 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
1616 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1617 std::swap(LHS, RHS);
1618 CC = DAG.getCondCode(InvCC);
1619 return true;
1620 }
1621 // Swapping operands didn't work. Try inverting the condition.
1622 InvCC = getSetCCInverse(CCCode, OpVT.isInteger());
1623 if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1624 // If inverting the condition is not enough, try swapping operands
1625 // on top of it.
1626 InvCC = ISD::getSetCCSwappedOperands(InvCC);
1627 NeedSwap = true;
1628 }
1629 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1630 CC = DAG.getCondCode(InvCC);
1631 NeedInvert = true;
1632 if (NeedSwap)
1633 std::swap(LHS, RHS);
1634 return true;
1635 }
1636
1637 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1638 unsigned Opc = 0;
1639 switch (CCCode) {
1640 default: llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1640)
;
1641 case ISD::SETO:
1642 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT)((TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && "If SETO is expanded, SETOEQ must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1643, __PRETTY_FUNCTION__))
1643 && "If SETO is expanded, SETOEQ must be legal!")((TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && "If SETO is expanded, SETOEQ must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1643, __PRETTY_FUNCTION__))
;
1644 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1645 case ISD::SETUO:
1646 assert(TLI.isCondCodeLegal(ISD::SETUNE, OpVT)((TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && "If SETUO is expanded, SETUNE must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1647, __PRETTY_FUNCTION__))
1647 && "If SETUO is expanded, SETUNE must be legal!")((TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && "If SETUO is expanded, SETUNE must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1647, __PRETTY_FUNCTION__))
;
1648 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break;
1649 case ISD::SETOEQ:
1650 case ISD::SETOGT:
1651 case ISD::SETOGE:
1652 case ISD::SETOLT:
1653 case ISD::SETOLE:
1654 case ISD::SETONE:
1655 case ISD::SETUEQ:
1656 case ISD::SETUNE:
1657 case ISD::SETUGT:
1658 case ISD::SETUGE:
1659 case ISD::SETULT:
1660 case ISD::SETULE:
1661 // If we are floating point, assign and break, otherwise fall through.
1662 if (!OpVT.isInteger()) {
1663 // We can use the 4th bit to tell if we are the unordered
1664 // or ordered version of the opcode.
1665 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
1666 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
1667 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1668 break;
1669 }
1670 // Fallthrough if we are unsigned integer.
1671 LLVM_FALLTHROUGH[[clang::fallthrough]];
1672 case ISD::SETLE:
1673 case ISD::SETGT:
1674 case ISD::SETGE:
1675 case ISD::SETLT:
1676 case ISD::SETNE:
1677 case ISD::SETEQ:
1678 // If all combinations of inverting the condition and swapping operands
1679 // didn't work then we have no means to expand the condition.
1680 llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1680)
;
1681 }
1682
1683 SDValue SetCC1, SetCC2;
1684 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
1685 // If we aren't the ordered or unorder operation,
1686 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1687 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1688 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1689 } else {
1690 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1691 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1692 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1693 }
1694 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1695 RHS = SDValue();
1696 CC = SDValue();
1697 return true;
1698 }
1699 }
1700 return false;
1701}
1702
1703/// Emit a store/load combination to the stack. This stores
1704/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1705/// a load from the stack slot to DestVT, extending it if needed.
1706/// The resultant code need not be legal.
1707SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1708 EVT DestVT, const SDLoc &dl) {
1709 // Create the stack frame object.
1710 unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1711 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1712 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1713
1714 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1715 int SPFI = StackPtrFI->getIndex();
1716 MachinePointerInfo PtrInfo =
1717 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1718
1719 unsigned SrcSize = SrcOp.getValueSizeInBits();
1720 unsigned SlotSize = SlotVT.getSizeInBits();
1721 unsigned DestSize = DestVT.getSizeInBits();
1722 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1723 unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1724
1725 // Emit a store to the stack slot. Use a truncstore if the input value is
1726 // later than DestVT.
1727 SDValue Store;
1728
1729 if (SrcSize > SlotSize)
1730 Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
1731 SlotVT, SrcAlign);
1732 else {
1733 assert(SrcSize == SlotSize && "Invalid store")((SrcSize == SlotSize && "Invalid store") ? static_cast
<void> (0) : __assert_fail ("SrcSize == SlotSize && \"Invalid store\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1733, __PRETTY_FUNCTION__))
;
1734 Store =
1735 DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1736 }
1737
1738 // Result is a load from the stack slot.
1739 if (SlotSize == DestSize)
1740 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1741
1742 assert(SlotSize < DestSize && "Unknown extension!")((SlotSize < DestSize && "Unknown extension!") ? static_cast
<void> (0) : __assert_fail ("SlotSize < DestSize && \"Unknown extension!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1742, __PRETTY_FUNCTION__))
;
1743 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1744 DestAlign);
1745}
1746
1747SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1748 SDLoc dl(Node);
1749 // Create a vector sized/aligned stack slot, store the value to element #0,
1750 // then load the whole vector back out.
1751 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1752
1753 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1754 int SPFI = StackPtrFI->getIndex();
1755
1756 SDValue Ch = DAG.getTruncStore(
1757 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1758 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1759 Node->getValueType(0).getVectorElementType());
1760 return DAG.getLoad(
1761 Node->getValueType(0), dl, Ch, StackPtr,
1762 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1763}
1764
1765static bool
1766ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1767 const TargetLowering &TLI, SDValue &Res) {
1768 unsigned NumElems = Node->getNumOperands();
1769 SDLoc dl(Node);
1770 EVT VT = Node->getValueType(0);
1771
1772 // Try to group the scalars into pairs, shuffle the pairs together, then
1773 // shuffle the pairs of pairs together, etc. until the vector has
1774 // been built. This will work only if all of the necessary shuffle masks
1775 // are legal.
1776
1777 // We do this in two phases; first to check the legality of the shuffles,
1778 // and next, assuming that all shuffles are legal, to create the new nodes.
1779 for (int Phase = 0; Phase < 2; ++Phase) {
1780 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1781 NewIntermedVals;
1782 for (unsigned i = 0; i < NumElems; ++i) {
1783 SDValue V = Node->getOperand(i);
1784 if (V.isUndef())
1785 continue;
1786
1787 SDValue Vec;
1788 if (Phase)
1789 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1790 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1791 }
1792
1793 while (IntermedVals.size() > 2) {
1794 NewIntermedVals.clear();
1795 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1796 // This vector and the next vector are shuffled together (simply to
1797 // append the one to the other).
1798 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1799
1800 SmallVector<int, 16> FinalIndices;
1801 FinalIndices.reserve(IntermedVals[i].second.size() +
1802 IntermedVals[i+1].second.size());
1803
1804 int k = 0;
1805 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1806 ++j, ++k) {
1807 ShuffleVec[k] = j;
1808 FinalIndices.push_back(IntermedVals[i].second[j]);
1809 }
1810 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1811 ++j, ++k) {
1812 ShuffleVec[k] = NumElems + j;
1813 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1814 }
1815
1816 SDValue Shuffle;
1817 if (Phase)
1818 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1819 IntermedVals[i+1].first,
1820 ShuffleVec);
1821 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1822 return false;
1823 NewIntermedVals.push_back(
1824 std::make_pair(Shuffle, std::move(FinalIndices)));
1825 }
1826
1827 // If we had an odd number of defined values, then append the last
1828 // element to the array of new vectors.
1829 if ((IntermedVals.size() & 1) != 0)
1830 NewIntermedVals.push_back(IntermedVals.back());
1831
1832 IntermedVals.swap(NewIntermedVals);
1833 }
1834
1835 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&((IntermedVals.size() <= 2 && IntermedVals.size() >
0 && "Invalid number of intermediate vectors") ? static_cast
<void> (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1836, __PRETTY_FUNCTION__))
1836 "Invalid number of intermediate vectors")((IntermedVals.size() <= 2 && IntermedVals.size() >
0 && "Invalid number of intermediate vectors") ? static_cast
<void> (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1836, __PRETTY_FUNCTION__))
;
1837 SDValue Vec1 = IntermedVals[0].first;
1838 SDValue Vec2;
1839 if (IntermedVals.size() > 1)
1840 Vec2 = IntermedVals[1].first;
1841 else if (Phase)
1842 Vec2 = DAG.getUNDEF(VT);
1843
1844 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1845 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1846 ShuffleVec[IntermedVals[0].second[i]] = i;
1847 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1848 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1849
1850 if (Phase)
1851 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1852 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1853 return false;
1854 }
1855
1856 return true;
1857}
1858
1859/// Expand a BUILD_VECTOR node on targets that don't
1860/// support the operation, but do support the resultant vector type.
1861SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1862 unsigned NumElems = Node->getNumOperands();
1863 SDValue Value1, Value2;
1864 SDLoc dl(Node);
1865 EVT VT = Node->getValueType(0);
1866 EVT OpVT = Node->getOperand(0).getValueType();
1867 EVT EltVT = VT.getVectorElementType();
1868
1869 // If the only non-undef value is the low element, turn this into a
1870 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1871 bool isOnlyLowElement = true;
1872 bool MoreThanTwoValues = false;
1873 bool isConstant = true;
1874 for (unsigned i = 0; i < NumElems; ++i) {
1875 SDValue V = Node->getOperand(i);
1876 if (V.isUndef())
1877 continue;
1878 if (i > 0)
1879 isOnlyLowElement = false;
1880 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1881 isConstant = false;
1882
1883 if (!Value1.getNode()) {
1884 Value1 = V;
1885 } else if (!Value2.getNode()) {
1886 if (V != Value1)
1887 Value2 = V;
1888 } else if (V != Value1 && V != Value2) {
1889 MoreThanTwoValues = true;
1890 }
1891 }
1892
1893 if (!Value1.getNode())
1894 return DAG.getUNDEF(VT);
1895
1896 if (isOnlyLowElement)
1897 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1898
1899 // If all elements are constants, create a load from the constant pool.
1900 if (isConstant) {
1901 SmallVector<Constant*, 16> CV;
1902 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1903 if (ConstantFPSDNode *V =
1904 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1905 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1906 } else if (ConstantSDNode *V =
1907 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1908 if (OpVT==EltVT)
1909 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1910 else {
1911 // If OpVT and EltVT don't match, EltVT is not legal and the
1912 // element values have been promoted/truncated earlier. Undo this;
1913 // we don't want a v16i8 to become a v16i32 for example.
1914 const ConstantInt *CI = V->getConstantIntValue();
1915 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1916 CI->getZExtValue()));
1917 }
1918 } else {
1919 assert(Node->getOperand(i).isUndef())((Node->getOperand(i).isUndef()) ? static_cast<void>
(0) : __assert_fail ("Node->getOperand(i).isUndef()", "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1919, __PRETTY_FUNCTION__))
;
1920 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1921 CV.push_back(UndefValue::get(OpNTy));
1922 }
1923 }
1924 Constant *CP = ConstantVector::get(CV);
1925 SDValue CPIdx =
1926 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1927 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1928 return DAG.getLoad(
1929 VT, dl, DAG.getEntryNode(), CPIdx,
1930 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1931 Alignment);
1932 }
1933
1934 SmallSet<SDValue, 16> DefinedValues;
1935 for (unsigned i = 0; i < NumElems; ++i) {
1936 if (Node->getOperand(i).isUndef())
1937 continue;
1938 DefinedValues.insert(Node->getOperand(i));
1939 }
1940
1941 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1942 if (!MoreThanTwoValues) {
1943 SmallVector<int, 8> ShuffleVec(NumElems, -1);
1944 for (unsigned i = 0; i < NumElems; ++i) {
1945 SDValue V = Node->getOperand(i);
1946 if (V.isUndef())
1947 continue;
1948 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1949 }
1950 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1951 // Get the splatted value into the low element of a vector register.
1952 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1953 SDValue Vec2;
1954 if (Value2.getNode())
1955 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1956 else
1957 Vec2 = DAG.getUNDEF(VT);
1958
1959 // Return shuffle(LowValVec, undef, <0,0,0,0>)
1960 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1961 }
1962 } else {
1963 SDValue Res;
1964 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1965 return Res;
1966 }
1967 }
1968
1969 // Otherwise, we can't handle this case efficiently.
1970 return ExpandVectorBuildThroughStack(Node);
1971}
1972
1973// Expand a node into a call to a libcall. If the result value
1974// does not fit into a register, return the lo part and set the hi part to the
1975// by-reg argument. If it does fit into a single register, return the result
1976// and leave the Hi part unset.
1977SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
1978 bool isSigned) {
1979 TargetLowering::ArgListTy Args;
1980 TargetLowering::ArgListEntry Entry;
1981 for (const SDValue &Op : Node->op_values()) {
1982 EVT ArgVT = Op.getValueType();
1983 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1984 Entry.Node = Op;
1985 Entry.Ty = ArgTy;
1986 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
1987 Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
1988 Args.push_back(Entry);
1989 }
1990 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
1991 TLI.getPointerTy(DAG.getDataLayout()));
1992
1993 EVT RetVT = Node->getValueType(0);
1994 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
1995
1996 // By default, the input chain to this libcall is the entry node of the
1997 // function. If the libcall is going to be emitted as a tail call then
1998 // TLI.isUsedByReturnOnly will change it to the right chain if the return
1999 // node which is being folded has a non-entry input chain.
2000 SDValue InChain = DAG.getEntryNode();
2001
2002 // isTailCall may be true since the callee does not reference caller stack
2003 // frame. Check if it's in the right position and that the return types match.
2004 SDValue TCChain = InChain;
2005 const Function &F = DAG.getMachineFunction().getFunction();
2006 bool isTailCall =
2007 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2008 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2009 if (isTailCall)
2010 InChain = TCChain;
2011
2012 TargetLowering::CallLoweringInfo CLI(DAG);
2013 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2014 CLI.setDebugLoc(SDLoc(Node))
2015 .setChain(InChain)
2016 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2017 std::move(Args))
2018 .setTailCall(isTailCall)
2019 .setSExtResult(signExtend)
2020 .setZExtResult(!signExtend)
2021 .setIsPostTypeLegalization(true);
2022
2023 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2024
2025 if (!CallInfo.second.getNode()) {
2026 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Created tailcall: "; DAG.
getRoot().dump(); } } while (false)
;
2027 // It's a tailcall, return the chain (which is the DAG root).
2028 return DAG.getRoot();
2029 }
2030
2031 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Created libcall: "; CallInfo
.first.dump(); } } while (false)
;
2032 return CallInfo.first;
2033}
2034
2035/// Generate a libcall taking the given operands as arguments
2036/// and returning a result of type RetVT.
2037SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
2038 const SDValue *Ops, unsigned NumOps,
2039 bool isSigned, const SDLoc &dl) {
2040 TargetLowering::ArgListTy Args;
2041 Args.reserve(NumOps);
2042
2043 TargetLowering::ArgListEntry Entry;
2044 for (unsigned i = 0; i != NumOps; ++i) {
2045 Entry.Node = Ops[i];
2046 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
2047 Entry.IsSExt = isSigned;
2048 Entry.IsZExt = !isSigned;
2049 Args.push_back(Entry);
2050 }
2051 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2052 TLI.getPointerTy(DAG.getDataLayout()));
2053
2054 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2055
2056 TargetLowering::CallLoweringInfo CLI(DAG);
2057 CLI.setDebugLoc(dl)
2058 .setChain(DAG.getEntryNode())
2059 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2060 std::move(Args))
2061 .setSExtResult(isSigned)
2062 .setZExtResult(!isSigned)
2063 .setIsPostTypeLegalization(true);
2064
2065 std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
2066
2067 return CallInfo.first;
2068}
2069
2070// Expand a node into a call to a libcall. Similar to
2071// ExpandLibCall except that the first operand is the in-chain.
2072std::pair<SDValue, SDValue>
2073SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2074 SDNode *Node,
2075 bool isSigned) {
2076 SDValue InChain = Node->getOperand(0);
2077
2078 TargetLowering::ArgListTy Args;
2079 TargetLowering::ArgListEntry Entry;
2080 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
2081 EVT ArgVT = Node->getOperand(i).getValueType();
2082 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2083 Entry.Node = Node->getOperand(i);
2084 Entry.Ty = ArgTy;
2085 Entry.IsSExt = isSigned;
2086 Entry.IsZExt = !isSigned;
2087 Args.push_back(Entry);
2088 }
2089 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2090 TLI.getPointerTy(DAG.getDataLayout()));
2091
2092 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2093
2094 TargetLowering::CallLoweringInfo CLI(DAG);
2095 CLI.setDebugLoc(SDLoc(Node))
2096 .setChain(InChain)
2097 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2098 std::move(Args))
2099 .setSExtResult(isSigned)
2100 .setZExtResult(!isSigned);
2101
2102 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2103
2104 return CallInfo;
2105}
2106
2107SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2108 RTLIB::Libcall Call_F32,
2109 RTLIB::Libcall Call_F64,
2110 RTLIB::Libcall Call_F80,
2111 RTLIB::Libcall Call_F128,
2112 RTLIB::Libcall Call_PPCF128) {
2113 if (Node->isStrictFPOpcode())
2114 Node = DAG.mutateStrictFPToFP(Node);
2115
2116 RTLIB::Libcall LC;
2117 switch (Node->getSimpleValueType(0).SimpleTy) {
2118 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2118)
;
2119 case MVT::f32: LC = Call_F32; break;
2120 case MVT::f64: LC = Call_F64; break;
2121 case MVT::f80: LC = Call_F80; break;
2122 case MVT::f128: LC = Call_F128; break;
2123 case MVT::ppcf128: LC = Call_PPCF128; break;
2124 }
2125 return ExpandLibCall(LC, Node, false);
2126}
2127
2128SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2129 RTLIB::Libcall Call_I8,
2130 RTLIB::Libcall Call_I16,
2131 RTLIB::Libcall Call_I32,
2132 RTLIB::Libcall Call_I64,
2133 RTLIB::Libcall Call_I128) {
2134 RTLIB::Libcall LC;
2135 switch (Node->getSimpleValueType(0).SimpleTy) {
2136 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2136)
;
2137 case MVT::i8: LC = Call_I8; break;
2138 case MVT::i16: LC = Call_I16; break;
2139 case MVT::i32: LC = Call_I32; break;
2140 case MVT::i64: LC = Call_I64; break;
2141 case MVT::i128: LC = Call_I128; break;
2142 }
2143 return ExpandLibCall(LC, Node, isSigned);
2144}
2145
2146/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2147void
2148SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2149 SmallVectorImpl<SDValue> &Results) {
2150 unsigned Opcode = Node->getOpcode();
2151 bool isSigned = Opcode == ISD::SDIVREM;
2152
2153 RTLIB::Libcall LC;
2154 switch (Node->getSimpleValueType(0).SimpleTy) {
2155 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2155)
;
2156 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2157 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2158 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2159 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2160 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2161 }
2162
2163 // The input chain to this libcall is the entry node of the function.
2164 // Legalizing the call will automatically add the previous call to the
2165 // dependence.
2166 SDValue InChain = DAG.getEntryNode();
2167
2168 EVT RetVT = Node->getValueType(0);
2169 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2170
2171 TargetLowering::ArgListTy Args;
2172 TargetLowering::ArgListEntry Entry;
2173 for (const SDValue &Op : Node->op_values()) {
2174 EVT ArgVT = Op.getValueType();
2175 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2176 Entry.Node = Op;
2177 Entry.Ty = ArgTy;
2178 Entry.IsSExt = isSigned;
2179 Entry.IsZExt = !isSigned;
2180 Args.push_back(Entry);
2181 }
2182
2183 // Also pass the return address of the remainder.
2184 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2185 Entry.Node = FIPtr;
2186 Entry.Ty = RetTy->getPointerTo();
2187 Entry.IsSExt = isSigned;
2188 Entry.IsZExt = !isSigned;
2189 Args.push_back(Entry);
2190
2191 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2192 TLI.getPointerTy(DAG.getDataLayout()));
2193
2194 SDLoc dl(Node);
2195 TargetLowering::CallLoweringInfo CLI(DAG);
2196 CLI.setDebugLoc(dl)
2197 .setChain(InChain)
2198 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2199 std::move(Args))
2200 .setSExtResult(isSigned)
2201 .setZExtResult(!isSigned);
2202
2203 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2204
2205 // Remainder is loaded back from the stack frame.
2206 SDValue Rem =
2207 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2208 Results.push_back(CallInfo.first);
2209 Results.push_back(Rem);
2210}
2211
2212/// Return true if sincos libcall is available.
2213static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2214 RTLIB::Libcall LC;
2215 switch (Node->getSimpleValueType(0).SimpleTy) {
2216 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2216)
;
2217 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2218 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2219 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2220 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2221 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2222 }
2223 return TLI.getLibcallName(LC) != nullptr;
2224}
2225
2226/// Only issue sincos libcall if both sin and cos are needed.
2227static bool useSinCos(SDNode *Node) {
2228 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2229 ? ISD::FCOS : ISD::FSIN;
2230
2231 SDValue Op0 = Node->getOperand(0);
2232 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2233 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2234 SDNode *User = *UI;
2235 if (User == Node)
2236 continue;
2237 // The other user might have been turned into sincos already.
2238 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2239 return true;
2240 }
2241 return false;
2242}
2243
2244/// Issue libcalls to sincos to compute sin / cos pairs.
2245void
2246SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2247 SmallVectorImpl<SDValue> &Results) {
2248 RTLIB::Libcall LC;
2249 switch (Node->getSimpleValueType(0).SimpleTy) {
2250 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2250)
;
2251 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2252 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2253 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2254 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2255 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2256 }
2257
2258 // The input chain to this libcall is the entry node of the function.
2259 // Legalizing the call will automatically add the previous call to the
2260 // dependence.
2261 SDValue InChain = DAG.getEntryNode();
2262
2263 EVT RetVT = Node->getValueType(0);
2264 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2265
2266 TargetLowering::ArgListTy Args;
2267 TargetLowering::ArgListEntry Entry;
2268
2269 // Pass the argument.
2270 Entry.Node = Node->getOperand(0);
2271 Entry.Ty = RetTy;
2272 Entry.IsSExt = false;
2273 Entry.IsZExt = false;
2274 Args.push_back(Entry);
2275
2276 // Pass the return address of sin.
2277 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2278 Entry.Node = SinPtr;
2279 Entry.Ty = RetTy->getPointerTo();
2280 Entry.IsSExt = false;
2281 Entry.IsZExt = false;
2282 Args.push_back(Entry);
2283
2284 // Also pass the return address of the cos.
2285 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2286 Entry.Node = CosPtr;
2287 Entry.Ty = RetTy->getPointerTo();
2288 Entry.IsSExt = false;
2289 Entry.IsZExt = false;
2290 Args.push_back(Entry);
2291
2292 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2293 TLI.getPointerTy(DAG.getDataLayout()));
2294
2295 SDLoc dl(Node);
2296 TargetLowering::CallLoweringInfo CLI(DAG);
2297 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2298 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2299 std::move(Args));
2300
2301 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2302
2303 Results.push_back(
2304 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2305 Results.push_back(
2306 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2307}
2308
2309/// This function is responsible for legalizing a
2310/// INT_TO_FP operation of the specified operand when the target requests that
2311/// we expand it. At this point, we know that the result and operand types are
2312/// legal for the target.
2313SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2314 EVT DestVT,
2315 const SDLoc &dl) {
2316 EVT SrcVT = Op0.getValueType();
2317 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2318
2319 // TODO: Should any fast-math-flags be set for the created nodes?
2320 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing INT_TO_FP\n"; }
} while (false)
;
2321 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
2322 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double "
"expansion\n"; } } while (false)
2323 "expansion\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double "
"expansion\n"; } } while (false)
;
2324
2325 // Get the stack frame index of a 8 byte buffer.
2326 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2327
2328 // word offset constant for Hi/Lo address computation
2329 SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2330 StackSlot.getValueType());
2331 // set up Hi and Lo (into buffer) address based on endian
2332 SDValue Hi = StackSlot;
2333 SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2334 StackSlot, WordOff);
2335 if (DAG.getDataLayout().isLittleEndian())
2336 std::swap(Hi, Lo);
2337
2338 // if signed map to unsigned space
2339 SDValue Op0Mapped;
2340 if (isSigned) {
2341 // constant used to invert sign bit (signed to unsigned mapping)
2342 SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2343 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2344 } else {
2345 Op0Mapped = Op0;
2346 }
2347 // store the lo of the constructed double - based on integer input
2348 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2349 MachinePointerInfo());
2350 // initial hi portion of constructed double
2351 SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2352 // store the hi of the constructed double - biased exponent
2353 SDValue Store2 =
2354 DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
2355 // load the constructed double
2356 SDValue Load =
2357 DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
2358 // FP constant to bias correct the final result
2359 SDValue Bias = DAG.getConstantFP(isSigned ?
2360 BitsToDouble(0x4330000080000000ULL) :
2361 BitsToDouble(0x4330000000000000ULL),
2362 dl, MVT::f64);
2363 // subtract the bias
2364 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2365 // final result
2366 SDValue Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2367 return Result;
2368 }
2369 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet")((!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"
) ? static_cast<void> (0) : __assert_fail ("!isSigned && \"Legalize cannot Expand SINT_TO_FP for i64 yet\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2369, __PRETTY_FUNCTION__))
;
2370 // Code below here assumes !isSigned without checking again.
2371
2372 // TODO: Generalize this for use with other types.
2373 if (SrcVT == MVT::i64 && DestVT == MVT::f32) {
2374 LLVM_DEBUG(dbgs() << "Converting unsigned i64 to f32\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Converting unsigned i64 to f32\n"
; } } while (false)
;
2375 // For unsigned conversions, convert them to signed conversions using the
2376 // algorithm from the x86_64 __floatundidf in compiler_rt.
2377 SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2378
2379 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2380 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst);
2381 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2382 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst);
2383 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
2384
2385 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or);
2386 SDValue Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);
2387
2388 // TODO: This really should be implemented using a branch rather than a
2389 // select. We happen to get lucky and machinesink does the right
2390 // thing most of the time. This would be a good candidate for a
2391 // pseudo-op, or, even better, for whole-function isel.
2392 SDValue SignBitTest =
2393 DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2394 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2395 return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2396 }
2397
2398 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2399
2400 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2401 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2402 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2403 Four = DAG.getIntPtrConstant(4, dl);
2404 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2405 SignSet, Four, Zero);
2406
2407 // If the sign bit of the integer is set, the large number will be treated
2408 // as a negative number. To counteract this, the dynamic code adds an
2409 // offset depending on the data type.
2410 uint64_t FF;
2411 switch (SrcVT.getSimpleVT().SimpleTy) {
2412 default: llvm_unreachable("Unsupported integer type!")::llvm::llvm_unreachable_internal("Unsupported integer type!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2412)
;
2413 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2414 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2415 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2416 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2417 }
2418 if (DAG.getDataLayout().isLittleEndian())
2419 FF <<= 32;
2420 Constant *FudgeFactor = ConstantInt::get(
2421 Type::getInt64Ty(*DAG.getContext()), FF);
2422
2423 SDValue CPIdx =
2424 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2425 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2426 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2427 Alignment = std::min(Alignment, 4u);
2428 SDValue FudgeInReg;
2429 if (DestVT == MVT::f32)
2430 FudgeInReg = DAG.getLoad(
2431 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2432 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2433 Alignment);
2434 else {
2435 SDValue Load = DAG.getExtLoad(
2436 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2437 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2438 Alignment);
2439 HandleSDNode Handle(Load);
2440 LegalizeOp(Load.getNode());
2441 FudgeInReg = Handle.getValue();
2442 }
2443
2444 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2445}
2446
2447/// This function is responsible for legalizing a
2448/// *INT_TO_FP operation of the specified operand when the target requests that
2449/// we promote it. At this point, we know that the result and operand types are
2450/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2451/// operation that takes a larger input.
2452SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2453 bool isSigned,
2454 const SDLoc &dl) {
2455 // First step, figure out the appropriate *INT_TO_FP operation to use.
2456 EVT NewInTy = LegalOp.getValueType();
2457
2458 unsigned OpToUse = 0;
2459
2460 // Scan for the appropriate larger type to use.
2461 while (true) {
2462 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2463 assert(NewInTy.isInteger() && "Ran out of possibilities!")((NewInTy.isInteger() && "Ran out of possibilities!")
? static_cast<void> (0) : __assert_fail ("NewInTy.isInteger() && \"Ran out of possibilities!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2463, __PRETTY_FUNCTION__))
;
2464
2465 // If the target supports SINT_TO_FP of this type, use it.
2466 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2467 OpToUse = ISD::SINT_TO_FP;
2468 break;
2469 }
2470 if (isSigned) continue;
2471
2472 // If the target supports UINT_TO_FP of this type, use it.
2473 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2474 OpToUse = ISD::UINT_TO_FP;
2475 break;
2476 }
2477
2478 // Otherwise, try a larger type.
2479 }
2480
2481 // Okay, we found the operation and type to use. Zero extend our input to the
2482 // desired type then run the operation on it.
2483 return DAG.getNode(OpToUse, dl, DestVT,
2484 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2485 dl, NewInTy, LegalOp));
2486}
2487
2488/// This function is responsible for legalizing a
2489/// FP_TO_*INT operation of the specified operand when the target requests that
2490/// we promote it. At this point, we know that the result and operand types are
2491/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2492/// operation that returns a larger result.
2493SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2494 bool isSigned,
2495 const SDLoc &dl) {
2496 // First step, figure out the appropriate FP_TO*INT operation to use.
2497 EVT NewOutTy = DestVT;
2498
2499 unsigned OpToUse = 0;
2500
2501 // Scan for the appropriate larger type to use.
2502 while (true) {
2503 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2504 assert(NewOutTy.isInteger() && "Ran out of possibilities!")((NewOutTy.isInteger() && "Ran out of possibilities!"
) ? static_cast<void> (0) : __assert_fail ("NewOutTy.isInteger() && \"Ran out of possibilities!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2504, __PRETTY_FUNCTION__))
;
2505
2506 // A larger signed type can hold all unsigned values of the requested type,
2507 // so using FP_TO_SINT is valid
2508 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2509 OpToUse = ISD::FP_TO_SINT;
2510 break;
2511 }
2512
2513 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2514 if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2515 OpToUse = ISD::FP_TO_UINT;
2516 break;
2517 }
2518
2519 // Otherwise, try a larger type.
2520 }
2521
2522 // Okay, we found the operation and type to use.
2523 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2524
2525 // Truncate the result of the extended FP_TO_*INT operation to the desired
2526 // size.
2527 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2528}
2529
2530/// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2531SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2532 EVT VT = Op.getValueType();
2533 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2534 unsigned Sz = VT.getScalarSizeInBits();
2535
2536 SDValue Tmp, Tmp2, Tmp3;
2537
2538 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2539 // and finally the i1 pairs.
2540 // TODO: We can easily support i4/i2 legal types if any target ever does.
2541 if (Sz >= 8 && isPowerOf2_32(Sz)) {
2542 // Create the masks - repeating the pattern every byte.
2543 APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0);
2544 APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0);
2545 for (unsigned J = 0; J != Sz; J += 8) {
2546 MaskHi4 = MaskHi4 | (0xF0ull << J);
2547 MaskLo4 = MaskLo4 | (0x0Full << J);
2548 MaskHi2 = MaskHi2 | (0xCCull << J);
2549 MaskLo2 = MaskLo2 | (0x33ull << J);
2550 MaskHi1 = MaskHi1 | (0xAAull << J);
2551 MaskLo1 = MaskLo1 | (0x55ull << J);
2552 }
2553
2554 // BSWAP if the type is wider than a single byte.
2555 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
2556
2557 // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2558 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
2559 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
2560 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, VT));
2561 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, VT));
2562 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2563
2564 // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2565 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
2566 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
2567 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, VT));
2568 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, VT));
2569 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2570
2571 // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2572 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
2573 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
2574 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, VT));
2575 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, VT));
2576 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2577 return Tmp;
2578 }
2579
2580 Tmp = DAG.getConstant(0, dl, VT);
2581 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
2582 if (I < J)
2583 Tmp2 =
2584 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
2585 else
2586 Tmp2 =
2587 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
2588
2589 APInt Shift(Sz, 1);
2590 Shift <<= J;
2591 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
2592 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
2593 }
2594
2595 return Tmp;
2596}
2597
2598/// Open code the operations for BSWAP of the specified operation.
2599SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2600 EVT VT = Op.getValueType();
2601 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2602 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2603 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
2604 default: llvm_unreachable("Unhandled Expand type in BSWAP!")::llvm::llvm_unreachable_internal("Unhandled Expand type in BSWAP!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2604)
;
2605 case MVT::i16:
2606 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2607 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2608 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2609 case MVT::i32:
2610 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2611 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2612 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2613 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2614 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2615 DAG.getConstant(0xFF0000, dl, VT));
2616 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2617 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2618 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2619 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2620 case MVT::i64:
2621 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2622 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2623 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2624 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2625 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2626 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2627 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2628 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2629 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2630 DAG.getConstant(255ULL<<48, dl, VT));
2631 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2632 DAG.getConstant(255ULL<<40, dl, VT));
2633 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2634 DAG.getConstant(255ULL<<32, dl, VT));
2635 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2636 DAG.getConstant(255ULL<<24, dl, VT));
2637 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2638 DAG.getConstant(255ULL<<16, dl, VT));
2639 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2640 DAG.getConstant(255ULL<<8 , dl, VT));
2641 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2642 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2643 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2644 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2645 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2646 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2647 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2648 }
2649}
2650
2651bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2652 LLVM_DEBUG(dbgs() << "Trying to expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to expand node\n";
} } while (false)
;
2653 SmallVector<SDValue, 8> Results;
2654 SDLoc dl(Node);
2655 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2656 bool NeedInvert;
2657 switch (Node->getOpcode()) {
2658 case ISD::CTPOP:
2659 if (TLI.expandCTPOP(Node, Tmp1, DAG))
2660 Results.push_back(Tmp1);
2661 break;
2662 case ISD::CTLZ:
2663 case ISD::CTLZ_ZERO_UNDEF:
2664 if (TLI.expandCTLZ(Node, Tmp1, DAG))
2665 Results.push_back(Tmp1);
2666 break;
2667 case ISD::CTTZ:
2668 case ISD::CTTZ_ZERO_UNDEF:
2669 if (TLI.expandCTTZ(Node, Tmp1, DAG))
2670 Results.push_back(Tmp1);
2671 break;
2672 case ISD::BITREVERSE:
2673 Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
2674 break;
2675 case ISD::BSWAP:
2676 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2677 break;
2678 case ISD::FRAMEADDR:
2679 case ISD::RETURNADDR:
2680 case ISD::FRAME_TO_ARGS_OFFSET:
2681 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2682 break;
2683 case ISD::EH_DWARF_CFA: {
2684 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2685 TLI.getPointerTy(DAG.getDataLayout()));
2686 SDValue Offset = DAG.getNode(ISD::ADD, dl,
2687 CfaArg.getValueType(),
2688 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2689 CfaArg.getValueType()),
2690 CfaArg);
2691 SDValue FA = DAG.getNode(
2692 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2693 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2694 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2695 FA, Offset));
2696 break;
2697 }
2698 case ISD::FLT_ROUNDS_:
2699 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2700 break;
2701 case ISD::EH_RETURN:
2702 case ISD::EH_LABEL:
2703 case ISD::PREFETCH:
2704 case ISD::VAEND:
2705 case ISD::EH_SJLJ_LONGJMP:
2706 // If the target didn't expand these, there's nothing to do, so just
2707 // preserve the chain and be done.
2708 Results.push_back(Node->getOperand(0));
2709 break;
2710 case ISD::READCYCLECOUNTER:
2711 // If the target didn't expand this, just return 'zero' and preserve the
2712 // chain.
2713 Results.append(Node->getNumValues() - 1,
2714 DAG.getConstant(0, dl, Node->getValueType(0)));
2715 Results.push_back(Node->getOperand(0));
2716 break;
2717 case ISD::EH_SJLJ_SETJMP:
2718 // If the target didn't expand this, just return 'zero' and preserve the
2719 // chain.
2720 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2721 Results.push_back(Node->getOperand(0));
2722 break;
2723 case ISD::ATOMIC_LOAD: {
2724 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2725 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2726 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2727 SDValue Swap = DAG.getAtomicCmpSwap(
2728 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2729 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2730 cast<AtomicSDNode>(Node)->getMemOperand());
2731 Results.push_back(Swap.getValue(0));
2732 Results.push_back(Swap.getValue(1));
2733 break;
2734 }
2735 case ISD::ATOMIC_STORE: {
2736 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2737 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2738 cast<AtomicSDNode>(Node)->getMemoryVT(),
2739 Node->getOperand(0),
2740 Node->getOperand(1), Node->getOperand(2),
2741 cast<AtomicSDNode>(Node)->getMemOperand());
2742 Results.push_back(Swap.getValue(1));
2743 break;
2744 }
2745 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2746 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2747 // splits out the success value as a comparison. Expanding the resulting
2748 // ATOMIC_CMP_SWAP will produce a libcall.
2749 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2750 SDValue Res = DAG.getAtomicCmpSwap(
2751 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2752 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2753 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2754
2755 SDValue ExtRes = Res;
2756 SDValue LHS = Res;
2757 SDValue RHS = Node->getOperand(1);
2758
2759 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2760 EVT OuterType = Node->getValueType(0);
2761 switch (TLI.getExtendForAtomicOps()) {
2762 case ISD::SIGN_EXTEND:
2763 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2764 DAG.getValueType(AtomicType));
2765 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2766 Node->getOperand(2), DAG.getValueType(AtomicType));
2767 ExtRes = LHS;
2768 break;
2769 case ISD::ZERO_EXTEND:
2770 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2771 DAG.getValueType(AtomicType));
2772 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2773 ExtRes = LHS;
2774 break;
2775 case ISD::ANY_EXTEND:
2776 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2777 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2778 break;
2779 default:
2780 llvm_unreachable("Invalid atomic op extension")::llvm::llvm_unreachable_internal("Invalid atomic op extension"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2780)
;
2781 }
2782
2783 SDValue Success =
2784 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2785
2786 Results.push_back(ExtRes.getValue(0));
2787 Results.push_back(Success);
2788 Results.push_back(Res.getValue(1));
2789 break;
2790 }
2791 case ISD::DYNAMIC_STACKALLOC:
2792 ExpandDYNAMIC_STACKALLOC(Node, Results);
2793 break;
2794 case ISD::MERGE_VALUES:
2795 for (unsigned i = 0; i < Node->getNumValues(); i++)
2796 Results.push_back(Node->getOperand(i));
2797 break;
2798 case ISD::UNDEF: {
2799 EVT VT = Node->getValueType(0);
2800 if (VT.isInteger())
2801 Results.push_back(DAG.getConstant(0, dl, VT));
2802 else {
2803 assert(VT.isFloatingPoint() && "Unknown value type!")((VT.isFloatingPoint() && "Unknown value type!") ? static_cast
<void> (0) : __assert_fail ("VT.isFloatingPoint() && \"Unknown value type!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2803, __PRETTY_FUNCTION__))
;
2804 Results.push_back(DAG.getConstantFP(0, dl, VT));
2805 }
2806 break;
2807 }
2808 case ISD::FP_ROUND:
2809 case ISD::BITCAST:
2810 Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2811 Node->getValueType(0), dl);
2812 Results.push_back(Tmp1);
2813 break;
2814 case ISD::FP_EXTEND:
2815 Tmp1 = EmitStackConvert(Node->getOperand(0),
2816 Node->getOperand(0).getValueType(),
2817 Node->getValueType(0), dl);
2818 Results.push_back(Tmp1);
2819 break;
2820 case ISD::SIGN_EXTEND_INREG: {
2821 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2822 EVT VT = Node->getValueType(0);
2823
2824 // An in-register sign-extend of a boolean is a negation:
2825 // 'true' (1) sign-extended is -1.
2826 // 'false' (0) sign-extended is 0.
2827 // However, we must mask the high bits of the source operand because the
2828 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2829
2830 // TODO: Do this for vectors too?
2831 if (ExtraVT.getSizeInBits() == 1) {
2832 SDValue One = DAG.getConstant(1, dl, VT);
2833 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2834 SDValue Zero = DAG.getConstant(0, dl, VT);
2835 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2836 Results.push_back(Neg);
2837 break;
2838 }
2839
2840 // NOTE: we could fall back on load/store here too for targets without
2841 // SRA. However, it is doubtful that any exist.
2842 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2843 unsigned BitsDiff = VT.getScalarSizeInBits() -
2844 ExtraVT.getScalarSizeInBits();
2845 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2846 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2847 Node->getOperand(0), ShiftCst);
2848 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2849 Results.push_back(Tmp1);
2850 break;
2851 }
2852 case ISD::FP_ROUND_INREG: {
2853 // The only way we can lower this is to turn it into a TRUNCSTORE,
2854 // EXTLOAD pair, targeting a temporary location (a stack slot).
2855
2856 // NOTE: there is a choice here between constantly creating new stack
2857 // slots and always reusing the same one. We currently always create
2858 // new ones, as reuse may inhibit scheduling.
2859 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2860 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
2861 Node->getValueType(0), dl);
2862 Results.push_back(Tmp1);
2863 break;
2864 }
2865 case ISD::UINT_TO_FP:
2866 if (TLI.expandUINT_TO_FP(Node, Tmp1, DAG)) {
2867 Results.push_back(Tmp1);
2868 break;
2869 }
2870 LLVM_FALLTHROUGH[[clang::fallthrough]];
2871 case ISD::SINT_TO_FP:
2872 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
2873 Node->getOperand(0), Node->getValueType(0), dl);
2874 Results.push_back(Tmp1);
2875 break;
2876 case ISD::FP_TO_SINT:
2877 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
2878 Results.push_back(Tmp1);
2879 break;
2880 case ISD::FP_TO_UINT:
2881 if (TLI.expandFP_TO_UINT(Node, Tmp1, DAG))
2882 Results.push_back(Tmp1);
2883 break;
2884 case ISD::VAARG:
2885 Results.push_back(DAG.expandVAArg(Node));
2886 Results.push_back(Results[0].getValue(1));
2887 break;
2888 case ISD::VACOPY:
2889 Results.push_back(DAG.expandVACopy(Node));
2890 break;
2891 case ISD::EXTRACT_VECTOR_ELT:
2892 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
2893 // This must be an access of the only element. Return it.
2894 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
2895 Node->getOperand(0));
2896 else
2897 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
2898 Results.push_back(Tmp1);
2899 break;
2900 case ISD::EXTRACT_SUBVECTOR:
2901 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
2902 break;
2903 case ISD::INSERT_SUBVECTOR:
2904 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
2905 break;
2906 case ISD::CONCAT_VECTORS:
2907 Results.push_back(ExpandVectorBuildThroughStack(Node));
2908 break;
2909 case ISD::SCALAR_TO_VECTOR:
2910 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
2911 break;
2912 case ISD::INSERT_VECTOR_ELT:
2913 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
2914 Node->getOperand(1),
2915 Node->getOperand(2), dl));
2916 break;
2917 case ISD::VECTOR_SHUFFLE: {
2918 SmallVector<int, 32> NewMask;
2919 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
2920
2921 EVT VT = Node->getValueType(0);
2922 EVT EltVT = VT.getVectorElementType();
2923 SDValue Op0 = Node->getOperand(0);
2924 SDValue Op1 = Node->getOperand(1);
2925 if (!TLI.isTypeLegal(EltVT)) {
2926 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
2927
2928 // BUILD_VECTOR operands are allowed to be wider than the element type.
2929 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
2930 // it.
2931 if (NewEltVT.bitsLT(EltVT)) {
2932 // Convert shuffle node.
2933 // If original node was v4i64 and the new EltVT is i32,
2934 // cast operands to v8i32 and re-build the mask.
2935
2936 // Calculate new VT, the size of the new VT should be equal to original.
2937 EVT NewVT =
2938 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
2939 VT.getSizeInBits() / NewEltVT.getSizeInBits());
2940 assert(NewVT.bitsEq(VT))((NewVT.bitsEq(VT)) ? static_cast<void> (0) : __assert_fail
("NewVT.bitsEq(VT)", "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2940, __PRETTY_FUNCTION__))
;
2941
2942 // cast operands to new VT
2943 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
2944 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
2945
2946 // Convert the shuffle mask
2947 unsigned int factor =
2948 NewVT.getVectorNumElements()/VT.getVectorNumElements();
2949
2950 // EltVT gets smaller
2951 assert(factor > 0)((factor > 0) ? static_cast<void> (0) : __assert_fail
("factor > 0", "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2951, __PRETTY_FUNCTION__))
;
2952
2953 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
2954 if (Mask[i] < 0) {
2955 for (unsigned fi = 0; fi < factor; ++fi)
2956 NewMask.push_back(Mask[i]);
2957 }
2958 else {
2959 for (unsigned fi = 0; fi < factor; ++fi)
2960 NewMask.push_back(Mask[i]*factor+fi);
2961 }
2962 }
2963 Mask = NewMask;
2964 VT = NewVT;
2965 }
2966 EltVT = NewEltVT;
2967 }
2968 unsigned NumElems = VT.getVectorNumElements();
2969 SmallVector<SDValue, 16> Ops;
2970 for (unsigned i = 0; i != NumElems; ++i) {
2971 if (Mask[i] < 0) {
2972 Ops.push_back(DAG.getUNDEF(EltVT));
2973 continue;
2974 }
2975 unsigned Idx = Mask[i];
2976 if (Idx < NumElems)
2977 Ops.push_back(DAG.getNode(
2978 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
2979 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2980 else
2981 Ops.push_back(DAG.getNode(
2982 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
2983 DAG.getConstant(Idx - NumElems, dl,
2984 TLI.getVectorIdxTy(DAG.getDataLayout()))));
2985 }
2986
2987 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
2988 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
2989 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
2990 Results.push_back(Tmp1);
2991 break;
2992 }
2993 case ISD::EXTRACT_ELEMENT: {
2994 EVT OpTy = Node->getOperand(0).getValueType();
2995 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
2996 // 1 -> Hi
2997 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
2998 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
2999 TLI.getShiftAmountTy(
3000 Node->getOperand(0).getValueType(),
3001 DAG.getDataLayout())));
3002 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3003 } else {
3004 // 0 -> Lo
3005 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3006 Node->getOperand(0));
3007 }
3008 Results.push_back(Tmp1);
3009 break;
3010 }
3011 case ISD::STACKSAVE:
3012 // Expand to CopyFromReg if the target set
3013 // StackPointerRegisterToSaveRestore.
3014 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3015 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3016 Node->getValueType(0)));
3017 Results.push_back(Results[0].getValue(1));
3018 } else {
3019 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3020 Results.push_back(Node->getOperand(0));
3021 }
3022 break;
3023 case ISD::STACKRESTORE:
3024 // Expand to CopyToReg if the target set
3025 // StackPointerRegisterToSaveRestore.
3026 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3027 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3028 Node->getOperand(1)));
3029 } else {
3030 Results.push_back(Node->getOperand(0));
3031 }
3032 break;
3033 case ISD::GET_DYNAMIC_AREA_OFFSET:
3034 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3035 Results.push_back(Results[0].getValue(0));
3036 break;
3037 case ISD::FCOPYSIGN:
3038 Results.push_back(ExpandFCOPYSIGN(Node));
3039 break;
3040 case ISD::FNEG:
3041 // Expand Y = FNEG(X) -> Y = SUB -0.0, X
3042 Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3043 // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3044 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3045 Node->getOperand(0));
3046 Results.push_back(Tmp1);
3047 break;
3048 case ISD::FABS:
3049 Results.push_back(ExpandFABS(Node));
3050 break;
3051 case ISD::SMIN:
3052 case ISD::SMAX:
3053 case ISD::UMIN:
3054 case ISD::UMAX: {
3055 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3056 ISD::CondCode Pred;
3057 switch (Node->getOpcode()) {
3058 default: llvm_unreachable("How did we get here?")::llvm::llvm_unreachable_internal("How did we get here?", "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3058)
;
3059 case ISD::SMAX: Pred = ISD::SETGT; break;
3060 case ISD::SMIN: Pred = ISD::SETLT; break;
3061 case ISD::UMAX: Pred = ISD::SETUGT; break;
3062 case ISD::UMIN: Pred = ISD::SETULT; break;
3063 }
3064 Tmp1 = Node->getOperand(0);
3065 Tmp2 = Node->getOperand(1);
3066 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3067 Results.push_back(Tmp1);
3068 break;
3069 }
3070 case ISD::FMINNUM:
3071 case ISD::FMAXNUM: {
3072 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3073 Results.push_back(Expanded);
3074 break;
3075 }
3076 case ISD::FSIN:
3077 case ISD::FCOS: {
3078 EVT VT = Node->getValueType(0);
3079 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3080 // fcos which share the same operand and both are used.
3081 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3082 isSinCosLibcallAvailable(Node, TLI))
3083 && useSinCos(Node)) {
3084 SDVTList VTs = DAG.getVTList(VT, VT);
3085 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3086 if (Node->getOpcode() == ISD::FCOS)
3087 Tmp1 = Tmp1.getValue(1);
3088 Results.push_back(Tmp1);
3089 }
3090 break;
3091 }
3092 case ISD::FMAD:
3093 llvm_unreachable("Illegal fmad should never be formed")::llvm::llvm_unreachable_internal("Illegal fmad should never be formed"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3093)
;
3094
3095 case ISD::FP16_TO_FP:
3096 if (Node->getValueType(0) != MVT::f32) {
3097 // We can extend to types bigger than f32 in two steps without changing
3098 // the result. Since "f16 -> f32" is much more commonly available, give
3099 // CodeGen the option of emitting that before resorting to a libcall.
3100 SDValue Res =
3101 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3102 Results.push_back(
3103 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3104 }
3105 break;
3106 case ISD::FP_TO_FP16:
3107 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing FP_TO_FP16\n";
} } while (false)
;
3108 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3109 SDValue Op = Node->getOperand(0);
3110 MVT SVT = Op.getSimpleValueType();
3111 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3112 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3113 // Under fastmath, we can expand this node into a fround followed by
3114 // a float-half conversion.
3115 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3116 DAG.getIntPtrConstant(0, dl));
3117 Results.push_back(
3118 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3119 }
3120 }
3121 break;
3122 case ISD::ConstantFP: {
3123 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3124 // Check to see if this FP immediate is already legal.
3125 // If this is a legal constant, turn it into a TargetConstantFP node.
3126 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
3127 Results.push_back(ExpandConstantFP(CFP, true));
3128 break;
3129 }
3130 case ISD::Constant: {
3131 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3132 Results.push_back(ExpandConstant(CP));
3133 break;
3134 }
3135 case ISD::FSUB: {
3136 EVT VT = Node->getValueType(0);
3137 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3138 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3139 const SDNodeFlags Flags = Node->getFlags();
3140 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3141 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3142 Results.push_back(Tmp1);
3143 }
3144 break;
3145 }
3146 case ISD::SUB: {
3147 EVT VT = Node->getValueType(0);
3148 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3150, __PRETTY_FUNCTION__))
3149 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3150, __PRETTY_FUNCTION__))
3150 "Don't know how to expand this subtraction!")((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3150, __PRETTY_FUNCTION__))
;
3151 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3152 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3153 VT));
3154 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3155 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3156 break;
3157 }
3158 case ISD::UREM:
3159 case ISD::SREM: {
3160 EVT VT = Node->getValueType(0);
3161 bool isSigned = Node->getOpcode() == ISD::SREM;
3162 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3163 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3164 Tmp2 = Node->getOperand(0);
3165 Tmp3 = Node->getOperand(1);
3166 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3167 SDVTList VTs = DAG.getVTList(VT, VT);
3168 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3169 Results.push_back(Tmp1);
3170 } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3171 // X % Y -> X-X/Y*Y
3172 Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3173 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3174 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3175 Results.push_back(Tmp1);
3176 }
3177 break;
3178 }
3179 case ISD::UDIV:
3180 case ISD::SDIV: {
3181 bool isSigned = Node->getOpcode() == ISD::SDIV;
3182 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3183 EVT VT = Node->getValueType(0);
3184 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3185 SDVTList VTs = DAG.getVTList(VT, VT);
3186 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3187 Node->getOperand(1));
3188 Results.push_back(Tmp1);
3189 }
3190 break;
3191 }
3192 case ISD::MULHU:
3193 case ISD::MULHS: {
3194 unsigned ExpandOpcode =
3195 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3196 EVT VT = Node->getValueType(0);
3197 SDVTList VTs = DAG.getVTList(VT, VT);
3198
3199 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3200 Node->getOperand(1));
3201 Results.push_back(Tmp1.getValue(1));
3202 break;
3203 }
3204 case ISD::UMUL_LOHI:
3205 case ISD::SMUL_LOHI: {
3206 SDValue LHS = Node->getOperand(0);
3207 SDValue RHS = Node->getOperand(1);
3208 MVT VT = LHS.getSimpleValueType();
3209 unsigned MULHOpcode =
3210 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3211
3212 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3213 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3214 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3215 break;
3216 }
3217
3218 SmallVector<SDValue, 4> Halves;
3219 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3220 assert(TLI.isTypeLegal(HalfType))((TLI.isTypeLegal(HalfType)) ? static_cast<void> (0) : __assert_fail
("TLI.isTypeLegal(HalfType)", "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3220, __PRETTY_FUNCTION__))
;
3221 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
3222 HalfType, DAG,
3223 TargetLowering::MulExpansionKind::Always)) {
3224 for (unsigned i = 0; i < 2; ++i) {
3225 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3226 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3227 SDValue Shift = DAG.getConstant(
3228 HalfType.getScalarSizeInBits(), dl,
3229 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3230 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3231 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3232 }
3233 break;
3234 }
3235 break;
3236 }
3237 case ISD::MUL: {
3238 EVT VT = Node->getValueType(0);
3239 SDVTList VTs = DAG.getVTList(VT, VT);
3240 // See if multiply or divide can be lowered using two-result operations.
3241 // We just need the low half of the multiply; try both the signed
3242 // and unsigned forms. If the target supports both SMUL_LOHI and
3243 // UMUL_LOHI, form a preference by checking which forms of plain
3244 // MULH it supports.
3245 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3246 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3247 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3248 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3249 unsigned OpToUse = 0;
3250 if (HasSMUL_LOHI && !HasMULHS) {
3251 OpToUse = ISD::SMUL_LOHI;
3252 } else if (HasUMUL_LOHI && !HasMULHU) {
3253 OpToUse = ISD::UMUL_LOHI;
3254 } else if (HasSMUL_LOHI) {
3255 OpToUse = ISD::SMUL_LOHI;
3256 } else if (HasUMUL_LOHI) {
3257 OpToUse = ISD::UMUL_LOHI;
3258 }
3259 if (OpToUse) {
3260 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3261 Node->getOperand(1)));
3262 break;
3263 }
3264
3265 SDValue Lo, Hi;
3266 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3267 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3268 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3269 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3270 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3271 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3272 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3273 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3274 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3275 SDValue Shift =
3276 DAG.getConstant(HalfType.getSizeInBits(), dl,
3277 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3278 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3279 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3280 }
3281 break;
3282 }
3283 case ISD::SADDSAT:
3284 case ISD::UADDSAT: {
3285 Results.push_back(TLI.getExpandedSaturationAddition(Node, DAG));
3286 break;
3287 }
3288 case ISD::SADDO:
3289 case ISD::SSUBO: {
3290 SDValue LHS = Node->getOperand(0);
3291 SDValue RHS = Node->getOperand(1);
3292 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3293 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3294 LHS, RHS);
3295 Results.push_back(Sum);
3296 EVT ResultType = Node->getValueType(1);
3297 EVT OType = getSetCCResultType(Node->getValueType(0));
3298
3299 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3300
3301 // LHSSign -> LHS >= 0
3302 // RHSSign -> RHS >= 0
3303 // SumSign -> Sum >= 0
3304 //
3305 // Add:
3306 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3307 // Sub:
3308 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3309 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3310 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3311 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3312 Node->getOpcode() == ISD::SADDO ?
3313 ISD::SETEQ : ISD::SETNE);
3314
3315 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3316 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3317
3318 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3319 Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
3320 break;
3321 }
3322 case ISD::UADDO:
3323 case ISD::USUBO: {
3324 SDValue LHS = Node->getOperand(0);
3325 SDValue RHS = Node->getOperand(1);
3326 bool IsAdd = Node->getOpcode() == ISD::UADDO;
3327 // If ADD/SUBCARRY is legal, use that instead.
3328 unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;
3329 if (TLI.isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
3330 SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
3331 SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
3332 { LHS, RHS, CarryIn });
3333 Results.push_back(SDValue(NodeCarry.getNode(), 0));
3334 Results.push_back(SDValue(NodeCarry.getNode(), 1));
3335 break;
3336 }
3337
3338 SDValue Sum = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
3339 LHS.getValueType(), LHS, RHS);
3340 Results.push_back(Sum);
3341
3342 EVT ResultType = Node->getValueType(1);
3343 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3344 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3345 SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3346
3347 Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
3348 break;
3349 }
3350 case ISD::UMULO:
3351 case ISD::SMULO: {
3352 EVT VT = Node->getValueType(0);
3353 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
3354 SDValue LHS = Node->getOperand(0);
3355 SDValue RHS = Node->getOperand(1);
3356 SDValue BottomHalf;
3357 SDValue TopHalf;
3358 static const unsigned Ops[2][3] =
3359 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
3360 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
3361 bool isSigned = Node->getOpcode() == ISD::SMULO;
3362 if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
3363 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3364 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
3365 } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
3366 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
3367 RHS);
3368 TopHalf = BottomHalf.getValue(1);
3369 } else if (TLI.isTypeLegal(WideVT)) {
3370 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
3371 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
3372 Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3373 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3374 DAG.getIntPtrConstant(0, dl));
3375 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3376 DAG.getIntPtrConstant(1, dl));
3377 } else {
3378 // We can fall back to a libcall with an illegal type for the MUL if we
3379 // have a libcall big enough.
3380 // Also, we can fall back to a division in some cases, but that's a big
3381 // performance hit in the general case.
3382 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3383 if (WideVT == MVT::i16)
3384 LC = RTLIB::MUL_I16;
3385 else if (WideVT == MVT::i32)
3386 LC = RTLIB::MUL_I32;
3387 else if (WideVT == MVT::i64)
3388 LC = RTLIB::MUL_I64;
3389 else if (WideVT == MVT::i128)
3390 LC = RTLIB::MUL_I128;
3391 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!")((LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!"
) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Cannot expand this operation!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3391, __PRETTY_FUNCTION__))
;
3392
3393 SDValue HiLHS;
3394 SDValue HiRHS;
3395 if (isSigned) {
3396 // The high part is obtained by SRA'ing all but one of the bits of low
3397 // part.
3398 unsigned LoSize = VT.getSizeInBits();
3399 HiLHS =
3400 DAG.getNode(ISD::SRA, dl, VT, LHS,
3401 DAG.getConstant(LoSize - 1, dl,
3402 TLI.getPointerTy(DAG.getDataLayout())));
3403 HiRHS =
3404 DAG.getNode(ISD::SRA, dl, VT, RHS,
3405 DAG.getConstant(LoSize - 1, dl,
3406 TLI.getPointerTy(DAG.getDataLayout())));
3407 } else {
3408 HiLHS = DAG.getConstant(0, dl, VT);
3409 HiRHS = DAG.getConstant(0, dl, VT);
3410 }
3411
3412 // Here we're passing the 2 arguments explicitly as 4 arguments that are
3413 // pre-lowered to the correct types. This all depends upon WideVT not
3414 // being a legal type for the architecture and thus has to be split to
3415 // two arguments.
3416 SDValue Ret;
3417 if(DAG.getDataLayout().isLittleEndian()) {
3418 // Halves of WideVT are packed into registers in different order
3419 // depending on platform endianness. This is usually handled by
3420 // the C calling convention, but we can't defer to it in
3421 // the legalizer.
3422 SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
3423 Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3424 } else {
3425 SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
3426 Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3427 }
3428 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&((Ret.getOpcode() == ISD::MERGE_VALUES && "Ret value is a collection of constituent nodes holding result."
) ? static_cast<void> (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3429, __PRETTY_FUNCTION__))
3429 "Ret value is a collection of constituent nodes holding result.")((Ret.getOpcode() == ISD::MERGE_VALUES && "Ret value is a collection of constituent nodes holding result."
) ? static_cast<void> (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3429, __PRETTY_FUNCTION__))
;
3430 BottomHalf = Ret.getOperand(0);
3431 TopHalf = Ret.getOperand(1);
3432 }
3433
3434 if (isSigned) {
3435 Tmp1 = DAG.getConstant(
3436 VT.getSizeInBits() - 1, dl,
3437 TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
3438 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3439 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
3440 ISD::SETNE);
3441 } else {
3442 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
3443 DAG.getConstant(0, dl, VT), ISD::SETNE);
3444 }
3445
3446 // Truncate the result if SetCC returns a larger type than needed.
3447 EVT RType = Node->getValueType(1);
3448 if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
3449 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
3450
3451 assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&((RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
"Unexpected result type for S/UMULO legalization") ? static_cast
<void> (0) : __assert_fail ("RType.getSizeInBits() == TopHalf.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3452, __PRETTY_FUNCTION__))
3452 "Unexpected result type for S/UMULO legalization")((RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
"Unexpected result type for S/UMULO legalization") ? static_cast
<void> (0) : __assert_fail ("RType.getSizeInBits() == TopHalf.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3452, __PRETTY_FUNCTION__))
;
3453
3454 Results.push_back(BottomHalf);
3455 Results.push_back(TopHalf);
3456 break;
3457 }
3458 case ISD::BUILD_PAIR: {
3459 EVT PairTy = Node->getValueType(0);
3460 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3461 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3462 Tmp2 = DAG.getNode(
3463 ISD::SHL, dl, PairTy, Tmp2,
3464 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3465 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3466 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3467 break;
3468 }
3469 case ISD::SELECT:
3470 Tmp1 = Node->getOperand(0);
3471 Tmp2 = Node->getOperand(1);
3472 Tmp3 = Node->getOperand(2);
3473 if (Tmp1.getOpcode() == ISD::SETCC) {
3474 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3475 Tmp2, Tmp3,
3476 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3477 } else {
3478 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3479 DAG.getConstant(0, dl, Tmp1.getValueType()),
3480 Tmp2, Tmp3, ISD::SETNE);
3481 }
3482 Results.push_back(Tmp1);
3483 break;
3484 case ISD::BR_JT: {
3485 SDValue Chain = Node->getOperand(0);
3486 SDValue Table = Node->getOperand(1);
3487 SDValue Index = Node->getOperand(2);
3488
3489 const DataLayout &TD = DAG.getDataLayout();
3490 EVT PTy = TLI.getPointerTy(TD);
3491
3492 unsigned EntrySize =
3493 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3494
3495 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3496 // This transformation needs to be done here since otherwise the MIPS
3497 // backend will end up emitting a three instruction multiply sequence
3498 // instead of a single shift and MSP430 will call a runtime function.
3499 if (llvm::isPowerOf2_32(EntrySize))
3500 Index = DAG.getNode(
3501 ISD::SHL, dl, Index.getValueType(), Index,
3502 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3503 else
3504 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3505 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3506 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3507 Index, Table);
3508
3509 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3510 SDValue LD = DAG.getExtLoad(
3511 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3512 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3513 Addr = LD;
3514 if (TLI.isJumpTableRelative()) {
3515 // For PIC, the sequence is:
3516 // BRIND(load(Jumptable + index) + RelocBase)
3517 // RelocBase can be JumpTable, GOT or some sort of global base.
3518 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3519 TLI.getPICJumpTableRelocBase(Table, DAG));
3520 }
3521
3522 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG);
3523 Results.push_back(Tmp1);
3524 break;
3525 }
3526 case ISD::BRCOND:
3527 // Expand brcond's setcc into its constituent parts and create a BR_CC
3528 // Node.
3529 Tmp1 = Node->getOperand(0);
3530 Tmp2 = Node->getOperand(1);
3531 if (Tmp2.getOpcode() == ISD::SETCC) {
3532 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3533 Tmp1, Tmp2.getOperand(2),
3534 Tmp2.getOperand(0), Tmp2.getOperand(1),
3535 Node->getOperand(2));
3536 } else {
3537 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3538 if (Tmp2.isUndef() ||
3539 (Tmp2.getOpcode() == ISD::AND &&
3540 isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3541 cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3542 Tmp3 = Tmp2;
3543 else
3544 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3545 DAG.getConstant(1, dl, Tmp2.getValueType()));
3546 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3547 DAG.getCondCode(ISD::SETNE), Tmp3,
3548 DAG.getConstant(0, dl, Tmp3.getValueType()),
3549 Node->getOperand(2));
3550 }
3551 Results.push_back(Tmp1);
3552 break;
3553 case ISD::SETCC: {
3554 Tmp1 = Node->getOperand(0);
3555 Tmp2 = Node->getOperand(1);
3556 Tmp3 = Node->getOperand(2);
3557 bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3558 Tmp3, NeedInvert, dl);
3559
3560 if (Legalized) {
3561 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3562 // condition code, create a new SETCC node.
3563 if (Tmp3.getNode())
3564 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3565 Tmp1, Tmp2, Tmp3);
3566
3567 // If we expanded the SETCC by inverting the condition code, then wrap
3568 // the existing SETCC in a NOT to restore the intended condition.
3569 if (NeedInvert)
3570 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3571
3572 Results.push_back(Tmp1);
3573 break;
3574 }
3575
3576 // Otherwise, SETCC for the given comparison type must be completely
3577 // illegal; expand it into a SELECT_CC.
3578 EVT VT = Node->getValueType(0);
3579 int TrueValue;
3580 switch (TLI.getBooleanContents(Tmp1.getValueType())) {
3581 case TargetLowering::ZeroOrOneBooleanContent:
3582 case TargetLowering::UndefinedBooleanContent:
3583 TrueValue = 1;
3584 break;
3585 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3586 TrueValue = -1;
3587 break;
3588 }
3589 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3590 DAG.getConstant(TrueValue, dl, VT),
3591 DAG.getConstant(0, dl, VT),
3592 Tmp3);
3593 Results.push_back(Tmp1);
3594 break;
3595 }
3596 case ISD::SELECT_CC: {
3597 Tmp1 = Node->getOperand(0); // LHS
3598 Tmp2 = Node->getOperand(1); // RHS
3599 Tmp3 = Node->getOperand(2); // True
3600 Tmp4 = Node->getOperand(3); // False
3601 EVT VT = Node->getValueType(0);
3602 SDValue CC = Node->getOperand(4);
3603 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3604
3605 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
3606 // If the condition code is legal, then we need to expand this
3607 // node using SETCC and SELECT.
3608 EVT CmpVT = Tmp1.getValueType();
3609 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3611, __PRETTY_FUNCTION__))
3610 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3611, __PRETTY_FUNCTION__))
3611 "expanded.")((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3611, __PRETTY_FUNCTION__))
;
3612 EVT CCVT =
3613 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
3614 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
3615 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3616 break;
3617 }
3618
3619 // SELECT_CC is legal, so the condition code must not be.
3620 bool Legalized = false;
3621 // Try to legalize by inverting the condition. This is for targets that
3622 // might support an ordered version of a condition, but not the unordered
3623 // version (or vice versa).
3624 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3625 Tmp1.getValueType().isInteger());
3626 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
3627 // Use the new condition code and swap true and false
3628 Legalized = true;
3629 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3630 } else {
3631 // If The inverse is not legal, then try to swap the arguments using
3632 // the inverse condition code.
3633 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3634 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
3635 // The swapped inverse condition is legal, so swap true and false,
3636 // lhs and rhs.
3637 Legalized = true;
3638 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3639 }
3640 }
3641
3642 if (!Legalized) {
3643 Legalized = LegalizeSetCCCondCode(
3644 getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3645 dl);
3646
3647 assert(Legalized && "Can't legalize SELECT_CC with legal condition!")((Legalized && "Can't legalize SELECT_CC with legal condition!"
) ? static_cast<void> (0) : __assert_fail ("Legalized && \"Can't legalize SELECT_CC with legal condition!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3647, __PRETTY_FUNCTION__))
;
3648
3649 // If we expanded the SETCC by inverting the condition code, then swap
3650 // the True/False operands to match.
3651 if (NeedInvert)
3652 std::swap(Tmp3, Tmp4);
3653
3654 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3655 // condition code, create a new SELECT_CC node.
3656 if (CC.getNode()) {
3657 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3658 Tmp1, Tmp2, Tmp3, Tmp4, CC);
3659 } else {
3660 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3661 CC = DAG.getCondCode(ISD::SETNE);
3662 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3663 Tmp2, Tmp3, Tmp4, CC);
3664 }
3665 }
3666 Results.push_back(Tmp1);
3667 break;
3668 }
3669 case ISD::BR_CC: {
3670 Tmp1 = Node->getOperand(0); // Chain
3671 Tmp2 = Node->getOperand(2); // LHS
3672 Tmp3 = Node->getOperand(3); // RHS
3673 Tmp4 = Node->getOperand(1); // CC
3674
3675 bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
3676 Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
3677 (void)Legalized;
3678 assert(Legalized && "Can't legalize BR_CC with legal condition!")((Legalized && "Can't legalize BR_CC with legal condition!"
) ? static_cast<void> (0) : __assert_fail ("Legalized && \"Can't legalize BR_CC with legal condition!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3678, __PRETTY_FUNCTION__))
;
3679
3680 // If we expanded the SETCC by inverting the condition code, then wrap
3681 // the existing SETCC in a NOT to restore the intended condition.
3682 if (NeedInvert)
3683 Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
3684
3685 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3686 // node.
3687 if (Tmp4.getNode()) {
3688 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3689 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3690 } else {
3691 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3692 Tmp4 = DAG.getCondCode(ISD::SETNE);
3693 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3694 Tmp2, Tmp3, Node->getOperand(4));
3695 }
3696 Results.push_back(Tmp1);
3697 break;
3698 }
3699 case ISD::BUILD_VECTOR:
3700 Results.push_back(ExpandBUILD_VECTOR(Node));
3701 break;
3702 case ISD::SRA:
3703 case ISD::SRL:
3704 case ISD::SHL: {
3705 // Scalarize vector SRA/SRL/SHL.
3706 EVT VT = Node->getValueType(0);
3707 assert(VT.isVector() && "Unable to legalize non-vector shift")((VT.isVector() && "Unable to legalize non-vector shift"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Unable to legalize non-vector shift\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3707, __PRETTY_FUNCTION__))
;
3708 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal")((TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"
) ? static_cast<void> (0) : __assert_fail ("TLI.isTypeLegal(VT.getScalarType())&& \"Element type must be legal\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3708, __PRETTY_FUNCTION__))
;
3709 unsigned NumElem = VT.getVectorNumElements();
3710
3711 SmallVector<SDValue, 8> Scalars;
3712 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3713 SDValue Ex = DAG.getNode(
3714 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3715 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3716 SDValue Sh = DAG.getNode(
3717 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3718 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3719 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3720 VT.getScalarType(), Ex, Sh));
3721 }
3722
3723 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3724 ReplaceNode(SDValue(Node, 0), Result);
3725 break;
3726 }
3727 case ISD::ROTL:
3728 case ISD::ROTR: {
3729 bool IsLeft = Node->getOpcode() == ISD::ROTL;
3730 SDValue Op0 = Node->getOperand(0), Op1 = Node->getOperand(1);
3731 EVT ResVT = Node->getValueType(0);
3732 EVT OpVT = Op0.getValueType();
3733 assert(OpVT == ResVT &&((OpVT == ResVT && "The result and the operand types of rotate should match"
) ? static_cast<void> (0) : __assert_fail ("OpVT == ResVT && \"The result and the operand types of rotate should match\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3734, __PRETTY_FUNCTION__))
3734 "The result and the operand types of rotate should match")((OpVT == ResVT && "The result and the operand types of rotate should match"
) ? static_cast<void> (0) : __assert_fail ("OpVT == ResVT && \"The result and the operand types of rotate should match\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3734, __PRETTY_FUNCTION__))
;
3735 EVT ShVT = Op1.getValueType();
3736 SDValue Width = DAG.getConstant(OpVT.getScalarSizeInBits(), dl, ShVT);
3737
3738 // If a rotate in the other direction is legal, use it.
3739 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
3740 if (TLI.isOperationLegal(RevRot, ResVT)) {
3741 SDValue Sub = DAG.getNode(ISD::SUB, dl, ShVT, Width, Op1);
3742 Results.push_back(DAG.getNode(RevRot, dl, ResVT, Op0, Sub));
3743 break;
3744 }
3745
3746 // Otherwise,
3747 // (rotl x, c) -> (or (shl x, (and c, w-1)), (srl x, (and w-c, w-1)))
3748 // (rotr x, c) -> (or (srl x, (and c, w-1)), (shl x, (and w-c, w-1)))
3749 //
3750 assert(isPowerOf2_32(OpVT.getScalarSizeInBits()) &&((isPowerOf2_32(OpVT.getScalarSizeInBits()) && "Expecting the type bitwidth to be a power of 2"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(OpVT.getScalarSizeInBits()) && \"Expecting the type bitwidth to be a power of 2\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3751, __PRETTY_FUNCTION__))
3751 "Expecting the type bitwidth to be a power of 2")((isPowerOf2_32(OpVT.getScalarSizeInBits()) && "Expecting the type bitwidth to be a power of 2"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(OpVT.getScalarSizeInBits()) && \"Expecting the type bitwidth to be a power of 2\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3751, __PRETTY_FUNCTION__))
;
3752 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
3753 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
3754 SDValue Width1 = DAG.getNode(ISD::SUB, dl, ShVT,
3755 Width, DAG.getConstant(1, dl, ShVT));
3756 SDValue NegOp1 = DAG.getNode(ISD::SUB, dl, ShVT, Width, Op1);
3757 SDValue And0 = DAG.getNode(ISD::AND, dl, ShVT, Op1, Width1);
3758 SDValue And1 = DAG.getNode(ISD::AND, dl, ShVT, NegOp1, Width1);
3759
3760 SDValue Or = DAG.getNode(ISD::OR, dl, ResVT,
3761 DAG.getNode(ShOpc, dl, ResVT, Op0, And0),
3762 DAG.getNode(HsOpc, dl, ResVT, Op0, And1));
3763 Results.push_back(Or);
3764 break;
3765 }
3766
3767 case ISD::GLOBAL_OFFSET_TABLE:
3768 case ISD::GlobalAddress:
3769 case ISD::GlobalTLSAddress:
3770 case ISD::ExternalSymbol:
3771 case ISD::ConstantPool:
3772 case ISD::JumpTable:
3773 case ISD::INTRINSIC_W_CHAIN:
3774 case ISD::INTRINSIC_WO_CHAIN:
3775 case ISD::INTRINSIC_VOID:
3776 // FIXME: Custom lowering for these operations shouldn't return null!
3777 break;
3778 }
3779
3780 // Replace the original node with the legalized result.
3781 if (Results.empty()) {
3782 LLVM_DEBUG(dbgs() << "Cannot expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Cannot expand node\n"; } }
while (false)
;
3783 return false;
3784 }
3785
3786 LLVM_DEBUG(dbgs() << "Successfully expanded node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully expanded node\n"
; } } while (false)
;
3787 ReplaceNode(Node, Results.data());
3788 return true;
3789}
3790
3791void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3792 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to convert node to libcall\n"
; } } while (false)
;
3793 SmallVector<SDValue, 8> Results;
3794 SDLoc dl(Node);
3795 // FIXME: Check flags on the node to see if we can use a finite call.
3796 bool CanUseFiniteLibCall = TM.Options.NoInfsFPMath && TM.Options.NoNaNsFPMath;
3797 unsigned Opc = Node->getOpcode();
3798 switch (Opc) {
3799 case ISD::ATOMIC_FENCE: {
3800 // If the target didn't lower this, lower it to '__sync_synchronize()' call
3801 // FIXME: handle "fence singlethread" more efficiently.
3802 TargetLowering::ArgListTy Args;
3803
3804 TargetLowering::CallLoweringInfo CLI(DAG);
3805 CLI.setDebugLoc(dl)
3806 .setChain(Node->getOperand(0))
3807 .setLibCallee(
3808 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3809 DAG.getExternalSymbol("__sync_synchronize",
3810 TLI.getPointerTy(DAG.getDataLayout())),
3811 std::move(Args));
3812
3813 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3814
3815 Results.push_back(CallResult.second);
3816 break;
3817 }
3818 // By default, atomic intrinsics are marked Legal and lowered. Targets
3819 // which don't support them directly, however, may want libcalls, in which
3820 // case they mark them Expand, and we get here.
3821 case ISD::ATOMIC_SWAP:
3822 case ISD::ATOMIC_LOAD_ADD:
3823 case ISD::ATOMIC_LOAD_SUB:
3824 case ISD::ATOMIC_LOAD_AND:
3825 case ISD::ATOMIC_LOAD_CLR:
3826 case ISD::ATOMIC_LOAD_OR:
3827 case ISD::ATOMIC_LOAD_XOR:
3828 case ISD::ATOMIC_LOAD_NAND:
3829 case ISD::ATOMIC_LOAD_MIN:
3830 case ISD::ATOMIC_LOAD_MAX:
3831 case ISD::ATOMIC_LOAD_UMIN:
3832 case ISD::ATOMIC_LOAD_UMAX:
3833 case ISD::ATOMIC_CMP_SWAP: {
3834 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3835 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
3836 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!")((LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"
) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected atomic op or value type!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3836, __PRETTY_FUNCTION__))
;
3837
3838 std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
3839 Results.push_back(Tmp.first);
3840 Results.push_back(Tmp.second);
3841 break;
3842 }
3843 case ISD::TRAP: {
3844 // If this operation is not supported, lower it to 'abort()' call
3845 TargetLowering::ArgListTy Args;
3846 TargetLowering::CallLoweringInfo CLI(DAG);
3847 CLI.setDebugLoc(dl)
3848 .setChain(Node->getOperand(0))
3849 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3850 DAG.getExternalSymbol(
3851 "abort", TLI.getPointerTy(DAG.getDataLayout())),
3852 std::move(Args));
3853 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3854
3855 Results.push_back(CallResult.second);
3856 break;
3857 }
3858 case ISD::FMINNUM:
3859 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3860 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3861 RTLIB::FMIN_PPCF128));
3862 break;
3863 case ISD::FMAXNUM:
3864 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3865 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3866 RTLIB::FMAX_PPCF128));
3867 break;
3868 case ISD::FSQRT:
3869 case ISD::STRICT_FSQRT:
3870 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3871 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3872 RTLIB::SQRT_PPCF128));
3873 break;
3874 case ISD::FCBRT:
3875 Results.push_back(ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
3876 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
3877 RTLIB::CBRT_PPCF128));
3878 break;
3879 case ISD::FSIN:
3880 case ISD::STRICT_FSIN:
3881 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
3882 RTLIB::SIN_F80, RTLIB::SIN_F128,
3883 RTLIB::SIN_PPCF128));
3884 break;
3885 case ISD::FCOS:
3886 case ISD::STRICT_FCOS:
3887 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
3888 RTLIB::COS_F80, RTLIB::COS_F128,
3889 RTLIB::COS_PPCF128));
3890 break;
3891 case ISD::FSINCOS:
3892 // Expand into sincos libcall.
3893 ExpandSinCosLibCall(Node, Results);
3894 break;
3895 case ISD::FLOG:
3896 case ISD::STRICT_FLOG:
3897 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log_finite))
3898 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_FINITE_F32,
3899 RTLIB::LOG_FINITE_F64,
3900 RTLIB::LOG_FINITE_F80,
3901 RTLIB::LOG_FINITE_F128,
3902 RTLIB::LOG_FINITE_PPCF128));
3903 else
3904 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
3905 RTLIB::LOG_F80, RTLIB::LOG_F128,
3906 RTLIB::LOG_PPCF128));
3907 break;
3908 case ISD::FLOG2:
3909 case ISD::STRICT_FLOG2:
3910 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log2_finite))
3911 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_FINITE_F32,
3912 RTLIB::LOG2_FINITE_F64,
3913 RTLIB::LOG2_FINITE_F80,
3914 RTLIB::LOG2_FINITE_F128,
3915 RTLIB::LOG2_FINITE_PPCF128));
3916 else
3917 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
3918 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
3919 RTLIB::LOG2_PPCF128));
3920 break;
3921 case ISD::FLOG10:
3922 case ISD::STRICT_FLOG10:
3923 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log10_finite))
3924 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_FINITE_F32,
3925 RTLIB::LOG10_FINITE_F64,
3926 RTLIB::LOG10_FINITE_F80,
3927 RTLIB::LOG10_FINITE_F128,
3928 RTLIB::LOG10_FINITE_PPCF128));
3929 else
3930 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
3931 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
3932 RTLIB::LOG10_PPCF128));
3933 break;
3934 case ISD::FEXP:
3935 case ISD::STRICT_FEXP:
3936 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp_finite))
3937 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_FINITE_F32,
3938 RTLIB::EXP_FINITE_F64,
3939 RTLIB::EXP_FINITE_F80,
3940 RTLIB::EXP_FINITE_F128,
3941 RTLIB::EXP_FINITE_PPCF128));
3942 else
3943 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
3944 RTLIB::EXP_F80, RTLIB::EXP_F128,
3945 RTLIB::EXP_PPCF128));
3946 break;
3947 case ISD::FEXP2:
3948 case ISD::STRICT_FEXP2:
3949 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp2_finite))
3950 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_FINITE_F32,
3951 RTLIB::EXP2_FINITE_F64,
3952 RTLIB::EXP2_FINITE_F80,
3953 RTLIB::EXP2_FINITE_F128,
3954 RTLIB::EXP2_FINITE_PPCF128));
3955 else
3956 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
3957 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
3958 RTLIB::EXP2_PPCF128));
3959 break;
3960 case ISD::FTRUNC:
3961 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
3962 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
3963 RTLIB::TRUNC_PPCF128));
3964 break;
3965 case ISD::FFLOOR:
3966 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
3967 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
3968 RTLIB::FLOOR_PPCF128));
3969 break;
3970 case ISD::FCEIL:
3971 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
3972 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
3973 RTLIB::CEIL_PPCF128));
3974 break;
3975 case ISD::FRINT:
3976 case ISD::STRICT_FRINT:
3977 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
3978 RTLIB::RINT_F80, RTLIB::RINT_F128,
3979 RTLIB::RINT_PPCF128));
3980 break;
3981 case ISD::FNEARBYINT:
3982 case ISD::STRICT_FNEARBYINT:
3983 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
3984 RTLIB::NEARBYINT_F64,
3985 RTLIB::NEARBYINT_F80,
3986 RTLIB::NEARBYINT_F128,
3987 RTLIB::NEARBYINT_PPCF128));
3988 break;
3989 case ISD::FROUND:
3990 Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
3991 RTLIB::ROUND_F64,
3992 RTLIB::ROUND_F80,
3993 RTLIB::ROUND_F128,
3994 RTLIB::ROUND_PPCF128));
3995 break;
3996 case ISD::FPOWI:
3997 case ISD::STRICT_FPOWI:
3998 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
3999 RTLIB::POWI_F80, RTLIB::POWI_F128,
4000 RTLIB::POWI_PPCF128));
4001 break;
4002 case ISD::FPOW:
4003 case ISD::STRICT_FPOW:
4004 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_pow_finite))
4005 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_FINITE_F32,
4006 RTLIB::POW_FINITE_F64,
4007 RTLIB::POW_FINITE_F80,
4008 RTLIB::POW_FINITE_F128,
4009 RTLIB::POW_FINITE_PPCF128));
4010 else
4011 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
4012 RTLIB::POW_F80, RTLIB::POW_F128,
4013 RTLIB::POW_PPCF128));
4014 break;
4015 case ISD::FDIV:
4016 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4017 RTLIB::DIV_F80, RTLIB::DIV_F128,
4018 RTLIB::DIV_PPCF128));
4019 break;
4020 case ISD::FREM:
4021 case ISD::STRICT_FREM:
4022 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4023 RTLIB::REM_F80, RTLIB::REM_F128,
4024 RTLIB::REM_PPCF128));
4025 break;
4026 case ISD::FMA:
4027 case ISD::STRICT_FMA:
4028 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4029 RTLIB::FMA_F80, RTLIB::FMA_F128,
4030 RTLIB::FMA_PPCF128));
4031 break;
4032 case ISD::FADD:
4033 Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4034 RTLIB::ADD_F80, RTLIB::ADD_F128,
4035 RTLIB::ADD_PPCF128));
4036 break;
4037 case ISD::FMUL:
4038 Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4039 RTLIB::MUL_F80, RTLIB::MUL_F128,
4040 RTLIB::MUL_PPCF128));
4041 break;
4042 case ISD::FP16_TO_FP:
4043 if (Node->getValueType(0) == MVT::f32) {
4044 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4045 }
4046 break;
4047 case ISD::FP_TO_FP16: {
4048 RTLIB::Libcall LC =
4049 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4050 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16")((LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16"
) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unable to expand fp_to_fp16\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4050, __PRETTY_FUNCTION__))
;
4051 Results.push_back(ExpandLibCall(LC, Node, false));
4052 break;
4053 }
4054 case ISD::FSUB:
4055 Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4056 RTLIB::SUB_F80, RTLIB::SUB_F128,
4057 RTLIB::SUB_PPCF128));
4058 break;
4059 case ISD::SREM:
4060 Results.push_back(ExpandIntLibCall(Node, true,
4061 RTLIB::SREM_I8,
4062 RTLIB::SREM_I16, RTLIB::SREM_I32,
4063 RTLIB::SREM_I64, RTLIB::SREM_I128));
4064 break;
4065 case ISD::UREM:
4066 Results.push_back(ExpandIntLibCall(Node, false,
4067 RTLIB::UREM_I8,
4068 RTLIB::UREM_I16, RTLIB::UREM_I32,
4069 RTLIB::UREM_I64, RTLIB::UREM_I128));
4070 break;
4071 case ISD::SDIV:
4072 Results.push_back(ExpandIntLibCall(Node, true,
4073 RTLIB::SDIV_I8,
4074 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4075 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4076 break;
4077 case ISD::UDIV:
4078 Results.push_back(ExpandIntLibCall(Node, false,
4079 RTLIB::UDIV_I8,
4080 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4081 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4082 break;
4083 case ISD::SDIVREM:
4084 case ISD::UDIVREM:
4085 // Expand into divrem libcall
4086 ExpandDivRemLibCall(Node, Results);
4087 break;
4088 case ISD::MUL:
4089 Results.push_back(ExpandIntLibCall(Node, false,
4090 RTLIB::MUL_I8,
4091 RTLIB::MUL_I16, RTLIB::MUL_I32,
4092 RTLIB::MUL_I64, RTLIB::MUL_I128));
4093 break;
4094 case ISD::CTLZ_ZERO_UNDEF:
4095 switch (Node->getSimpleValueType(0).SimpleTy) {
4096 default:
4097 llvm_unreachable("LibCall explicitly requested, but not available")::llvm::llvm_unreachable_internal("LibCall explicitly requested, but not available"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4097)
;
4098 case MVT::i32:
4099 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false));
4100 break;
4101 case MVT::i64:
4102 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false));
4103 break;
4104 case MVT::i128:
4105 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false));
4106 break;
4107 }
4108 break;
4109 }
4110
4111 // Replace the original node with the legalized result.
4112 if (!Results.empty()) {
4113 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully converted node to libcall\n"
; } } while (false)
;
4114 ReplaceNode(Node, Results.data());
4115 } else
4116 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not convert node to libcall\n"
; } } while (false)
;
4117}
4118
4119// Determine the vector type to use in place of an original scalar element when
4120// promoting equally sized vectors.
4121static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4122 MVT EltVT, MVT NewEltVT) {
4123 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4124 MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4125 assert(TLI.isTypeLegal(MidVT) && "unexpected")((TLI.isTypeLegal(MidVT) && "unexpected") ? static_cast
<void> (0) : __assert_fail ("TLI.isTypeLegal(MidVT) && \"unexpected\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4125, __PRETTY_FUNCTION__))
;
4126 return MidVT;
4127}
4128
4129void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4130 LLVM_DEBUG(dbgs() << "Trying to promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to promote node\n"
; } } while (false)
;
4131 SmallVector<SDValue, 8> Results;
4132 MVT OVT = Node->getSimpleValueType(0);
4133 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4134 Node->getOpcode() == ISD::SINT_TO_FP ||
4135 Node->getOpcode() == ISD::SETCC ||
4136 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4137 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4138 OVT = Node->getOperand(0).getSimpleValueType();
4139 }
4140 if (Node->getOpcode() == ISD::BR_CC)
4141 OVT = Node->getOperand(2).getSimpleValueType();
4142 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4143 SDLoc dl(Node);
4144 SDValue Tmp1, Tmp2, Tmp3;
4145 switch (Node->getOpcode()) {
4146 case ISD::CTTZ:
4147 case ISD::CTTZ_ZERO_UNDEF:
4148 case ISD::CTLZ:
4149 case ISD::CTLZ_ZERO_UNDEF:
4150 case ISD::CTPOP:
4151 // Zero extend the argument.
4152 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4153 if (Node->getOpcode() == ISD::CTTZ) {
4154 // The count is the same in the promoted type except if the original
4155 // value was zero. This can be handled by setting the bit just off
4156 // the top of the original type.
4157 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4158 OVT.getSizeInBits());
4159 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4160 DAG.getConstant(TopBit, dl, NVT));
4161 }
4162 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4163 // already the correct result.
4164 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4165 if (Node->getOpcode() == ISD::CTLZ ||
4166 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4167 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4168 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4169 DAG.getConstant(NVT.getSizeInBits() -
4170 OVT.getSizeInBits(), dl, NVT));
4171 }
4172 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4173 break;
4174 case ISD::BITREVERSE:
4175 case ISD::BSWAP: {
4176 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4177 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4178 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4179 Tmp1 = DAG.getNode(
4180 ISD::SRL, dl, NVT, Tmp1,
4181 DAG.getConstant(DiffBits, dl,
4182 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4183
4184 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4185 break;
4186 }
4187 case ISD::FP_TO_UINT:
4188 case ISD::FP_TO_SINT:
4189 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4190 Node->getOpcode() == ISD::FP_TO_SINT, dl);
4191 Results.push_back(Tmp1);
4192 break;
4193 case ISD::UINT_TO_FP:
4194 case ISD::SINT_TO_FP:
4195 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4196 Node->getOpcode() == ISD::SINT_TO_FP, dl);
4197 Results.push_back(Tmp1);
4198 break;
4199 case ISD::VAARG: {
4200 SDValue Chain = Node->getOperand(0); // Get the chain.
4201 SDValue Ptr = Node->getOperand(1); // Get the pointer.
4202
4203 unsigned TruncOp;
4204 if (OVT.isVector()) {
4205 TruncOp = ISD::BITCAST;
4206 } else {
4207 assert(OVT.isInteger()((OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4208, __PRETTY_FUNCTION__))
4208 && "VAARG promotion is supported only for vectors or integer types")((OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4208, __PRETTY_FUNCTION__))
;
4209 TruncOp = ISD::TRUNCATE;
4210 }
4211
4212 // Perform the larger operation, then convert back
4213 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4214 Node->getConstantOperandVal(3));
4215 Chain = Tmp1.getValue(1);
4216
4217 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4218
4219 // Modified the chain result - switch anything that used the old chain to
4220 // use the new one.
4221 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4222 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4223 if (UpdatedNodes) {
4224 UpdatedNodes->insert(Tmp2.getNode());
4225 UpdatedNodes->insert(Chain.getNode());
4226 }
4227 ReplacedNode(Node);
4228 break;
4229 }
4230 case ISD::MUL:
4231 case ISD::SDIV:
4232 case ISD::SREM:
4233 case ISD::UDIV:
4234 case ISD::UREM:
4235 case ISD::AND:
4236 case ISD::OR:
4237 case ISD::XOR: {
4238 unsigned ExtOp, TruncOp;
4239 if (OVT.isVector()) {
4240 ExtOp = ISD::BITCAST;
4241 TruncOp = ISD::BITCAST;
4242 } else {
4243 assert(OVT.isInteger() && "Cannot promote logic operation")((OVT.isInteger() && "Cannot promote logic operation"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"Cannot promote logic operation\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4243, __PRETTY_FUNCTION__))
;
4244
4245 switch (Node->getOpcode()) {
4246 default:
4247 ExtOp = ISD::ANY_EXTEND;
4248 break;
4249 case ISD::SDIV:
4250 case ISD::SREM:
4251 ExtOp = ISD::SIGN_EXTEND;
4252 break;
4253 case ISD::UDIV:
4254 case ISD::UREM:
4255 ExtOp = ISD::ZERO_EXTEND;
4256 break;
4257 }
4258 TruncOp = ISD::TRUNCATE;
4259 }
4260 // Promote each of the values to the new type.
4261 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4262 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4263 // Perform the larger operation, then convert back
4264 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4265 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4266 break;
4267 }
4268 case ISD::UMUL_LOHI:
4269 case ISD::SMUL_LOHI: {
4270 // Promote to a multiply in a wider integer type.
4271 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4272 : ISD::SIGN_EXTEND;
4273 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4274 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4275 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4276
4277 auto &DL = DAG.getDataLayout();
4278 unsigned OriginalSize = OVT.getScalarSizeInBits();
4279 Tmp2 = DAG.getNode(
4280 ISD::SRL, dl, NVT, Tmp1,
4281 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4282 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4283 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4284 break;
4285 }
4286 case ISD::SELECT: {
4287 unsigned ExtOp, TruncOp;
4288 if (Node->getValueType(0).isVector() ||
4289 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4290 ExtOp = ISD::BITCAST;
4291 TruncOp = ISD::BITCAST;
4292 } else if (Node->getValueType(0).isInteger()) {
4293 ExtOp = ISD::ANY_EXTEND;
4294 TruncOp = ISD::TRUNCATE;
4295 } else {
4296 ExtOp = ISD::FP_EXTEND;
4297 TruncOp = ISD::FP_ROUND;
4298 }
4299 Tmp1 = Node->getOperand(0);
4300 // Promote each of the values to the new type.
4301 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4302 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4303 // Perform the larger operation, then round down.
4304 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4305 if (TruncOp != ISD::FP_ROUND)
4306 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4307 else
4308 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4309 DAG.getIntPtrConstant(0, dl));
4310 Results.push_back(Tmp1);
4311 break;
4312 }
4313 case ISD::VECTOR_SHUFFLE: {
4314 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4315
4316 // Cast the two input vectors.
4317 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4318 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4319
4320 // Convert the shuffle mask to the right # elements.
4321 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4322 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4323 Results.push_back(Tmp1);
4324 break;
4325 }
4326 case ISD::SETCC: {
4327 unsigned ExtOp = ISD::FP_EXTEND;
4328 if (NVT.isInteger()) {
4329 ISD::CondCode CCCode =
4330 cast<CondCodeSDNode>(Node->getOperand(2))->get();
4331 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4332 }
4333 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4334 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4335 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
4336 Tmp1, Tmp2, Node->getOperand(2)));
4337 break;
4338 }
4339 case ISD::BR_CC: {
4340 unsigned ExtOp = ISD::FP_EXTEND;
4341 if (NVT.isInteger()) {
4342 ISD::CondCode CCCode =
4343 cast<CondCodeSDNode>(Node->getOperand(1))->get();
4344 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4345 }
4346 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4347 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4348 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4349 Node->getOperand(0), Node->getOperand(1),
4350 Tmp1, Tmp2, Node->getOperand(4)));
4351 break;
4352 }
4353 case ISD::FADD:
4354 case ISD::FSUB:
4355 case ISD::FMUL:
4356 case ISD::FDIV:
4357 case ISD::FREM:
4358 case ISD::FMINNUM:
4359 case ISD::FMAXNUM:
4360 case ISD::FPOW:
4361 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4362 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4363 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4364 Node->getFlags());
4365 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4366 Tmp3, DAG.getIntPtrConstant(0, dl)));
4367 break;
4368 case ISD::FMA:
4369 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4370 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4371 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4372 Results.push_back(
4373 DAG.getNode(ISD::FP_ROUND, dl, OVT,
4374 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4375 DAG.getIntPtrConstant(0, dl)));
4376 break;
4377 case ISD::FCOPYSIGN:
4378 case ISD::FPOWI: {
4379 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4380 Tmp2 = Node->getOperand(1);
4381 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4382
4383 // fcopysign doesn't change anything but the sign bit, so
4384 // (fp_round (fcopysign (fpext a), b))
4385 // is as precise as
4386 // (fp_round (fpext a))
4387 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4388 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4389 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4390 Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4391 break;
4392 }
4393 case ISD::FFLOOR:
4394 case ISD::FCEIL:
4395 case ISD::FRINT:
4396 case ISD::FNEARBYINT:
4397 case ISD::FROUND:
4398 case ISD::FTRUNC:
4399 case ISD::FNEG:
4400 case ISD::FSQRT:
4401 case ISD::FSIN:
4402 case ISD::FCOS:
4403 case ISD::FLOG:
4404 case ISD::FLOG2:
4405 case ISD::FLOG10:
4406 case ISD::FABS:
4407 case ISD::FEXP:
4408 case ISD::FEXP2:
4409 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4410 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4411 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4412 Tmp2, DAG.getIntPtrConstant(0, dl)));
4413 break;
4414 case ISD::BUILD_VECTOR: {
4415 MVT EltVT = OVT.getVectorElementType();
4416 MVT NewEltVT = NVT.getVectorElementType();
4417
4418 // Handle bitcasts to a different vector type with the same total bit size
4419 //
4420 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4421 // =>
4422 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4423
4424 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for build_vector") ? static_cast
<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4425, __PRETTY_FUNCTION__))
4425 "Invalid promote type for build_vector")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for build_vector") ? static_cast
<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4425, __PRETTY_FUNCTION__))
;
4426 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4426, __PRETTY_FUNCTION__))
;
4427
4428 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4429
4430 SmallVector<SDValue, 8> NewOps;
4431 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4432 SDValue Op = Node->getOperand(I);
4433 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4434 }
4435
4436 SDLoc SL(Node);
4437 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4438 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4439 Results.push_back(CvtVec);
4440 break;
4441 }
4442 case ISD::EXTRACT_VECTOR_ELT: {
4443 MVT EltVT = OVT.getVectorElementType();
4444 MVT NewEltVT = NVT.getVectorElementType();
4445
4446 // Handle bitcasts to a different vector type with the same total bit size.
4447 //
4448 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4449 // =>
4450 // v4i32:castx = bitcast x:v2i64
4451 //
4452 // i64 = bitcast
4453 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4454 // (i32 (extract_vector_elt castx, (2 * y + 1)))
4455 //
4456
4457 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for extract_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4458, __PRETTY_FUNCTION__))
4458 "Invalid promote type for extract_vector_elt")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for extract_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4458, __PRETTY_FUNCTION__))
;
4459 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4459, __PRETTY_FUNCTION__))
;
4460
4461 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4462 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4463
4464 SDValue Idx = Node->getOperand(1);
4465 EVT IdxVT = Idx.getValueType();
4466 SDLoc SL(Node);
4467 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4468 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4469
4470 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4471
4472 SmallVector<SDValue, 8> NewOps;
4473 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4474 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4475 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4476
4477 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4478 CastVec, TmpIdx);
4479 NewOps.push_back(Elt);
4480 }
4481
4482 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4483 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4484 break;
4485 }
4486 case ISD::INSERT_VECTOR_ELT: {
4487 MVT EltVT = OVT.getVectorElementType();
4488 MVT NewEltVT = NVT.getVectorElementType();
4489
4490 // Handle bitcasts to a different vector type with the same total bit size
4491 //
4492 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4493 // =>
4494 // v4i32:castx = bitcast x:v2i64
4495 // v2i32:casty = bitcast y:i64
4496 //
4497 // v2i64 = bitcast
4498 // (v4i32 insert_vector_elt
4499 // (v4i32 insert_vector_elt v4i32:castx,
4500 // (extract_vector_elt casty, 0), 2 * z),
4501 // (extract_vector_elt casty, 1), (2 * z + 1))
4502
4503 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for insert_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4504, __PRETTY_FUNCTION__))
4504 "Invalid promote type for insert_vector_elt")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for insert_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4504, __PRETTY_FUNCTION__))
;
4505 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4505, __PRETTY_FUNCTION__))
;
4506
4507 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4508 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4509
4510 SDValue Val = Node->getOperand(1);
4511 SDValue Idx = Node->getOperand(2);
4512 EVT IdxVT = Idx.getValueType();
4513 SDLoc SL(Node);
4514
4515 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4516 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4517
4518 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4519 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4520
4521 SDValue NewVec = CastVec;
4522 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4523 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4524 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4525
4526 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4527 CastVal, IdxOffset);
4528
4529 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4530 NewVec, Elt, InEltIdx);
4531 }
4532
4533 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4534 break;
4535 }
4536 case ISD::SCALAR_TO_VECTOR: {
4537 MVT EltVT = OVT.getVectorElementType();
4538 MVT NewEltVT = NVT.getVectorElementType();
4539
4540 // Handle bitcasts to different vector type with the same total bit size.
4541 //
4542 // e.g. v2i64 = scalar_to_vector x:i64
4543 // =>
4544 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4545 //
4546
4547 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4548 SDValue Val = Node->getOperand(0);
4549 SDLoc SL(Node);
4550
4551 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4552 SDValue Undef = DAG.getUNDEF(MidVT);
4553
4554 SmallVector<SDValue, 8> NewElts;
4555 NewElts.push_back(CastVal);
4556 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4557 NewElts.push_back(Undef);
4558
4559 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4560 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4561 Results.push_back(CvtVec);
4562 break;
4563 }
4564 }
4565
4566 // Replace the original node with the legalized result.
4567 if (!Results.empty()) {
4568 LLVM_DEBUG(dbgs() << "Successfully promoted node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully promoted node\n"
; } } while (false)
;
4569 ReplaceNode(Node, Results.data());
4570 } else
4571 LLVM_DEBUG(dbgs() << "Could not promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not promote node\n"
; } } while (false)
;
4572}
4573
4574/// This is the entry point for the file.
4575void SelectionDAG::Legalize() {
4576 AssignTopologicalOrder();
4577
4578 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4579 // Use a delete listener to remove nodes which were deleted during
4580 // legalization from LegalizeNodes. This is needed to handle the situation
4581 // where a new node is allocated by the object pool to the same address of a
4582 // previously deleted node.
4583 DAGNodeDeletedListener DeleteListener(
4584 *this,
4585 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4586
4587 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4588
4589 // Visit all the nodes. We start in topological order, so that we see
4590 // nodes with their original operands intact. Legalization can produce
4591 // new nodes which may themselves need to be legalized. Iterate until all
4592 // nodes have been legalized.
4593 while (true) {
4594 bool AnyLegalized = false;
4595 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4596 --NI;
4597
4598 SDNode *N = &*NI;
4599 if (N->use_empty() && N != getRoot().getNode()) {
4600 ++NI;
4601 DeleteNode(N);
4602 continue;
4603 }
4604
4605 if (LegalizedNodes.insert(N).second) {
4606 AnyLegalized = true;
4607 Legalizer.LegalizeOp(N);
4608
4609 if (N->use_empty() && N != getRoot().getNode()) {
4610 ++NI;
4611 DeleteNode(N);
4612 }
4613 }
4614 }
4615 if (!AnyLegalized)
4616 break;
4617
4618 }
4619
4620 // Remove dead nodes now.
4621 RemoveDeadNodes();
4622}
4623
4624bool SelectionDAG::LegalizeOp(SDNode *N,
4625 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4626 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4627 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4628
4629 // Directly insert the node in question, and legalize it. This will recurse
4630 // as needed through operands.
4631 LegalizedNodes.insert(N);
4632 Legalizer.LegalizeOp(N);
4633
4634 return LegalizedNodes.count(N);
4635}

/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/MathExtras.h

1//===-- llvm/Support/MathExtras.h - Useful math functions -------*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file contains some functions that are useful for math stuff.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_SUPPORT_MATHEXTRAS_H
15#define LLVM_SUPPORT_MATHEXTRAS_H
16
17#include "llvm/Support/Compiler.h"
18#include "llvm/Support/SwapByteOrder.h"
19#include <algorithm>
20#include <cassert>
21#include <climits>
22#include <cstring>
23#include <limits>
24#include <type_traits>
25
26#ifdef __ANDROID_NDK__
27#include <android/api-level.h>
28#endif
29
30#ifdef _MSC_VER
31// Declare these intrinsics manually rather including intrin.h. It's very
32// expensive, and MathExtras.h is popular.
33// #include <intrin.h>
34extern "C" {
35unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask);
36unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask);
37unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask);
38unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask);
39}
40#endif
41
42namespace llvm {
43/// The behavior an operation has on an input of 0.
44enum ZeroBehavior {
45 /// The returned value is undefined.
46 ZB_Undefined,
47 /// The returned value is numeric_limits<T>::max()
48 ZB_Max,
49 /// The returned value is numeric_limits<T>::digits
50 ZB_Width
51};
52
53namespace detail {
54template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter {
55 static std::size_t count(T Val, ZeroBehavior) {
56 if (!Val)
57 return std::numeric_limits<T>::digits;
58 if (Val & 0x1)
59 return 0;
60
61 // Bisection method.
62 std::size_t ZeroBits = 0;
63 T Shift = std::numeric_limits<T>::digits >> 1;
64 T Mask = std::numeric_limits<T>::max() >> Shift;
65 while (Shift) {
66 if ((Val & Mask) == 0) {
67 Val >>= Shift;
68 ZeroBits |= Shift;
69 }
70 Shift >>= 1;
71 Mask >>= Shift;
72 }
73 return ZeroBits;
74 }
75};
76
77#if __GNUC__4 >= 4 || defined(_MSC_VER)
78template <typename T> struct TrailingZerosCounter<T, 4> {
79 static std::size_t count(T Val, ZeroBehavior ZB) {
80 if (ZB != ZB_Undefined && Val == 0)
81 return 32;
82
83#if __has_builtin(__builtin_ctz)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
84 return __builtin_ctz(Val);
85#elif defined(_MSC_VER)
86 unsigned long Index;
87 _BitScanForward(&Index, Val);
88 return Index;
89#endif
90 }
91};
92
93#if !defined(_MSC_VER) || defined(_M_X64)
94template <typename T> struct TrailingZerosCounter<T, 8> {
95 static std::size_t count(T Val, ZeroBehavior ZB) {
96 if (ZB != ZB_Undefined && Val == 0)
97 return 64;
98
99#if __has_builtin(__builtin_ctzll)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
100 return __builtin_ctzll(Val);
101#elif defined(_MSC_VER)
102 unsigned long Index;
103 _BitScanForward64(&Index, Val);
104 return Index;
105#endif
106 }
107};
108#endif
109#endif
110} // namespace detail
111
112/// Count number of 0's from the least significant bit to the most
113/// stopping at the first 1.
114///
115/// Only unsigned integral types are allowed.
116///
117/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
118/// valid arguments.
119template <typename T>
120std::size_t countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
121 static_assert(std::numeric_limits<T>::is_integer &&
122 !std::numeric_limits<T>::is_signed,
123 "Only unsigned integral types are allowed.");
124 return llvm::detail::TrailingZerosCounter<T, sizeof(T)>::count(Val, ZB);
125}
126
127namespace detail {
128template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter {
129 static std::size_t count(T Val, ZeroBehavior) {
130 if (!Val)
131 return std::numeric_limits<T>::digits;
132
133 // Bisection method.
134 std::size_t ZeroBits = 0;
135 for (T Shift = std::numeric_limits<T>::digits >> 1; Shift; Shift >>= 1) {
136 T Tmp = Val >> Shift;
137 if (Tmp)
138 Val = Tmp;
139 else
140 ZeroBits |= Shift;
141 }
142 return ZeroBits;
143 }
144};
145
146#if __GNUC__4 >= 4 || defined(_MSC_VER)
147template <typename T> struct LeadingZerosCounter<T, 4> {
148 static std::size_t count(T Val, ZeroBehavior ZB) {
149 if (ZB != ZB_Undefined && Val == 0)
150 return 32;
151
152#if __has_builtin(__builtin_clz)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
153 return __builtin_clz(Val);
154#elif defined(_MSC_VER)
155 unsigned long Index;
156 _BitScanReverse(&Index, Val);
157 return Index ^ 31;
158#endif
159 }
160};
161
162#if !defined(_MSC_VER) || defined(_M_X64)
163template <typename T> struct LeadingZerosCounter<T, 8> {
164 static std::size_t count(T Val, ZeroBehavior ZB) {
165 if (ZB != ZB_Undefined && Val == 0)
166 return 64;
167
168#if __has_builtin(__builtin_clzll)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
169 return __builtin_clzll(Val);
170#elif defined(_MSC_VER)
171 unsigned long Index;
172 _BitScanReverse64(&Index, Val);
173 return Index ^ 63;
174#endif
175 }
176};
177#endif
178#endif
179} // namespace detail
180
181/// Count number of 0's from the most significant bit to the least
182/// stopping at the first 1.
183///
184/// Only unsigned integral types are allowed.
185///
186/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
187/// valid arguments.
188template <typename T>
189std::size_t countLeadingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
190 static_assert(std::numeric_limits<T>::is_integer &&
191 !std::numeric_limits<T>::is_signed,
192 "Only unsigned integral types are allowed.");
193 return llvm::detail::LeadingZerosCounter<T, sizeof(T)>::count(Val, ZB);
194}
195
196/// Get the index of the first set bit starting from the least
197/// significant bit.
198///
199/// Only unsigned integral types are allowed.
200///
201/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
202/// valid arguments.
203template <typename T> T findFirstSet(T Val, ZeroBehavior ZB = ZB_Max) {
204 if (ZB == ZB_Max && Val == 0)
205 return std::numeric_limits<T>::max();
206
207 return countTrailingZeros(Val, ZB_Undefined);
208}
209
210/// Create a bitmask with the N right-most bits set to 1, and all other
211/// bits set to 0. Only unsigned types are allowed.
212template <typename T> T maskTrailingOnes(unsigned N) {
213 static_assert(std::is_unsigned<T>::value, "Invalid type!");
214 const unsigned Bits = CHAR_BIT8 * sizeof(T);
215 assert(N <= Bits && "Invalid bit index")((N <= Bits && "Invalid bit index") ? static_cast<
void> (0) : __assert_fail ("N <= Bits && \"Invalid bit index\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/MathExtras.h"
, 215, __PRETTY_FUNCTION__))
;
216 return N == 0 ? 0 : (T(-1) >> (Bits - N));
217}
218
219/// Create a bitmask with the N left-most bits set to 1, and all other
220/// bits set to 0. Only unsigned types are allowed.
221template <typename T> T maskLeadingOnes(unsigned N) {
222 return ~maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
223}
224
225/// Create a bitmask with the N right-most bits set to 0, and all other
226/// bits set to 1. Only unsigned types are allowed.
227template <typename T> T maskTrailingZeros(unsigned N) {
228 return maskLeadingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
229}
230
231/// Create a bitmask with the N left-most bits set to 0, and all other
232/// bits set to 1. Only unsigned types are allowed.
233template <typename T> T maskLeadingZeros(unsigned N) {
234 return maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
235}
236
237/// Get the index of the last set bit starting from the least
238/// significant bit.
239///
240/// Only unsigned integral types are allowed.
241///
242/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
243/// valid arguments.
244template <typename T> T findLastSet(T Val, ZeroBehavior ZB = ZB_Max) {
245 if (ZB == ZB_Max && Val == 0)
246 return std::numeric_limits<T>::max();
247
248 // Use ^ instead of - because both gcc and llvm can remove the associated ^
249 // in the __builtin_clz intrinsic on x86.
250 return countLeadingZeros(Val, ZB_Undefined) ^
251 (std::numeric_limits<T>::digits - 1);
252}
253
254/// Macro compressed bit reversal table for 256 bits.
255///
256/// http://graphics.stanford.edu/~seander/bithacks.html#BitReverseTable
257static const unsigned char BitReverseTable256[256] = {
258#define R2(n) n, n + 2 * 64, n + 1 * 64, n + 3 * 64
259#define R4(n) R2(n), R2(n + 2 * 16), R2(n + 1 * 16), R2(n + 3 * 16)
260#define R6(n) R4(n), R4(n + 2 * 4), R4(n + 1 * 4), R4(n + 3 * 4)
261 R6(0), R6(2), R6(1), R6(3)
262#undef R2
263#undef R4
264#undef R6
265};
266
267/// Reverse the bits in \p Val.
268template <typename T>
269T reverseBits(T Val) {
270 unsigned char in[sizeof(Val)];
271 unsigned char out[sizeof(Val)];
272 std::memcpy(in, &Val, sizeof(Val));
273 for (unsigned i = 0; i < sizeof(Val); ++i)
274 out[(sizeof(Val) - i) - 1] = BitReverseTable256[in[i]];
275 std::memcpy(&Val, out, sizeof(Val));
276 return Val;
277}
278
279// NOTE: The following support functions use the _32/_64 extensions instead of
280// type overloading so that signed and unsigned integers can be used without
281// ambiguity.
282
283/// Return the high 32 bits of a 64 bit value.
284constexpr inline uint32_t Hi_32(uint64_t Value) {
285 return static_cast<uint32_t>(Value >> 32);
286}
287
288/// Return the low 32 bits of a 64 bit value.
289constexpr inline uint32_t Lo_32(uint64_t Value) {
290 return static_cast<uint32_t>(Value);
291}
292
293/// Make a 64-bit integer from a high / low pair of 32-bit integers.
294constexpr inline uint64_t Make_64(uint32_t High, uint32_t Low) {
295 return ((uint64_t)High << 32) | (uint64_t)Low;
296}
297
298/// Checks if an integer fits into the given bit width.
299template <unsigned N> constexpr inline bool isInt(int64_t x) {
300 return N >= 64 || (-(INT64_C(1)1L<<(N-1)) <= x && x < (INT64_C(1)1L<<(N-1)));
301}
302// Template specializations to get better code for common cases.
303template <> constexpr inline bool isInt<8>(int64_t x) {
304 return static_cast<int8_t>(x) == x;
305}
306template <> constexpr inline bool isInt<16>(int64_t x) {
307 return static_cast<int16_t>(x) == x;
308}
309template <> constexpr inline bool isInt<32>(int64_t x) {
310 return static_cast<int32_t>(x) == x;
311}
312
313/// Checks if a signed integer is an N bit number shifted left by S.
314template <unsigned N, unsigned S>
315constexpr inline bool isShiftedInt(int64_t x) {
316 static_assert(
317 N > 0, "isShiftedInt<0> doesn't make sense (refers to a 0-bit number.");
318 static_assert(N + S <= 64, "isShiftedInt<N, S> with N + S > 64 is too wide.");
319 return isInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
320}
321
322/// Checks if an unsigned integer fits into the given bit width.
323///
324/// This is written as two functions rather than as simply
325///
326/// return N >= 64 || X < (UINT64_C(1) << N);
327///
328/// to keep MSVC from (incorrectly) warning on isUInt<64> that we're shifting
329/// left too many places.
330template <unsigned N>
331constexpr inline typename std::enable_if<(N < 64), bool>::type
332isUInt(uint64_t X) {
333 static_assert(N > 0, "isUInt<0> doesn't make sense");
334 return X < (UINT64_C(1)1UL << (N));
335}
336template <unsigned N>
337constexpr inline typename std::enable_if<N >= 64, bool>::type
338isUInt(uint64_t X) {
339 return true;
340}
341
342// Template specializations to get better code for common cases.
343template <> constexpr inline bool isUInt<8>(uint64_t x) {
344 return static_cast<uint8_t>(x) == x;
345}
346template <> constexpr inline bool isUInt<16>(uint64_t x) {
347 return static_cast<uint16_t>(x) == x;
348}
349template <> constexpr inline bool isUInt<32>(uint64_t x) {
350 return static_cast<uint32_t>(x) == x;
351}
352
353/// Checks if a unsigned integer is an N bit number shifted left by S.
354template <unsigned N, unsigned S>
355constexpr inline bool isShiftedUInt(uint64_t x) {
356 static_assert(
357 N > 0, "isShiftedUInt<0> doesn't make sense (refers to a 0-bit number)");
358 static_assert(N + S <= 64,
359 "isShiftedUInt<N, S> with N + S > 64 is too wide.");
360 // Per the two static_asserts above, S must be strictly less than 64. So
361 // 1 << S is not undefined behavior.
362 return isUInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
363}
364
365/// Gets the maximum value for a N-bit unsigned integer.
366inline uint64_t maxUIntN(uint64_t N) {
367 assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/MathExtras.h"
, 367, __PRETTY_FUNCTION__))
;
368
369 // uint64_t(1) << 64 is undefined behavior, so we can't do
370 // (uint64_t(1) << N) - 1
371 // without checking first that N != 64. But this works and doesn't have a
372 // branch.
373 return UINT64_MAX(18446744073709551615UL) >> (64 - N);
374}
375
376/// Gets the minimum value for a N-bit signed integer.
377inline int64_t minIntN(int64_t N) {
378 assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/MathExtras.h"
, 378, __PRETTY_FUNCTION__))
;
379
380 return -(UINT64_C(1)1UL<<(N-1));
381}
382
383/// Gets the maximum value for a N-bit signed integer.
384inline int64_t maxIntN(int64_t N) {
385 assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/MathExtras.h"
, 385, __PRETTY_FUNCTION__))
;
386
387 // This relies on two's complement wraparound when N == 64, so we convert to
388 // int64_t only at the very end to avoid UB.
389 return (UINT64_C(1)1UL << (N - 1)) - 1;
390}
391
392/// Checks if an unsigned integer fits into the given (dynamic) bit width.
393inline bool isUIntN(unsigned N, uint64_t x) {
394 return N >= 64 || x <= maxUIntN(N);
395}
396
397/// Checks if an signed integer fits into the given (dynamic) bit width.
398inline bool isIntN(unsigned N, int64_t x) {
399 return N >= 64 || (minIntN(N) <= x && x <= maxIntN(N));
400}
401
402/// Return true if the argument is a non-empty sequence of ones starting at the
403/// least significant bit with the remainder zero (32 bit version).
404/// Ex. isMask_32(0x0000FFFFU) == true.
405constexpr inline bool isMask_32(uint32_t Value) {
406 return Value && ((Value + 1) & Value) == 0;
407}
408
409/// Return true if the argument is a non-empty sequence of ones starting at the
410/// least significant bit with the remainder zero (64 bit version).
411constexpr inline bool isMask_64(uint64_t Value) {
412 return Value && ((Value + 1) & Value) == 0;
413}
414
415/// Return true if the argument contains a non-empty sequence of ones with the
416/// remainder zero (32 bit version.) Ex. isShiftedMask_32(0x0000FF00U) == true.
417constexpr inline bool isShiftedMask_32(uint32_t Value) {
418 return Value && isMask_32((Value - 1) | Value);
419}
420
421/// Return true if the argument contains a non-empty sequence of ones with the
422/// remainder zero (64 bit version.)
423constexpr inline bool isShiftedMask_64(uint64_t Value) {
424 return Value && isMask_64((Value - 1) | Value);
425}
426
427/// Return true if the argument is a power of two > 0.
428/// Ex. isPowerOf2_32(0x00100000U) == true (32 bit edition.)
429constexpr inline bool isPowerOf2_32(uint32_t Value) {
430 return Value && !(Value & (Value - 1));
431}
432
433/// Return true if the argument is a power of two > 0 (64 bit edition.)
434constexpr inline bool isPowerOf2_64(uint64_t Value) {
435 return Value && !(Value & (Value - 1));
436}
437
438/// Return a byte-swapped representation of the 16-bit argument.
439inline uint16_t ByteSwap_16(uint16_t Value) {
440 return sys::SwapByteOrder_16(Value);
441}
442
443/// Return a byte-swapped representation of the 32-bit argument.
444inline uint32_t ByteSwap_32(uint32_t Value) {
445 return sys::SwapByteOrder_32(Value);
446}
447
448/// Return a byte-swapped representation of the 64-bit argument.
449inline uint64_t ByteSwap_64(uint64_t Value) {
450 return sys::SwapByteOrder_64(Value);
451}
452
453/// Count the number of ones from the most significant bit to the first
454/// zero bit.
455///
456/// Ex. countLeadingOnes(0xFF0FFF00) == 8.
457/// Only unsigned integral types are allowed.
458///
459/// \param ZB the behavior on an input of all ones. Only ZB_Width and
460/// ZB_Undefined are valid arguments.
461template <typename T>
462std::size_t countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
463 static_assert(std::numeric_limits<T>::is_integer &&
464 !std::numeric_limits<T>::is_signed,
465 "Only unsigned integral types are allowed.");
466 return countLeadingZeros<T>(~Value, ZB);
467}
468
469/// Count the number of ones from the least significant bit to the first
470/// zero bit.
471///
472/// Ex. countTrailingOnes(0x00FF00FF) == 8.
473/// Only unsigned integral types are allowed.
474///
475/// \param ZB the behavior on an input of all ones. Only ZB_Width and
476/// ZB_Undefined are valid arguments.
477template <typename T>
478std::size_t countTrailingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
479 static_assert(std::numeric_limits<T>::is_integer &&
480 !std::numeric_limits<T>::is_signed,
481 "Only unsigned integral types are allowed.");
482 return countTrailingZeros<T>(~Value, ZB);
483}
484
485namespace detail {
486template <typename T, std::size_t SizeOfT> struct PopulationCounter {
487 static unsigned count(T Value) {
488 // Generic version, forward to 32 bits.
489 static_assert(SizeOfT <= 4, "Not implemented!");
490#if __GNUC__4 >= 4
491 return __builtin_popcount(Value);
492#else
493 uint32_t v = Value;
494 v = v - ((v >> 1) & 0x55555555);
495 v = (v & 0x33333333) + ((v >> 2) & 0x33333333);
496 return ((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24;
497#endif
498 }
499};
500
501template <typename T> struct PopulationCounter<T, 8> {
502 static unsigned count(T Value) {
503#if __GNUC__4 >= 4
504 return __builtin_popcountll(Value);
505#else
506 uint64_t v = Value;
507 v = v - ((v >> 1) & 0x5555555555555555ULL);
508 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
509 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
510 return unsigned((uint64_t)(v * 0x0101010101010101ULL) >> 56);
511#endif
512 }
513};
514} // namespace detail
515
516/// Count the number of set bits in a value.
517/// Ex. countPopulation(0xF000F000) = 8
518/// Returns 0 if the word is zero.
519template <typename T>
520inline unsigned countPopulation(T Value) {
521 static_assert(std::numeric_limits<T>::is_integer &&
522 !std::numeric_limits<T>::is_signed,
523 "Only unsigned integral types are allowed.");
524 return detail::PopulationCounter<T, sizeof(T)>::count(Value);
525}
526
527/// Return the log base 2 of the specified value.
528inline double Log2(double Value) {
529#if defined(__ANDROID_API__) && __ANDROID_API__ < 18
530 return __builtin_log(Value) / __builtin_log(2.0);
531#else
532 return log2(Value);
533#endif
534}
535
536/// Return the floor log base 2 of the specified value, -1 if the value is zero.
537/// (32 bit edition.)
538/// Ex. Log2_32(32) == 5, Log2_32(1) == 0, Log2_32(0) == -1, Log2_32(6) == 2
539inline unsigned Log2_32(uint32_t Value) {
540 return 31 - countLeadingZeros(Value);
18
Returning the value 4294967295
541}
542
543/// Return the floor log base 2 of the specified value, -1 if the value is zero.
544/// (64 bit edition.)
545inline unsigned Log2_64(uint64_t Value) {
546 return 63 - countLeadingZeros(Value);
547}
548
549/// Return the ceil log base 2 of the specified value, 32 if the value is zero.
550/// (32 bit edition).
551/// Ex. Log2_32_Ceil(32) == 5, Log2_32_Ceil(1) == 0, Log2_32_Ceil(6) == 3
552inline unsigned Log2_32_Ceil(uint32_t Value) {
553 return 32 - countLeadingZeros(Value - 1);
554}
555
556/// Return the ceil log base 2 of the specified value, 64 if the value is zero.
557/// (64 bit edition.)
558inline unsigned Log2_64_Ceil(uint64_t Value) {
559 return 64 - countLeadingZeros(Value - 1);
560}
561
562/// Return the greatest common divisor of the values using Euclid's algorithm.
563inline uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B) {
564 while (B) {
565 uint64_t T = B;
566 B = A % B;
567 A = T;
568 }
569 return A;
570}
571
572/// This function takes a 64-bit integer and returns the bit equivalent double.
573inline double BitsToDouble(uint64_t Bits) {
574 double D;
575 static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
576 memcpy(&D, &Bits, sizeof(Bits));
577 return D;
578}
579
580/// This function takes a 32-bit integer and returns the bit equivalent float.
581inline float BitsToFloat(uint32_t Bits) {
582 float F;
583 static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
584 memcpy(&F, &Bits, sizeof(Bits));
585 return F;
586}
587
588/// This function takes a double and returns the bit equivalent 64-bit integer.
589/// Note that copying doubles around changes the bits of NaNs on some hosts,
590/// notably x86, so this routine cannot be used if these bits are needed.
591inline uint64_t DoubleToBits(double Double) {
592 uint64_t Bits;
593 static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
594 memcpy(&Bits, &Double, sizeof(Double));
595 return Bits;
596}
597
598/// This function takes a float and returns the bit equivalent 32-bit integer.
599/// Note that copying floats around changes the bits of NaNs on some hosts,
600/// notably x86, so this routine cannot be used if these bits are needed.
601inline uint32_t FloatToBits(float Float) {
602 uint32_t Bits;
603 static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
604 memcpy(&Bits, &Float, sizeof(Float));
605 return Bits;
606}
607
608/// A and B are either alignments or offsets. Return the minimum alignment that
609/// may be assumed after adding the two together.
610constexpr inline uint64_t MinAlign(uint64_t A, uint64_t B) {
611 // The largest power of 2 that divides both A and B.
612 //
613 // Replace "-Value" by "1+~Value" in the following commented code to avoid
614 // MSVC warning C4146
615 // return (A | B) & -(A | B);
616 return (A | B) & (1 + ~(A | B));
617}
618
619/// Aligns \c Addr to \c Alignment bytes, rounding up.
620///
621/// Alignment should be a power of two. This method rounds up, so
622/// alignAddr(7, 4) == 8 and alignAddr(8, 4) == 8.
623inline uintptr_t alignAddr(const void *Addr, size_t Alignment) {
624 assert(Alignment && isPowerOf2_64((uint64_t)Alignment) &&((Alignment && isPowerOf2_64((uint64_t)Alignment) &&
"Alignment is not a power of two!") ? static_cast<void>
(0) : __assert_fail ("Alignment && isPowerOf2_64((uint64_t)Alignment) && \"Alignment is not a power of two!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/MathExtras.h"
, 625, __PRETTY_FUNCTION__))
625 "Alignment is not a power of two!")((Alignment && isPowerOf2_64((uint64_t)Alignment) &&
"Alignment is not a power of two!") ? static_cast<void>
(0) : __assert_fail ("Alignment && isPowerOf2_64((uint64_t)Alignment) && \"Alignment is not a power of two!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/MathExtras.h"
, 625, __PRETTY_FUNCTION__))
;
626
627 assert((uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr)(((uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr) ? static_cast
<void> (0) : __assert_fail ("(uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr"
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/MathExtras.h"
, 627, __PRETTY_FUNCTION__))
;
628
629 return (((uintptr_t)Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1));
630}
631
632/// Returns the necessary adjustment for aligning \c Ptr to \c Alignment
633/// bytes, rounding up.
634inline size_t alignmentAdjustment(const void *Ptr, size_t Alignment) {
635 return alignAddr(Ptr, Alignment) - (uintptr_t)Ptr;
636}
637
638/// Returns the next power of two (in 64-bits) that is strictly greater than A.
639/// Returns zero on overflow.
640inline uint64_t NextPowerOf2(uint64_t A) {
641 A |= (A >> 1);
642 A |= (A >> 2);
643 A |= (A >> 4);
644 A |= (A >> 8);
645 A |= (A >> 16);
646 A |= (A >> 32);
647 return A + 1;
648}
649
650/// Returns the power of two which is less than or equal to the given value.
651/// Essentially, it is a floor operation across the domain of powers of two.
652inline uint64_t PowerOf2Floor(uint64_t A) {
653 if (!A) return 0;
654 return 1ull << (63 - countLeadingZeros(A, ZB_Undefined));
655}
656
657/// Returns the power of two which is greater than or equal to the given value.
658/// Essentially, it is a ceil operation across the domain of powers of two.
659inline uint64_t PowerOf2Ceil(uint64_t A) {
660 if (!A)
661 return 0;
662 return NextPowerOf2(A - 1);
663}
664
665/// Returns the next integer (mod 2**64) that is greater than or equal to
666/// \p Value and is a multiple of \p Align. \p Align must be non-zero.
667///
668/// If non-zero \p Skew is specified, the return value will be a minimal
669/// integer that is greater than or equal to \p Value and equal to
670/// \p Align * N + \p Skew for some integer N. If \p Skew is larger than
671/// \p Align, its value is adjusted to '\p Skew mod \p Align'.
672///
673/// Examples:
674/// \code
675/// alignTo(5, 8) = 8
676/// alignTo(17, 8) = 24
677/// alignTo(~0LL, 8) = 0
678/// alignTo(321, 255) = 510
679///
680/// alignTo(5, 8, 7) = 7
681/// alignTo(17, 8, 1) = 17
682/// alignTo(~0LL, 8, 3) = 3
683/// alignTo(321, 255, 42) = 552
684/// \endcode
685inline uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
686 assert(Align != 0u && "Align can't be 0.")((Align != 0u && "Align can't be 0.") ? static_cast<
void> (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/MathExtras.h"
, 686, __PRETTY_FUNCTION__))
;
687 Skew %= Align;
688 return (Value + Align - 1 - Skew) / Align * Align + Skew;
689}
690
691/// Returns the next integer (mod 2**64) that is greater than or equal to
692/// \p Value and is a multiple of \c Align. \c Align must be non-zero.
693template <uint64_t Align> constexpr inline uint64_t alignTo(uint64_t Value) {
694 static_assert(Align != 0u, "Align must be non-zero");
695 return (Value + Align - 1) / Align * Align;
696}
697
698/// Returns the integer ceil(Numerator / Denominator).
699inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) {
700 return alignTo(Numerator, Denominator) / Denominator;
701}
702
703/// \c alignTo for contexts where a constant expression is required.
704/// \sa alignTo
705///
706/// \todo FIXME: remove when \c constexpr becomes really \c constexpr
707template <uint64_t Align>
708struct AlignTo {
709 static_assert(Align != 0u, "Align must be non-zero");
710 template <uint64_t Value>
711 struct from_value {
712 static const uint64_t value = (Value + Align - 1) / Align * Align;
713 };
714};
715
716/// Returns the largest uint64_t less than or equal to \p Value and is
717/// \p Skew mod \p Align. \p Align must be non-zero
718inline uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
719 assert(Align != 0u && "Align can't be 0.")((Align != 0u && "Align can't be 0.") ? static_cast<
void> (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/MathExtras.h"
, 719, __PRETTY_FUNCTION__))
;
720 Skew %= Align;
721 return (Value - Skew) / Align * Align + Skew;
722}
723
724/// Returns the offset to the next integer (mod 2**64) that is greater than
725/// or equal to \p Value and is a multiple of \p Align. \p Align must be
726/// non-zero.
727inline uint64_t OffsetToAlignment(uint64_t Value, uint64_t Align) {
728 return alignTo(Value, Align) - Value;
729}
730
731/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
732/// Requires 0 < B <= 32.
733template <unsigned B> constexpr inline int32_t SignExtend32(uint32_t X) {
734 static_assert(B > 0, "Bit width can't be 0.");
735 static_assert(B <= 32, "Bit width out of range.");
736 return int32_t(X << (32 - B)) >> (32 - B);
737}
738
739/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
740/// Requires 0 < B < 32.
741inline int32_t SignExtend32(uint32_t X, unsigned B) {
742 assert(B > 0 && "Bit width can't be 0.")((B > 0 && "Bit width can't be 0.") ? static_cast<
void> (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/MathExtras.h"
, 742, __PRETTY_FUNCTION__))
;
743 assert(B <= 32 && "Bit width out of range.")((B <= 32 && "Bit width out of range.") ? static_cast
<void> (0) : __assert_fail ("B <= 32 && \"Bit width out of range.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/MathExtras.h"
, 743, __PRETTY_FUNCTION__))
;
744 return int32_t(X << (32 - B)) >> (32 - B);
745}
746
747/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
748/// Requires 0 < B < 64.
749template <unsigned B> constexpr inline int64_t SignExtend64(uint64_t x) {
750 static_assert(B > 0, "Bit width can't be 0.");
751 static_assert(B <= 64, "Bit width out of range.");
752 return int64_t(x << (64 - B)) >> (64 - B);
753}
754
755/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
756/// Requires 0 < B < 64.
757inline int64_t SignExtend64(uint64_t X, unsigned B) {
758 assert(B > 0 && "Bit width can't be 0.")((B > 0 && "Bit width can't be 0.") ? static_cast<
void> (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/MathExtras.h"
, 758, __PRETTY_FUNCTION__))
;
759 assert(B <= 64 && "Bit width out of range.")((B <= 64 && "Bit width out of range.") ? static_cast
<void> (0) : __assert_fail ("B <= 64 && \"Bit width out of range.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/MathExtras.h"
, 759, __PRETTY_FUNCTION__))
;
760 return int64_t(X << (64 - B)) >> (64 - B);
761}
762
763/// Subtract two unsigned integers, X and Y, of type T and return the absolute
764/// value of the result.
765template <typename T>
766typename std::enable_if<std::is_unsigned<T>::value, T>::type
767AbsoluteDifference(T X, T Y) {
768 return std::max(X, Y) - std::min(X, Y);
769}
770
771/// Add two unsigned integers, X and Y, of type T. Clamp the result to the
772/// maximum representable value of T on overflow. ResultOverflowed indicates if
773/// the result is larger than the maximum representable value of type T.
774template <typename T>
775typename std::enable_if<std::is_unsigned<T>::value, T>::type
776SaturatingAdd(T X, T Y, bool *ResultOverflowed = nullptr) {
777 bool Dummy;
778 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
779 // Hacker's Delight, p. 29
780 T Z = X + Y;
781 Overflowed = (Z < X || Z < Y);
782 if (Overflowed)
783 return std::numeric_limits<T>::max();
784 else
785 return Z;
786}
787
788/// Multiply two unsigned integers, X and Y, of type T. Clamp the result to the
789/// maximum representable value of T on overflow. ResultOverflowed indicates if
790/// the result is larger than the maximum representable value of type T.
791template <typename T>
792typename std::enable_if<std::is_unsigned<T>::value, T>::type
793SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) {
794 bool Dummy;
795 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
796
797 // Hacker's Delight, p. 30 has a different algorithm, but we don't use that
798 // because it fails for uint16_t (where multiplication can have undefined
799 // behavior due to promotion to int), and requires a division in addition
800 // to the multiplication.
801
802 Overflowed = false;
803
804 // Log2(Z) would be either Log2Z or Log2Z + 1.
805 // Special case: if X or Y is 0, Log2_64 gives -1, and Log2Z
806 // will necessarily be less than Log2Max as desired.
807 int Log2Z = Log2_64(X) + Log2_64(Y);
808 const T Max = std::numeric_limits<T>::max();
809 int Log2Max = Log2_64(Max);
810 if (Log2Z < Log2Max) {
811 return X * Y;
812 }
813 if (Log2Z > Log2Max) {
814 Overflowed = true;
815 return Max;
816 }
817
818 // We're going to use the top bit, and maybe overflow one
819 // bit past it. Multiply all but the bottom bit then add
820 // that on at the end.
821 T Z = (X >> 1) * Y;
822 if (Z & ~(Max >> 1)) {
823 Overflowed = true;
824 return Max;
825 }
826 Z <<= 1;
827 if (X & 1)
828 return SaturatingAdd(Z, Y, ResultOverflowed);
829
830 return Z;
831}
832
833/// Multiply two unsigned integers, X and Y, and add the unsigned integer, A to
834/// the product. Clamp the result to the maximum representable value of T on
835/// overflow. ResultOverflowed indicates if the result is larger than the
836/// maximum representable value of type T.
837template <typename T>
838typename std::enable_if<std::is_unsigned<T>::value, T>::type
839SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed = nullptr) {
840 bool Dummy;
841 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
842
843 T Product = SaturatingMultiply(X, Y, &Overflowed);
844 if (Overflowed)
845 return Product;
846
847 return SaturatingAdd(A, Product, &Overflowed);
848}
849
850/// Use this rather than HUGE_VALF; the latter causes warnings on MSVC.
851extern const float huge_valf;
852} // End llvm namespace
853
854#endif