Bug Summary

File:llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
Warning:line 567, column 29
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizeDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/CodeGen/SelectionDAG -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/include -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/include -D NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/CodeGen/SelectionDAG -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-09-04-040900-46481-1 -x c++ /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/SmallPtrSet.h"
18#include "llvm/ADT/SmallSet.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/Analysis/TargetLibraryInfo.h"
21#include "llvm/CodeGen/ISDOpcodes.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineMemOperand.h"
25#include "llvm/CodeGen/RuntimeLibcalls.h"
26#include "llvm/CodeGen/SelectionDAG.h"
27#include "llvm/CodeGen/SelectionDAGNodes.h"
28#include "llvm/CodeGen/TargetFrameLowering.h"
29#include "llvm/CodeGen/TargetLowering.h"
30#include "llvm/CodeGen/TargetSubtargetInfo.h"
31#include "llvm/CodeGen/ValueTypes.h"
32#include "llvm/IR/CallingConv.h"
33#include "llvm/IR/Constants.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/DerivedTypes.h"
36#include "llvm/IR/Function.h"
37#include "llvm/IR/Metadata.h"
38#include "llvm/IR/Type.h"
39#include "llvm/Support/Casting.h"
40#include "llvm/Support/Compiler.h"
41#include "llvm/Support/Debug.h"
42#include "llvm/Support/ErrorHandling.h"
43#include "llvm/Support/MachineValueType.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/raw_ostream.h"
46#include "llvm/Target/TargetMachine.h"
47#include "llvm/Target/TargetOptions.h"
48#include <algorithm>
49#include <cassert>
50#include <cstdint>
51#include <tuple>
52#include <utility>
53
54using namespace llvm;
55
56#define DEBUG_TYPE"legalizedag" "legalizedag"
57
58namespace {
59
60/// Keeps track of state when getting the sign of a floating-point value as an
61/// integer.
62struct FloatSignAsInt {
63 EVT FloatVT;
64 SDValue Chain;
65 SDValue FloatPtr;
66 SDValue IntPtr;
67 MachinePointerInfo IntPointerInfo;
68 MachinePointerInfo FloatPointerInfo;
69 SDValue IntValue;
70 APInt SignMask;
71 uint8_t SignBit;
72};
73
74//===----------------------------------------------------------------------===//
75/// This takes an arbitrary SelectionDAG as input and
76/// hacks on it until the target machine can handle it. This involves
77/// eliminating value sizes the machine cannot handle (promoting small sizes to
78/// large sizes or splitting up large values into small values) as well as
79/// eliminating operations the machine cannot handle.
80///
81/// This code also does a small amount of optimization and recognition of idioms
82/// as part of its processing. For example, if a target does not support a
83/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
84/// will attempt merge setcc and brc instructions into brcc's.
85class SelectionDAGLegalize {
86 const TargetMachine &TM;
87 const TargetLowering &TLI;
88 SelectionDAG &DAG;
89
90 /// The set of nodes which have already been legalized. We hold a
91 /// reference to it in order to update as necessary on node deletion.
92 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93
94 /// A set of all the nodes updated during legalization.
95 SmallSetVector<SDNode *, 16> *UpdatedNodes;
96
97 EVT getSetCCResultType(EVT VT) const {
98 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
99 }
100
101 // Libcall insertion helpers.
102
103public:
104 SelectionDAGLegalize(SelectionDAG &DAG,
105 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
106 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
107 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
108 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109
110 /// Legalizes the given operation.
111 void LegalizeOp(SDNode *Node);
112
113private:
114 SDValue OptimizeFloatStore(StoreSDNode *ST);
115
116 void LegalizeLoadOps(SDNode *Node);
117 void LegalizeStoreOps(SDNode *Node);
118
119 /// Some targets cannot handle a variable
120 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
121 /// is necessary to spill the vector being inserted into to memory, perform
122 /// the insert there, and then read the result back.
123 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
124 const SDLoc &dl);
125 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
126 const SDLoc &dl);
127
128 /// Return a vector shuffle operation which
129 /// performs the same shuffe in terms of order or result bytes, but on a type
130 /// whose vector element type is narrower than the original shuffle type.
131 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
132 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133 SDValue N1, SDValue N2,
134 ArrayRef<int> Mask) const;
135
136 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
137
138 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
139 SmallVectorImpl<SDValue> &Results);
140 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
141 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
142 RTLIB::Libcall Call_F128,
143 RTLIB::Libcall Call_PPCF128,
144 SmallVectorImpl<SDValue> &Results);
145 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
146 RTLIB::Libcall Call_I8,
147 RTLIB::Libcall Call_I16,
148 RTLIB::Libcall Call_I32,
149 RTLIB::Libcall Call_I64,
150 RTLIB::Libcall Call_I128);
151 void ExpandArgFPLibCall(SDNode *Node,
152 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
153 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
154 RTLIB::Libcall Call_PPCF128,
155 SmallVectorImpl<SDValue> &Results);
156 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
158
159 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
160 const SDLoc &dl);
161 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
162 const SDLoc &dl, SDValue ChainIn);
163 SDValue ExpandBUILD_VECTOR(SDNode *Node);
164 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
165 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
166 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
167 SmallVectorImpl<SDValue> &Results);
168 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
169 SDValue Value) const;
170 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
171 SDValue NewIntValue) const;
172 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
173 SDValue ExpandFABS(SDNode *Node) const;
174 SDValue ExpandFNEG(SDNode *Node) const;
175 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
176 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
177 SmallVectorImpl<SDValue> &Results);
178 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
179 SmallVectorImpl<SDValue> &Results);
180 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
181
182 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
183
184 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
185 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
186 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
187
188 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
189 SDValue ExpandConstant(ConstantSDNode *CP);
190
191 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
192 bool ExpandNode(SDNode *Node);
193 void ConvertNodeToLibcall(SDNode *Node);
194 void PromoteNode(SDNode *Node);
195
196public:
197 // Node replacement helpers
198
199 void ReplacedNode(SDNode *N) {
200 LegalizedNodes.erase(N);
201 if (UpdatedNodes)
202 UpdatedNodes->insert(N);
203 }
204
205 void ReplaceNode(SDNode *Old, SDNode *New) {
206 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { } while (false)
207 dbgs() << " with: "; New->dump(&DAG))do { } while (false);
208
209 assert(Old->getNumValues() == New->getNumValues() &&(static_cast<void> (0))
210 "Replacing one node with another that produces a different number "(static_cast<void> (0))
211 "of values!")(static_cast<void> (0));
212 DAG.ReplaceAllUsesWith(Old, New);
213 if (UpdatedNodes)
214 UpdatedNodes->insert(New);
215 ReplacedNode(Old);
216 }
217
218 void ReplaceNode(SDValue Old, SDValue New) {
219 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { } while (false)
220 dbgs() << " with: "; New->dump(&DAG))do { } while (false);
221
222 DAG.ReplaceAllUsesWith(Old, New);
223 if (UpdatedNodes)
224 UpdatedNodes->insert(New.getNode());
225 ReplacedNode(Old.getNode());
226 }
227
228 void ReplaceNode(SDNode *Old, const SDValue *New) {
229 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG))do { } while (false);
230
231 DAG.ReplaceAllUsesWith(Old, New);
232 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
233 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");do { } while (false)
234 New[i]->dump(&DAG))do { } while (false);
235 if (UpdatedNodes)
236 UpdatedNodes->insert(New[i].getNode());
237 }
238 ReplacedNode(Old);
239 }
240
241 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
242 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { } while (false)
243 dbgs() << " with: "; New->dump(&DAG))do { } while (false);
244
245 DAG.ReplaceAllUsesOfValueWith(Old, New);
246 if (UpdatedNodes)
247 UpdatedNodes->insert(New.getNode());
248 ReplacedNode(Old.getNode());
249 }
250};
251
252} // end anonymous namespace
253
254/// Return a vector shuffle operation which
255/// performs the same shuffle in terms of order or result bytes, but on a type
256/// whose vector element type is narrower than the original shuffle type.
257/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
258SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
259 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
260 ArrayRef<int> Mask) const {
261 unsigned NumMaskElts = VT.getVectorNumElements();
262 unsigned NumDestElts = NVT.getVectorNumElements();
263 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
264
265 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!")(static_cast<void> (0));
266
267 if (NumEltsGrowth == 1)
268 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
269
270 SmallVector<int, 8> NewMask;
271 for (unsigned i = 0; i != NumMaskElts; ++i) {
272 int Idx = Mask[i];
273 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
274 if (Idx < 0)
275 NewMask.push_back(-1);
276 else
277 NewMask.push_back(Idx * NumEltsGrowth + j);
278 }
279 }
280 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?")(static_cast<void> (0));
281 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?")(static_cast<void> (0));
282 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
283}
284
285/// Expands the ConstantFP node to an integer constant or
286/// a load from the constant pool.
287SDValue
288SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
289 bool Extend = false;
290 SDLoc dl(CFP);
291
292 // If a FP immediate is precise when represented as a float and if the
293 // target can do an extending load from float to double, we put it into
294 // the constant pool as a float, even if it's is statically typed as a
295 // double. This shrinks FP constants and canonicalizes them for targets where
296 // an FP extending load is the same cost as a normal load (such as on the x87
297 // fp stack or PPC FP unit).
298 EVT VT = CFP->getValueType(0);
299 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
300 if (!UseCP) {
301 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion")(static_cast<void> (0));
302 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
303 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
304 }
305
306 APFloat APF = CFP->getValueAPF();
307 EVT OrigVT = VT;
308 EVT SVT = VT;
309
310 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
311 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
312 if (!APF.isSignaling()) {
313 while (SVT != MVT::f32 && SVT != MVT::f16) {
314 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
315 if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
316 // Only do this if the target has a native EXTLOAD instruction from
317 // smaller type.
318 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
319 TLI.ShouldShrinkFPConstant(OrigVT)) {
320 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
321 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
322 VT = SVT;
323 Extend = true;
324 }
325 }
326 }
327
328 SDValue CPIdx =
329 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
330 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
331 if (Extend) {
332 SDValue Result = DAG.getExtLoad(
333 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
334 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
335 Alignment);
336 return Result;
337 }
338 SDValue Result = DAG.getLoad(
339 OrigVT, dl, DAG.getEntryNode(), CPIdx,
340 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
341 return Result;
342}
343
344/// Expands the Constant node to a load from the constant pool.
345SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
346 SDLoc dl(CP);
347 EVT VT = CP->getValueType(0);
348 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
349 TLI.getPointerTy(DAG.getDataLayout()));
350 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
351 SDValue Result = DAG.getLoad(
352 VT, dl, DAG.getEntryNode(), CPIdx,
353 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
354 return Result;
355}
356
357/// Some target cannot handle a variable insertion index for the
358/// INSERT_VECTOR_ELT instruction. In this case, it
359/// is necessary to spill the vector being inserted into to memory, perform
360/// the insert there, and then read the result back.
361SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
362 SDValue Val,
363 SDValue Idx,
364 const SDLoc &dl) {
365 SDValue Tmp1 = Vec;
366 SDValue Tmp2 = Val;
367 SDValue Tmp3 = Idx;
368
369 // If the target doesn't support this, we have to spill the input vector
370 // to a temporary stack slot, update the element, then reload it. This is
371 // badness. We could also load the value into a vector register (either
372 // with a "move to register" or "extload into register" instruction, then
373 // permute it into place, if the idx is a constant and if the idx is
374 // supported by the target.
375 EVT VT = Tmp1.getValueType();
376 EVT EltVT = VT.getVectorElementType();
377 SDValue StackPtr = DAG.CreateStackTemporary(VT);
378
379 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
380
381 // Store the vector.
382 SDValue Ch = DAG.getStore(
383 DAG.getEntryNode(), dl, Tmp1, StackPtr,
384 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
385
386 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
387
388 // Store the scalar value.
389 Ch = DAG.getTruncStore(
390 Ch, dl, Tmp2, StackPtr2,
391 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
392 // Load the updated vector.
393 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
394 DAG.getMachineFunction(), SPFI));
395}
396
397SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
398 SDValue Idx,
399 const SDLoc &dl) {
400 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
401 // SCALAR_TO_VECTOR requires that the type of the value being inserted
402 // match the element type of the vector being created, except for
403 // integers in which case the inserted value can be over width.
404 EVT EltVT = Vec.getValueType().getVectorElementType();
405 if (Val.getValueType() == EltVT ||
406 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
407 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
408 Vec.getValueType(), Val);
409
410 unsigned NumElts = Vec.getValueType().getVectorNumElements();
411 // We generate a shuffle of InVec and ScVec, so the shuffle mask
412 // should be 0,1,2,3,4,5... with the appropriate element replaced with
413 // elt 0 of the RHS.
414 SmallVector<int, 8> ShufOps;
415 for (unsigned i = 0; i != NumElts; ++i)
416 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
417
418 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
419 }
420 }
421 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
422}
423
424SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
425 if (!ISD::isNormalStore(ST))
426 return SDValue();
427
428 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n")do { } while (false);
429 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
430 // FIXME: move this to the DAG Combiner! Note that we can't regress due
431 // to phase ordering between legalized code and the dag combiner. This
432 // probably means that we need to integrate dag combiner and legalizer
433 // together.
434 // We generally can't do this one for long doubles.
435 SDValue Chain = ST->getChain();
436 SDValue Ptr = ST->getBasePtr();
437 SDValue Value = ST->getValue();
438 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
439 AAMDNodes AAInfo = ST->getAAInfo();
440 SDLoc dl(ST);
441
442 // Don't optimise TargetConstantFP
443 if (Value.getOpcode() == ISD::TargetConstantFP)
444 return SDValue();
445
446 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
447 if (CFP->getValueType(0) == MVT::f32 &&
448 TLI.isTypeLegal(MVT::i32)) {
449 SDValue Con = DAG.getConstant(CFP->getValueAPF().
450 bitcastToAPInt().zextOrTrunc(32),
451 SDLoc(CFP), MVT::i32);
452 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
453 ST->getOriginalAlign(), MMOFlags, AAInfo);
454 }
455
456 if (CFP->getValueType(0) == MVT::f64) {
457 // If this target supports 64-bit registers, do a single 64-bit store.
458 if (TLI.isTypeLegal(MVT::i64)) {
459 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
460 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
461 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
462 ST->getOriginalAlign(), MMOFlags, AAInfo);
463 }
464
465 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
466 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
467 // stores. If the target supports neither 32- nor 64-bits, this
468 // xform is certainly not worth it.
469 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
470 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
471 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
472 if (DAG.getDataLayout().isBigEndian())
473 std::swap(Lo, Hi);
474
475 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(),
476 ST->getOriginalAlign(), MMOFlags, AAInfo);
477 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(4), dl);
478 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
479 ST->getPointerInfo().getWithOffset(4),
480 ST->getOriginalAlign(), MMOFlags, AAInfo);
481
482 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
483 }
484 }
485 }
486 return SDValue();
487}
488
489void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
490 StoreSDNode *ST = cast<StoreSDNode>(Node);
1
'Node' is a 'StoreSDNode'
491 SDValue Chain = ST->getChain();
492 SDValue Ptr = ST->getBasePtr();
493 SDLoc dl(Node);
494
495 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
496 AAMDNodes AAInfo = ST->getAAInfo();
497
498 if (!ST->isTruncatingStore()) {
2
Assuming the condition is false
3
Taking false branch
499 LLVM_DEBUG(dbgs() << "Legalizing store operation\n")do { } while (false);
500 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
501 ReplaceNode(ST, OptStore);
502 return;
503 }
504
505 SDValue Value = ST->getValue();
506 MVT VT = Value.getSimpleValueType();
507 switch (TLI.getOperationAction(ISD::STORE, VT)) {
508 default: llvm_unreachable("This action is not supported yet!")__builtin_unreachable();
509 case TargetLowering::Legal: {
510 // If this is an unaligned store and the target doesn't support it,
511 // expand it.
512 EVT MemVT = ST->getMemoryVT();
513 const DataLayout &DL = DAG.getDataLayout();
514 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
515 *ST->getMemOperand())) {
516 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n")do { } while (false);
517 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
518 ReplaceNode(SDValue(ST, 0), Result);
519 } else
520 LLVM_DEBUG(dbgs() << "Legal store\n")do { } while (false);
521 break;
522 }
523 case TargetLowering::Custom: {
524 LLVM_DEBUG(dbgs() << "Trying custom lowering\n")do { } while (false);
525 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
526 if (Res && Res != SDValue(Node, 0))
527 ReplaceNode(SDValue(Node, 0), Res);
528 return;
529 }
530 case TargetLowering::Promote: {
531 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
532 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&(static_cast<void> (0))
533 "Can only promote stores to same size type")(static_cast<void> (0));
534 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
535 SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
536 ST->getOriginalAlign(), MMOFlags, AAInfo);
537 ReplaceNode(SDValue(Node, 0), Result);
538 break;
539 }
540 }
541 return;
542 }
543
544 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n")do { } while (false);
4
Loop condition is false. Exiting loop
545 SDValue Value = ST->getValue();
546 EVT StVT = ST->getMemoryVT();
547 TypeSize StWidth = StVT.getSizeInBits();
548 TypeSize StSize = StVT.getStoreSizeInBits();
549 auto &DL = DAG.getDataLayout();
550
551 if (StWidth != StSize) {
5
Calling 'UnivariateLinearPolyBase::operator!='
12
Returning from 'UnivariateLinearPolyBase::operator!='
552 // Promote to a byte-sized store with upper bits zero if not
553 // storing an integral number of bytes. For example, promote
554 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
555 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedSize());
556 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
557 SDValue Result =
558 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
559 ST->getOriginalAlign(), MMOFlags, AAInfo);
560 ReplaceNode(SDValue(Node, 0), Result);
561 } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedSize())) {
13
Calling 'EVT::isVector'
16
Returning from 'EVT::isVector'
17
Assuming the condition is true
18
Calling 'isPowerOf2_64'
21
Returning from 'isPowerOf2_64'
22
Taking true branch
562 // If not storing a power-of-2 number of bits, expand as two stores.
563 assert(!StVT.isVector() && "Unsupported truncstore!")(static_cast<void> (0));
564 unsigned StWidthBits = StWidth.getFixedSize();
565 unsigned LogStWidth = Log2_32(StWidthBits);
23
Calling 'Log2_32'
25
Returning from 'Log2_32'
26
'LogStWidth' initialized to 4294967295
566 assert(LogStWidth < 32)(static_cast<void> (0));
567 unsigned RoundWidth = 1 << LogStWidth;
27
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'
568 assert(RoundWidth < StWidthBits)(static_cast<void> (0));
569 unsigned ExtraWidth = StWidthBits - RoundWidth;
570 assert(ExtraWidth < RoundWidth)(static_cast<void> (0));
571 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&(static_cast<void> (0))
572 "Store size not an integral number of bytes!")(static_cast<void> (0));
573 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
574 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
575 SDValue Lo, Hi;
576 unsigned IncrementSize;
577
578 if (DL.isLittleEndian()) {
579 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
580 // Store the bottom RoundWidth bits.
581 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
582 RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
583
584 // Store the remaining ExtraWidth bits.
585 IncrementSize = RoundWidth / 8;
586 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
587 Hi = DAG.getNode(
588 ISD::SRL, dl, Value.getValueType(), Value,
589 DAG.getConstant(RoundWidth, dl,
590 TLI.getShiftAmountTy(Value.getValueType(), DL)));
591 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
592 ST->getPointerInfo().getWithOffset(IncrementSize),
593 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
594 } else {
595 // Big endian - avoid unaligned stores.
596 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
597 // Store the top RoundWidth bits.
598 Hi = DAG.getNode(
599 ISD::SRL, dl, Value.getValueType(), Value,
600 DAG.getConstant(ExtraWidth, dl,
601 TLI.getShiftAmountTy(Value.getValueType(), DL)));
602 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT,
603 ST->getOriginalAlign(), MMOFlags, AAInfo);
604
605 // Store the remaining ExtraWidth bits.
606 IncrementSize = RoundWidth / 8;
607 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
608 DAG.getConstant(IncrementSize, dl,
609 Ptr.getValueType()));
610 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,
611 ST->getPointerInfo().getWithOffset(IncrementSize),
612 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
613 }
614
615 // The order of the stores doesn't matter.
616 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
617 ReplaceNode(SDValue(Node, 0), Result);
618 } else {
619 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
620 default: llvm_unreachable("This action is not supported yet!")__builtin_unreachable();
621 case TargetLowering::Legal: {
622 EVT MemVT = ST->getMemoryVT();
623 // If this is an unaligned store and the target doesn't support it,
624 // expand it.
625 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
626 *ST->getMemOperand())) {
627 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
628 ReplaceNode(SDValue(ST, 0), Result);
629 }
630 break;
631 }
632 case TargetLowering::Custom: {
633 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
634 if (Res && Res != SDValue(Node, 0))
635 ReplaceNode(SDValue(Node, 0), Res);
636 return;
637 }
638 case TargetLowering::Expand:
639 assert(!StVT.isVector() &&(static_cast<void> (0))
640 "Vector Stores are handled in LegalizeVectorOps")(static_cast<void> (0));
641
642 SDValue Result;
643
644 // TRUNCSTORE:i16 i32 -> STORE i16
645 if (TLI.isTypeLegal(StVT)) {
646 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
647 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
648 ST->getOriginalAlign(), MMOFlags, AAInfo);
649 } else {
650 // The in-memory type isn't legal. Truncate to the type it would promote
651 // to, and then do a truncstore.
652 Value = DAG.getNode(ISD::TRUNCATE, dl,
653 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
654 Value);
655 Result =
656 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT,
657 ST->getOriginalAlign(), MMOFlags, AAInfo);
658 }
659
660 ReplaceNode(SDValue(Node, 0), Result);
661 break;
662 }
663 }
664}
665
666void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
667 LoadSDNode *LD = cast<LoadSDNode>(Node);
668 SDValue Chain = LD->getChain(); // The chain.
669 SDValue Ptr = LD->getBasePtr(); // The base pointer.
670 SDValue Value; // The value returned by the load op.
671 SDLoc dl(Node);
672
673 ISD::LoadExtType ExtType = LD->getExtensionType();
674 if (ExtType == ISD::NON_EXTLOAD) {
675 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n")do { } while (false);
676 MVT VT = Node->getSimpleValueType(0);
677 SDValue RVal = SDValue(Node, 0);
678 SDValue RChain = SDValue(Node, 1);
679
680 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
681 default: llvm_unreachable("This action is not supported yet!")__builtin_unreachable();
682 case TargetLowering::Legal: {
683 EVT MemVT = LD->getMemoryVT();
684 const DataLayout &DL = DAG.getDataLayout();
685 // If this is an unaligned load and the target doesn't support it,
686 // expand it.
687 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
688 *LD->getMemOperand())) {
689 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
690 }
691 break;
692 }
693 case TargetLowering::Custom:
694 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
695 RVal = Res;
696 RChain = Res.getValue(1);
697 }
698 break;
699
700 case TargetLowering::Promote: {
701 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
702 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&(static_cast<void> (0))
703 "Can only promote loads to same size type")(static_cast<void> (0));
704
705 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
706 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
707 RChain = Res.getValue(1);
708 break;
709 }
710 }
711 if (RChain.getNode() != Node) {
712 assert(RVal.getNode() != Node && "Load must be completely replaced")(static_cast<void> (0));
713 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
714 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
715 if (UpdatedNodes) {
716 UpdatedNodes->insert(RVal.getNode());
717 UpdatedNodes->insert(RChain.getNode());
718 }
719 ReplacedNode(Node);
720 }
721 return;
722 }
723
724 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n")do { } while (false);
725 EVT SrcVT = LD->getMemoryVT();
726 TypeSize SrcWidth = SrcVT.getSizeInBits();
727 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
728 AAMDNodes AAInfo = LD->getAAInfo();
729
730 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
731 // Some targets pretend to have an i1 loading operation, and actually
732 // load an i8. This trick is correct for ZEXTLOAD because the top 7
733 // bits are guaranteed to be zero; it helps the optimizers understand
734 // that these bits are zero. It is also useful for EXTLOAD, since it
735 // tells the optimizers that those bits are undefined. It would be
736 // nice to have an effective generic way of getting these benefits...
737 // Until such a way is found, don't insist on promoting i1 here.
738 (SrcVT != MVT::i1 ||
739 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
740 TargetLowering::Promote)) {
741 // Promote to a byte-sized load if not loading an integral number of
742 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
743 unsigned NewWidth = SrcVT.getStoreSizeInBits();
744 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
745 SDValue Ch;
746
747 // The extra bits are guaranteed to be zero, since we stored them that
748 // way. A zext load from NVT thus automatically gives zext from SrcVT.
749
750 ISD::LoadExtType NewExtType =
751 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
752
753 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
754 Chain, Ptr, LD->getPointerInfo(), NVT,
755 LD->getOriginalAlign(), MMOFlags, AAInfo);
756
757 Ch = Result.getValue(1); // The chain.
758
759 if (ExtType == ISD::SEXTLOAD)
760 // Having the top bits zero doesn't help when sign extending.
761 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
762 Result.getValueType(),
763 Result, DAG.getValueType(SrcVT));
764 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
765 // All the top bits are guaranteed to be zero - inform the optimizers.
766 Result = DAG.getNode(ISD::AssertZext, dl,
767 Result.getValueType(), Result,
768 DAG.getValueType(SrcVT));
769
770 Value = Result;
771 Chain = Ch;
772 } else if (!isPowerOf2_64(SrcWidth.getKnownMinSize())) {
773 // If not loading a power-of-2 number of bits, expand as two loads.
774 assert(!SrcVT.isVector() && "Unsupported extload!")(static_cast<void> (0));
775 unsigned SrcWidthBits = SrcWidth.getFixedSize();
776 unsigned LogSrcWidth = Log2_32(SrcWidthBits);
777 assert(LogSrcWidth < 32)(static_cast<void> (0));
778 unsigned RoundWidth = 1 << LogSrcWidth;
779 assert(RoundWidth < SrcWidthBits)(static_cast<void> (0));
780 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
781 assert(ExtraWidth < RoundWidth)(static_cast<void> (0));
782 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&(static_cast<void> (0))
783 "Load size not an integral number of bytes!")(static_cast<void> (0));
784 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
785 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
786 SDValue Lo, Hi, Ch;
787 unsigned IncrementSize;
788 auto &DL = DAG.getDataLayout();
789
790 if (DL.isLittleEndian()) {
791 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
792 // Load the bottom RoundWidth bits.
793 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
794 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
795 MMOFlags, AAInfo);
796
797 // Load the remaining ExtraWidth bits.
798 IncrementSize = RoundWidth / 8;
799 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
800 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
801 LD->getPointerInfo().getWithOffset(IncrementSize),
802 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
803
804 // Build a factor node to remember that this load is independent of
805 // the other one.
806 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
807 Hi.getValue(1));
808
809 // Move the top bits to the right place.
810 Hi = DAG.getNode(
811 ISD::SHL, dl, Hi.getValueType(), Hi,
812 DAG.getConstant(RoundWidth, dl,
813 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
814
815 // Join the hi and lo parts.
816 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
817 } else {
818 // Big endian - avoid unaligned loads.
819 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
820 // Load the top RoundWidth bits.
821 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
822 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
823 MMOFlags, AAInfo);
824
825 // Load the remaining ExtraWidth bits.
826 IncrementSize = RoundWidth / 8;
827 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
828 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
829 LD->getPointerInfo().getWithOffset(IncrementSize),
830 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
831
832 // Build a factor node to remember that this load is independent of
833 // the other one.
834 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
835 Hi.getValue(1));
836
837 // Move the top bits to the right place.
838 Hi = DAG.getNode(
839 ISD::SHL, dl, Hi.getValueType(), Hi,
840 DAG.getConstant(ExtraWidth, dl,
841 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
842
843 // Join the hi and lo parts.
844 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
845 }
846
847 Chain = Ch;
848 } else {
849 bool isCustom = false;
850 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
851 SrcVT.getSimpleVT())) {
852 default: llvm_unreachable("This action is not supported yet!")__builtin_unreachable();
853 case TargetLowering::Custom:
854 isCustom = true;
855 LLVM_FALLTHROUGH[[gnu::fallthrough]];
856 case TargetLowering::Legal:
857 Value = SDValue(Node, 0);
858 Chain = SDValue(Node, 1);
859
860 if (isCustom) {
861 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
862 Value = Res;
863 Chain = Res.getValue(1);
864 }
865 } else {
866 // If this is an unaligned load and the target doesn't support it,
867 // expand it.
868 EVT MemVT = LD->getMemoryVT();
869 const DataLayout &DL = DAG.getDataLayout();
870 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
871 *LD->getMemOperand())) {
872 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
873 }
874 }
875 break;
876
877 case TargetLowering::Expand: {
878 EVT DestVT = Node->getValueType(0);
879 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
880 // If the source type is not legal, see if there is a legal extload to
881 // an intermediate type that we can then extend further.
882 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
883 if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
884 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
885 // If we are loading a legal type, this is a non-extload followed by a
886 // full extend.
887 ISD::LoadExtType MidExtType =
888 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
889
890 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
891 SrcVT, LD->getMemOperand());
892 unsigned ExtendOp =
893 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
894 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
895 Chain = Load.getValue(1);
896 break;
897 }
898
899 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
900 // normal undefined upper bits behavior to allow using an in-reg extend
901 // with the illegal FP type, so load as an integer and do the
902 // from-integer conversion.
903 if (SrcVT.getScalarType() == MVT::f16) {
904 EVT ISrcVT = SrcVT.changeTypeToInteger();
905 EVT IDestVT = DestVT.changeTypeToInteger();
906 EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
907
908 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
909 Ptr, ISrcVT, LD->getMemOperand());
910 Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
911 Chain = Result.getValue(1);
912 break;
913 }
914 }
915
916 assert(!SrcVT.isVector() &&(static_cast<void> (0))
917 "Vector Loads are handled in LegalizeVectorOps")(static_cast<void> (0));
918
919 // FIXME: This does not work for vectors on most targets. Sign-
920 // and zero-extend operations are currently folded into extending
921 // loads, whether they are legal or not, and then we end up here
922 // without any support for legalizing them.
923 assert(ExtType != ISD::EXTLOAD &&(static_cast<void> (0))
924 "EXTLOAD should always be supported!")(static_cast<void> (0));
925 // Turn the unsupported load into an EXTLOAD followed by an
926 // explicit zero/sign extend inreg.
927 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
928 Node->getValueType(0),
929 Chain, Ptr, SrcVT,
930 LD->getMemOperand());
931 SDValue ValRes;
932 if (ExtType == ISD::SEXTLOAD)
933 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
934 Result.getValueType(),
935 Result, DAG.getValueType(SrcVT));
936 else
937 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
938 Value = ValRes;
939 Chain = Result.getValue(1);
940 break;
941 }
942 }
943 }
944
945 // Since loads produce two values, make sure to remember that we legalized
946 // both of them.
947 if (Chain.getNode() != Node) {
948 assert(Value.getNode() != Node && "Load must be completely replaced")(static_cast<void> (0));
949 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
950 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
951 if (UpdatedNodes) {
952 UpdatedNodes->insert(Value.getNode());
953 UpdatedNodes->insert(Chain.getNode());
954 }
955 ReplacedNode(Node);
956 }
957}
958
959/// Return a legal replacement for the given operation, with all legal operands.
960void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
961 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG))do { } while (false);
962
963 // Allow illegal target nodes and illegal registers.
964 if (Node->getOpcode() == ISD::TargetConstant ||
965 Node->getOpcode() == ISD::Register)
966 return;
967
968#ifndef NDEBUG1
969 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
970 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==(static_cast<void> (0))
971 TargetLowering::TypeLegal &&(static_cast<void> (0))
972 "Unexpected illegal type!")(static_cast<void> (0));
973
974 for (const SDValue &Op : Node->op_values())
975 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==(static_cast<void> (0))
976 TargetLowering::TypeLegal ||(static_cast<void> (0))
977 Op.getOpcode() == ISD::TargetConstant ||(static_cast<void> (0))
978 Op.getOpcode() == ISD::Register) &&(static_cast<void> (0))
979 "Unexpected illegal type!")(static_cast<void> (0));
980#endif
981
982 // Figure out the correct action; the way to query this varies by opcode
983 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
984 bool SimpleFinishLegalizing = true;
985 switch (Node->getOpcode()) {
986 case ISD::INTRINSIC_W_CHAIN:
987 case ISD::INTRINSIC_WO_CHAIN:
988 case ISD::INTRINSIC_VOID:
989 case ISD::STACKSAVE:
990 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
991 break;
992 case ISD::GET_DYNAMIC_AREA_OFFSET:
993 Action = TLI.getOperationAction(Node->getOpcode(),
994 Node->getValueType(0));
995 break;
996 case ISD::VAARG:
997 Action = TLI.getOperationAction(Node->getOpcode(),
998 Node->getValueType(0));
999 if (Action != TargetLowering::Promote)
1000 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1001 break;
1002 case ISD::FP_TO_FP16:
1003 case ISD::SINT_TO_FP:
1004 case ISD::UINT_TO_FP:
1005 case ISD::EXTRACT_VECTOR_ELT:
1006 case ISD::LROUND:
1007 case ISD::LLROUND:
1008 case ISD::LRINT:
1009 case ISD::LLRINT:
1010 Action = TLI.getOperationAction(Node->getOpcode(),
1011 Node->getOperand(0).getValueType());
1012 break;
1013 case ISD::STRICT_FP_TO_FP16:
1014 case ISD::STRICT_SINT_TO_FP:
1015 case ISD::STRICT_UINT_TO_FP:
1016 case ISD::STRICT_LRINT:
1017 case ISD::STRICT_LLRINT:
1018 case ISD::STRICT_LROUND:
1019 case ISD::STRICT_LLROUND:
1020 // These pseudo-ops are the same as the other STRICT_ ops except
1021 // they are registered with setOperationAction() using the input type
1022 // instead of the output type.
1023 Action = TLI.getOperationAction(Node->getOpcode(),
1024 Node->getOperand(1).getValueType());
1025 break;
1026 case ISD::SIGN_EXTEND_INREG: {
1027 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1028 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1029 break;
1030 }
1031 case ISD::ATOMIC_STORE:
1032 Action = TLI.getOperationAction(Node->getOpcode(),
1033 Node->getOperand(2).getValueType());
1034 break;
1035 case ISD::SELECT_CC:
1036 case ISD::STRICT_FSETCC:
1037 case ISD::STRICT_FSETCCS:
1038 case ISD::SETCC:
1039 case ISD::BR_CC: {
1040 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1041 Node->getOpcode() == ISD::STRICT_FSETCC ? 3 :
1042 Node->getOpcode() == ISD::STRICT_FSETCCS ? 3 :
1043 Node->getOpcode() == ISD::SETCC ? 2 : 1;
1044 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 :
1045 Node->getOpcode() == ISD::STRICT_FSETCC ? 1 :
1046 Node->getOpcode() == ISD::STRICT_FSETCCS ? 1 : 0;
1047 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1048 ISD::CondCode CCCode =
1049 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1050 Action = TLI.getCondCodeAction(CCCode, OpVT);
1051 if (Action == TargetLowering::Legal) {
1052 if (Node->getOpcode() == ISD::SELECT_CC)
1053 Action = TLI.getOperationAction(Node->getOpcode(),
1054 Node->getValueType(0));
1055 else
1056 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1057 }
1058 break;
1059 }
1060 case ISD::LOAD:
1061 case ISD::STORE:
1062 // FIXME: Model these properly. LOAD and STORE are complicated, and
1063 // STORE expects the unlegalized operand in some cases.
1064 SimpleFinishLegalizing = false;
1065 break;
1066 case ISD::CALLSEQ_START:
1067 case ISD::CALLSEQ_END:
1068 // FIXME: This shouldn't be necessary. These nodes have special properties
1069 // dealing with the recursive nature of legalization. Removing this
1070 // special case should be done as part of making LegalizeDAG non-recursive.
1071 SimpleFinishLegalizing = false;
1072 break;
1073 case ISD::EXTRACT_ELEMENT:
1074 case ISD::FLT_ROUNDS_:
1075 case ISD::MERGE_VALUES:
1076 case ISD::EH_RETURN:
1077 case ISD::FRAME_TO_ARGS_OFFSET:
1078 case ISD::EH_DWARF_CFA:
1079 case ISD::EH_SJLJ_SETJMP:
1080 case ISD::EH_SJLJ_LONGJMP:
1081 case ISD::EH_SJLJ_SETUP_DISPATCH:
1082 // These operations lie about being legal: when they claim to be legal,
1083 // they should actually be expanded.
1084 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1085 if (Action == TargetLowering::Legal)
1086 Action = TargetLowering::Expand;
1087 break;
1088 case ISD::INIT_TRAMPOLINE:
1089 case ISD::ADJUST_TRAMPOLINE:
1090 case ISD::FRAMEADDR:
1091 case ISD::RETURNADDR:
1092 case ISD::ADDROFRETURNADDR:
1093 case ISD::SPONENTRY:
1094 // These operations lie about being legal: when they claim to be legal,
1095 // they should actually be custom-lowered.
1096 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1097 if (Action == TargetLowering::Legal)
1098 Action = TargetLowering::Custom;
1099 break;
1100 case ISD::READCYCLECOUNTER:
1101 // READCYCLECOUNTER returns an i64, even if type legalization might have
1102 // expanded that to several smaller types.
1103 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1104 break;
1105 case ISD::READ_REGISTER:
1106 case ISD::WRITE_REGISTER:
1107 // Named register is legal in the DAG, but blocked by register name
1108 // selection if not implemented by target (to chose the correct register)
1109 // They'll be converted to Copy(To/From)Reg.
1110 Action = TargetLowering::Legal;
1111 break;
1112 case ISD::UBSANTRAP:
1113 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1114 if (Action == TargetLowering::Expand) {
1115 // replace ISD::UBSANTRAP with ISD::TRAP
1116 SDValue NewVal;
1117 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1118 Node->getOperand(0));
1119 ReplaceNode(Node, NewVal.getNode());
1120 LegalizeOp(NewVal.getNode());
1121 return;
1122 }
1123 break;
1124 case ISD::DEBUGTRAP:
1125 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1126 if (Action == TargetLowering::Expand) {
1127 // replace ISD::DEBUGTRAP with ISD::TRAP
1128 SDValue NewVal;
1129 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1130 Node->getOperand(0));
1131 ReplaceNode(Node, NewVal.getNode());
1132 LegalizeOp(NewVal.getNode());
1133 return;
1134 }
1135 break;
1136 case ISD::SADDSAT:
1137 case ISD::UADDSAT:
1138 case ISD::SSUBSAT:
1139 case ISD::USUBSAT:
1140 case ISD::SSHLSAT:
1141 case ISD::USHLSAT:
1142 case ISD::FP_TO_SINT_SAT:
1143 case ISD::FP_TO_UINT_SAT:
1144 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1145 break;
1146 case ISD::SMULFIX:
1147 case ISD::SMULFIXSAT:
1148 case ISD::UMULFIX:
1149 case ISD::UMULFIXSAT:
1150 case ISD::SDIVFIX:
1151 case ISD::SDIVFIXSAT:
1152 case ISD::UDIVFIX:
1153 case ISD::UDIVFIXSAT: {
1154 unsigned Scale = Node->getConstantOperandVal(2);
1155 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1156 Node->getValueType(0), Scale);
1157 break;
1158 }
1159 case ISD::MSCATTER:
1160 Action = TLI.getOperationAction(Node->getOpcode(),
1161 cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1162 break;
1163 case ISD::MSTORE:
1164 Action = TLI.getOperationAction(Node->getOpcode(),
1165 cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1166 break;
1167 case ISD::VECREDUCE_FADD:
1168 case ISD::VECREDUCE_FMUL:
1169 case ISD::VECREDUCE_ADD:
1170 case ISD::VECREDUCE_MUL:
1171 case ISD::VECREDUCE_AND:
1172 case ISD::VECREDUCE_OR:
1173 case ISD::VECREDUCE_XOR:
1174 case ISD::VECREDUCE_SMAX:
1175 case ISD::VECREDUCE_SMIN:
1176 case ISD::VECREDUCE_UMAX:
1177 case ISD::VECREDUCE_UMIN:
1178 case ISD::VECREDUCE_FMAX:
1179 case ISD::VECREDUCE_FMIN:
1180 Action = TLI.getOperationAction(
1181 Node->getOpcode(), Node->getOperand(0).getValueType());
1182 break;
1183 case ISD::VECREDUCE_SEQ_FADD:
1184 case ISD::VECREDUCE_SEQ_FMUL:
1185 Action = TLI.getOperationAction(
1186 Node->getOpcode(), Node->getOperand(1).getValueType());
1187 break;
1188 default:
1189 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1190 Action = TargetLowering::Legal;
1191 } else {
1192 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1193 }
1194 break;
1195 }
1196
1197 if (SimpleFinishLegalizing) {
1198 SDNode *NewNode = Node;
1199 switch (Node->getOpcode()) {
1200 default: break;
1201 case ISD::SHL:
1202 case ISD::SRL:
1203 case ISD::SRA:
1204 case ISD::ROTL:
1205 case ISD::ROTR: {
1206 // Legalizing shifts/rotates requires adjusting the shift amount
1207 // to the appropriate width.
1208 SDValue Op0 = Node->getOperand(0);
1209 SDValue Op1 = Node->getOperand(1);
1210 if (!Op1.getValueType().isVector()) {
1211 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1212 // The getShiftAmountOperand() may create a new operand node or
1213 // return the existing one. If new operand is created we need
1214 // to update the parent node.
1215 // Do not try to legalize SAO here! It will be automatically legalized
1216 // in the next round.
1217 if (SAO != Op1)
1218 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1219 }
1220 }
1221 break;
1222 case ISD::FSHL:
1223 case ISD::FSHR:
1224 case ISD::SRL_PARTS:
1225 case ISD::SRA_PARTS:
1226 case ISD::SHL_PARTS: {
1227 // Legalizing shifts/rotates requires adjusting the shift amount
1228 // to the appropriate width.
1229 SDValue Op0 = Node->getOperand(0);
1230 SDValue Op1 = Node->getOperand(1);
1231 SDValue Op2 = Node->getOperand(2);
1232 if (!Op2.getValueType().isVector()) {
1233 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1234 // The getShiftAmountOperand() may create a new operand node or
1235 // return the existing one. If new operand is created we need
1236 // to update the parent node.
1237 if (SAO != Op2)
1238 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1239 }
1240 break;
1241 }
1242 }
1243
1244 if (NewNode != Node) {
1245 ReplaceNode(Node, NewNode);
1246 Node = NewNode;
1247 }
1248 switch (Action) {
1249 case TargetLowering::Legal:
1250 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n")do { } while (false);
1251 return;
1252 case TargetLowering::Custom:
1253 LLVM_DEBUG(dbgs() << "Trying custom legalization\n")do { } while (false);
1254 // FIXME: The handling for custom lowering with multiple results is
1255 // a complete mess.
1256 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1257 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1258 return;
1259
1260 if (Node->getNumValues() == 1) {
1261 // Verify the new types match the original. Glue is waived because
1262 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1263 assert((Res.getValueType() == Node->getValueType(0) ||(static_cast<void> (0))
1264 Node->getValueType(0) == MVT::Glue) &&(static_cast<void> (0))
1265 "Type mismatch for custom legalized operation")(static_cast<void> (0));
1266 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n")do { } while (false);
1267 // We can just directly replace this node with the lowered value.
1268 ReplaceNode(SDValue(Node, 0), Res);
1269 return;
1270 }
1271
1272 SmallVector<SDValue, 8> ResultVals;
1273 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1274 // Verify the new types match the original. Glue is waived because
1275 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1276 assert((Res->getValueType(i) == Node->getValueType(i) ||(static_cast<void> (0))
1277 Node->getValueType(i) == MVT::Glue) &&(static_cast<void> (0))
1278 "Type mismatch for custom legalized operation")(static_cast<void> (0));
1279 ResultVals.push_back(Res.getValue(i));
1280 }
1281 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n")do { } while (false);
1282 ReplaceNode(Node, ResultVals.data());
1283 return;
1284 }
1285 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n")do { } while (false);
1286 LLVM_FALLTHROUGH[[gnu::fallthrough]];
1287 case TargetLowering::Expand:
1288 if (ExpandNode(Node))
1289 return;
1290 LLVM_FALLTHROUGH[[gnu::fallthrough]];
1291 case TargetLowering::LibCall:
1292 ConvertNodeToLibcall(Node);
1293 return;
1294 case TargetLowering::Promote:
1295 PromoteNode(Node);
1296 return;
1297 }
1298 }
1299
1300 switch (Node->getOpcode()) {
1301 default:
1302#ifndef NDEBUG1
1303 dbgs() << "NODE: ";
1304 Node->dump( &DAG);
1305 dbgs() << "\n";
1306#endif
1307 llvm_unreachable("Do not know how to legalize this operator!")__builtin_unreachable();
1308
1309 case ISD::CALLSEQ_START:
1310 case ISD::CALLSEQ_END:
1311 break;
1312 case ISD::LOAD:
1313 return LegalizeLoadOps(Node);
1314 case ISD::STORE:
1315 return LegalizeStoreOps(Node);
1316 }
1317}
1318
1319SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1320 SDValue Vec = Op.getOperand(0);
1321 SDValue Idx = Op.getOperand(1);
1322 SDLoc dl(Op);
1323
1324 // Before we generate a new store to a temporary stack slot, see if there is
1325 // already one that we can use. There often is because when we scalarize
1326 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1327 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1328 // the vector. If all are expanded here, we don't want one store per vector
1329 // element.
1330
1331 // Caches for hasPredecessorHelper
1332 SmallPtrSet<const SDNode *, 32> Visited;
1333 SmallVector<const SDNode *, 16> Worklist;
1334 Visited.insert(Op.getNode());
1335 Worklist.push_back(Idx.getNode());
1336 SDValue StackPtr, Ch;
1337 for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1338 UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1339 SDNode *User = *UI;
1340 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1341 if (ST->isIndexed() || ST->isTruncatingStore() ||
1342 ST->getValue() != Vec)
1343 continue;
1344
1345 // Make sure that nothing else could have stored into the destination of
1346 // this store.
1347 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1348 continue;
1349
1350 // If the index is dependent on the store we will introduce a cycle when
1351 // creating the load (the load uses the index, and by replacing the chain
1352 // we will make the index dependent on the load). Also, the store might be
1353 // dependent on the extractelement and introduce a cycle when creating
1354 // the load.
1355 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1356 ST->hasPredecessor(Op.getNode()))
1357 continue;
1358
1359 StackPtr = ST->getBasePtr();
1360 Ch = SDValue(ST, 0);
1361 break;
1362 }
1363 }
1364
1365 EVT VecVT = Vec.getValueType();
1366
1367 if (!Ch.getNode()) {
1368 // Store the value to a temporary stack slot, then LOAD the returned part.
1369 StackPtr = DAG.CreateStackTemporary(VecVT);
1370 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1371 MachinePointerInfo());
1372 }
1373
1374 SDValue NewLoad;
1375
1376 if (Op.getValueType().isVector()) {
1377 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1378 Op.getValueType(), Idx);
1379 NewLoad =
1380 DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1381 } else {
1382 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1383 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1384 MachinePointerInfo(),
1385 VecVT.getVectorElementType());
1386 }
1387
1388 // Replace the chain going out of the store, by the one out of the load.
1389 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1390
1391 // We introduced a cycle though, so update the loads operands, making sure
1392 // to use the original store's chain as an incoming chain.
1393 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1394 NewLoad->op_end());
1395 NewLoadOperands[0] = Ch;
1396 NewLoad =
1397 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1398 return NewLoad;
1399}
1400
1401SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1402 assert(Op.getValueType().isVector() && "Non-vector insert subvector!")(static_cast<void> (0));
1403
1404 SDValue Vec = Op.getOperand(0);
1405 SDValue Part = Op.getOperand(1);
1406 SDValue Idx = Op.getOperand(2);
1407 SDLoc dl(Op);
1408
1409 // Store the value to a temporary stack slot, then LOAD the returned part.
1410 EVT VecVT = Vec.getValueType();
1411 EVT SubVecVT = Part.getValueType();
1412 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1413 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1414 MachinePointerInfo PtrInfo =
1415 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1416
1417 // First store the whole vector.
1418 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1419
1420 // Then store the inserted part.
1421 SDValue SubStackPtr =
1422 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1423
1424 // Store the subvector.
1425 Ch = DAG.getStore(
1426 Ch, dl, Part, SubStackPtr,
1427 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1428
1429 // Finally, load the updated vector.
1430 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1431}
1432
1433SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1434 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||(static_cast<void> (0))
1435 Node->getOpcode() == ISD::CONCAT_VECTORS) &&(static_cast<void> (0))
1436 "Unexpected opcode!")(static_cast<void> (0));
1437
1438 // We can't handle this case efficiently. Allocate a sufficiently
1439 // aligned object on the stack, store each operand into it, then load
1440 // the result as a vector.
1441 // Create the stack frame object.
1442 EVT VT = Node->getValueType(0);
1443 EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
1444 : Node->getOperand(0).getValueType();
1445 SDLoc dl(Node);
1446 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1447 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1448 MachinePointerInfo PtrInfo =
1449 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1450
1451 // Emit a store of each element to the stack slot.
1452 SmallVector<SDValue, 8> Stores;
1453 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1454 assert(TypeByteSize > 0 && "Vector element type too small for stack store!")(static_cast<void> (0));
1455
1456 // If the destination vector element type of a BUILD_VECTOR is narrower than
1457 // the source element type, only store the bits necessary.
1458 bool Truncate = isa<BuildVectorSDNode>(Node) &&
1459 MemVT.bitsLT(Node->getOperand(0).getValueType());
1460
1461 // Store (in the right endianness) the elements to memory.
1462 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1463 // Ignore undef elements.
1464 if (Node->getOperand(i).isUndef()) continue;
1465
1466 unsigned Offset = TypeByteSize*i;
1467
1468 SDValue Idx = DAG.getMemBasePlusOffset(FIPtr, TypeSize::Fixed(Offset), dl);
1469
1470 if (Truncate)
1471 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1472 Node->getOperand(i), Idx,
1473 PtrInfo.getWithOffset(Offset), MemVT));
1474 else
1475 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1476 Idx, PtrInfo.getWithOffset(Offset)));
1477 }
1478
1479 SDValue StoreChain;
1480 if (!Stores.empty()) // Not all undef elements?
1481 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1482 else
1483 StoreChain = DAG.getEntryNode();
1484
1485 // Result is a load from the stack slot.
1486 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1487}
1488
1489/// Bitcast a floating-point value to an integer value. Only bitcast the part
1490/// containing the sign bit if the target has no integer value capable of
1491/// holding all bits of the floating-point value.
1492void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1493 const SDLoc &DL,
1494 SDValue Value) const {
1495 EVT FloatVT = Value.getValueType();
1496 unsigned NumBits = FloatVT.getScalarSizeInBits();
1497 State.FloatVT = FloatVT;
1498 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1499 // Convert to an integer of the same size.
1500 if (TLI.isTypeLegal(IVT)) {
1501 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1502 State.SignMask = APInt::getSignMask(NumBits);
1503 State.SignBit = NumBits - 1;
1504 return;
1505 }
1506
1507 auto &DataLayout = DAG.getDataLayout();
1508 // Store the float to memory, then load the sign part out as an integer.
1509 MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1510 // First create a temporary that is aligned for both the load and store.
1511 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1512 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1513 // Then store the float to it.
1514 State.FloatPtr = StackPtr;
1515 MachineFunction &MF = DAG.getMachineFunction();
1516 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1517 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1518 State.FloatPointerInfo);
1519
1520 SDValue IntPtr;
1521 if (DataLayout.isBigEndian()) {
1522 assert(FloatVT.isByteSized() && "Unsupported floating point type!")(static_cast<void> (0));
1523 // Load out a legal integer with the same sign bit as the float.
1524 IntPtr = StackPtr;
1525 State.IntPointerInfo = State.FloatPointerInfo;
1526 } else {
1527 // Advance the pointer so that the loaded byte will contain the sign bit.
1528 unsigned ByteOffset = (NumBits / 8) - 1;
1529 IntPtr =
1530 DAG.getMemBasePlusOffset(StackPtr, TypeSize::Fixed(ByteOffset), DL);
1531 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1532 ByteOffset);
1533 }
1534
1535 State.IntPtr = IntPtr;
1536 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1537 State.IntPointerInfo, MVT::i8);
1538 State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);
1539 State.SignBit = 7;
1540}
1541
1542/// Replace the integer value produced by getSignAsIntValue() with a new value
1543/// and cast the result back to a floating-point type.
1544SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1545 const SDLoc &DL,
1546 SDValue NewIntValue) const {
1547 if (!State.Chain)
1548 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1549
1550 // Override the part containing the sign bit in the value stored on the stack.
1551 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1552 State.IntPointerInfo, MVT::i8);
1553 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1554 State.FloatPointerInfo);
1555}
1556
1557SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1558 SDLoc DL(Node);
1559 SDValue Mag = Node->getOperand(0);
1560 SDValue Sign = Node->getOperand(1);
1561
1562 // Get sign bit into an integer value.
1563 FloatSignAsInt SignAsInt;
1564 getSignAsIntValue(SignAsInt, DL, Sign);
1565
1566 EVT IntVT = SignAsInt.IntValue.getValueType();
1567 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1568 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1569 SignMask);
1570
1571 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1572 EVT FloatVT = Mag.getValueType();
1573 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1574 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1575 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1576 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1577 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1578 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1579 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1580 }
1581
1582 // Transform Mag value to integer, and clear the sign bit.
1583 FloatSignAsInt MagAsInt;
1584 getSignAsIntValue(MagAsInt, DL, Mag);
1585 EVT MagVT = MagAsInt.IntValue.getValueType();
1586 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1587 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1588 ClearSignMask);
1589
1590 // Get the signbit at the right position for MagAsInt.
1591 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1592 EVT ShiftVT = IntVT;
1593 if (SignBit.getScalarValueSizeInBits() <
1594 ClearedSign.getScalarValueSizeInBits()) {
1595 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1596 ShiftVT = MagVT;
1597 }
1598 if (ShiftAmount > 0) {
1599 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1600 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1601 } else if (ShiftAmount < 0) {
1602 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1603 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1604 }
1605 if (SignBit.getScalarValueSizeInBits() >
1606 ClearedSign.getScalarValueSizeInBits()) {
1607 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1608 }
1609
1610 // Store the part with the modified sign and convert back to float.
1611 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1612 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1613}
1614
1615SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1616 // Get the sign bit as an integer.
1617 SDLoc DL(Node);
1618 FloatSignAsInt SignAsInt;
1619 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
1620 EVT IntVT = SignAsInt.IntValue.getValueType();
1621
1622 // Flip the sign.
1623 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1624 SDValue SignFlip =
1625 DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask);
1626
1627 // Convert back to float.
1628 return modifySignAsInt(SignAsInt, DL, SignFlip);
1629}
1630
1631SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1632 SDLoc DL(Node);
1633 SDValue Value = Node->getOperand(0);
1634
1635 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1636 EVT FloatVT = Value.getValueType();
1637 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1638 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1639 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1640 }
1641
1642 // Transform value to integer, clear the sign bit and transform back.
1643 FloatSignAsInt ValueAsInt;
1644 getSignAsIntValue(ValueAsInt, DL, Value);
1645 EVT IntVT = ValueAsInt.IntValue.getValueType();
1646 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1647 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1648 ClearSignMask);
1649 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1650}
1651
1652void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1653 SmallVectorImpl<SDValue> &Results) {
1654 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1655 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"(static_cast<void> (0))
1656 " not tell us which reg is the stack pointer!")(static_cast<void> (0));
1657 SDLoc dl(Node);
1658 EVT VT = Node->getValueType(0);
1659 SDValue Tmp1 = SDValue(Node, 0);
1660 SDValue Tmp2 = SDValue(Node, 1);
1661 SDValue Tmp3 = Node->getOperand(2);
1662 SDValue Chain = Tmp1.getOperand(0);
1663
1664 // Chain the dynamic stack allocation so that it doesn't modify the stack
1665 // pointer when other instructions are using the stack.
1666 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1667
1668 SDValue Size = Tmp2.getOperand(1);
1669 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1670 Chain = SP.getValue(1);
1671 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1672 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1673 unsigned Opc =
1674 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1675 ISD::ADD : ISD::SUB;
1676
1677 Align StackAlign = TFL->getStackAlign();
1678 Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); // Value
1679 if (Alignment > StackAlign)
1680 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1681 DAG.getConstant(-Alignment.value(), dl, VT));
1682 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1683
1684 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1685 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1686
1687 Results.push_back(Tmp1);
1688 Results.push_back(Tmp2);
1689}
1690
1691/// Emit a store/load combination to the stack. This stores
1692/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1693/// a load from the stack slot to DestVT, extending it if needed.
1694/// The resultant code need not be legal.
1695SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1696 EVT DestVT, const SDLoc &dl) {
1697 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1698}
1699
1700SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1701 EVT DestVT, const SDLoc &dl,
1702 SDValue Chain) {
1703 unsigned SrcSize = SrcOp.getValueSizeInBits();
1704 unsigned SlotSize = SlotVT.getSizeInBits();
1705 unsigned DestSize = DestVT.getSizeInBits();
1706 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1707 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1708
1709 // Don't convert with stack if the load/store is expensive.
1710 if ((SrcSize > SlotSize &&
1711 !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||
1712 (SlotSize < DestSize &&
1713 !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))
1714 return SDValue();
1715
1716 // Create the stack frame object.
1717 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1718 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1719 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);
1720
1721 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1722 int SPFI = StackPtrFI->getIndex();
1723 MachinePointerInfo PtrInfo =
1724 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1725
1726 // Emit a store to the stack slot. Use a truncstore if the input value is
1727 // later than DestVT.
1728 SDValue Store;
1729
1730 if (SrcSize > SlotSize)
1731 Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
1732 SlotVT, SrcAlign);
1733 else {
1734 assert(SrcSize == SlotSize && "Invalid store")(static_cast<void> (0));
1735 Store =
1736 DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1737 }
1738
1739 // Result is a load from the stack slot.
1740 if (SlotSize == DestSize)
1741 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1742
1743 assert(SlotSize < DestSize && "Unknown extension!")(static_cast<void> (0));
1744 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1745 DestAlign);
1746}
1747
1748SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1749 SDLoc dl(Node);
1750 // Create a vector sized/aligned stack slot, store the value to element #0,
1751 // then load the whole vector back out.
1752 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1753
1754 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1755 int SPFI = StackPtrFI->getIndex();
1756
1757 SDValue Ch = DAG.getTruncStore(
1758 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1759 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1760 Node->getValueType(0).getVectorElementType());
1761 return DAG.getLoad(
1762 Node->getValueType(0), dl, Ch, StackPtr,
1763 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1764}
1765
1766static bool
1767ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1768 const TargetLowering &TLI, SDValue &Res) {
1769 unsigned NumElems = Node->getNumOperands();
1770 SDLoc dl(Node);
1771 EVT VT = Node->getValueType(0);
1772
1773 // Try to group the scalars into pairs, shuffle the pairs together, then
1774 // shuffle the pairs of pairs together, etc. until the vector has
1775 // been built. This will work only if all of the necessary shuffle masks
1776 // are legal.
1777
1778 // We do this in two phases; first to check the legality of the shuffles,
1779 // and next, assuming that all shuffles are legal, to create the new nodes.
1780 for (int Phase = 0; Phase < 2; ++Phase) {
1781 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1782 NewIntermedVals;
1783 for (unsigned i = 0; i < NumElems; ++i) {
1784 SDValue V = Node->getOperand(i);
1785 if (V.isUndef())
1786 continue;
1787
1788 SDValue Vec;
1789 if (Phase)
1790 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1791 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1792 }
1793
1794 while (IntermedVals.size() > 2) {
1795 NewIntermedVals.clear();
1796 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1797 // This vector and the next vector are shuffled together (simply to
1798 // append the one to the other).
1799 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1800
1801 SmallVector<int, 16> FinalIndices;
1802 FinalIndices.reserve(IntermedVals[i].second.size() +
1803 IntermedVals[i+1].second.size());
1804
1805 int k = 0;
1806 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1807 ++j, ++k) {
1808 ShuffleVec[k] = j;
1809 FinalIndices.push_back(IntermedVals[i].second[j]);
1810 }
1811 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1812 ++j, ++k) {
1813 ShuffleVec[k] = NumElems + j;
1814 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1815 }
1816
1817 SDValue Shuffle;
1818 if (Phase)
1819 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1820 IntermedVals[i+1].first,
1821 ShuffleVec);
1822 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1823 return false;
1824 NewIntermedVals.push_back(
1825 std::make_pair(Shuffle, std::move(FinalIndices)));
1826 }
1827
1828 // If we had an odd number of defined values, then append the last
1829 // element to the array of new vectors.
1830 if ((IntermedVals.size() & 1) != 0)
1831 NewIntermedVals.push_back(IntermedVals.back());
1832
1833 IntermedVals.swap(NewIntermedVals);
1834 }
1835
1836 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&(static_cast<void> (0))
1837 "Invalid number of intermediate vectors")(static_cast<void> (0));
1838 SDValue Vec1 = IntermedVals[0].first;
1839 SDValue Vec2;
1840 if (IntermedVals.size() > 1)
1841 Vec2 = IntermedVals[1].first;
1842 else if (Phase)
1843 Vec2 = DAG.getUNDEF(VT);
1844
1845 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1846 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1847 ShuffleVec[IntermedVals[0].second[i]] = i;
1848 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1849 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1850
1851 if (Phase)
1852 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1853 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1854 return false;
1855 }
1856
1857 return true;
1858}
1859
1860/// Expand a BUILD_VECTOR node on targets that don't
1861/// support the operation, but do support the resultant vector type.
1862SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1863 unsigned NumElems = Node->getNumOperands();
1864 SDValue Value1, Value2;
1865 SDLoc dl(Node);
1866 EVT VT = Node->getValueType(0);
1867 EVT OpVT = Node->getOperand(0).getValueType();
1868 EVT EltVT = VT.getVectorElementType();
1869
1870 // If the only non-undef value is the low element, turn this into a
1871 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1872 bool isOnlyLowElement = true;
1873 bool MoreThanTwoValues = false;
1874 bool isConstant = true;
1875 for (unsigned i = 0; i < NumElems; ++i) {
1876 SDValue V = Node->getOperand(i);
1877 if (V.isUndef())
1878 continue;
1879 if (i > 0)
1880 isOnlyLowElement = false;
1881 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1882 isConstant = false;
1883
1884 if (!Value1.getNode()) {
1885 Value1 = V;
1886 } else if (!Value2.getNode()) {
1887 if (V != Value1)
1888 Value2 = V;
1889 } else if (V != Value1 && V != Value2) {
1890 MoreThanTwoValues = true;
1891 }
1892 }
1893
1894 if (!Value1.getNode())
1895 return DAG.getUNDEF(VT);
1896
1897 if (isOnlyLowElement)
1898 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1899
1900 // If all elements are constants, create a load from the constant pool.
1901 if (isConstant) {
1902 SmallVector<Constant*, 16> CV;
1903 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1904 if (ConstantFPSDNode *V =
1905 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1906 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1907 } else if (ConstantSDNode *V =
1908 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1909 if (OpVT==EltVT)
1910 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1911 else {
1912 // If OpVT and EltVT don't match, EltVT is not legal and the
1913 // element values have been promoted/truncated earlier. Undo this;
1914 // we don't want a v16i8 to become a v16i32 for example.
1915 const ConstantInt *CI = V->getConstantIntValue();
1916 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1917 CI->getZExtValue()));
1918 }
1919 } else {
1920 assert(Node->getOperand(i).isUndef())(static_cast<void> (0));
1921 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1922 CV.push_back(UndefValue::get(OpNTy));
1923 }
1924 }
1925 Constant *CP = ConstantVector::get(CV);
1926 SDValue CPIdx =
1927 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1928 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1929 return DAG.getLoad(
1930 VT, dl, DAG.getEntryNode(), CPIdx,
1931 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1932 Alignment);
1933 }
1934
1935 SmallSet<SDValue, 16> DefinedValues;
1936 for (unsigned i = 0; i < NumElems; ++i) {
1937 if (Node->getOperand(i).isUndef())
1938 continue;
1939 DefinedValues.insert(Node->getOperand(i));
1940 }
1941
1942 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1943 if (!MoreThanTwoValues) {
1944 SmallVector<int, 8> ShuffleVec(NumElems, -1);
1945 for (unsigned i = 0; i < NumElems; ++i) {
1946 SDValue V = Node->getOperand(i);
1947 if (V.isUndef())
1948 continue;
1949 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1950 }
1951 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1952 // Get the splatted value into the low element of a vector register.
1953 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1954 SDValue Vec2;
1955 if (Value2.getNode())
1956 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1957 else
1958 Vec2 = DAG.getUNDEF(VT);
1959
1960 // Return shuffle(LowValVec, undef, <0,0,0,0>)
1961 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1962 }
1963 } else {
1964 SDValue Res;
1965 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1966 return Res;
1967 }
1968 }
1969
1970 // Otherwise, we can't handle this case efficiently.
1971 return ExpandVectorBuildThroughStack(Node);
1972}
1973
1974SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
1975 SDLoc DL(Node);
1976 EVT VT = Node->getValueType(0);
1977 SDValue SplatVal = Node->getOperand(0);
1978
1979 return DAG.getSplatBuildVector(VT, DL, SplatVal);
1980}
1981
1982// Expand a node into a call to a libcall. If the result value
1983// does not fit into a register, return the lo part and set the hi part to the
1984// by-reg argument. If it does fit into a single register, return the result
1985// and leave the Hi part unset.
1986SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
1987 bool isSigned) {
1988 TargetLowering::ArgListTy Args;
1989 TargetLowering::ArgListEntry Entry;
1990 for (const SDValue &Op : Node->op_values()) {
1991 EVT ArgVT = Op.getValueType();
1992 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1993 Entry.Node = Op;
1994 Entry.Ty = ArgTy;
1995 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
1996 Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
1997 Args.push_back(Entry);
1998 }
1999 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2000 TLI.getPointerTy(DAG.getDataLayout()));
2001
2002 EVT RetVT = Node->getValueType(0);
2003 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2004
2005 // By default, the input chain to this libcall is the entry node of the
2006 // function. If the libcall is going to be emitted as a tail call then
2007 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2008 // node which is being folded has a non-entry input chain.
2009 SDValue InChain = DAG.getEntryNode();
2010
2011 // isTailCall may be true since the callee does not reference caller stack
2012 // frame. Check if it's in the right position and that the return types match.
2013 SDValue TCChain = InChain;
2014 const Function &F = DAG.getMachineFunction().getFunction();
2015 bool isTailCall =
2016 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2017 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2018 if (isTailCall)
2019 InChain = TCChain;
2020
2021 TargetLowering::CallLoweringInfo CLI(DAG);
2022 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2023 CLI.setDebugLoc(SDLoc(Node))
2024 .setChain(InChain)
2025 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2026 std::move(Args))
2027 .setTailCall(isTailCall)
2028 .setSExtResult(signExtend)
2029 .setZExtResult(!signExtend)
2030 .setIsPostTypeLegalization(true);
2031
2032 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2033
2034 if (!CallInfo.second.getNode()) {
2035 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG))do { } while (false);
2036 // It's a tailcall, return the chain (which is the DAG root).
2037 return DAG.getRoot();
2038 }
2039
2040 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG))do { } while (false);
2041 return CallInfo.first;
2042}
2043
2044void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2045 RTLIB::Libcall LC,
2046 SmallVectorImpl<SDValue> &Results) {
2047 if (LC == RTLIB::UNKNOWN_LIBCALL)
2048 llvm_unreachable("Can't create an unknown libcall!")__builtin_unreachable();
2049
2050 if (Node->isStrictFPOpcode()) {
2051 EVT RetVT = Node->getValueType(0);
2052 SmallVector<SDValue, 4> Ops(drop_begin(Node->ops()));
2053 TargetLowering::MakeLibCallOptions CallOptions;
2054 // FIXME: This doesn't support tail calls.
2055 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2056 Ops, CallOptions,
2057 SDLoc(Node),
2058 Node->getOperand(0));
2059 Results.push_back(Tmp.first);
2060 Results.push_back(Tmp.second);
2061 } else {
2062 SDValue Tmp = ExpandLibCall(LC, Node, false);
2063 Results.push_back(Tmp);
2064 }
2065}
2066
2067/// Expand the node to a libcall based on the result type.
2068void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2069 RTLIB::Libcall Call_F32,
2070 RTLIB::Libcall Call_F64,
2071 RTLIB::Libcall Call_F80,
2072 RTLIB::Libcall Call_F128,
2073 RTLIB::Libcall Call_PPCF128,
2074 SmallVectorImpl<SDValue> &Results) {
2075 RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0),
2076 Call_F32, Call_F64, Call_F80,
2077 Call_F128, Call_PPCF128);
2078 ExpandFPLibCall(Node, LC, Results);
2079}
2080
2081SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2082 RTLIB::Libcall Call_I8,
2083 RTLIB::Libcall Call_I16,
2084 RTLIB::Libcall Call_I32,
2085 RTLIB::Libcall Call_I64,
2086 RTLIB::Libcall Call_I128) {
2087 RTLIB::Libcall LC;
2088 switch (Node->getSimpleValueType(0).SimpleTy) {
2089 default: llvm_unreachable("Unexpected request for libcall!")__builtin_unreachable();
2090 case MVT::i8: LC = Call_I8; break;
2091 case MVT::i16: LC = Call_I16; break;
2092 case MVT::i32: LC = Call_I32; break;
2093 case MVT::i64: LC = Call_I64; break;
2094 case MVT::i128: LC = Call_I128; break;
2095 }
2096 return ExpandLibCall(LC, Node, isSigned);
2097}
2098
2099/// Expand the node to a libcall based on first argument type (for instance
2100/// lround and its variant).
2101void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2102 RTLIB::Libcall Call_F32,
2103 RTLIB::Libcall Call_F64,
2104 RTLIB::Libcall Call_F80,
2105 RTLIB::Libcall Call_F128,
2106 RTLIB::Libcall Call_PPCF128,
2107 SmallVectorImpl<SDValue> &Results) {
2108 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2109 RTLIB::Libcall LC = RTLIB::getFPLibCall(InVT.getSimpleVT(),
2110 Call_F32, Call_F64, Call_F80,
2111 Call_F128, Call_PPCF128);
2112 ExpandFPLibCall(Node, LC, Results);
2113}
2114
2115/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2116void
2117SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2118 SmallVectorImpl<SDValue> &Results) {
2119 unsigned Opcode = Node->getOpcode();
2120 bool isSigned = Opcode == ISD::SDIVREM;
2121
2122 RTLIB::Libcall LC;
2123 switch (Node->getSimpleValueType(0).SimpleTy) {
2124 default: llvm_unreachable("Unexpected request for libcall!")__builtin_unreachable();
2125 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2126 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2127 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2128 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2129 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2130 }
2131
2132 // The input chain to this libcall is the entry node of the function.
2133 // Legalizing the call will automatically add the previous call to the
2134 // dependence.
2135 SDValue InChain = DAG.getEntryNode();
2136
2137 EVT RetVT = Node->getValueType(0);
2138 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2139
2140 TargetLowering::ArgListTy Args;
2141 TargetLowering::ArgListEntry Entry;
2142 for (const SDValue &Op : Node->op_values()) {
2143 EVT ArgVT = Op.getValueType();
2144 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2145 Entry.Node = Op;
2146 Entry.Ty = ArgTy;
2147 Entry.IsSExt = isSigned;
2148 Entry.IsZExt = !isSigned;
2149 Args.push_back(Entry);
2150 }
2151
2152 // Also pass the return address of the remainder.
2153 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2154 Entry.Node = FIPtr;
2155 Entry.Ty = RetTy->getPointerTo();
2156 Entry.IsSExt = isSigned;
2157 Entry.IsZExt = !isSigned;
2158 Args.push_back(Entry);
2159
2160 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2161 TLI.getPointerTy(DAG.getDataLayout()));
2162
2163 SDLoc dl(Node);
2164 TargetLowering::CallLoweringInfo CLI(DAG);
2165 CLI.setDebugLoc(dl)
2166 .setChain(InChain)
2167 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2168 std::move(Args))
2169 .setSExtResult(isSigned)
2170 .setZExtResult(!isSigned);
2171
2172 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2173
2174 // Remainder is loaded back from the stack frame.
2175 SDValue Rem =
2176 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2177 Results.push_back(CallInfo.first);
2178 Results.push_back(Rem);
2179}
2180
2181/// Return true if sincos libcall is available.
2182static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2183 RTLIB::Libcall LC;
2184 switch (Node->getSimpleValueType(0).SimpleTy) {
2185 default: llvm_unreachable("Unexpected request for libcall!")__builtin_unreachable();
2186 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2187 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2188 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2189 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2190 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2191 }
2192 return TLI.getLibcallName(LC) != nullptr;
2193}
2194
2195/// Only issue sincos libcall if both sin and cos are needed.
2196static bool useSinCos(SDNode *Node) {
2197 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2198 ? ISD::FCOS : ISD::FSIN;
2199
2200 SDValue Op0 = Node->getOperand(0);
2201 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2202 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2203 SDNode *User = *UI;
2204 if (User == Node)
2205 continue;
2206 // The other user might have been turned into sincos already.
2207 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2208 return true;
2209 }
2210 return false;
2211}
2212
2213/// Issue libcalls to sincos to compute sin / cos pairs.
2214void
2215SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2216 SmallVectorImpl<SDValue> &Results) {
2217 RTLIB::Libcall LC;
2218 switch (Node->getSimpleValueType(0).SimpleTy) {
2219 default: llvm_unreachable("Unexpected request for libcall!")__builtin_unreachable();
2220 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2221 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2222 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2223 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2224 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2225 }
2226
2227 // The input chain to this libcall is the entry node of the function.
2228 // Legalizing the call will automatically add the previous call to the
2229 // dependence.
2230 SDValue InChain = DAG.getEntryNode();
2231
2232 EVT RetVT = Node->getValueType(0);
2233 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2234
2235 TargetLowering::ArgListTy Args;
2236 TargetLowering::ArgListEntry Entry;
2237
2238 // Pass the argument.
2239 Entry.Node = Node->getOperand(0);
2240 Entry.Ty = RetTy;
2241 Entry.IsSExt = false;
2242 Entry.IsZExt = false;
2243 Args.push_back(Entry);
2244
2245 // Pass the return address of sin.
2246 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2247 Entry.Node = SinPtr;
2248 Entry.Ty = RetTy->getPointerTo();
2249 Entry.IsSExt = false;
2250 Entry.IsZExt = false;
2251 Args.push_back(Entry);
2252
2253 // Also pass the return address of the cos.
2254 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2255 Entry.Node = CosPtr;
2256 Entry.Ty = RetTy->getPointerTo();
2257 Entry.IsSExt = false;
2258 Entry.IsZExt = false;
2259 Args.push_back(Entry);
2260
2261 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2262 TLI.getPointerTy(DAG.getDataLayout()));
2263
2264 SDLoc dl(Node);
2265 TargetLowering::CallLoweringInfo CLI(DAG);
2266 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2267 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2268 std::move(Args));
2269
2270 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2271
2272 Results.push_back(
2273 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2274 Results.push_back(
2275 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2276}
2277
2278/// This function is responsible for legalizing a
2279/// INT_TO_FP operation of the specified operand when the target requests that
2280/// we expand it. At this point, we know that the result and operand types are
2281/// legal for the target.
2282SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2283 SDValue &Chain) {
2284 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2285 Node->getOpcode() == ISD::SINT_TO_FP);
2286 EVT DestVT = Node->getValueType(0);
2287 SDLoc dl(Node);
2288 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2289 SDValue Op0 = Node->getOperand(OpNo);
2290 EVT SrcVT = Op0.getValueType();
2291
2292 // TODO: Should any fast-math-flags be set for the created nodes?
2293 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n")do { } while (false);
2294 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2295 (DestVT.bitsLE(MVT::f64) ||
2296 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2297 : ISD::FP_EXTEND,
2298 DestVT))) {
2299 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "do { } while (false)
2300 "expansion\n")do { } while (false);
2301
2302 // Get the stack frame index of a 8 byte buffer.
2303 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2304
2305 SDValue Lo = Op0;
2306 // if signed map to unsigned space
2307 if (isSigned) {
2308 // Invert sign bit (signed to unsigned mapping).
2309 Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,
2310 DAG.getConstant(0x80000000u, dl, MVT::i32));
2311 }
2312 // Initial hi portion of constructed double.
2313 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2314
2315 // If this a big endian target, swap the lo and high data.
2316 if (DAG.getDataLayout().isBigEndian())
2317 std::swap(Lo, Hi);
2318
2319 SDValue MemChain = DAG.getEntryNode();
2320
2321 // Store the lo of the constructed double.
2322 SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot,
2323 MachinePointerInfo());
2324 // Store the hi of the constructed double.
2325 SDValue HiPtr = DAG.getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), dl);
2326 SDValue Store2 =
2327 DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo());
2328 MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
2329
2330 // load the constructed double
2331 SDValue Load =
2332 DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2333 // FP constant to bias correct the final result
2334 SDValue Bias = DAG.getConstantFP(isSigned ?
2335 BitsToDouble(0x4330000080000000ULL) :
2336 BitsToDouble(0x4330000000000000ULL),
2337 dl, MVT::f64);
2338 // Subtract the bias and get the final result.
2339 SDValue Sub;
2340 SDValue Result;
2341 if (Node->isStrictFPOpcode()) {
2342 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
2343 {Node->getOperand(0), Load, Bias});
2344 Chain = Sub.getValue(1);
2345 if (DestVT != Sub.getValueType()) {
2346 std::pair<SDValue, SDValue> ResultPair;
2347 ResultPair =
2348 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2349 Result = ResultPair.first;
2350 Chain = ResultPair.second;
2351 }
2352 else
2353 Result = Sub;
2354 } else {
2355 Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2356 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2357 }
2358 return Result;
2359 }
2360
2361 if (isSigned)
2362 return SDValue();
2363
2364 // TODO: Generalize this for use with other types.
2365 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2366 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2367 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n")do { } while (false);
2368 // For unsigned conversions, convert them to signed conversions using the
2369 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2370 // should be valid for i32->f32 as well.
2371
2372 // More generally this transform should be valid if there are 3 more bits
2373 // in the integer type than the significand. Rounding uses the first bit
2374 // after the width of the significand and the OR of all bits after that. So
2375 // we need to be able to OR the shifted out bit into one of the bits that
2376 // participate in the OR.
2377
2378 // TODO: This really should be implemented using a branch rather than a
2379 // select. We happen to get lucky and machinesink does the right
2380 // thing most of the time. This would be a good candidate for a
2381 // pseudo-op, or, even better, for whole-function isel.
2382 EVT SetCCVT = getSetCCResultType(SrcVT);
2383
2384 SDValue SignBitTest = DAG.getSetCC(
2385 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2386
2387 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2388 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2389 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst);
2390 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2391 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst);
2392 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
2393
2394 SDValue Slow, Fast;
2395 if (Node->isStrictFPOpcode()) {
2396 // In strict mode, we must avoid spurious exceptions, and therefore
2397 // must make sure to only emit a single STRICT_SINT_TO_FP.
2398 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0);
2399 Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2400 { Node->getOperand(0), InCvt });
2401 Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2402 { Fast.getValue(1), Fast, Fast });
2403 Chain = Slow.getValue(1);
2404 // The STRICT_SINT_TO_FP inherits the exception mode from the
2405 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2406 // never raise any exception.
2407 SDNodeFlags Flags;
2408 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2409 Fast->setFlags(Flags);
2410 Flags.setNoFPExcept(true);
2411 Slow->setFlags(Flags);
2412 } else {
2413 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or);
2414 Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);
2415 Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2416 }
2417
2418 return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2419 }
2420
2421 // Don't expand it if there isn't cheap fadd.
2422 if (!TLI.isOperationLegalOrCustom(
2423 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT))
2424 return SDValue();
2425
2426 // The following optimization is valid only if every value in SrcVT (when
2427 // treated as signed) is representable in DestVT. Check that the mantissa
2428 // size of DestVT is >= than the number of bits in SrcVT -1.
2429 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=(static_cast<void> (0))
2430 SrcVT.getSizeInBits() - 1 &&(static_cast<void> (0))
2431 "Cannot perform lossless SINT_TO_FP!")(static_cast<void> (0));
2432
2433 SDValue Tmp1;
2434 if (Node->isStrictFPOpcode()) {
2435 Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2436 { Node->getOperand(0), Op0 });
2437 } else
2438 Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2439
2440 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2441 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2442 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2443 Four = DAG.getIntPtrConstant(4, dl);
2444 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2445 SignSet, Four, Zero);
2446
2447 // If the sign bit of the integer is set, the large number will be treated
2448 // as a negative number. To counteract this, the dynamic code adds an
2449 // offset depending on the data type.
2450 uint64_t FF;
2451 switch (SrcVT.getSimpleVT().SimpleTy) {
2452 default:
2453 return SDValue();
2454 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2455 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2456 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2457 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2458 }
2459 if (DAG.getDataLayout().isLittleEndian())
2460 FF <<= 32;
2461 Constant *FudgeFactor = ConstantInt::get(
2462 Type::getInt64Ty(*DAG.getContext()), FF);
2463
2464 SDValue CPIdx =
2465 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2466 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2467 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2468 Alignment = commonAlignment(Alignment, 4);
2469 SDValue FudgeInReg;
2470 if (DestVT == MVT::f32)
2471 FudgeInReg = DAG.getLoad(
2472 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2473 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2474 Alignment);
2475 else {
2476 SDValue Load = DAG.getExtLoad(
2477 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2478 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2479 Alignment);
2480 HandleSDNode Handle(Load);
2481 LegalizeOp(Load.getNode());
2482 FudgeInReg = Handle.getValue();
2483 }
2484
2485 if (Node->isStrictFPOpcode()) {
2486 SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2487 { Tmp1.getValue(1), Tmp1, FudgeInReg });
2488 Chain = Result.getValue(1);
2489 return Result;
2490 }
2491
2492 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2493}
2494
2495/// This function is responsible for legalizing a
2496/// *INT_TO_FP operation of the specified operand when the target requests that
2497/// we promote it. At this point, we know that the result and operand types are
2498/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2499/// operation that takes a larger input.
2500void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2501 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
2502 bool IsStrict = N->isStrictFPOpcode();
2503 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2504 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2505 EVT DestVT = N->getValueType(0);
2506 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2507 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2508 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2509
2510 // First step, figure out the appropriate *INT_TO_FP operation to use.
2511 EVT NewInTy = LegalOp.getValueType();
2512
2513 unsigned OpToUse = 0;
2514
2515 // Scan for the appropriate larger type to use.
2516 while (true) {
2517 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2518 assert(NewInTy.isInteger() && "Ran out of possibilities!")(static_cast<void> (0));
2519
2520 // If the target supports SINT_TO_FP of this type, use it.
2521 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2522 OpToUse = SIntOp;
2523 break;
2524 }
2525 if (IsSigned)
2526 continue;
2527
2528 // If the target supports UINT_TO_FP of this type, use it.
2529 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2530 OpToUse = UIntOp;
2531 break;
2532 }
2533
2534 // Otherwise, try a larger type.
2535 }
2536
2537 // Okay, we found the operation and type to use. Zero extend our input to the
2538 // desired type then run the operation on it.
2539 if (IsStrict) {
2540 SDValue Res =
2541 DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
2542 {N->getOperand(0),
2543 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2544 dl, NewInTy, LegalOp)});
2545 Results.push_back(Res);
2546 Results.push_back(Res.getValue(1));
2547 return;
2548 }
2549
2550 Results.push_back(
2551 DAG.getNode(OpToUse, dl, DestVT,
2552 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2553 dl, NewInTy, LegalOp)));
2554}
2555
2556/// This function is responsible for legalizing a
2557/// FP_TO_*INT operation of the specified operand when the target requests that
2558/// we promote it. At this point, we know that the result and operand types are
2559/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2560/// operation that returns a larger result.
2561void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2562 SmallVectorImpl<SDValue> &Results) {
2563 bool IsStrict = N->isStrictFPOpcode();
2564 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2565 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2566 EVT DestVT = N->getValueType(0);
2567 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2568 // First step, figure out the appropriate FP_TO*INT operation to use.
2569 EVT NewOutTy = DestVT;
2570
2571 unsigned OpToUse = 0;
2572
2573 // Scan for the appropriate larger type to use.
2574 while (true) {
2575 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2576 assert(NewOutTy.isInteger() && "Ran out of possibilities!")(static_cast<void> (0));
2577
2578 // A larger signed type can hold all unsigned values of the requested type,
2579 // so using FP_TO_SINT is valid
2580 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2581 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2582 break;
2583
2584 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2585 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2586 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2587 break;
2588
2589 // Otherwise, try a larger type.
2590 }
2591
2592 // Okay, we found the operation and type to use.
2593 SDValue Operation;
2594 if (IsStrict) {
2595 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2596 Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp);
2597 } else
2598 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2599
2600 // Truncate the result of the extended FP_TO_*INT operation to the desired
2601 // size.
2602 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2603 Results.push_back(Trunc);
2604 if (IsStrict)
2605 Results.push_back(Operation.getValue(1));
2606}
2607
2608/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2609/// the result and operand types are legal and there must be a legal
2610/// FP_TO_*INT_SAT operation for a larger result type.
2611SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2612 const SDLoc &dl) {
2613 unsigned Opcode = Node->getOpcode();
2614
2615 // Scan for the appropriate larger type to use.
2616 EVT NewOutTy = Node->getValueType(0);
2617 while (true) {
2618 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2619 assert(NewOutTy.isInteger() && "Ran out of possibilities!")(static_cast<void> (0));
2620
2621 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2622 break;
2623 }
2624
2625 // Saturation width is determined by second operand, so we don't have to
2626 // perform any fixup and can directly truncate the result.
2627 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0),
2628 Node->getOperand(1));
2629 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result);
2630}
2631
2632/// Open code the operations for PARITY of the specified operation.
2633SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2634 EVT VT = Op.getValueType();
2635 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2636 unsigned Sz = VT.getScalarSizeInBits();
2637
2638 // If CTPOP is legal, use it. Otherwise use shifts and xor.
2639 SDValue Result;
2640 if (TLI.isOperationLegal(ISD::CTPOP, VT)) {
2641 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
2642 } else {
2643 Result = Op;
2644 for (unsigned i = Log2_32_Ceil(Sz); i != 0;) {
2645 SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result,
2646 DAG.getConstant(1ULL << (--i), dl, ShVT));
2647 Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift);
2648 }
2649 }
2650
2651 return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2652}
2653
2654bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2655 LLVM_DEBUG(dbgs() << "Trying to expand node\n")do { } while (false);
2656 SmallVector<SDValue, 8> Results;
2657 SDLoc dl(Node);
2658 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2659 bool NeedInvert;
2660 switch (Node->getOpcode()) {
2661 case ISD::ABS:
2662 if (TLI.expandABS(Node, Tmp1, DAG))
2663 Results.push_back(Tmp1);
2664 break;
2665 case ISD::CTPOP:
2666 if (TLI.expandCTPOP(Node, Tmp1, DAG))
2667 Results.push_back(Tmp1);
2668 break;
2669 case ISD::CTLZ:
2670 case ISD::CTLZ_ZERO_UNDEF:
2671 if (TLI.expandCTLZ(Node, Tmp1, DAG))
2672 Results.push_back(Tmp1);
2673 break;
2674 case ISD::CTTZ:
2675 case ISD::CTTZ_ZERO_UNDEF:
2676 if (TLI.expandCTTZ(Node, Tmp1, DAG))
2677 Results.push_back(Tmp1);
2678 break;
2679 case ISD::BITREVERSE:
2680 if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))
2681 Results.push_back(Tmp1);
2682 break;
2683 case ISD::BSWAP:
2684 if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))
2685 Results.push_back(Tmp1);
2686 break;
2687 case ISD::PARITY:
2688 Results.push_back(ExpandPARITY(Node->getOperand(0), dl));
2689 break;
2690 case ISD::FRAMEADDR:
2691 case ISD::RETURNADDR:
2692 case ISD::FRAME_TO_ARGS_OFFSET:
2693 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2694 break;
2695 case ISD::EH_DWARF_CFA: {
2696 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2697 TLI.getPointerTy(DAG.getDataLayout()));
2698 SDValue Offset = DAG.getNode(ISD::ADD, dl,
2699 CfaArg.getValueType(),
2700 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2701 CfaArg.getValueType()),
2702 CfaArg);
2703 SDValue FA = DAG.getNode(
2704 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2705 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2706 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2707 FA, Offset));
2708 break;
2709 }
2710 case ISD::FLT_ROUNDS_:
2711 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2712 Results.push_back(Node->getOperand(0));
2713 break;
2714 case ISD::EH_RETURN:
2715 case ISD::EH_LABEL:
2716 case ISD::PREFETCH:
2717 case ISD::VAEND:
2718 case ISD::EH_SJLJ_LONGJMP:
2719 // If the target didn't expand these, there's nothing to do, so just
2720 // preserve the chain and be done.
2721 Results.push_back(Node->getOperand(0));
2722 break;
2723 case ISD::READCYCLECOUNTER:
2724 // If the target didn't expand this, just return 'zero' and preserve the
2725 // chain.
2726 Results.append(Node->getNumValues() - 1,
2727 DAG.getConstant(0, dl, Node->getValueType(0)));
2728 Results.push_back(Node->getOperand(0));
2729 break;
2730 case ISD::EH_SJLJ_SETJMP:
2731 // If the target didn't expand this, just return 'zero' and preserve the
2732 // chain.
2733 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2734 Results.push_back(Node->getOperand(0));
2735 break;
2736 case ISD::ATOMIC_LOAD: {
2737 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2738 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2739 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2740 SDValue Swap = DAG.getAtomicCmpSwap(
2741 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2742 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2743 cast<AtomicSDNode>(Node)->getMemOperand());
2744 Results.push_back(Swap.getValue(0));
2745 Results.push_back(Swap.getValue(1));
2746 break;
2747 }
2748 case ISD::ATOMIC_STORE: {
2749 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2750 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2751 cast<AtomicSDNode>(Node)->getMemoryVT(),
2752 Node->getOperand(0),
2753 Node->getOperand(1), Node->getOperand(2),
2754 cast<AtomicSDNode>(Node)->getMemOperand());
2755 Results.push_back(Swap.getValue(1));
2756 break;
2757 }
2758 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2759 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2760 // splits out the success value as a comparison. Expanding the resulting
2761 // ATOMIC_CMP_SWAP will produce a libcall.
2762 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2763 SDValue Res = DAG.getAtomicCmpSwap(
2764 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2765 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2766 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2767
2768 SDValue ExtRes = Res;
2769 SDValue LHS = Res;
2770 SDValue RHS = Node->getOperand(1);
2771
2772 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2773 EVT OuterType = Node->getValueType(0);
2774 switch (TLI.getExtendForAtomicOps()) {
2775 case ISD::SIGN_EXTEND:
2776 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2777 DAG.getValueType(AtomicType));
2778 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2779 Node->getOperand(2), DAG.getValueType(AtomicType));
2780 ExtRes = LHS;
2781 break;
2782 case ISD::ZERO_EXTEND:
2783 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2784 DAG.getValueType(AtomicType));
2785 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2786 ExtRes = LHS;
2787 break;
2788 case ISD::ANY_EXTEND:
2789 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2790 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2791 break;
2792 default:
2793 llvm_unreachable("Invalid atomic op extension")__builtin_unreachable();
2794 }
2795
2796 SDValue Success =
2797 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2798
2799 Results.push_back(ExtRes.getValue(0));
2800 Results.push_back(Success);
2801 Results.push_back(Res.getValue(1));
2802 break;
2803 }
2804 case ISD::DYNAMIC_STACKALLOC:
2805 ExpandDYNAMIC_STACKALLOC(Node, Results);
2806 break;
2807 case ISD::MERGE_VALUES:
2808 for (unsigned i = 0; i < Node->getNumValues(); i++)
2809 Results.push_back(Node->getOperand(i));
2810 break;
2811 case ISD::UNDEF: {
2812 EVT VT = Node->getValueType(0);
2813 if (VT.isInteger())
2814 Results.push_back(DAG.getConstant(0, dl, VT));
2815 else {
2816 assert(VT.isFloatingPoint() && "Unknown value type!")(static_cast<void> (0));
2817 Results.push_back(DAG.getConstantFP(0, dl, VT));
2818 }
2819 break;
2820 }
2821 case ISD::STRICT_FP_ROUND:
2822 // When strict mode is enforced we can't do expansion because it
2823 // does not honor the "strict" properties. Only libcall is allowed.
2824 if (TLI.isStrictFPEnabled())
2825 break;
2826 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
2827 // since this operation is more efficient than stack operation.
2828 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
2829 Node->getValueType(0))
2830 == TargetLowering::Legal)
2831 break;
2832 // We fall back to use stack operation when the FP_ROUND operation
2833 // isn't available.
2834 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),
2835 Node->getValueType(0), dl,
2836 Node->getOperand(0)))) {
2837 ReplaceNode(Node, Tmp1.getNode());
2838 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n")do { } while (false);
2839 return true;
2840 }
2841 break;
2842 case ISD::FP_ROUND:
2843 case ISD::BITCAST:
2844 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2845 Node->getValueType(0), dl)))
2846 Results.push_back(Tmp1);
2847 break;
2848 case ISD::STRICT_FP_EXTEND:
2849 // When strict mode is enforced we can't do expansion because it
2850 // does not honor the "strict" properties. Only libcall is allowed.
2851 if (TLI.isStrictFPEnabled())
2852 break;
2853 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
2854 // since this operation is more efficient than stack operation.
2855 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
2856 Node->getValueType(0))
2857 == TargetLowering::Legal)
2858 break;
2859 // We fall back to use stack operation when the FP_EXTEND operation
2860 // isn't available.
2861 if ((Tmp1 = EmitStackConvert(
2862 Node->getOperand(1), Node->getOperand(1).getValueType(),
2863 Node->getValueType(0), dl, Node->getOperand(0)))) {
2864 ReplaceNode(Node, Tmp1.getNode());
2865 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n")do { } while (false);
2866 return true;
2867 }
2868 break;
2869 case ISD::FP_EXTEND:
2870 if ((Tmp1 = EmitStackConvert(Node->getOperand(0),
2871 Node->getOperand(0).getValueType(),
2872 Node->getValueType(0), dl)))
2873 Results.push_back(Tmp1);
2874 break;
2875 case ISD::SIGN_EXTEND_INREG: {
2876 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2877 EVT VT = Node->getValueType(0);
2878
2879 // An in-register sign-extend of a boolean is a negation:
2880 // 'true' (1) sign-extended is -1.
2881 // 'false' (0) sign-extended is 0.
2882 // However, we must mask the high bits of the source operand because the
2883 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2884
2885 // TODO: Do this for vectors too?
2886 if (ExtraVT.getSizeInBits() == 1) {
2887 SDValue One = DAG.getConstant(1, dl, VT);
2888 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2889 SDValue Zero = DAG.getConstant(0, dl, VT);
2890 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2891 Results.push_back(Neg);
2892 break;
2893 }
2894
2895 // NOTE: we could fall back on load/store here too for targets without
2896 // SRA. However, it is doubtful that any exist.
2897 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2898 unsigned BitsDiff = VT.getScalarSizeInBits() -
2899 ExtraVT.getScalarSizeInBits();
2900 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2901 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2902 Node->getOperand(0), ShiftCst);
2903 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2904 Results.push_back(Tmp1);
2905 break;
2906 }
2907 case ISD::UINT_TO_FP:
2908 case ISD::STRICT_UINT_TO_FP:
2909 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) {
2910 Results.push_back(Tmp1);
2911 if (Node->isStrictFPOpcode())
2912 Results.push_back(Tmp2);
2913 break;
2914 }
2915 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2916 case ISD::SINT_TO_FP:
2917 case ISD::STRICT_SINT_TO_FP:
2918 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
2919 Results.push_back(Tmp1);
2920 if (Node->isStrictFPOpcode())
2921 Results.push_back(Tmp2);
2922 }
2923 break;
2924 case ISD::FP_TO_SINT:
2925 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
2926 Results.push_back(Tmp1);
2927 break;
2928 case ISD::STRICT_FP_TO_SINT:
2929 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {
2930 ReplaceNode(Node, Tmp1.getNode());
2931 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n")do { } while (false);
2932 return true;
2933 }
2934 break;
2935 case ISD::FP_TO_UINT:
2936 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))
2937 Results.push_back(Tmp1);
2938 break;
2939 case ISD::STRICT_FP_TO_UINT:
2940 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {
2941 // Relink the chain.
2942 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2);
2943 // Replace the new UINT result.
2944 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);
2945 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n")do { } while (false);
2946 return true;
2947 }
2948 break;
2949 case ISD::FP_TO_SINT_SAT:
2950 case ISD::FP_TO_UINT_SAT:
2951 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG));
2952 break;
2953 case ISD::VAARG:
2954 Results.push_back(DAG.expandVAArg(Node));
2955 Results.push_back(Results[0].getValue(1));
2956 break;
2957 case ISD::VACOPY:
2958 Results.push_back(DAG.expandVACopy(Node));
2959 break;
2960 case ISD::EXTRACT_VECTOR_ELT:
2961 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
2962 // This must be an access of the only element. Return it.
2963 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
2964 Node->getOperand(0));
2965 else
2966 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
2967 Results.push_back(Tmp1);
2968 break;
2969 case ISD::EXTRACT_SUBVECTOR:
2970 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
2971 break;
2972 case ISD::INSERT_SUBVECTOR:
2973 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
2974 break;
2975 case ISD::CONCAT_VECTORS:
2976 Results.push_back(ExpandVectorBuildThroughStack(Node));
2977 break;
2978 case ISD::SCALAR_TO_VECTOR:
2979 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
2980 break;
2981 case ISD::INSERT_VECTOR_ELT:
2982 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
2983 Node->getOperand(1),
2984 Node->getOperand(2), dl));
2985 break;
2986 case ISD::VECTOR_SHUFFLE: {
2987 SmallVector<int, 32> NewMask;
2988 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
2989
2990 EVT VT = Node->getValueType(0);
2991 EVT EltVT = VT.getVectorElementType();
2992 SDValue Op0 = Node->getOperand(0);
2993 SDValue Op1 = Node->getOperand(1);
2994 if (!TLI.isTypeLegal(EltVT)) {
2995 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
2996
2997 // BUILD_VECTOR operands are allowed to be wider than the element type.
2998 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
2999 // it.
3000 if (NewEltVT.bitsLT(EltVT)) {
3001 // Convert shuffle node.
3002 // If original node was v4i64 and the new EltVT is i32,
3003 // cast operands to v8i32 and re-build the mask.
3004
3005 // Calculate new VT, the size of the new VT should be equal to original.
3006 EVT NewVT =
3007 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3008 VT.getSizeInBits() / NewEltVT.getSizeInBits());
3009 assert(NewVT.bitsEq(VT))(static_cast<void> (0));
3010
3011 // cast operands to new VT
3012 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3013 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3014
3015 // Convert the shuffle mask
3016 unsigned int factor =
3017 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3018
3019 // EltVT gets smaller
3020 assert(factor > 0)(static_cast<void> (0));
3021
3022 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3023 if (Mask[i] < 0) {
3024 for (unsigned fi = 0; fi < factor; ++fi)
3025 NewMask.push_back(Mask[i]);
3026 }
3027 else {
3028 for (unsigned fi = 0; fi < factor; ++fi)
3029 NewMask.push_back(Mask[i]*factor+fi);
3030 }
3031 }
3032 Mask = NewMask;
3033 VT = NewVT;
3034 }
3035 EltVT = NewEltVT;
3036 }
3037 unsigned NumElems = VT.getVectorNumElements();
3038 SmallVector<SDValue, 16> Ops;
3039 for (unsigned i = 0; i != NumElems; ++i) {
3040 if (Mask[i] < 0) {
3041 Ops.push_back(DAG.getUNDEF(EltVT));
3042 continue;
3043 }
3044 unsigned Idx = Mask[i];
3045 if (Idx < NumElems)
3046 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3047 DAG.getVectorIdxConstant(Idx, dl)));
3048 else
3049 Ops.push_back(
3050 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3051 DAG.getVectorIdxConstant(Idx - NumElems, dl)));
3052 }
3053
3054 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3055 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3056 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3057 Results.push_back(Tmp1);
3058 break;
3059 }
3060 case ISD::VECTOR_SPLICE: {
3061 Results.push_back(TLI.expandVectorSplice(Node, DAG));
3062 break;
3063 }
3064 case ISD::EXTRACT_ELEMENT: {
3065 EVT OpTy = Node->getOperand(0).getValueType();
3066 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3067 // 1 -> Hi
3068 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3069 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3070 TLI.getShiftAmountTy(
3071 Node->getOperand(0).getValueType(),
3072 DAG.getDataLayout())));
3073 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3074 } else {
3075 // 0 -> Lo
3076 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3077 Node->getOperand(0));
3078 }
3079 Results.push_back(Tmp1);
3080 break;
3081 }
3082 case ISD::STACKSAVE:
3083 // Expand to CopyFromReg if the target set
3084 // StackPointerRegisterToSaveRestore.
3085 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3086 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3087 Node->getValueType(0)));
3088 Results.push_back(Results[0].getValue(1));
3089 } else {
3090 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3091 Results.push_back(Node->getOperand(0));
3092 }
3093 break;
3094 case ISD::STACKRESTORE:
3095 // Expand to CopyToReg if the target set
3096 // StackPointerRegisterToSaveRestore.
3097 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3098 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3099 Node->getOperand(1)));
3100 } else {
3101 Results.push_back(Node->getOperand(0));
3102 }
3103 break;
3104 case ISD::GET_DYNAMIC_AREA_OFFSET:
3105 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3106 Results.push_back(Results[0].getValue(0));
3107 break;
3108 case ISD::FCOPYSIGN:
3109 Results.push_back(ExpandFCOPYSIGN(Node));
3110 break;
3111 case ISD::FNEG:
3112 Results.push_back(ExpandFNEG(Node));
3113 break;
3114 case ISD::FABS:
3115 Results.push_back(ExpandFABS(Node));
3116 break;
3117 case ISD::SMIN:
3118 case ISD::SMAX:
3119 case ISD::UMIN:
3120 case ISD::UMAX: {
3121 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3122 ISD::CondCode Pred;
3123 switch (Node->getOpcode()) {
3124 default: llvm_unreachable("How did we get here?")__builtin_unreachable();
3125 case ISD::SMAX: Pred = ISD::SETGT; break;
3126 case ISD::SMIN: Pred = ISD::SETLT; break;
3127 case ISD::UMAX: Pred = ISD::SETUGT; break;
3128 case ISD::UMIN: Pred = ISD::SETULT; break;
3129 }
3130 Tmp1 = Node->getOperand(0);
3131 Tmp2 = Node->getOperand(1);
3132 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3133 Results.push_back(Tmp1);
3134 break;
3135 }
3136 case ISD::FMINNUM:
3137 case ISD::FMAXNUM: {
3138 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3139 Results.push_back(Expanded);
3140 break;
3141 }
3142 case ISD::FSIN:
3143 case ISD::FCOS: {
3144 EVT VT = Node->getValueType(0);
3145 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3146 // fcos which share the same operand and both are used.
3147 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3148 isSinCosLibcallAvailable(Node, TLI))
3149 && useSinCos(Node)) {
3150 SDVTList VTs = DAG.getVTList(VT, VT);
3151 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3152 if (Node->getOpcode() == ISD::FCOS)
3153 Tmp1 = Tmp1.getValue(1);
3154 Results.push_back(Tmp1);
3155 }
3156 break;
3157 }
3158 case ISD::FMAD:
3159 llvm_unreachable("Illegal fmad should never be formed")__builtin_unreachable();
3160
3161 case ISD::FP16_TO_FP:
3162 if (Node->getValueType(0) != MVT::f32) {
3163 // We can extend to types bigger than f32 in two steps without changing
3164 // the result. Since "f16 -> f32" is much more commonly available, give
3165 // CodeGen the option of emitting that before resorting to a libcall.
3166 SDValue Res =
3167 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3168 Results.push_back(
3169 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3170 }
3171 break;
3172 case ISD::STRICT_FP16_TO_FP:
3173 if (Node->getValueType(0) != MVT::f32) {
3174 // We can extend to types bigger than f32 in two steps without changing
3175 // the result. Since "f16 -> f32" is much more commonly available, give
3176 // CodeGen the option of emitting that before resorting to a libcall.
3177 SDValue Res =
3178 DAG.getNode(ISD::STRICT_FP16_TO_FP, dl, {MVT::f32, MVT::Other},
3179 {Node->getOperand(0), Node->getOperand(1)});
3180 Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
3181 {Node->getValueType(0), MVT::Other},
3182 {Res.getValue(1), Res});
3183 Results.push_back(Res);
3184 Results.push_back(Res.getValue(1));
3185 }
3186 break;
3187 case ISD::FP_TO_FP16:
3188 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n")do { } while (false);
3189 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3190 SDValue Op = Node->getOperand(0);
3191 MVT SVT = Op.getSimpleValueType();
3192 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3193 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3194 // Under fastmath, we can expand this node into a fround followed by
3195 // a float-half conversion.
3196 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3197 DAG.getIntPtrConstant(0, dl));
3198 Results.push_back(
3199 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3200 }
3201 }
3202 break;
3203 case ISD::ConstantFP: {
3204 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3205 // Check to see if this FP immediate is already legal.
3206 // If this is a legal constant, turn it into a TargetConstantFP node.
3207 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3208 DAG.shouldOptForSize()))
3209 Results.push_back(ExpandConstantFP(CFP, true));
3210 break;
3211 }
3212 case ISD::Constant: {
3213 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3214 Results.push_back(ExpandConstant(CP));
3215 break;
3216 }
3217 case ISD::FSUB: {
3218 EVT VT = Node->getValueType(0);
3219 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3220 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3221 const SDNodeFlags Flags = Node->getFlags();
3222 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3223 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3224 Results.push_back(Tmp1);
3225 }
3226 break;
3227 }
3228 case ISD::SUB: {
3229 EVT VT = Node->getValueType(0);
3230 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&(static_cast<void> (0))
3231 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&(static_cast<void> (0))
3232 "Don't know how to expand this subtraction!")(static_cast<void> (0));
3233 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3234 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3235 VT));
3236 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3237 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3238 break;
3239 }
3240 case ISD::UREM:
3241 case ISD::SREM:
3242 if (TLI.expandREM(Node, Tmp1, DAG))
3243 Results.push_back(Tmp1);
3244 break;
3245 case ISD::UDIV:
3246 case ISD::SDIV: {
3247 bool isSigned = Node->getOpcode() == ISD::SDIV;
3248 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3249 EVT VT = Node->getValueType(0);
3250 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3251 SDVTList VTs = DAG.getVTList(VT, VT);
3252 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3253 Node->getOperand(1));
3254 Results.push_back(Tmp1);
3255 }
3256 break;
3257 }
3258 case ISD::MULHU:
3259 case ISD::MULHS: {
3260 unsigned ExpandOpcode =
3261 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3262 EVT VT = Node->getValueType(0);
3263 SDVTList VTs = DAG.getVTList(VT, VT);
3264
3265 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3266 Node->getOperand(1));
3267 Results.push_back(Tmp1.getValue(1));
3268 break;
3269 }
3270 case ISD::UMUL_LOHI:
3271 case ISD::SMUL_LOHI: {
3272 SDValue LHS = Node->getOperand(0);
3273 SDValue RHS = Node->getOperand(1);
3274 MVT VT = LHS.getSimpleValueType();
3275 unsigned MULHOpcode =
3276 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3277
3278 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3279 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3280 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3281 break;
3282 }
3283
3284 SmallVector<SDValue, 4> Halves;
3285 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3286 assert(TLI.isTypeLegal(HalfType))(static_cast<void> (0));
3287 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3288 HalfType, DAG,
3289 TargetLowering::MulExpansionKind::Always)) {
3290 for (unsigned i = 0; i < 2; ++i) {
3291 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3292 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3293 SDValue Shift = DAG.getConstant(
3294 HalfType.getScalarSizeInBits(), dl,
3295 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3296 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3297 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3298 }
3299 break;
3300 }
3301 break;
3302 }
3303 case ISD::MUL: {
3304 EVT VT = Node->getValueType(0);
3305 SDVTList VTs = DAG.getVTList(VT, VT);
3306 // See if multiply or divide can be lowered using two-result operations.
3307 // We just need the low half of the multiply; try both the signed
3308 // and unsigned forms. If the target supports both SMUL_LOHI and
3309 // UMUL_LOHI, form a preference by checking which forms of plain
3310 // MULH it supports.
3311 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3312 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3313 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3314 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3315 unsigned OpToUse = 0;
3316 if (HasSMUL_LOHI && !HasMULHS) {
3317 OpToUse = ISD::SMUL_LOHI;
3318 } else if (HasUMUL_LOHI && !HasMULHU) {
3319 OpToUse = ISD::UMUL_LOHI;
3320 } else if (HasSMUL_LOHI) {
3321 OpToUse = ISD::SMUL_LOHI;
3322 } else if (HasUMUL_LOHI) {
3323 OpToUse = ISD::UMUL_LOHI;
3324 }
3325 if (OpToUse) {
3326 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3327 Node->getOperand(1)));
3328 break;
3329 }
3330
3331 SDValue Lo, Hi;
3332 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3333 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3334 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3335 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3336 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3337 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3338 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3339 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3340 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3341 SDValue Shift =
3342 DAG.getConstant(HalfType.getSizeInBits(), dl,
3343 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3344 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3345 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3346 }
3347 break;
3348 }
3349 case ISD::FSHL:
3350 case ISD::FSHR:
3351 if (TLI.expandFunnelShift(Node, Tmp1, DAG))
3352 Results.push_back(Tmp1);
3353 break;
3354 case ISD::ROTL:
3355 case ISD::ROTR:
3356 if (TLI.expandROT(Node, true /*AllowVectorOps*/, Tmp1, DAG))
3357 Results.push_back(Tmp1);
3358 break;
3359 case ISD::SADDSAT:
3360 case ISD::UADDSAT:
3361 case ISD::SSUBSAT:
3362 case ISD::USUBSAT:
3363 Results.push_back(TLI.expandAddSubSat(Node, DAG));
3364 break;
3365 case ISD::SSHLSAT:
3366 case ISD::USHLSAT:
3367 Results.push_back(TLI.expandShlSat(Node, DAG));
3368 break;
3369 case ISD::SMULFIX:
3370 case ISD::SMULFIXSAT:
3371 case ISD::UMULFIX:
3372 case ISD::UMULFIXSAT:
3373 Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3374 break;
3375 case ISD::SDIVFIX:
3376 case ISD::SDIVFIXSAT:
3377 case ISD::UDIVFIX:
3378 case ISD::UDIVFIXSAT:
3379 if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node),
3380 Node->getOperand(0),
3381 Node->getOperand(1),
3382 Node->getConstantOperandVal(2),
3383 DAG)) {
3384 Results.push_back(V);
3385 break;
3386 }
3387 // FIXME: We might want to retry here with a wider type if we fail, if that
3388 // type is legal.
3389 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3390 // <= 128 (which is the case for all of the default Embedded-C types),
3391 // we will only get here with types and scales that we could always expand
3392 // if we were allowed to generate libcalls to division functions of illegal
3393 // type. But we cannot do that.
3394 llvm_unreachable("Cannot expand DIVFIX!")__builtin_unreachable();
3395 case ISD::ADDCARRY:
3396 case ISD::SUBCARRY: {
3397 SDValue LHS = Node->getOperand(0);
3398 SDValue RHS = Node->getOperand(1);
3399 SDValue Carry = Node->getOperand(2);
3400
3401 bool IsAdd = Node->getOpcode() == ISD::ADDCARRY;
3402
3403 // Initial add of the 2 operands.
3404 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3405 EVT VT = LHS.getValueType();
3406 SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3407
3408 // Initial check for overflow.
3409 EVT CarryType = Node->getValueType(1);
3410 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3411 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3412 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3413
3414 // Add of the sum and the carry.
3415 SDValue One = DAG.getConstant(1, dl, VT);
3416 SDValue CarryExt =
3417 DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3418 SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3419
3420 // Second check for overflow. If we are adding, we can only overflow if the
3421 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3422 // If we are subtracting, we can only overflow if the initial sum is 0 and
3423 // the carry is set, resulting in a new sum of all 1s.
3424 SDValue Zero = DAG.getConstant(0, dl, VT);
3425 SDValue Overflow2 =
3426 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)
3427 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ);
3428 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
3429 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3430
3431 SDValue ResultCarry =
3432 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
3433
3434 Results.push_back(Sum2);
3435 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3436 break;
3437 }
3438 case ISD::SADDO:
3439 case ISD::SSUBO: {
3440 SDValue Result, Overflow;
3441 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3442 Results.push_back(Result);
3443 Results.push_back(Overflow);
3444 break;
3445 }
3446 case ISD::UADDO:
3447 case ISD::USUBO: {
3448 SDValue Result, Overflow;
3449 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3450 Results.push_back(Result);
3451 Results.push_back(Overflow);
3452 break;
3453 }
3454 case ISD::UMULO:
3455 case ISD::SMULO: {
3456 SDValue Result, Overflow;
3457 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3458 Results.push_back(Result);
3459 Results.push_back(Overflow);
3460 }
3461 break;
3462 }
3463 case ISD::BUILD_PAIR: {
3464 EVT PairTy = Node->getValueType(0);
3465 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3466 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3467 Tmp2 = DAG.getNode(
3468 ISD::SHL, dl, PairTy, Tmp2,
3469 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3470 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3471 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3472 break;
3473 }
3474 case ISD::SELECT:
3475 Tmp1 = Node->getOperand(0);
3476 Tmp2 = Node->getOperand(1);
3477 Tmp3 = Node->getOperand(2);
3478 if (Tmp1.getOpcode() == ISD::SETCC) {
3479 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3480 Tmp2, Tmp3,
3481 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3482 } else {
3483 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3484 DAG.getConstant(0, dl, Tmp1.getValueType()),
3485 Tmp2, Tmp3, ISD::SETNE);
3486 }
3487 Tmp1->setFlags(Node->getFlags());
3488 Results.push_back(Tmp1);
3489 break;
3490 case ISD::BR_JT: {
3491 SDValue Chain = Node->getOperand(0);
3492 SDValue Table = Node->getOperand(1);
3493 SDValue Index = Node->getOperand(2);
3494
3495 const DataLayout &TD = DAG.getDataLayout();
3496 EVT PTy = TLI.getPointerTy(TD);
3497
3498 unsigned EntrySize =
3499 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3500
3501 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3502 // This transformation needs to be done here since otherwise the MIPS
3503 // backend will end up emitting a three instruction multiply sequence
3504 // instead of a single shift and MSP430 will call a runtime function.
3505 if (llvm::isPowerOf2_32(EntrySize))
3506 Index = DAG.getNode(
3507 ISD::SHL, dl, Index.getValueType(), Index,
3508 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3509 else
3510 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3511 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3512 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3513 Index, Table);
3514
3515 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3516 SDValue LD = DAG.getExtLoad(
3517 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3518 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3519 Addr = LD;
3520 if (TLI.isJumpTableRelative()) {
3521 // For PIC, the sequence is:
3522 // BRIND(load(Jumptable + index) + RelocBase)
3523 // RelocBase can be JumpTable, GOT or some sort of global base.
3524 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3525 TLI.getPICJumpTableRelocBase(Table, DAG));
3526 }
3527
3528 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG);
3529 Results.push_back(Tmp1);
3530 break;
3531 }
3532 case ISD::BRCOND:
3533 // Expand brcond's setcc into its constituent parts and create a BR_CC
3534 // Node.
3535 Tmp1 = Node->getOperand(0);
3536 Tmp2 = Node->getOperand(1);
3537 if (Tmp2.getOpcode() == ISD::SETCC) {
3538 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3539 Tmp1, Tmp2.getOperand(2),
3540 Tmp2.getOperand(0), Tmp2.getOperand(1),
3541 Node->getOperand(2));
3542 } else {
3543 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3544 if (Tmp2.isUndef() ||
3545 (Tmp2.getOpcode() == ISD::AND &&
3546 isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3547 cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3548 Tmp3 = Tmp2;
3549 else
3550 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3551 DAG.getConstant(1, dl, Tmp2.getValueType()));
3552 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3553 DAG.getCondCode(ISD::SETNE), Tmp3,
3554 DAG.getConstant(0, dl, Tmp3.getValueType()),
3555 Node->getOperand(2));
3556 }
3557 Results.push_back(Tmp1);
3558 break;
3559 case ISD::SETCC:
3560 case ISD::STRICT_FSETCC:
3561 case ISD::STRICT_FSETCCS: {
3562 bool IsStrict = Node->getOpcode() != ISD::SETCC;
3563 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
3564 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
3565 unsigned Offset = IsStrict ? 1 : 0;
3566 Tmp1 = Node->getOperand(0 + Offset);
3567 Tmp2 = Node->getOperand(1 + Offset);
3568 Tmp3 = Node->getOperand(2 + Offset);
3569 bool Legalized =
3570 TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3,
3571 NeedInvert, dl, Chain, IsSignaling);
3572
3573 if (Legalized) {
3574 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3575 // condition code, create a new SETCC node.
3576 if (Tmp3.getNode())
3577 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3578 Tmp1, Tmp2, Tmp3, Node->getFlags());
3579
3580 // If we expanded the SETCC by inverting the condition code, then wrap
3581 // the existing SETCC in a NOT to restore the intended condition.
3582 if (NeedInvert)
3583 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3584
3585 Results.push_back(Tmp1);
3586 if (IsStrict)
3587 Results.push_back(Chain);
3588
3589 break;
3590 }
3591
3592 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
3593 // understand if this code is useful for strict nodes.
3594 assert(!IsStrict && "Don't know how to expand for strict nodes.")(static_cast<void> (0));
3595
3596 // Otherwise, SETCC for the given comparison type must be completely
3597 // illegal; expand it into a SELECT_CC.
3598 EVT VT = Node->getValueType(0);
3599 int TrueValue;
3600 switch (TLI.getBooleanContents(Tmp1.getValueType())) {
3601 case TargetLowering::ZeroOrOneBooleanContent:
3602 case TargetLowering::UndefinedBooleanContent:
3603 TrueValue = 1;
3604 break;
3605 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3606 TrueValue = -1;
3607 break;
3608 }
3609 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3610 DAG.getConstant(TrueValue, dl, VT),
3611 DAG.getConstant(0, dl, VT),
3612 Tmp3);
3613 Tmp1->setFlags(Node->getFlags());
3614 Results.push_back(Tmp1);
3615 break;
3616 }
3617 case ISD::SELECT_CC: {
3618 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
3619 Tmp1 = Node->getOperand(0); // LHS
3620 Tmp2 = Node->getOperand(1); // RHS
3621 Tmp3 = Node->getOperand(2); // True
3622 Tmp4 = Node->getOperand(3); // False
3623 EVT VT = Node->getValueType(0);
3624 SDValue Chain;
3625 SDValue CC = Node->getOperand(4);
3626 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3627
3628 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
3629 // If the condition code is legal, then we need to expand this
3630 // node using SETCC and SELECT.
3631 EVT CmpVT = Tmp1.getValueType();
3632 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&(static_cast<void> (0))
3633 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "(static_cast<void> (0))
3634 "expanded.")(static_cast<void> (0));
3635 EVT CCVT = getSetCCResultType(CmpVT);
3636 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
3637 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3638 break;
3639 }
3640
3641 // SELECT_CC is legal, so the condition code must not be.
3642 bool Legalized = false;
3643 // Try to legalize by inverting the condition. This is for targets that
3644 // might support an ordered version of a condition, but not the unordered
3645 // version (or vice versa).
3646 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType());
3647 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
3648 // Use the new condition code and swap true and false
3649 Legalized = true;
3650 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3651 Tmp1->setFlags(Node->getFlags());
3652 } else {
3653 // If The inverse is not legal, then try to swap the arguments using
3654 // the inverse condition code.
3655 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3656 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
3657 // The swapped inverse condition is legal, so swap true and false,
3658 // lhs and rhs.
3659 Legalized = true;
3660 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3661 Tmp1->setFlags(Node->getFlags());
3662 }
3663 }
3664
3665 if (!Legalized) {
3666 Legalized = TLI.LegalizeSetCCCondCode(
3667 DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,
3668 NeedInvert, dl, Chain);
3669
3670 assert(Legalized && "Can't legalize SELECT_CC with legal condition!")(static_cast<void> (0));
3671
3672 // If we expanded the SETCC by inverting the condition code, then swap
3673 // the True/False operands to match.
3674 if (NeedInvert)
3675 std::swap(Tmp3, Tmp4);
3676
3677 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3678 // condition code, create a new SELECT_CC node.
3679 if (CC.getNode()) {
3680 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3681 Tmp1, Tmp2, Tmp3, Tmp4, CC);
3682 } else {
3683 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3684 CC = DAG.getCondCode(ISD::SETNE);
3685 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3686 Tmp2, Tmp3, Tmp4, CC);
3687 }
3688 Tmp1->setFlags(Node->getFlags());
3689 }
3690 Results.push_back(Tmp1);
3691 break;
3692 }
3693 case ISD::BR_CC: {
3694 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
3695 SDValue Chain;
3696 Tmp1 = Node->getOperand(0); // Chain
3697 Tmp2 = Node->getOperand(2); // LHS
3698 Tmp3 = Node->getOperand(3); // RHS
3699 Tmp4 = Node->getOperand(1); // CC
3700
3701 bool Legalized =
3702 TLI.LegalizeSetCCCondCode(DAG, getSetCCResultType(Tmp2.getValueType()),
3703 Tmp2, Tmp3, Tmp4, NeedInvert, dl, Chain);
3704 (void)Legalized;
3705 assert(Legalized && "Can't legalize BR_CC with legal condition!")(static_cast<void> (0));
3706
3707 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3708 // node.
3709 if (Tmp4.getNode()) {
3710 assert(!NeedInvert && "Don't know how to invert BR_CC!")(static_cast<void> (0));
3711
3712 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3713 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3714 } else {
3715 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3716 Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE);
3717 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3718 Tmp2, Tmp3, Node->getOperand(4));
3719 }
3720 Results.push_back(Tmp1);
3721 break;
3722 }
3723 case ISD::BUILD_VECTOR:
3724 Results.push_back(ExpandBUILD_VECTOR(Node));
3725 break;
3726 case ISD::SPLAT_VECTOR:
3727 Results.push_back(ExpandSPLAT_VECTOR(Node));
3728 break;
3729 case ISD::SRA:
3730 case ISD::SRL:
3731 case ISD::SHL: {
3732 // Scalarize vector SRA/SRL/SHL.
3733 EVT VT = Node->getValueType(0);
3734 assert(VT.isVector() && "Unable to legalize non-vector shift")(static_cast<void> (0));
3735 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal")(static_cast<void> (0));
3736 unsigned NumElem = VT.getVectorNumElements();
3737
3738 SmallVector<SDValue, 8> Scalars;
3739 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3740 SDValue Ex =
3741 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(),
3742 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl));
3743 SDValue Sh =
3744 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(),
3745 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl));
3746 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3747 VT.getScalarType(), Ex, Sh));
3748 }
3749
3750 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3751 Results.push_back(Result);
3752 break;
3753 }
3754 case ISD::VECREDUCE_FADD:
3755 case ISD::VECREDUCE_FMUL:
3756 case ISD::VECREDUCE_ADD:
3757 case ISD::VECREDUCE_MUL:
3758 case ISD::VECREDUCE_AND:
3759 case ISD::VECREDUCE_OR:
3760 case ISD::VECREDUCE_XOR:
3761 case ISD::VECREDUCE_SMAX:
3762 case ISD::VECREDUCE_SMIN:
3763 case ISD::VECREDUCE_UMAX:
3764 case ISD::VECREDUCE_UMIN:
3765 case ISD::VECREDUCE_FMAX:
3766 case ISD::VECREDUCE_FMIN:
3767 Results.push_back(TLI.expandVecReduce(Node, DAG));
3768 break;
3769 case ISD::GLOBAL_OFFSET_TABLE:
3770 case ISD::GlobalAddress:
3771 case ISD::GlobalTLSAddress:
3772 case ISD::ExternalSymbol:
3773 case ISD::ConstantPool:
3774 case ISD::JumpTable:
3775 case ISD::INTRINSIC_W_CHAIN:
3776 case ISD::INTRINSIC_WO_CHAIN:
3777 case ISD::INTRINSIC_VOID:
3778 // FIXME: Custom lowering for these operations shouldn't return null!
3779 // Return true so that we don't call ConvertNodeToLibcall which also won't
3780 // do anything.
3781 return true;
3782 }
3783
3784 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
3785 // FIXME: We were asked to expand a strict floating-point operation,
3786 // but there is currently no expansion implemented that would preserve
3787 // the "strict" properties. For now, we just fall back to the non-strict
3788 // version if that is legal on the target. The actual mutation of the
3789 // operation will happen in SelectionDAGISel::DoInstructionSelection.
3790 switch (Node->getOpcode()) {
3791 default:
3792 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3793 Node->getValueType(0))
3794 == TargetLowering::Legal)
3795 return true;
3796 break;
3797 case ISD::STRICT_FSUB: {
3798 if (TLI.getStrictFPOperationAction(
3799 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal)
3800 return true;
3801 if (TLI.getStrictFPOperationAction(
3802 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal)
3803 break;
3804
3805 EVT VT = Node->getValueType(0);
3806 const SDNodeFlags Flags = Node->getFlags();
3807 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);
3808 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(),
3809 {Node->getOperand(0), Node->getOperand(1), Neg},
3810 Flags);
3811
3812 Results.push_back(Fadd);
3813 Results.push_back(Fadd.getValue(1));
3814 break;
3815 }
3816 case ISD::STRICT_SINT_TO_FP:
3817 case ISD::STRICT_UINT_TO_FP:
3818 case ISD::STRICT_LRINT:
3819 case ISD::STRICT_LLRINT:
3820 case ISD::STRICT_LROUND:
3821 case ISD::STRICT_LLROUND:
3822 // These are registered by the operand type instead of the value
3823 // type. Reflect that here.
3824 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3825 Node->getOperand(1).getValueType())
3826 == TargetLowering::Legal)
3827 return true;
3828 break;
3829 }
3830 }
3831
3832 // Replace the original node with the legalized result.
3833 if (Results.empty()) {
3834 LLVM_DEBUG(dbgs() << "Cannot expand node\n")do { } while (false);
3835 return false;
3836 }
3837
3838 LLVM_DEBUG(dbgs() << "Successfully expanded node\n")do { } while (false);
3839 ReplaceNode(Node, Results.data());
3840 return true;
3841}
3842
3843void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3844 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n")do { } while (false);
3845 SmallVector<SDValue, 8> Results;
3846 SDLoc dl(Node);
3847 // FIXME: Check flags on the node to see if we can use a finite call.
3848 unsigned Opc = Node->getOpcode();
3849 switch (Opc) {
3850 case ISD::ATOMIC_FENCE: {
3851 // If the target didn't lower this, lower it to '__sync_synchronize()' call
3852 // FIXME: handle "fence singlethread" more efficiently.
3853 TargetLowering::ArgListTy Args;
3854
3855 TargetLowering::CallLoweringInfo CLI(DAG);
3856 CLI.setDebugLoc(dl)
3857 .setChain(Node->getOperand(0))
3858 .setLibCallee(
3859 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3860 DAG.getExternalSymbol("__sync_synchronize",
3861 TLI.getPointerTy(DAG.getDataLayout())),
3862 std::move(Args));
3863
3864 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3865
3866 Results.push_back(CallResult.second);
3867 break;
3868 }
3869 // By default, atomic intrinsics are marked Legal and lowered. Targets
3870 // which don't support them directly, however, may want libcalls, in which
3871 // case they mark them Expand, and we get here.
3872 case ISD::ATOMIC_SWAP:
3873 case ISD::ATOMIC_LOAD_ADD:
3874 case ISD::ATOMIC_LOAD_SUB:
3875 case ISD::ATOMIC_LOAD_AND:
3876 case ISD::ATOMIC_LOAD_CLR:
3877 case ISD::ATOMIC_LOAD_OR:
3878 case ISD::ATOMIC_LOAD_XOR:
3879 case ISD::ATOMIC_LOAD_NAND:
3880 case ISD::ATOMIC_LOAD_MIN:
3881 case ISD::ATOMIC_LOAD_MAX:
3882 case ISD::ATOMIC_LOAD_UMIN:
3883 case ISD::ATOMIC_LOAD_UMAX:
3884 case ISD::ATOMIC_CMP_SWAP: {
3885 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3886 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering();
3887 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
3888 EVT RetVT = Node->getValueType(0);
3889 TargetLowering::MakeLibCallOptions CallOptions;
3890 SmallVector<SDValue, 4> Ops;
3891 if (TLI.getLibcallName(LC)) {
3892 // If outline atomic available, prepare its arguments and expand.
3893 Ops.append(Node->op_begin() + 2, Node->op_end());
3894 Ops.push_back(Node->getOperand(1));
3895
3896 } else {
3897 LC = RTLIB::getSYNC(Opc, VT);
3898 assert(LC != RTLIB::UNKNOWN_LIBCALL &&(static_cast<void> (0))
3899 "Unexpected atomic op or value type!")(static_cast<void> (0));
3900 // Arguments for expansion to sync libcall
3901 Ops.append(Node->op_begin() + 1, Node->op_end());
3902 }
3903 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
3904 Ops, CallOptions,
3905 SDLoc(Node),
3906 Node->getOperand(0));
3907 Results.push_back(Tmp.first);
3908 Results.push_back(Tmp.second);
3909 break;
3910 }
3911 case ISD::TRAP: {
3912 // If this operation is not supported, lower it to 'abort()' call
3913 TargetLowering::ArgListTy Args;
3914 TargetLowering::CallLoweringInfo CLI(DAG);
3915 CLI.setDebugLoc(dl)
3916 .setChain(Node->getOperand(0))
3917 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3918 DAG.getExternalSymbol(
3919 "abort", TLI.getPointerTy(DAG.getDataLayout())),
3920 std::move(Args));
3921 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3922
3923 Results.push_back(CallResult.second);
3924 break;
3925 }
3926 case ISD::FMINNUM:
3927 case ISD::STRICT_FMINNUM:
3928 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3929 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3930 RTLIB::FMIN_PPCF128, Results);
3931 break;
3932 case ISD::FMAXNUM:
3933 case ISD::STRICT_FMAXNUM:
3934 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3935 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3936 RTLIB::FMAX_PPCF128, Results);
3937 break;
3938 case ISD::FSQRT:
3939 case ISD::STRICT_FSQRT:
3940 ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3941 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3942 RTLIB::SQRT_PPCF128, Results);
3943 break;
3944 case ISD::FCBRT:
3945 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
3946 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
3947 RTLIB::CBRT_PPCF128, Results);
3948 break;
3949 case ISD::FSIN:
3950 case ISD::STRICT_FSIN:
3951 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
3952 RTLIB::SIN_F80, RTLIB::SIN_F128,
3953 RTLIB::SIN_PPCF128, Results);
3954 break;
3955 case ISD::FCOS:
3956 case ISD::STRICT_FCOS:
3957 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
3958 RTLIB::COS_F80, RTLIB::COS_F128,
3959 RTLIB::COS_PPCF128, Results);
3960 break;
3961 case ISD::FSINCOS:
3962 // Expand into sincos libcall.
3963 ExpandSinCosLibCall(Node, Results);
3964 break;
3965 case ISD::FLOG:
3966 case ISD::STRICT_FLOG:
3967 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
3968 RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results);
3969 break;
3970 case ISD::FLOG2:
3971 case ISD::STRICT_FLOG2:
3972 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
3973 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results);
3974 break;
3975 case ISD::FLOG10:
3976 case ISD::STRICT_FLOG10:
3977 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
3978 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results);
3979 break;
3980 case ISD::FEXP:
3981 case ISD::STRICT_FEXP:
3982 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
3983 RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results);
3984 break;
3985 case ISD::FEXP2:
3986 case ISD::STRICT_FEXP2:
3987 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
3988 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results);
3989 break;
3990 case ISD::FTRUNC:
3991 case ISD::STRICT_FTRUNC:
3992 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
3993 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
3994 RTLIB::TRUNC_PPCF128, Results);
3995 break;
3996 case ISD::FFLOOR:
3997 case ISD::STRICT_FFLOOR:
3998 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
3999 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4000 RTLIB::FLOOR_PPCF128, Results);
4001 break;
4002 case ISD::FCEIL:
4003 case ISD::STRICT_FCEIL:
4004 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4005 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4006 RTLIB::CEIL_PPCF128, Results);
4007 break;
4008 case ISD::FRINT:
4009 case ISD::STRICT_FRINT:
4010 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4011 RTLIB::RINT_F80, RTLIB::RINT_F128,
4012 RTLIB::RINT_PPCF128, Results);
4013 break;
4014 case ISD::FNEARBYINT:
4015 case ISD::STRICT_FNEARBYINT:
4016 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4017 RTLIB::NEARBYINT_F64,
4018 RTLIB::NEARBYINT_F80,
4019 RTLIB::NEARBYINT_F128,
4020 RTLIB::NEARBYINT_PPCF128, Results);
4021 break;
4022 case ISD::FROUND:
4023 case ISD::STRICT_FROUND:
4024 ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4025 RTLIB::ROUND_F64,
4026 RTLIB::ROUND_F80,
4027 RTLIB::ROUND_F128,
4028 RTLIB::ROUND_PPCF128, Results);
4029 break;
4030 case ISD::FROUNDEVEN:
4031 case ISD::STRICT_FROUNDEVEN:
4032 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,
4033 RTLIB::ROUNDEVEN_F64,
4034 RTLIB::ROUNDEVEN_F80,
4035 RTLIB::ROUNDEVEN_F128,
4036 RTLIB::ROUNDEVEN_PPCF128, Results);
4037 break;
4038 case ISD::FPOWI:
4039 case ISD::STRICT_FPOWI: {
4040 RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0));
4041 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.")(static_cast<void> (0));
4042 if (!TLI.getLibcallName(LC)) {
4043 // Some targets don't have a powi libcall; use pow instead.
4044 SDValue Exponent = DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node),
4045 Node->getValueType(0),
4046 Node->getOperand(1));
4047 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node),
4048 Node->getValueType(0), Node->getOperand(0),
4049 Exponent));
4050 break;
4051 }
4052 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4053 bool ExponentHasSizeOfInt =
4054 DAG.getLibInfo().getIntSize() ==
4055 Node->getOperand(1 + Offset).getValueType().getSizeInBits();
4056 if (!ExponentHasSizeOfInt) {
4057 // If the exponent does not match with sizeof(int) a libcall to
4058 // RTLIB::POWI would use the wrong type for the argument.
4059 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
4060 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
4061 break;
4062 }
4063 ExpandFPLibCall(Node, LC, Results);
4064 break;
4065 }
4066 case ISD::FPOW:
4067 case ISD::STRICT_FPOW:
4068 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4069 RTLIB::POW_F128, RTLIB::POW_PPCF128, Results);
4070 break;
4071 case ISD::LROUND:
4072 case ISD::STRICT_LROUND:
4073 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
4074 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4075 RTLIB::LROUND_F128,
4076 RTLIB::LROUND_PPCF128, Results);
4077 break;
4078 case ISD::LLROUND:
4079 case ISD::STRICT_LLROUND:
4080 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
4081 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4082 RTLIB::LLROUND_F128,
4083 RTLIB::LLROUND_PPCF128, Results);
4084 break;
4085 case ISD::LRINT:
4086 case ISD::STRICT_LRINT:
4087 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
4088 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4089 RTLIB::LRINT_F128,
4090 RTLIB::LRINT_PPCF128, Results);
4091 break;
4092 case ISD::LLRINT:
4093 case ISD::STRICT_LLRINT:
4094 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
4095 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4096 RTLIB::LLRINT_F128,
4097 RTLIB::LLRINT_PPCF128, Results);
4098 break;
4099 case ISD::FDIV:
4100 case ISD::STRICT_FDIV:
4101 ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4102 RTLIB::DIV_F80, RTLIB::DIV_F128,
4103 RTLIB::DIV_PPCF128, Results);
4104 break;
4105 case ISD::FREM:
4106 case ISD::STRICT_FREM:
4107 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4108 RTLIB::REM_F80, RTLIB::REM_F128,
4109 RTLIB::REM_PPCF128, Results);
4110 break;
4111 case ISD::FMA:
4112 case ISD::STRICT_FMA:
4113 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4114 RTLIB::FMA_F80, RTLIB::FMA_F128,
4115 RTLIB::FMA_PPCF128, Results);
4116 break;
4117 case ISD::FADD:
4118 case ISD::STRICT_FADD:
4119 ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4120 RTLIB::ADD_F80, RTLIB::ADD_F128,
4121 RTLIB::ADD_PPCF128, Results);
4122 break;
4123 case ISD::FMUL:
4124 case ISD::STRICT_FMUL:
4125 ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4126 RTLIB::MUL_F80, RTLIB::MUL_F128,
4127 RTLIB::MUL_PPCF128, Results);
4128 break;
4129 case ISD::FP16_TO_FP:
4130 if (Node->getValueType(0) == MVT::f32) {
4131 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4132 }
4133 break;
4134 case ISD::STRICT_FP16_TO_FP: {
4135 if (Node->getValueType(0) == MVT::f32) {
4136 TargetLowering::MakeLibCallOptions CallOptions;
4137 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4138 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
4139 SDLoc(Node), Node->getOperand(0));
4140 Results.push_back(Tmp.first);
4141 Results.push_back(Tmp.second);
4142 }
4143 break;
4144 }
4145 case ISD::FP_TO_FP16: {
4146 RTLIB::Libcall LC =
4147 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4148 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16")(static_cast<void> (0));
4149 Results.push_back(ExpandLibCall(LC, Node, false));
4150 break;
4151 }
4152 case ISD::STRICT_SINT_TO_FP:
4153 case ISD::STRICT_UINT_TO_FP:
4154 case ISD::SINT_TO_FP:
4155 case ISD::UINT_TO_FP: {
4156 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
4157 bool IsStrict = Node->isStrictFPOpcode();
4158 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
4159 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
4160 EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType();
4161 EVT RVT = Node->getValueType(0);
4162 EVT NVT = EVT();
4163 SDLoc dl(Node);
4164
4165 // Even if the input is legal, no libcall may exactly match, eg. we don't
4166 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
4167 // eg: i13 -> fp. Then, look for an appropriate libcall.
4168 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4169 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4170 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4171 ++t) {
4172 NVT = (MVT::SimpleValueType)t;
4173 // The source needs to big enough to hold the operand.
4174 if (NVT.bitsGE(SVT))
4175 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT)
4176 : RTLIB::getUINTTOFP(NVT, RVT);
4177 }
4178 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall")(static_cast<void> (0));
4179
4180 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4181 // Sign/zero extend the argument if the libcall takes a larger type.
4182 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
4183 NVT, Node->getOperand(IsStrict ? 1 : 0));
4184 TargetLowering::MakeLibCallOptions CallOptions;
4185 CallOptions.setSExt(Signed);
4186 std::pair<SDValue, SDValue> Tmp =
4187 TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain);
4188 Results.push_back(Tmp.first);
4189 if (IsStrict)
4190 Results.push_back(Tmp.second);
4191 break;
4192 }
4193 case ISD::FP_TO_SINT:
4194 case ISD::FP_TO_UINT:
4195 case ISD::STRICT_FP_TO_SINT:
4196 case ISD::STRICT_FP_TO_UINT: {
4197 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
4198 bool IsStrict = Node->isStrictFPOpcode();
4199 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
4200 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
4201
4202 SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4203 EVT SVT = Op.getValueType();
4204 EVT RVT = Node->getValueType(0);
4205 EVT NVT = EVT();
4206 SDLoc dl(Node);
4207
4208 // Even if the result is legal, no libcall may exactly match, eg. we don't
4209 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
4210 // eg: fp -> i32. Then, look for an appropriate libcall.
4211 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4212 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4213 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4214 ++IntVT) {
4215 NVT = (MVT::SimpleValueType)IntVT;
4216 // The type needs to big enough to hold the result.
4217 if (NVT.bitsGE(RVT))
4218 LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT)
4219 : RTLIB::getFPTOUINT(SVT, NVT);
4220 }
4221 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall")(static_cast<void> (0));
4222
4223 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4224 TargetLowering::MakeLibCallOptions CallOptions;
4225 std::pair<SDValue, SDValue> Tmp =
4226 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
4227
4228 // Truncate the result if the libcall returns a larger type.
4229 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first));
4230 if (IsStrict)
4231 Results.push_back(Tmp.second);
4232 break;
4233 }
4234
4235 case ISD::FP_ROUND:
4236 case ISD::STRICT_FP_ROUND: {
4237 // X = FP_ROUND(Y, TRUNC)
4238 // TRUNC is a flag, which is always an integer that is zero or one.
4239 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
4240 // is known to not change the value of Y.
4241 // We can only expand it into libcall if the TRUNC is 0.
4242 bool IsStrict = Node->isStrictFPOpcode();
4243 SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4244 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4245 EVT VT = Node->getValueType(0);
4246 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))(static_cast<void> (0))
4247 ->isNullValue() &&(static_cast<void> (0))
4248 "Unable to expand as libcall if it is not normal rounding")(static_cast<void> (0));
4249
4250 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), VT);
4251 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall")(static_cast<void> (0));
4252
4253 TargetLowering::MakeLibCallOptions CallOptions;
4254 std::pair<SDValue, SDValue> Tmp =
4255 TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain);
4256 Results.push_back(Tmp.first);
4257 if (IsStrict)
4258 Results.push_back(Tmp.second);
4259 break;
4260 }
4261 case ISD::FP_EXTEND: {
4262 Results.push_back(
4263 ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(),
4264 Node->getValueType(0)),
4265 Node, false));
4266 break;
4267 }
4268 case ISD::STRICT_FP_EXTEND:
4269 case ISD::STRICT_FP_TO_FP16: {
4270 RTLIB::Libcall LC =
4271 Node->getOpcode() == ISD::STRICT_FP_TO_FP16
4272 ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16)
4273 : RTLIB::getFPEXT(Node->getOperand(1).getValueType(),
4274 Node->getValueType(0));
4275 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall")(static_cast<void> (0));
4276
4277 TargetLowering::MakeLibCallOptions CallOptions;
4278 std::pair<SDValue, SDValue> Tmp =
4279 TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1),
4280 CallOptions, SDLoc(Node), Node->getOperand(0));
4281 Results.push_back(Tmp.first);
4282 Results.push_back(Tmp.second);
4283 break;
4284 }
4285 case ISD::FSUB:
4286 case ISD::STRICT_FSUB:
4287 ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4288 RTLIB::SUB_F80, RTLIB::SUB_F128,
4289 RTLIB::SUB_PPCF128, Results);
4290 break;
4291 case ISD::SREM:
4292 Results.push_back(ExpandIntLibCall(Node, true,
4293 RTLIB::SREM_I8,
4294 RTLIB::SREM_I16, RTLIB::SREM_I32,
4295 RTLIB::SREM_I64, RTLIB::SREM_I128));
4296 break;
4297 case ISD::UREM:
4298 Results.push_back(ExpandIntLibCall(Node, false,
4299 RTLIB::UREM_I8,
4300 RTLIB::UREM_I16, RTLIB::UREM_I32,
4301 RTLIB::UREM_I64, RTLIB::UREM_I128));
4302 break;
4303 case ISD::SDIV:
4304 Results.push_back(ExpandIntLibCall(Node, true,
4305 RTLIB::SDIV_I8,
4306 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4307 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4308 break;
4309 case ISD::UDIV:
4310 Results.push_back(ExpandIntLibCall(Node, false,
4311 RTLIB::UDIV_I8,
4312 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4313 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4314 break;
4315 case ISD::SDIVREM:
4316 case ISD::UDIVREM:
4317 // Expand into divrem libcall
4318 ExpandDivRemLibCall(Node, Results);
4319 break;
4320 case ISD::MUL:
4321 Results.push_back(ExpandIntLibCall(Node, false,
4322 RTLIB::MUL_I8,
4323 RTLIB::MUL_I16, RTLIB::MUL_I32,
4324 RTLIB::MUL_I64, RTLIB::MUL_I128));
4325 break;
4326 case ISD::CTLZ_ZERO_UNDEF:
4327 switch (Node->getSimpleValueType(0).SimpleTy) {
4328 default:
4329 llvm_unreachable("LibCall explicitly requested, but not available")__builtin_unreachable();
4330 case MVT::i32:
4331 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false));
4332 break;
4333 case MVT::i64:
4334 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false));
4335 break;
4336 case MVT::i128:
4337 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false));
4338 break;
4339 }
4340 break;
4341 }
4342
4343 // Replace the original node with the legalized result.
4344 if (!Results.empty()) {
4345 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n")do { } while (false);
4346 ReplaceNode(Node, Results.data());
4347 } else
4348 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n")do { } while (false);
4349}
4350
4351// Determine the vector type to use in place of an original scalar element when
4352// promoting equally sized vectors.
4353static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4354 MVT EltVT, MVT NewEltVT) {
4355 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4356 MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4357 assert(TLI.isTypeLegal(MidVT) && "unexpected")(static_cast<void> (0));
4358 return MidVT;
4359}
4360
4361void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4362 LLVM_DEBUG(dbgs() << "Trying to promote node\n")do { } while (false);
4363 SmallVector<SDValue, 8> Results;
4364 MVT OVT = Node->getSimpleValueType(0);
4365 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4366 Node->getOpcode() == ISD::SINT_TO_FP ||
4367 Node->getOpcode() == ISD::SETCC ||
4368 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4369 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4370 OVT = Node->getOperand(0).getSimpleValueType();
4371 }
4372 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
4373 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
4374 Node->getOpcode() == ISD::STRICT_FSETCC ||
4375 Node->getOpcode() == ISD::STRICT_FSETCCS)
4376 OVT = Node->getOperand(1).getSimpleValueType();
4377 if (Node->getOpcode() == ISD::BR_CC ||
4378 Node->getOpcode() == ISD::SELECT_CC)
4379 OVT = Node->getOperand(2).getSimpleValueType();
4380 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4381 SDLoc dl(Node);
4382 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4383 switch (Node->getOpcode()) {
4384 case ISD::CTTZ:
4385 case ISD::CTTZ_ZERO_UNDEF:
4386 case ISD::CTLZ:
4387 case ISD::CTLZ_ZERO_UNDEF:
4388 case ISD::CTPOP:
4389 // Zero extend the argument unless its cttz, then use any_extend.
4390 if (Node->getOpcode() == ISD::CTTZ ||
4391 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
4392 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
4393 else
4394 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4395
4396 if (Node->getOpcode() == ISD::CTTZ) {
4397 // The count is the same in the promoted type except if the original
4398 // value was zero. This can be handled by setting the bit just off
4399 // the top of the original type.
4400 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4401 OVT.getSizeInBits());
4402 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4403 DAG.getConstant(TopBit, dl, NVT));
4404 }
4405 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4406 // already the correct result.
4407 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4408 if (Node->getOpcode() == ISD::CTLZ ||
4409 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4410 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4411 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4412 DAG.getConstant(NVT.getSizeInBits() -
4413 OVT.getSizeInBits(), dl, NVT));
4414 }
4415 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4416 break;
4417 case ISD::BITREVERSE:
4418 case ISD::BSWAP: {
4419 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4420 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4421 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4422 Tmp1 = DAG.getNode(
4423 ISD::SRL, dl, NVT, Tmp1,
4424 DAG.getConstant(DiffBits, dl,
4425 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4426
4427 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4428 break;
4429 }
4430 case ISD::FP_TO_UINT:
4431 case ISD::STRICT_FP_TO_UINT:
4432 case ISD::FP_TO_SINT:
4433 case ISD::STRICT_FP_TO_SINT:
4434 PromoteLegalFP_TO_INT(Node, dl, Results);
4435 break;
4436 case ISD::FP_TO_UINT_SAT:
4437 case ISD::FP_TO_SINT_SAT:
4438 Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl));
4439 break;
4440 case ISD::UINT_TO_FP:
4441 case ISD::STRICT_UINT_TO_FP:
4442 case ISD::SINT_TO_FP:
4443 case ISD::STRICT_SINT_TO_FP:
4444 PromoteLegalINT_TO_FP(Node, dl, Results);
4445 break;
4446 case ISD::VAARG: {
4447 SDValue Chain = Node->getOperand(0); // Get the chain.
4448 SDValue Ptr = Node->getOperand(1); // Get the pointer.
4449
4450 unsigned TruncOp;
4451 if (OVT.isVector()) {
4452 TruncOp = ISD::BITCAST;
4453 } else {
4454 assert(OVT.isInteger()(static_cast<void> (0))
4455 && "VAARG promotion is supported only for vectors or integer types")(static_cast<void> (0));
4456 TruncOp = ISD::TRUNCATE;
4457 }
4458
4459 // Perform the larger operation, then convert back
4460 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4461 Node->getConstantOperandVal(3));
4462 Chain = Tmp1.getValue(1);
4463
4464 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4465
4466 // Modified the chain result - switch anything that used the old chain to
4467 // use the new one.
4468 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4469 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4470 if (UpdatedNodes) {
4471 UpdatedNodes->insert(Tmp2.getNode());
4472 UpdatedNodes->insert(Chain.getNode());
4473 }
4474 ReplacedNode(Node);
4475 break;
4476 }
4477 case ISD::MUL:
4478 case ISD::SDIV:
4479 case ISD::SREM:
4480 case ISD::UDIV:
4481 case ISD::UREM:
4482 case ISD::AND:
4483 case ISD::OR:
4484 case ISD::XOR: {
4485 unsigned ExtOp, TruncOp;
4486 if (OVT.isVector()) {
4487 ExtOp = ISD::BITCAST;
4488 TruncOp = ISD::BITCAST;
4489 } else {
4490 assert(OVT.isInteger() && "Cannot promote logic operation")(static_cast<void> (0));
4491
4492 switch (Node->getOpcode()) {
4493 default:
4494 ExtOp = ISD::ANY_EXTEND;
4495 break;
4496 case ISD::SDIV:
4497 case ISD::SREM:
4498 ExtOp = ISD::SIGN_EXTEND;
4499 break;
4500 case ISD::UDIV:
4501 case ISD::UREM:
4502 ExtOp = ISD::ZERO_EXTEND;
4503 break;
4504 }
4505 TruncOp = ISD::TRUNCATE;
4506 }
4507 // Promote each of the values to the new type.
4508 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4509 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4510 // Perform the larger operation, then convert back
4511 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4512 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4513 break;
4514 }
4515 case ISD::UMUL_LOHI:
4516 case ISD::SMUL_LOHI: {
4517 // Promote to a multiply in a wider integer type.
4518 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4519 : ISD::SIGN_EXTEND;
4520 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4521 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4522 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4523
4524 auto &DL = DAG.getDataLayout();
4525 unsigned OriginalSize = OVT.getScalarSizeInBits();
4526 Tmp2 = DAG.getNode(
4527 ISD::SRL, dl, NVT, Tmp1,
4528 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4529 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4530 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4531 break;
4532 }
4533 case ISD::SELECT: {
4534 unsigned ExtOp, TruncOp;
4535 if (Node->getValueType(0).isVector() ||
4536 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4537 ExtOp = ISD::BITCAST;
4538 TruncOp = ISD::BITCAST;
4539 } else if (Node->getValueType(0).isInteger()) {
4540 ExtOp = ISD::ANY_EXTEND;
4541 TruncOp = ISD::TRUNCATE;
4542 } else {
4543 ExtOp = ISD::FP_EXTEND;
4544 TruncOp = ISD::FP_ROUND;
4545 }
4546 Tmp1 = Node->getOperand(0);
4547 // Promote each of the values to the new type.
4548 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4549 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4550 // Perform the larger operation, then round down.
4551 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4552 Tmp1->setFlags(Node->getFlags());
4553 if (TruncOp != ISD::FP_ROUND)
4554 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4555 else
4556 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4557 DAG.getIntPtrConstant(0, dl));
4558 Results.push_back(Tmp1);
4559 break;
4560 }
4561 case ISD::VECTOR_SHUFFLE: {
4562 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4563
4564 // Cast the two input vectors.
4565 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4566 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4567
4568 // Convert the shuffle mask to the right # elements.
4569 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4570 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4571 Results.push_back(Tmp1);
4572 break;
4573 }
4574 case ISD::VECTOR_SPLICE: {
4575 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
4576 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1));
4577 Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2,
4578 Node->getOperand(2));
4579 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3));
4580 break;
4581 }
4582 case ISD::SELECT_CC: {
4583 SDValue Cond = Node->getOperand(4);
4584 ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get();
4585 // Type of the comparison operands.
4586 MVT CVT = Node->getSimpleValueType(0);
4587 assert(CVT == OVT && "not handled")(static_cast<void> (0));
4588
4589 unsigned ExtOp = ISD::FP_EXTEND;
4590 if (NVT.isInteger()) {
4591 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4592 }
4593
4594 // Promote the comparison operands, if needed.
4595 if (TLI.isCondCodeLegal(CCCode, CVT)) {
4596 Tmp1 = Node->getOperand(0);
4597 Tmp2 = Node->getOperand(1);
4598 } else {
4599 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4600 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4601 }
4602 // Cast the true/false operands.
4603 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4604 Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4605
4606 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
4607 Node->getFlags());
4608
4609 // Cast the result back to the original type.
4610 if (ExtOp != ISD::FP_EXTEND)
4611 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1);
4612 else
4613 Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1,
4614 DAG.getIntPtrConstant(0, dl));
4615
4616 Results.push_back(Tmp1);
4617 break;
4618 }
4619 case ISD::SETCC:
4620 case ISD::STRICT_FSETCC:
4621 case ISD::STRICT_FSETCCS: {
4622 unsigned ExtOp = ISD::FP_EXTEND;
4623 if (NVT.isInteger()) {
4624 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
4625 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4626 }
4627 if (Node->isStrictFPOpcode()) {
4628 SDValue InChain = Node->getOperand(0);
4629 std::tie(Tmp1, std::ignore) =
4630 DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT);
4631 std::tie(Tmp2, std::ignore) =
4632 DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT);
4633 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(1), Tmp2.getValue(1)};
4634 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
4635 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
4636 Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs,
4637 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
4638 Node->getFlags()));
4639 Results.push_back(Results.back().getValue(1));
4640 break;
4641 }
4642 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4643 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4644 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1,
4645 Tmp2, Node->getOperand(2), Node->getFlags()));
4646 break;
4647 }
4648 case ISD::BR_CC: {
4649 unsigned ExtOp = ISD::FP_EXTEND;
4650 if (NVT.isInteger()) {
4651 ISD::CondCode CCCode =
4652 cast<CondCodeSDNode>(Node->getOperand(1))->get();
4653 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4654 }
4655 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4656 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4657 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4658 Node->getOperand(0), Node->getOperand(1),
4659 Tmp1, Tmp2, Node->getOperand(4)));
4660 break;
4661 }
4662 case ISD::FADD:
4663 case ISD::FSUB:
4664 case ISD::FMUL:
4665 case ISD::FDIV:
4666 case ISD::FREM:
4667 case ISD::FMINNUM:
4668 case ISD::FMAXNUM:
4669 case ISD::FPOW:
4670 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4671 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4672 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4673 Node->getFlags());
4674 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4675 Tmp3, DAG.getIntPtrConstant(0, dl)));
4676 break;
4677 case ISD::STRICT_FREM:
4678 case ISD::STRICT_FPOW:
4679 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4680 {Node->getOperand(0), Node->getOperand(1)});
4681 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4682 {Node->getOperand(0), Node->getOperand(2)});
4683 Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
4684 Tmp2.getValue(1));
4685 Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4686 {Tmp3, Tmp1, Tmp2});
4687 Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
4688 {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
4689 Results.push_back(Tmp1);
4690 Results.push_back(Tmp1.getValue(1));
4691 break;
4692 case ISD::FMA:
4693 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4694 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4695 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4696 Results.push_back(
4697 DAG.getNode(ISD::FP_ROUND, dl, OVT,
4698 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4699 DAG.getIntPtrConstant(0, dl)));
4700 break;
4701 case ISD::FCOPYSIGN:
4702 case ISD::FPOWI: {
4703 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4704 Tmp2 = Node->getOperand(1);
4705 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4706
4707 // fcopysign doesn't change anything but the sign bit, so
4708 // (fp_round (fcopysign (fpext a), b))
4709 // is as precise as
4710 // (fp_round (fpext a))
4711 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4712 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4713 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4714 Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4715 break;
4716 }
4717 case ISD::FFLOOR:
4718 case ISD::FCEIL:
4719 case ISD::FRINT:
4720 case ISD::FNEARBYINT:
4721 case ISD::FROUND:
4722 case ISD::FROUNDEVEN:
4723 case ISD::FTRUNC:
4724 case ISD::FNEG:
4725 case ISD::FSQRT:
4726 case ISD::FSIN:
4727 case ISD::FCOS:
4728 case ISD::FLOG:
4729 case ISD::FLOG2:
4730 case ISD::FLOG10:
4731 case ISD::FABS:
4732 case ISD::FEXP:
4733 case ISD::FEXP2:
4734 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4735 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4736 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4737 Tmp2, DAG.getIntPtrConstant(0, dl)));
4738 break;
4739 case ISD::STRICT_FFLOOR:
4740 case ISD::STRICT_FCEIL:
4741 case ISD::STRICT_FSIN:
4742 case ISD::STRICT_FCOS:
4743 case ISD::STRICT_FLOG:
4744 case ISD::STRICT_FLOG10:
4745 case ISD::STRICT_FEXP:
4746 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4747 {Node->getOperand(0), Node->getOperand(1)});
4748 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4749 {Tmp1.getValue(1), Tmp1});
4750 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
4751 {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
4752 Results.push_back(Tmp3);
4753 Results.push_back(Tmp3.getValue(1));
4754 break;
4755 case ISD::BUILD_VECTOR: {
4756 MVT EltVT = OVT.getVectorElementType();
4757 MVT NewEltVT = NVT.getVectorElementType();
4758
4759 // Handle bitcasts to a different vector type with the same total bit size
4760 //
4761 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4762 // =>
4763 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4764
4765 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&(static_cast<void> (0))
4766 "Invalid promote type for build_vector")(static_cast<void> (0));
4767 assert(NewEltVT.bitsLT(EltVT) && "not handled")(static_cast<void> (0));
4768
4769 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4770
4771 SmallVector<SDValue, 8> NewOps;
4772 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4773 SDValue Op = Node->getOperand(I);
4774 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4775 }
4776
4777 SDLoc SL(Node);
4778 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4779 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4780 Results.push_back(CvtVec);
4781 break;
4782 }
4783 case ISD::EXTRACT_VECTOR_ELT: {
4784 MVT EltVT = OVT.getVectorElementType();
4785 MVT NewEltVT = NVT.getVectorElementType();
4786
4787 // Handle bitcasts to a different vector type with the same total bit size.
4788 //
4789 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4790 // =>
4791 // v4i32:castx = bitcast x:v2i64
4792 //
4793 // i64 = bitcast
4794 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4795 // (i32 (extract_vector_elt castx, (2 * y + 1)))
4796 //
4797
4798 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&(static_cast<void> (0))
4799 "Invalid promote type for extract_vector_elt")(static_cast<void> (0));
4800 assert(NewEltVT.bitsLT(EltVT) && "not handled")(static_cast<void> (0));
4801
4802 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4803 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4804
4805 SDValue Idx = Node->getOperand(1);
4806 EVT IdxVT = Idx.getValueType();
4807 SDLoc SL(Node);
4808 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4809 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4810
4811 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4812
4813 SmallVector<SDValue, 8> NewOps;
4814 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4815 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4816 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4817
4818 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4819 CastVec, TmpIdx);
4820 NewOps.push_back(Elt);
4821 }
4822
4823 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4824 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4825 break;
4826 }
4827 case ISD::INSERT_VECTOR_ELT: {
4828 MVT EltVT = OVT.getVectorElementType();
4829 MVT NewEltVT = NVT.getVectorElementType();
4830
4831 // Handle bitcasts to a different vector type with the same total bit size
4832 //
4833 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4834 // =>
4835 // v4i32:castx = bitcast x:v2i64
4836 // v2i32:casty = bitcast y:i64
4837 //
4838 // v2i64 = bitcast
4839 // (v4i32 insert_vector_elt
4840 // (v4i32 insert_vector_elt v4i32:castx,
4841 // (extract_vector_elt casty, 0), 2 * z),
4842 // (extract_vector_elt casty, 1), (2 * z + 1))
4843
4844 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&(static_cast<void> (0))
4845 "Invalid promote type for insert_vector_elt")(static_cast<void> (0));
4846 assert(NewEltVT.bitsLT(EltVT) && "not handled")(static_cast<void> (0));
4847
4848 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4849 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4850
4851 SDValue Val = Node->getOperand(1);
4852 SDValue Idx = Node->getOperand(2);
4853 EVT IdxVT = Idx.getValueType();
4854 SDLoc SL(Node);
4855
4856 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4857 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4858
4859 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4860 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4861
4862 SDValue NewVec = CastVec;
4863 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4864 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4865 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4866
4867 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4868 CastVal, IdxOffset);
4869
4870 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4871 NewVec, Elt, InEltIdx);
4872 }
4873
4874 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4875 break;
4876 }
4877 case ISD::SCALAR_TO_VECTOR: {
4878 MVT EltVT = OVT.getVectorElementType();
4879 MVT NewEltVT = NVT.getVectorElementType();
4880
4881 // Handle bitcasts to different vector type with the same total bit size.
4882 //
4883 // e.g. v2i64 = scalar_to_vector x:i64
4884 // =>
4885 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4886 //
4887
4888 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4889 SDValue Val = Node->getOperand(0);
4890 SDLoc SL(Node);
4891
4892 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4893 SDValue Undef = DAG.getUNDEF(MidVT);
4894
4895 SmallVector<SDValue, 8> NewElts;
4896 NewElts.push_back(CastVal);
4897 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4898 NewElts.push_back(Undef);
4899
4900 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4901 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4902 Results.push_back(CvtVec);
4903 break;
4904 }
4905 case ISD::ATOMIC_SWAP: {
4906 AtomicSDNode *AM = cast<AtomicSDNode>(Node);
4907 SDLoc SL(Node);
4908 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
4909 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&(static_cast<void> (0))
4910 "unexpected promotion type")(static_cast<void> (0));
4911 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&(static_cast<void> (0))
4912 "unexpected atomic_swap with illegal type")(static_cast<void> (0));
4913
4914 SDValue NewAtomic
4915 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
4916 DAG.getVTList(NVT, MVT::Other),
4917 { AM->getChain(), AM->getBasePtr(), CastVal },
4918 AM->getMemOperand());
4919 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
4920 Results.push_back(NewAtomic.getValue(1));
4921 break;
4922 }
4923 }
4924
4925 // Replace the original node with the legalized result.
4926 if (!Results.empty()) {
4927 LLVM_DEBUG(dbgs() << "Successfully promoted node\n")do { } while (false);
4928 ReplaceNode(Node, Results.data());
4929 } else
4930 LLVM_DEBUG(dbgs() << "Could not promote node\n")do { } while (false);
4931}
4932
4933/// This is the entry point for the file.
4934void SelectionDAG::Legalize() {
4935 AssignTopologicalOrder();
4936
4937 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4938 // Use a delete listener to remove nodes which were deleted during
4939 // legalization from LegalizeNodes. This is needed to handle the situation
4940 // where a new node is allocated by the object pool to the same address of a
4941 // previously deleted node.
4942 DAGNodeDeletedListener DeleteListener(
4943 *this,
4944 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4945
4946 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4947
4948 // Visit all the nodes. We start in topological order, so that we see
4949 // nodes with their original operands intact. Legalization can produce
4950 // new nodes which may themselves need to be legalized. Iterate until all
4951 // nodes have been legalized.
4952 while (true) {
4953 bool AnyLegalized = false;
4954 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4955 --NI;
4956
4957 SDNode *N = &*NI;
4958 if (N->use_empty() && N != getRoot().getNode()) {
4959 ++NI;
4960 DeleteNode(N);
4961 continue;
4962 }
4963
4964 if (LegalizedNodes.insert(N).second) {
4965 AnyLegalized = true;
4966 Legalizer.LegalizeOp(N);
4967
4968 if (N->use_empty() && N != getRoot().getNode()) {
4969 ++NI;
4970 DeleteNode(N);
4971 }
4972 }
4973 }
4974 if (!AnyLegalized)
4975 break;
4976
4977 }
4978
4979 // Remove dead nodes now.
4980 RemoveDeadNodes();
4981}
4982
4983bool SelectionDAG::LegalizeOp(SDNode *N,
4984 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4985 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4986 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4987
4988 // Directly insert the node in question, and legalize it. This will recurse
4989 // as needed through operands.
4990 LegalizedNodes.insert(N);
4991 Legalizer.LegalizeOp(N);
4992
4993 return LegalizedNodes.count(N);
4994}

/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/include/llvm/Support/TypeSize.h

1//===- TypeSize.h - Wrapper around type sizes -------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides a struct that can be used to query the size of IR types
10// which may be scalable vectors. It provides convenience operators so that
11// it can be used in much the same way as a single scalar value.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_SUPPORT_TYPESIZE_H
16#define LLVM_SUPPORT_TYPESIZE_H
17
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/Support/MathExtras.h"
20#include "llvm/Support/WithColor.h"
21
22#include <algorithm>
23#include <array>
24#include <cassert>
25#include <cstdint>
26#include <type_traits>
27
28namespace llvm {
29
30/// Reports a diagnostic message to indicate an invalid size request has been
31/// done on a scalable vector. This function may not return.
32void reportInvalidSizeRequest(const char *Msg);
33
34template <typename LeafTy> struct LinearPolyBaseTypeTraits {};
35
36//===----------------------------------------------------------------------===//
37// LinearPolyBase - a base class for linear polynomials with multiple
38// dimensions. This can e.g. be used to describe offsets that are have both a
39// fixed and scalable component.
40//===----------------------------------------------------------------------===//
41
42/// LinearPolyBase describes a linear polynomial:
43/// c0 * scale0 + c1 * scale1 + ... + cK * scaleK
44/// where the scale is implicit, so only the coefficients are encoded.
45template <typename LeafTy>
46class LinearPolyBase {
47public:
48 using ScalarTy = typename LinearPolyBaseTypeTraits<LeafTy>::ScalarTy;
49 static constexpr auto Dimensions = LinearPolyBaseTypeTraits<LeafTy>::Dimensions;
50 static_assert(Dimensions != std::numeric_limits<unsigned>::max(),
51 "Dimensions out of range");
52
53private:
54 std::array<ScalarTy, Dimensions> Coefficients;
55
56protected:
57 LinearPolyBase(ArrayRef<ScalarTy> Values) {
58 std::copy(Values.begin(), Values.end(), Coefficients.begin());
59 }
60
61public:
62 friend LeafTy &operator+=(LeafTy &LHS, const LeafTy &RHS) {
63 for (unsigned I=0; I<Dimensions; ++I)
64 LHS.Coefficients[I] += RHS.Coefficients[I];
65 return LHS;
66 }
67
68 friend LeafTy &operator-=(LeafTy &LHS, const LeafTy &RHS) {
69 for (unsigned I=0; I<Dimensions; ++I)
70 LHS.Coefficients[I] -= RHS.Coefficients[I];
71 return LHS;
72 }
73
74 friend LeafTy &operator*=(LeafTy &LHS, ScalarTy RHS) {
75 for (auto &C : LHS.Coefficients)
76 C *= RHS;
77 return LHS;
78 }
79
80 friend LeafTy operator+(const LeafTy &LHS, const LeafTy &RHS) {
81 LeafTy Copy = LHS;
82 return Copy += RHS;
83 }
84
85 friend LeafTy operator-(const LeafTy &LHS, const LeafTy &RHS) {
86 LeafTy Copy = LHS;
87 return Copy -= RHS;
88 }
89
90 friend LeafTy operator*(const LeafTy &LHS, ScalarTy RHS) {
91 LeafTy Copy = LHS;
92 return Copy *= RHS;
93 }
94
95 template <typename U = ScalarTy>
96 friend typename std::enable_if_t<std::is_signed<U>::value, LeafTy>
97 operator-(const LeafTy &LHS) {
98 LeafTy Copy = LHS;
99 return Copy *= -1;
100 }
101
102 bool operator==(const LinearPolyBase &RHS) const {
103 return std::equal(Coefficients.begin(), Coefficients.end(),
104 RHS.Coefficients.begin());
105 }
106
107 bool operator!=(const LinearPolyBase &RHS) const {
108 return !(*this == RHS);
109 }
110
111 bool isZero() const {
112 return all_of(Coefficients, [](const ScalarTy &C) { return C == 0; });
113 }
114 bool isNonZero() const { return !isZero(); }
115 explicit operator bool() const { return isNonZero(); }
116
117 ScalarTy getValue(unsigned Dim) const { return Coefficients[Dim]; }
118};
119
120//===----------------------------------------------------------------------===//
121// StackOffset - Represent an offset with named fixed and scalable components.
122//===----------------------------------------------------------------------===//
123
124class StackOffset;
125template <> struct LinearPolyBaseTypeTraits<StackOffset> {
126 using ScalarTy = int64_t;
127 static constexpr unsigned Dimensions = 2;
128};
129
130/// StackOffset is a class to represent an offset with 2 dimensions,
131/// named fixed and scalable, respectively. This class allows a value for both
132/// dimensions to depict e.g. "8 bytes and 16 scalable bytes", which is needed
133/// to represent stack offsets.
134class StackOffset : public LinearPolyBase<StackOffset> {
135protected:
136 StackOffset(ScalarTy Fixed, ScalarTy Scalable)
137 : LinearPolyBase<StackOffset>({Fixed, Scalable}) {}
138
139public:
140 StackOffset() : StackOffset({0, 0}) {}
141 StackOffset(const LinearPolyBase<StackOffset> &Other)
142 : LinearPolyBase<StackOffset>(Other) {}
143 static StackOffset getFixed(ScalarTy Fixed) { return {Fixed, 0}; }
144 static StackOffset getScalable(ScalarTy Scalable) { return {0, Scalable}; }
145 static StackOffset get(ScalarTy Fixed, ScalarTy Scalable) {
146 return {Fixed, Scalable};
147 }
148
149 ScalarTy getFixed() const { return this->getValue(0); }
150 ScalarTy getScalable() const { return this->getValue(1); }
151};
152
153//===----------------------------------------------------------------------===//
154// UnivariateLinearPolyBase - a base class for linear polynomials with multiple
155// dimensions, but where only one dimension can be set at any time.
156// This can e.g. be used to describe sizes that are either fixed or scalable.
157//===----------------------------------------------------------------------===//
158
159/// UnivariateLinearPolyBase is a base class for ElementCount and TypeSize.
160/// Like LinearPolyBase it tries to represent a linear polynomial
161/// where only one dimension can be set at any time, e.g.
162/// 0 * scale0 + 0 * scale1 + ... + cJ * scaleJ + ... + 0 * scaleK
163/// The dimension that is set is the univariate dimension.
164template <typename LeafTy>
165class UnivariateLinearPolyBase {
166public:
167 using ScalarTy = typename LinearPolyBaseTypeTraits<LeafTy>::ScalarTy;
168 static constexpr auto Dimensions = LinearPolyBaseTypeTraits<LeafTy>::Dimensions;
169 static_assert(Dimensions != std::numeric_limits<unsigned>::max(),
170 "Dimensions out of range");
171
172protected:
173 ScalarTy Value; // The value at the univeriate dimension.
174 unsigned UnivariateDim; // The univeriate dimension.
175
176 UnivariateLinearPolyBase(ScalarTy Val, unsigned UnivariateDim)
177 : Value(Val), UnivariateDim(UnivariateDim) {
178 assert(UnivariateDim < Dimensions && "Dimension out of range")(static_cast<void> (0));
179 }
180
181 friend LeafTy &operator+=(LeafTy &LHS, const LeafTy &RHS) {
182 assert(LHS.UnivariateDim == RHS.UnivariateDim && "Invalid dimensions")(static_cast<void> (0));
183 LHS.Value += RHS.Value;
184 return LHS;
185 }
186
187 friend LeafTy &operator-=(LeafTy &LHS, const LeafTy &RHS) {
188 assert(LHS.UnivariateDim == RHS.UnivariateDim && "Invalid dimensions")(static_cast<void> (0));
189 LHS.Value -= RHS.Value;
190 return LHS;
191 }
192
193 friend LeafTy &operator*=(LeafTy &LHS, ScalarTy RHS) {
194 LHS.Value *= RHS;
195 return LHS;
196 }
197
198 friend LeafTy operator+(const LeafTy &LHS, const LeafTy &RHS) {
199 LeafTy Copy = LHS;
200 return Copy += RHS;
201 }
202
203 friend LeafTy operator-(const LeafTy &LHS, const LeafTy &RHS) {
204 LeafTy Copy = LHS;
205 return Copy -= RHS;
206 }
207
208 friend LeafTy operator*(const LeafTy &LHS, ScalarTy RHS) {
209 LeafTy Copy = LHS;
210 return Copy *= RHS;
211 }
212
213 template <typename U = ScalarTy>
214 friend typename std::enable_if<std::is_signed<U>::value, LeafTy>::type
215 operator-(const LeafTy &LHS) {
216 LeafTy Copy = LHS;
217 return Copy *= -1;
218 }
219
220public:
221 bool operator==(const UnivariateLinearPolyBase &RHS) const {
222 return Value == RHS.Value && UnivariateDim == RHS.UnivariateDim;
7
Assuming 'Value' is equal to 'RHS.Value'
8
Assuming 'UnivariateDim' is equal to 'RHS.UnivariateDim'
9
Returning the value 1, which participates in a condition later
223 }
224
225 bool operator!=(const UnivariateLinearPolyBase &RHS) const {
226 return !(*this == RHS);
6
Calling 'UnivariateLinearPolyBase::operator=='
10
Returning from 'UnivariateLinearPolyBase::operator=='
11
Returning zero, which participates in a condition later
227 }
228
229 bool isZero() const { return !Value; }
230 bool isNonZero() const { return !isZero(); }
231 explicit operator bool() const { return isNonZero(); }
232 ScalarTy getValue() const { return Value; }
233 ScalarTy getValue(unsigned Dim) const {
234 return Dim == UnivariateDim ? Value : 0;
235 }
236
237 /// Add \p RHS to the value at the univariate dimension.
238 LeafTy getWithIncrement(ScalarTy RHS) const {
239 return static_cast<LeafTy>(
240 UnivariateLinearPolyBase(Value + RHS, UnivariateDim));
241 }
242
243 /// Subtract \p RHS from the value at the univariate dimension.
244 LeafTy getWithDecrement(ScalarTy RHS) const {
245 return static_cast<LeafTy>(
246 UnivariateLinearPolyBase(Value - RHS, UnivariateDim));
247 }
248};
249
250
251//===----------------------------------------------------------------------===//
252// LinearPolySize - base class for fixed- or scalable sizes.
253// ^ ^
254// | |
255// | +----- ElementCount - Leaf class to represent an element count
256// | (vscale x unsigned)
257// |
258// +-------- TypeSize - Leaf class to represent a type size
259// (vscale x uint64_t)
260//===----------------------------------------------------------------------===//
261
262/// LinearPolySize is a base class to represent sizes. It is either
263/// fixed-sized or it is scalable-sized, but it cannot be both.
264template <typename LeafTy>
265class LinearPolySize : public UnivariateLinearPolyBase<LeafTy> {
266 // Make the parent class a friend, so that it can access the protected
267 // conversion/copy-constructor for UnivariatePolyBase<LeafTy> ->
268 // LinearPolySize<LeafTy>.
269 friend class UnivariateLinearPolyBase<LeafTy>;
270
271public:
272 using ScalarTy = typename UnivariateLinearPolyBase<LeafTy>::ScalarTy;
273 enum Dims : unsigned { FixedDim = 0, ScalableDim = 1 };
274
275protected:
276 LinearPolySize(ScalarTy MinVal, Dims D)
277 : UnivariateLinearPolyBase<LeafTy>(MinVal, D) {}
278
279 LinearPolySize(const UnivariateLinearPolyBase<LeafTy> &V)
280 : UnivariateLinearPolyBase<LeafTy>(V) {}
281
282public:
283
284 static LeafTy getFixed(ScalarTy MinVal) {
285 return static_cast<LeafTy>(LinearPolySize(MinVal, FixedDim));
286 }
287 static LeafTy getScalable(ScalarTy MinVal) {
288 return static_cast<LeafTy>(LinearPolySize(MinVal, ScalableDim));
289 }
290 static LeafTy get(ScalarTy MinVal, bool Scalable) {
291 return static_cast<LeafTy>(
292 LinearPolySize(MinVal, Scalable ? ScalableDim : FixedDim));
293 }
294 static LeafTy getNull() { return get(0, false); }
295
296 /// Returns the minimum value this size can represent.
297 ScalarTy getKnownMinValue() const { return this->getValue(); }
298 /// Returns whether the size is scaled by a runtime quantity (vscale).
299 bool isScalable() const { return this->UnivariateDim == ScalableDim; }
300 /// A return value of true indicates we know at compile time that the number
301 /// of elements (vscale * Min) is definitely even. However, returning false
302 /// does not guarantee that the total number of elements is odd.
303 bool isKnownEven() const { return (getKnownMinValue() & 0x1) == 0; }
304 /// This function tells the caller whether the element count is known at
305 /// compile time to be a multiple of the scalar value RHS.
306 bool isKnownMultipleOf(ScalarTy RHS) const {
307 return getKnownMinValue() % RHS == 0;
308 }
309
310 // Return the minimum value with the assumption that the count is exact.
311 // Use in places where a scalable count doesn't make sense (e.g. non-vector
312 // types, or vectors in backends which don't support scalable vectors).
313 ScalarTy getFixedValue() const {
314 assert(!isScalable() &&(static_cast<void> (0))
315 "Request for a fixed element count on a scalable object")(static_cast<void> (0));
316 return getKnownMinValue();
317 }
318
319 // For some cases, size ordering between scalable and fixed size types cannot
320 // be determined at compile time, so such comparisons aren't allowed.
321 //
322 // e.g. <vscale x 2 x i16> could be bigger than <4 x i32> with a runtime
323 // vscale >= 5, equal sized with a vscale of 4, and smaller with
324 // a vscale <= 3.
325 //
326 // All the functions below make use of the fact vscale is always >= 1, which
327 // means that <vscale x 4 x i32> is guaranteed to be >= <4 x i32>, etc.
328
329 static bool isKnownLT(const LinearPolySize &LHS, const LinearPolySize &RHS) {
330 if (!LHS.isScalable() || RHS.isScalable())
331 return LHS.getKnownMinValue() < RHS.getKnownMinValue();
332 return false;
333 }
334
335 static bool isKnownGT(const LinearPolySize &LHS, const LinearPolySize &RHS) {
336 if (LHS.isScalable() || !RHS.isScalable())
337 return LHS.getKnownMinValue() > RHS.getKnownMinValue();
338 return false;
339 }
340
341 static bool isKnownLE(const LinearPolySize &LHS, const LinearPolySize &RHS) {
342 if (!LHS.isScalable() || RHS.isScalable())
343 return LHS.getKnownMinValue() <= RHS.getKnownMinValue();
344 return false;
345 }
346
347 static bool isKnownGE(const LinearPolySize &LHS, const LinearPolySize &RHS) {
348 if (LHS.isScalable() || !RHS.isScalable())
349 return LHS.getKnownMinValue() >= RHS.getKnownMinValue();
350 return false;
351 }
352
353 /// We do not provide the '/' operator here because division for polynomial
354 /// types does not work in the same way as for normal integer types. We can
355 /// only divide the minimum value (or coefficient) by RHS, which is not the
356 /// same as
357 /// (Min * Vscale) / RHS
358 /// The caller is recommended to use this function in combination with
359 /// isKnownMultipleOf(RHS), which lets the caller know if it's possible to
360 /// perform a lossless divide by RHS.
361 LeafTy divideCoefficientBy(ScalarTy RHS) const {
362 return static_cast<LeafTy>(
363 LinearPolySize::get(getKnownMinValue() / RHS, isScalable()));
364 }
365
366 LeafTy coefficientNextPowerOf2() const {
367 return static_cast<LeafTy>(LinearPolySize::get(
368 static_cast<ScalarTy>(llvm::NextPowerOf2(getKnownMinValue())),
369 isScalable()));
370 }
371
372 /// Printing function.
373 void print(raw_ostream &OS) const {
374 if (isScalable())
375 OS << "vscale x ";
376 OS << getKnownMinValue();
377 }
378};
379
380class ElementCount;
381template <> struct LinearPolyBaseTypeTraits<ElementCount> {
382 using ScalarTy = unsigned;
383 static constexpr unsigned Dimensions = 2;
384};
385
386class ElementCount : public LinearPolySize<ElementCount> {
387public:
388 ElementCount() : LinearPolySize(LinearPolySize::getNull()) {}
389
390 ElementCount(const LinearPolySize<ElementCount> &V) : LinearPolySize(V) {}
391
392 /// Counting predicates.
393 ///
394 ///@{ Number of elements..
395 /// Exactly one element.
396 bool isScalar() const { return !isScalable() && getKnownMinValue() == 1; }
397 /// One or more elements.
398 bool isVector() const {
399 return (isScalable() && getKnownMinValue() != 0) || getKnownMinValue() > 1;
400 }
401 ///@}
402};
403
404// This class is used to represent the size of types. If the type is of fixed
405class TypeSize;
406template <> struct LinearPolyBaseTypeTraits<TypeSize> {
407 using ScalarTy = uint64_t;
408 static constexpr unsigned Dimensions = 2;
409};
410
411// TODO: Most functionality in this class will gradually be phased out
412// so it will resemble LinearPolySize as much as possible.
413//
414// TypeSize is used to represent the size of types. If the type is of fixed
415// size, it will represent the exact size. If the type is a scalable vector,
416// it will represent the known minimum size.
417class TypeSize : public LinearPolySize<TypeSize> {
418public:
419 TypeSize(const LinearPolySize<TypeSize> &V) : LinearPolySize(V) {}
420 TypeSize(ScalarTy MinVal, bool IsScalable)
421 : LinearPolySize(LinearPolySize::get(MinVal, IsScalable)) {}
422
423 static TypeSize Fixed(ScalarTy MinVal) { return TypeSize(MinVal, false); }
424 static TypeSize Scalable(ScalarTy MinVal) { return TypeSize(MinVal, true); }
425
426 ScalarTy getFixedSize() const { return getFixedValue(); }
427 ScalarTy getKnownMinSize() const { return getKnownMinValue(); }
428
429 // All code for this class below this point is needed because of the
430 // temporary implicit conversion to uint64_t. The operator overloads are
431 // needed because otherwise the conversion of the parent class
432 // UnivariateLinearPolyBase -> TypeSize is ambiguous.
433 // TODO: Remove the implicit conversion.
434
435 // Casts to a uint64_t if this is a fixed-width size.
436 //
437 // This interface is deprecated and will be removed in a future version
438 // of LLVM in favour of upgrading uses that rely on this implicit conversion
439 // to uint64_t. Calls to functions that return a TypeSize should use the
440 // proper interfaces to TypeSize.
441 // In practice this is mostly calls to MVT/EVT::getSizeInBits().
442 //
443 // To determine how to upgrade the code:
444 //
445 // if (<algorithm works for both scalable and fixed-width vectors>)
446 // use getKnownMinValue()
447 // else if (<algorithm works only for fixed-width vectors>) {
448 // if <algorithm can be adapted for both scalable and fixed-width vectors>
449 // update the algorithm and use getKnownMinValue()
450 // else
451 // bail out early for scalable vectors and use getFixedValue()
452 // }
453 operator ScalarTy() const;
454
455 // Additional operators needed to avoid ambiguous parses
456 // because of the implicit conversion hack.
457 friend TypeSize operator*(const TypeSize &LHS, const int RHS) {
458 return LHS * (ScalarTy)RHS;
459 }
460 friend TypeSize operator*(const TypeSize &LHS, const unsigned RHS) {
461 return LHS * (ScalarTy)RHS;
462 }
463 friend TypeSize operator*(const TypeSize &LHS, const int64_t RHS) {
464 return LHS * (ScalarTy)RHS;
465 }
466 friend TypeSize operator*(const int LHS, const TypeSize &RHS) {
467 return RHS * LHS;
468 }
469 friend TypeSize operator*(const unsigned LHS, const TypeSize &RHS) {
470 return RHS * LHS;
471 }
472 friend TypeSize operator*(const int64_t LHS, const TypeSize &RHS) {
473 return RHS * LHS;
474 }
475 friend TypeSize operator*(const uint64_t LHS, const TypeSize &RHS) {
476 return RHS * LHS;
477 }
478};
479
480//===----------------------------------------------------------------------===//
481// Utilities
482//===----------------------------------------------------------------------===//
483
484/// Returns a TypeSize with a known minimum size that is the next integer
485/// (mod 2**64) that is greater than or equal to \p Value and is a multiple
486/// of \p Align. \p Align must be non-zero.
487///
488/// Similar to the alignTo functions in MathExtras.h
489inline TypeSize alignTo(TypeSize Size, uint64_t Align) {
490 assert(Align != 0u && "Align must be non-zero")(static_cast<void> (0));
491 return {(Size.getKnownMinValue() + Align - 1) / Align * Align,
492 Size.isScalable()};
493}
494
495/// Stream operator function for `LinearPolySize`.
496template <typename LeafTy>
497inline raw_ostream &operator<<(raw_ostream &OS,
498 const LinearPolySize<LeafTy> &PS) {
499 PS.print(OS);
500 return OS;
501}
502
503template <typename T> struct DenseMapInfo;
504template <> struct DenseMapInfo<ElementCount> {
505 static inline ElementCount getEmptyKey() {
506 return ElementCount::getScalable(~0U);
507 }
508 static inline ElementCount getTombstoneKey() {
509 return ElementCount::getFixed(~0U - 1);
510 }
511 static unsigned getHashValue(const ElementCount &EltCnt) {
512 unsigned HashVal = EltCnt.getKnownMinValue() * 37U;
513 if (EltCnt.isScalable())
514 return (HashVal - 1U);
515
516 return HashVal;
517 }
518
519 static bool isEqual(const ElementCount &LHS, const ElementCount &RHS) {
520 return LHS == RHS;
521 }
522};
523
524} // end namespace llvm
525
526#endif // LLVM_SUPPORT_TYPESIZE_H

/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/include/llvm/CodeGen/ValueTypes.h

1//===- CodeGen/ValueTypes.h - Low-Level Target independ. types --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the set of low-level target independent types which various
10// values in the code generator are. This allows the target specific behavior
11// of instructions to be described to target independent passes.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_CODEGEN_VALUETYPES_H
16#define LLVM_CODEGEN_VALUETYPES_H
17
18#include "llvm/Support/Compiler.h"
19#include "llvm/Support/MachineValueType.h"
20#include "llvm/Support/MathExtras.h"
21#include "llvm/Support/TypeSize.h"
22#include "llvm/Support/WithColor.h"
23#include <cassert>
24#include <cstdint>
25#include <string>
26
27namespace llvm {
28
29 class LLVMContext;
30 class Type;
31
32 /// Extended Value Type. Capable of holding value types which are not native
33 /// for any processor (such as the i12345 type), as well as the types an MVT
34 /// can represent.
35 struct EVT {
36 private:
37 MVT V = MVT::INVALID_SIMPLE_VALUE_TYPE;
38 Type *LLVMTy = nullptr;
39
40 public:
41 constexpr EVT() = default;
42 constexpr EVT(MVT::SimpleValueType SVT) : V(SVT) {}
43 constexpr EVT(MVT S) : V(S) {}
44
45 bool operator==(EVT VT) const {
46 return !(*this != VT);
47 }
48 bool operator!=(EVT VT) const {
49 if (V.SimpleTy != VT.V.SimpleTy)
50 return true;
51 if (V.SimpleTy == MVT::INVALID_SIMPLE_VALUE_TYPE)
52 return LLVMTy != VT.LLVMTy;
53 return false;
54 }
55
56 /// Returns the EVT that represents a floating-point type with the given
57 /// number of bits. There are two floating-point types with 128 bits - this
58 /// returns f128 rather than ppcf128.
59 static EVT getFloatingPointVT(unsigned BitWidth) {
60 return MVT::getFloatingPointVT(BitWidth);
61 }
62
63 /// Returns the EVT that represents an integer with the given number of
64 /// bits.
65 static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth) {
66 MVT M = MVT::getIntegerVT(BitWidth);
67 if (M.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE)
68 return M;
69 return getExtendedIntegerVT(Context, BitWidth);
70 }
71
72 /// Returns the EVT that represents a vector NumElements in length, where
73 /// each element is of type VT.
74 static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements,
75 bool IsScalable = false) {
76 MVT M = MVT::getVectorVT(VT.V, NumElements, IsScalable);
77 if (M.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE)
78 return M;
79 return getExtendedVectorVT(Context, VT, NumElements, IsScalable);
80 }
81
82 /// Returns the EVT that represents a vector EC.Min elements in length,
83 /// where each element is of type VT.
84 static EVT getVectorVT(LLVMContext &Context, EVT VT, ElementCount EC) {
85 MVT M = MVT::getVectorVT(VT.V, EC);
86 if (M.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE)
87 return M;
88 return getExtendedVectorVT(Context, VT, EC);
89 }
90
91 /// Return a vector with the same number of elements as this vector, but
92 /// with the element type converted to an integer type with the same
93 /// bitwidth.
94 EVT changeVectorElementTypeToInteger() const {
95 if (isSimple())
96 return getSimpleVT().changeVectorElementTypeToInteger();
97 return changeExtendedVectorElementTypeToInteger();
98 }
99
100 /// Return a VT for a vector type whose attributes match ourselves
101 /// with the exception of the element type that is chosen by the caller.
102 EVT changeVectorElementType(EVT EltVT) const {
103 if (isSimple()) {
104 assert(EltVT.isSimple() &&(static_cast<void> (0))
105 "Can't change simple vector VT to have extended element VT")(static_cast<void> (0));
106 return getSimpleVT().changeVectorElementType(EltVT.getSimpleVT());
107 }
108 return changeExtendedVectorElementType(EltVT);
109 }
110
111 /// Return the type converted to an equivalently sized integer or vector
112 /// with integer element type. Similar to changeVectorElementTypeToInteger,
113 /// but also handles scalars.
114 EVT changeTypeToInteger() {
115 if (isVector())
116 return changeVectorElementTypeToInteger();
117
118 if (isSimple())
119 return getSimpleVT().changeTypeToInteger();
120 return changeExtendedTypeToInteger();
121 }
122
123 /// Test if the given EVT has zero size, this will fail if called on a
124 /// scalable type
125 bool isZeroSized() const {
126 return !isScalableVector() && getSizeInBits() == 0;
127 }
128
129 /// Test if the given EVT is simple (as opposed to being extended).
130 bool isSimple() const {
131 return V.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE;
132 }
133
134 /// Test if the given EVT is extended (as opposed to being simple).
135 bool isExtended() const {
136 return !isSimple();
137 }
138
139 /// Return true if this is a FP or a vector FP type.
140 bool isFloatingPoint() const {
141 return isSimple() ? V.isFloatingPoint() : isExtendedFloatingPoint();
142 }
143
144 /// Return true if this is an integer or a vector integer type.
145 bool isInteger() const {
146 return isSimple() ? V.isInteger() : isExtendedInteger();
147 }
148
149 /// Return true if this is an integer, but not a vector.
150 bool isScalarInteger() const {
151 return isSimple() ? V.isScalarInteger() : isExtendedScalarInteger();
152 }
153
154 /// Return true if this is a vector value type.
155 bool isVector() const {
156 return isSimple() ? V.isVector() : isExtendedVector();
14
'?' condition is false
15
Returning value, which participates in a condition later
157 }
158
159 /// Return true if this is a vector type where the runtime
160 /// length is machine dependent
161 bool isScalableVector() const {
162 return isSimple() ? V.isScalableVector() : isExtendedScalableVector();
163 }
164
165 bool isFixedLengthVector() const {
166 return isSimple() ? V.isFixedLengthVector()
167 : isExtendedFixedLengthVector();
168 }
169
170 /// Return true if this is a 16-bit vector type.
171 bool is16BitVector() const {
172 return isSimple() ? V.is16BitVector() : isExtended16BitVector();
173 }
174
175 /// Return true if this is a 32-bit vector type.
176 bool is32BitVector() const {
177 return isSimple() ? V.is32BitVector() : isExtended32BitVector();
178 }
179
180 /// Return true if this is a 64-bit vector type.
181 bool is64BitVector() const {
182 return isSimple() ? V.is64BitVector() : isExtended64BitVector();
183 }
184
185 /// Return true if this is a 128-bit vector type.
186 bool is128BitVector() const {
187 return isSimple() ? V.is128BitVector() : isExtended128BitVector();
188 }
189
190 /// Return true if this is a 256-bit vector type.
191 bool is256BitVector() const {
192 return isSimple() ? V.is256BitVector() : isExtended256BitVector();
193 }
194
195 /// Return true if this is a 512-bit vector type.
196 bool is512BitVector() const {
197 return isSimple() ? V.is512BitVector() : isExtended512BitVector();
198 }
199
200 /// Return true if this is a 1024-bit vector type.
201 bool is1024BitVector() const {
202 return isSimple() ? V.is1024BitVector() : isExtended1024BitVector();
203 }
204
205 /// Return true if this is a 2048-bit vector type.
206 bool is2048BitVector() const {
207 return isSimple() ? V.is2048BitVector() : isExtended2048BitVector();
208 }
209
210 /// Return true if this is an overloaded type for TableGen.
211 bool isOverloaded() const {
212 return (V==MVT::iAny || V==MVT::fAny || V==MVT::vAny || V==MVT::iPTRAny);
213 }
214
215 /// Return true if the bit size is a multiple of 8.
216 bool isByteSized() const {
217 return !isZeroSized() && getSizeInBits().isKnownMultipleOf(8);
218 }
219
220 /// Return true if the size is a power-of-two number of bytes.
221 bool isRound() const {
222 if (isScalableVector())
223 return false;
224 unsigned BitSize = getSizeInBits();
225 return BitSize >= 8 && !(BitSize & (BitSize - 1));
226 }
227
228 /// Return true if this has the same number of bits as VT.
229 bool bitsEq(EVT VT) const {
230 if (EVT::operator==(VT)) return true;
231 return getSizeInBits() == VT.getSizeInBits();
232 }
233
234 /// Return true if we know at compile time this has more bits than VT.
235 bool knownBitsGT(EVT VT) const {
236 return TypeSize::isKnownGT(getSizeInBits(), VT.getSizeInBits());
237 }
238
239 /// Return true if we know at compile time this has more than or the same
240 /// bits as VT.
241 bool knownBitsGE(EVT VT) const {
242 return TypeSize::isKnownGE(getSizeInBits(), VT.getSizeInBits());
243 }
244
245 /// Return true if we know at compile time this has fewer bits than VT.
246 bool knownBitsLT(EVT VT) const {
247 return TypeSize::isKnownLT(getSizeInBits(), VT.getSizeInBits());
248 }
249
250 /// Return true if we know at compile time this has fewer than or the same
251 /// bits as VT.
252 bool knownBitsLE(EVT VT) const {
253 return TypeSize::isKnownLE(getSizeInBits(), VT.getSizeInBits());
254 }
255
256 /// Return true if this has more bits than VT.
257 bool bitsGT(EVT VT) const {
258 if (EVT::operator==(VT)) return false;
259 assert(isScalableVector() == VT.isScalableVector() &&(static_cast<void> (0))
260 "Comparison between scalable and fixed types")(static_cast<void> (0));
261 return knownBitsGT(VT);
262 }
263
264 /// Return true if this has no less bits than VT.
265 bool bitsGE(EVT VT) const {
266 if (EVT::operator==(VT)) return true;
267 assert(isScalableVector() == VT.isScalableVector() &&(static_cast<void> (0))
268 "Comparison between scalable and fixed types")(static_cast<void> (0));
269 return knownBitsGE(VT);
270 }
271
272 /// Return true if this has less bits than VT.
273 bool bitsLT(EVT VT) const {
274 if (EVT::operator==(VT)) return false;
275 assert(isScalableVector() == VT.isScalableVector() &&(static_cast<void> (0))
276 "Comparison between scalable and fixed types")(static_cast<void> (0));
277 return knownBitsLT(VT);
278 }
279
280 /// Return true if this has no more bits than VT.
281 bool bitsLE(EVT VT) const {
282 if (EVT::operator==(VT)) return true;
283 assert(isScalableVector() == VT.isScalableVector() &&(static_cast<void> (0))
284 "Comparison between scalable and fixed types")(static_cast<void> (0));
285 return knownBitsLE(VT);
286 }
287
288 /// Return the SimpleValueType held in the specified simple EVT.
289 MVT getSimpleVT() const {
290 assert(isSimple() && "Expected a SimpleValueType!")(static_cast<void> (0));
291 return V;
292 }
293
294 /// If this is a vector type, return the element type, otherwise return
295 /// this.
296 EVT getScalarType() const {
297 return isVector() ? getVectorElementType() : *this;
298 }
299
300 /// Given a vector type, return the type of each element.
301 EVT getVectorElementType() const {
302 assert(isVector() && "Invalid vector type!")(static_cast<void> (0));
303 if (isSimple())
304 return V.getVectorElementType();
305 return getExtendedVectorElementType();
306 }
307
308 /// Given a vector type, return the number of elements it contains.
309 unsigned getVectorNumElements() const {
310 assert(isVector() && "Invalid vector type!")(static_cast<void> (0));
311
312 if (isScalableVector())
313 llvm::reportInvalidSizeRequest(
314 "Possible incorrect use of EVT::getVectorNumElements() for "
315 "scalable vector. Scalable flag may be dropped, use "
316 "EVT::getVectorElementCount() instead");
317
318 return isSimple() ? V.getVectorNumElements()
319 : getExtendedVectorNumElements();
320 }
321
322 // Given a (possibly scalable) vector type, return the ElementCount
323 ElementCount getVectorElementCount() const {
324 assert((isVector()) && "Invalid vector type!")(static_cast<void> (0));
325 if (isSimple())
326 return V.getVectorElementCount();
327
328 return getExtendedVectorElementCount();
329 }
330
331 /// Given a vector type, return the minimum number of elements it contains.
332 unsigned getVectorMinNumElements() const {
333 return getVectorElementCount().getKnownMinValue();
334 }
335
336 /// Return the size of the specified value type in bits.
337 ///
338 /// If the value type is a scalable vector type, the scalable property will
339 /// be set and the runtime size will be a positive integer multiple of the
340 /// base size.
341 TypeSize getSizeInBits() const {
342 if (isSimple())
343 return V.getSizeInBits();
344 return getExtendedSizeInBits();
345 }
346
347 /// Return the size of the specified fixed width value type in bits. The
348 /// function will assert if the type is scalable.
349 uint64_t getFixedSizeInBits() const {
350 return getSizeInBits().getFixedSize();
351 }
352
353 uint64_t getScalarSizeInBits() const {
354 return getScalarType().getSizeInBits().getFixedSize();
355 }
356
357 /// Return the number of bytes overwritten by a store of the specified value
358 /// type.
359 ///
360 /// If the value type is a scalable vector type, the scalable property will
361 /// be set and the runtime size will be a positive integer multiple of the
362 /// base size.
363 TypeSize getStoreSize() const {
364 TypeSize BaseSize = getSizeInBits();
365 return {(BaseSize.getKnownMinSize() + 7) / 8, BaseSize.isScalable()};
366 }
367
368 /// Return the number of bits overwritten by a store of the specified value
369 /// type.
370 ///
371 /// If the value type is a scalable vector type, the scalable property will
372 /// be set and the runtime size will be a positive integer multiple of the
373 /// base size.
374 TypeSize getStoreSizeInBits() const {
375 return getStoreSize() * 8;
376 }
377
378 /// Rounds the bit-width of the given integer EVT up to the nearest power of
379 /// two (and at least to eight), and returns the integer EVT with that
380 /// number of bits.
381 EVT getRoundIntegerType(LLVMContext &Context) const {
382 assert(isInteger() && !isVector() && "Invalid integer type!")(static_cast<void> (0));
383 unsigned BitWidth = getSizeInBits();
384 if (BitWidth <= 8)
385 return EVT(MVT::i8);
386 return getIntegerVT(Context, 1 << Log2_32_Ceil(BitWidth));
387 }
388
389 /// Finds the smallest simple value type that is greater than or equal to
390 /// half the width of this EVT. If no simple value type can be found, an
391 /// extended integer value type of half the size (rounded up) is returned.
392 EVT getHalfSizedIntegerVT(LLVMContext &Context) const {
393 assert(isInteger() && !isVector() && "Invalid integer type!")(static_cast<void> (0));
394 unsigned EVTSize = getSizeInBits();
395 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
396 IntVT <= MVT::LAST_INTEGER_VALUETYPE; ++IntVT) {
397 EVT HalfVT = EVT((MVT::SimpleValueType)IntVT);
398 if (HalfVT.getSizeInBits() * 2 >= EVTSize)
399 return HalfVT;
400 }
401 return getIntegerVT(Context, (EVTSize + 1) / 2);
402 }
403
404 /// Return a VT for an integer vector type with the size of the
405 /// elements doubled. The typed returned may be an extended type.
406 EVT widenIntegerVectorElementType(LLVMContext &Context) const {
407 EVT EltVT = getVectorElementType();
408 EltVT = EVT::getIntegerVT(Context, 2 * EltVT.getSizeInBits());
409 return EVT::getVectorVT(Context, EltVT, getVectorElementCount());
410 }
411
412 // Return a VT for a vector type with the same element type but
413 // half the number of elements. The type returned may be an
414 // extended type.
415 EVT getHalfNumVectorElementsVT(LLVMContext &Context) const {
416 EVT EltVT = getVectorElementType();
417 auto EltCnt = getVectorElementCount();
418 assert(EltCnt.isKnownEven() && "Splitting vector, but not in half!")(static_cast<void> (0));
419 return EVT::getVectorVT(Context, EltVT, EltCnt.divideCoefficientBy(2));
420 }
421
422 // Return a VT for a vector type with the same element type but
423 // double the number of elements. The type returned may be an
424 // extended type.
425 EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const {
426 EVT EltVT = getVectorElementType();
427 auto EltCnt = getVectorElementCount();
428 return EVT::getVectorVT(Context, EltVT, EltCnt * 2);
429 }
430
431 /// Returns true if the given vector is a power of 2.
432 bool isPow2VectorType() const {
433 unsigned NElts = getVectorMinNumElements();
434 return !(NElts & (NElts - 1));
435 }
436
437 /// Widens the length of the given vector EVT up to the nearest power of 2
438 /// and returns that type.
439 EVT getPow2VectorType(LLVMContext &Context) const {
440 if (!isPow2VectorType()) {
441 ElementCount NElts = getVectorElementCount();
442 unsigned NewMinCount = 1 << Log2_32_Ceil(NElts.getKnownMinValue());
443 NElts = ElementCount::get(NewMinCount, NElts.isScalable());
444 return EVT::getVectorVT(Context, getVectorElementType(), NElts);
445 }
446 else {
447 return *this;
448 }
449 }
450
451 /// This function returns value type as a string, e.g. "i32".
452 std::string getEVTString() const;
453
454 /// This method returns an LLVM type corresponding to the specified EVT.
455 /// For integer types, this returns an unsigned type. Note that this will
456 /// abort for types that cannot be represented.
457 Type *getTypeForEVT(LLVMContext &Context) const;
458
459 /// Return the value type corresponding to the specified type.
460 /// This returns all pointers as iPTR. If HandleUnknown is true, unknown
461 /// types are returned as Other, otherwise they are invalid.
462 static EVT getEVT(Type *Ty, bool HandleUnknown = false);
463
464 intptr_t getRawBits() const {
465 if (isSimple())
466 return V.SimpleTy;
467 else
468 return (intptr_t)(LLVMTy);
469 }
470
471 /// A meaningless but well-behaved order, useful for constructing
472 /// containers.
473 struct compareRawBits {
474 bool operator()(EVT L, EVT R) const {
475 if (L.V.SimpleTy == R.V.SimpleTy)
476 return L.LLVMTy < R.LLVMTy;
477 else
478 return L.V.SimpleTy < R.V.SimpleTy;
479 }
480 };
481
482 private:
483 // Methods for handling the Extended-type case in functions above.
484 // These are all out-of-line to prevent users of this header file
485 // from having a dependency on Type.h.
486 EVT changeExtendedTypeToInteger() const;
487 EVT changeExtendedVectorElementType(EVT EltVT) const;
488 EVT changeExtendedVectorElementTypeToInteger() const;
489 static EVT getExtendedIntegerVT(LLVMContext &C, unsigned BitWidth);
490 static EVT getExtendedVectorVT(LLVMContext &C, EVT VT, unsigned NumElements,
491 bool IsScalable);
492 static EVT getExtendedVectorVT(LLVMContext &Context, EVT VT,
493 ElementCount EC);
494 bool isExtendedFloatingPoint() const LLVM_READONLY__attribute__((__pure__));
495 bool isExtendedInteger() const LLVM_READONLY__attribute__((__pure__));
496 bool isExtendedScalarInteger() const LLVM_READONLY__attribute__((__pure__));
497 bool isExtendedVector() const LLVM_READONLY__attribute__((__pure__));
498 bool isExtended16BitVector() const LLVM_READONLY__attribute__((__pure__));
499 bool isExtended32BitVector() const LLVM_READONLY__attribute__((__pure__));
500 bool isExtended64BitVector() const LLVM_READONLY__attribute__((__pure__));
501 bool isExtended128BitVector() const LLVM_READONLY__attribute__((__pure__));
502 bool isExtended256BitVector() const LLVM_READONLY__attribute__((__pure__));
503 bool isExtended512BitVector() const LLVM_READONLY__attribute__((__pure__));
504 bool isExtended1024BitVector() const LLVM_READONLY__attribute__((__pure__));
505 bool isExtended2048BitVector() const LLVM_READONLY__attribute__((__pure__));
506 bool isExtendedFixedLengthVector() const LLVM_READONLY__attribute__((__pure__));
507 bool isExtendedScalableVector() const LLVM_READONLY__attribute__((__pure__));
508 EVT getExtendedVectorElementType() const;
509 unsigned getExtendedVectorNumElements() const LLVM_READONLY__attribute__((__pure__));
510 ElementCount getExtendedVectorElementCount() const LLVM_READONLY__attribute__((__pure__));
511 TypeSize getExtendedSizeInBits() const LLVM_READONLY__attribute__((__pure__));
512 };
513
514} // end namespace llvm
515
516#endif // LLVM_CODEGEN_VALUETYPES_H

/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/include/llvm/Support/MathExtras.h

1//===-- llvm/Support/MathExtras.h - Useful math functions -------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains some functions that are useful for math stuff.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_SUPPORT_MATHEXTRAS_H
14#define LLVM_SUPPORT_MATHEXTRAS_H
15
16#include "llvm/Support/Compiler.h"
17#include <cassert>
18#include <climits>
19#include <cmath>
20#include <cstdint>
21#include <cstring>
22#include <limits>
23#include <type_traits>
24
25#ifdef __ANDROID_NDK__
26#include <android/api-level.h>
27#endif
28
29#ifdef _MSC_VER
30// Declare these intrinsics manually rather including intrin.h. It's very
31// expensive, and MathExtras.h is popular.
32// #include <intrin.h>
33extern "C" {
34unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask);
35unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask);
36unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask);
37unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask);
38}
39#endif
40
41namespace llvm {
42
43/// The behavior an operation has on an input of 0.
44enum ZeroBehavior {
45 /// The returned value is undefined.
46 ZB_Undefined,
47 /// The returned value is numeric_limits<T>::max()
48 ZB_Max,
49 /// The returned value is numeric_limits<T>::digits
50 ZB_Width
51};
52
53/// Mathematical constants.
54namespace numbers {
55// TODO: Track C++20 std::numbers.
56// TODO: Favor using the hexadecimal FP constants (requires C++17).
57constexpr double e = 2.7182818284590452354, // (0x1.5bf0a8b145749P+1) https://oeis.org/A001113
58 egamma = .57721566490153286061, // (0x1.2788cfc6fb619P-1) https://oeis.org/A001620
59 ln2 = .69314718055994530942, // (0x1.62e42fefa39efP-1) https://oeis.org/A002162
60 ln10 = 2.3025850929940456840, // (0x1.24bb1bbb55516P+1) https://oeis.org/A002392
61 log2e = 1.4426950408889634074, // (0x1.71547652b82feP+0)
62 log10e = .43429448190325182765, // (0x1.bcb7b1526e50eP-2)
63 pi = 3.1415926535897932385, // (0x1.921fb54442d18P+1) https://oeis.org/A000796
64 inv_pi = .31830988618379067154, // (0x1.45f306bc9c883P-2) https://oeis.org/A049541
65 sqrtpi = 1.7724538509055160273, // (0x1.c5bf891b4ef6bP+0) https://oeis.org/A002161
66 inv_sqrtpi = .56418958354775628695, // (0x1.20dd750429b6dP-1) https://oeis.org/A087197
67 sqrt2 = 1.4142135623730950488, // (0x1.6a09e667f3bcdP+0) https://oeis.org/A00219
68 inv_sqrt2 = .70710678118654752440, // (0x1.6a09e667f3bcdP-1)
69 sqrt3 = 1.7320508075688772935, // (0x1.bb67ae8584caaP+0) https://oeis.org/A002194
70 inv_sqrt3 = .57735026918962576451, // (0x1.279a74590331cP-1)
71 phi = 1.6180339887498948482; // (0x1.9e3779b97f4a8P+0) https://oeis.org/A001622
72constexpr float ef = 2.71828183F, // (0x1.5bf0a8P+1) https://oeis.org/A001113
73 egammaf = .577215665F, // (0x1.2788d0P-1) https://oeis.org/A001620
74 ln2f = .693147181F, // (0x1.62e430P-1) https://oeis.org/A002162
75 ln10f = 2.30258509F, // (0x1.26bb1cP+1) https://oeis.org/A002392
76 log2ef = 1.44269504F, // (0x1.715476P+0)
77 log10ef = .434294482F, // (0x1.bcb7b2P-2)
78 pif = 3.14159265F, // (0x1.921fb6P+1) https://oeis.org/A000796
79 inv_pif = .318309886F, // (0x1.45f306P-2) https://oeis.org/A049541
80 sqrtpif = 1.77245385F, // (0x1.c5bf8aP+0) https://oeis.org/A002161
81 inv_sqrtpif = .564189584F, // (0x1.20dd76P-1) https://oeis.org/A087197
82 sqrt2f = 1.41421356F, // (0x1.6a09e6P+0) https://oeis.org/A002193
83 inv_sqrt2f = .707106781F, // (0x1.6a09e6P-1)
84 sqrt3f = 1.73205081F, // (0x1.bb67aeP+0) https://oeis.org/A002194
85 inv_sqrt3f = .577350269F, // (0x1.279a74P-1)
86 phif = 1.61803399F; // (0x1.9e377aP+0) https://oeis.org/A001622
87} // namespace numbers
88
89namespace detail {
90template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter {
91 static unsigned count(T Val, ZeroBehavior) {
92 if (!Val)
93 return std::numeric_limits<T>::digits;
94 if (Val & 0x1)
95 return 0;
96
97 // Bisection method.
98 unsigned ZeroBits = 0;
99 T Shift = std::numeric_limits<T>::digits >> 1;
100 T Mask = std::numeric_limits<T>::max() >> Shift;
101 while (Shift) {
102 if ((Val & Mask) == 0) {
103 Val >>= Shift;
104 ZeroBits |= Shift;
105 }
106 Shift >>= 1;
107 Mask >>= Shift;
108 }
109 return ZeroBits;
110 }
111};
112
113#if defined(__GNUC__4) || defined(_MSC_VER)
114template <typename T> struct TrailingZerosCounter<T, 4> {
115 static unsigned count(T Val, ZeroBehavior ZB) {
116 if (ZB != ZB_Undefined && Val == 0)
117 return 32;
118
119#if __has_builtin(__builtin_ctz)1 || defined(__GNUC__4)
120 return __builtin_ctz(Val);
121#elif defined(_MSC_VER)
122 unsigned long Index;
123 _BitScanForward(&Index, Val);
124 return Index;
125#endif
126 }
127};
128
129#if !defined(_MSC_VER) || defined(_M_X64)
130template <typename T> struct TrailingZerosCounter<T, 8> {
131 static unsigned count(T Val, ZeroBehavior ZB) {
132 if (ZB != ZB_Undefined && Val == 0)
133 return 64;
134
135#if __has_builtin(__builtin_ctzll)1 || defined(__GNUC__4)
136 return __builtin_ctzll(Val);
137#elif defined(_MSC_VER)
138 unsigned long Index;
139 _BitScanForward64(&Index, Val);
140 return Index;
141#endif
142 }
143};
144#endif
145#endif
146} // namespace detail
147
148/// Count number of 0's from the least significant bit to the most
149/// stopping at the first 1.
150///
151/// Only unsigned integral types are allowed.
152///
153/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
154/// valid arguments.
155template <typename T>
156unsigned countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
157 static_assert(std::numeric_limits<T>::is_integer &&
158 !std::numeric_limits<T>::is_signed,
159 "Only unsigned integral types are allowed.");
160 return llvm::detail::TrailingZerosCounter<T, sizeof(T)>::count(Val, ZB);
161}
162
163namespace detail {
164template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter {
165 static unsigned count(T Val, ZeroBehavior) {
166 if (!Val)
167 return std::numeric_limits<T>::digits;
168
169 // Bisection method.
170 unsigned ZeroBits = 0;
171 for (T Shift = std::numeric_limits<T>::digits >> 1; Shift; Shift >>= 1) {
172 T Tmp = Val >> Shift;
173 if (Tmp)
174 Val = Tmp;
175 else
176 ZeroBits |= Shift;
177 }
178 return ZeroBits;
179 }
180};
181
182#if defined(__GNUC__4) || defined(_MSC_VER)
183template <typename T> struct LeadingZerosCounter<T, 4> {
184 static unsigned count(T Val, ZeroBehavior ZB) {
185 if (ZB != ZB_Undefined && Val == 0)
186 return 32;
187
188#if __has_builtin(__builtin_clz)1 || defined(__GNUC__4)
189 return __builtin_clz(Val);
190#elif defined(_MSC_VER)
191 unsigned long Index;
192 _BitScanReverse(&Index, Val);
193 return Index ^ 31;
194#endif
195 }
196};
197
198#if !defined(_MSC_VER) || defined(_M_X64)
199template <typename T> struct LeadingZerosCounter<T, 8> {
200 static unsigned count(T Val, ZeroBehavior ZB) {
201 if (ZB != ZB_Undefined && Val == 0)
202 return 64;
203
204#if __has_builtin(__builtin_clzll)1 || defined(__GNUC__4)
205 return __builtin_clzll(Val);
206#elif defined(_MSC_VER)
207 unsigned long Index;
208 _BitScanReverse64(&Index, Val);
209 return Index ^ 63;
210#endif
211 }
212};
213#endif
214#endif
215} // namespace detail
216
217/// Count number of 0's from the most significant bit to the least
218/// stopping at the first 1.
219///
220/// Only unsigned integral types are allowed.
221///
222/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
223/// valid arguments.
224template <typename T>
225unsigned countLeadingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
226 static_assert(std::numeric_limits<T>::is_integer &&
227 !std::numeric_limits<T>::is_signed,
228 "Only unsigned integral types are allowed.");
229 return llvm::detail::LeadingZerosCounter<T, sizeof(T)>::count(Val, ZB);
230}
231
232/// Get the index of the first set bit starting from the least
233/// significant bit.
234///
235/// Only unsigned integral types are allowed.
236///
237/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
238/// valid arguments.
239template <typename T> T findFirstSet(T Val, ZeroBehavior ZB = ZB_Max) {
240 if (ZB == ZB_Max && Val == 0)
241 return std::numeric_limits<T>::max();
242
243 return countTrailingZeros(Val, ZB_Undefined);
244}
245
246/// Create a bitmask with the N right-most bits set to 1, and all other
247/// bits set to 0. Only unsigned types are allowed.
248template <typename T> T maskTrailingOnes(unsigned N) {
249 static_assert(std::is_unsigned<T>::value, "Invalid type!");
250 const unsigned Bits = CHAR_BIT8 * sizeof(T);
251 assert(N <= Bits && "Invalid bit index")(static_cast<void> (0));
252 return N == 0 ? 0 : (T(-1) >> (Bits - N));
253}
254
255/// Create a bitmask with the N left-most bits set to 1, and all other
256/// bits set to 0. Only unsigned types are allowed.
257template <typename T> T maskLeadingOnes(unsigned N) {
258 return ~maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
259}
260
261/// Create a bitmask with the N right-most bits set to 0, and all other
262/// bits set to 1. Only unsigned types are allowed.
263template <typename T> T maskTrailingZeros(unsigned N) {
264 return maskLeadingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
265}
266
267/// Create a bitmask with the N left-most bits set to 0, and all other
268/// bits set to 1. Only unsigned types are allowed.
269template <typename T> T maskLeadingZeros(unsigned N) {
270 return maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
271}
272
273/// Get the index of the last set bit starting from the least
274/// significant bit.
275///
276/// Only unsigned integral types are allowed.
277///
278/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
279/// valid arguments.
280template <typename T> T findLastSet(T Val, ZeroBehavior ZB = ZB_Max) {
281 if (ZB == ZB_Max && Val == 0)
282 return std::numeric_limits<T>::max();
283
284 // Use ^ instead of - because both gcc and llvm can remove the associated ^
285 // in the __builtin_clz intrinsic on x86.
286 return countLeadingZeros(Val, ZB_Undefined) ^
287 (std::numeric_limits<T>::digits - 1);
288}
289
290/// Macro compressed bit reversal table for 256 bits.
291///
292/// http://graphics.stanford.edu/~seander/bithacks.html#BitReverseTable
293static const unsigned char BitReverseTable256[256] = {
294#define R2(n) n, n + 2 * 64, n + 1 * 64, n + 3 * 64
295#define R4(n) R2(n), R2(n + 2 * 16), R2(n + 1 * 16), R2(n + 3 * 16)
296#define R6(n) R4(n), R4(n + 2 * 4), R4(n + 1 * 4), R4(n + 3 * 4)
297 R6(0), R6(2), R6(1), R6(3)
298#undef R2
299#undef R4
300#undef R6
301};
302
303/// Reverse the bits in \p Val.
304template <typename T>
305T reverseBits(T Val) {
306 unsigned char in[sizeof(Val)];
307 unsigned char out[sizeof(Val)];
308 std::memcpy(in, &Val, sizeof(Val));
309 for (unsigned i = 0; i < sizeof(Val); ++i)
310 out[(sizeof(Val) - i) - 1] = BitReverseTable256[in[i]];
311 std::memcpy(&Val, out, sizeof(Val));
312 return Val;
313}
314
315#if __has_builtin(__builtin_bitreverse8)1
316template<>
317inline uint8_t reverseBits<uint8_t>(uint8_t Val) {
318 return __builtin_bitreverse8(Val);
319}
320#endif
321
322#if __has_builtin(__builtin_bitreverse16)1
323template<>
324inline uint16_t reverseBits<uint16_t>(uint16_t Val) {
325 return __builtin_bitreverse16(Val);
326}
327#endif
328
329#if __has_builtin(__builtin_bitreverse32)1
330template<>
331inline uint32_t reverseBits<uint32_t>(uint32_t Val) {
332 return __builtin_bitreverse32(Val);
333}
334#endif
335
336#if __has_builtin(__builtin_bitreverse64)1
337template<>
338inline uint64_t reverseBits<uint64_t>(uint64_t Val) {
339 return __builtin_bitreverse64(Val);
340}
341#endif
342
343// NOTE: The following support functions use the _32/_64 extensions instead of
344// type overloading so that signed and unsigned integers can be used without
345// ambiguity.
346
347/// Return the high 32 bits of a 64 bit value.
348constexpr inline uint32_t Hi_32(uint64_t Value) {
349 return static_cast<uint32_t>(Value >> 32);
350}
351
352/// Return the low 32 bits of a 64 bit value.
353constexpr inline uint32_t Lo_32(uint64_t Value) {
354 return static_cast<uint32_t>(Value);
355}
356
357/// Make a 64-bit integer from a high / low pair of 32-bit integers.
358constexpr inline uint64_t Make_64(uint32_t High, uint32_t Low) {
359 return ((uint64_t)High << 32) | (uint64_t)Low;
360}
361
362/// Checks if an integer fits into the given bit width.
363template <unsigned N> constexpr inline bool isInt(int64_t x) {
364 return N >= 64 || (-(INT64_C(1)1L<<(N-1)) <= x && x < (INT64_C(1)1L<<(N-1)));
365}
366// Template specializations to get better code for common cases.
367template <> constexpr inline bool isInt<8>(int64_t x) {
368 return static_cast<int8_t>(x) == x;
369}
370template <> constexpr inline bool isInt<16>(int64_t x) {
371 return static_cast<int16_t>(x) == x;
372}
373template <> constexpr inline bool isInt<32>(int64_t x) {
374 return static_cast<int32_t>(x) == x;
375}
376
377/// Checks if a signed integer is an N bit number shifted left by S.
378template <unsigned N, unsigned S>
379constexpr inline bool isShiftedInt(int64_t x) {
380 static_assert(
381 N > 0, "isShiftedInt<0> doesn't make sense (refers to a 0-bit number.");
382 static_assert(N + S <= 64, "isShiftedInt<N, S> with N + S > 64 is too wide.");
383 return isInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
384}
385
386/// Checks if an unsigned integer fits into the given bit width.
387///
388/// This is written as two functions rather than as simply
389///
390/// return N >= 64 || X < (UINT64_C(1) << N);
391///
392/// to keep MSVC from (incorrectly) warning on isUInt<64> that we're shifting
393/// left too many places.
394template <unsigned N>
395constexpr inline std::enable_if_t<(N < 64), bool> isUInt(uint64_t X) {
396 static_assert(N > 0, "isUInt<0> doesn't make sense");
397 return X < (UINT64_C(1)1UL << (N));
398}
399template <unsigned N>
400constexpr inline std::enable_if_t<N >= 64, bool> isUInt(uint64_t) {
401 return true;
402}
403
404// Template specializations to get better code for common cases.
405template <> constexpr inline bool isUInt<8>(uint64_t x) {
406 return static_cast<uint8_t>(x) == x;
407}
408template <> constexpr inline bool isUInt<16>(uint64_t x) {
409 return static_cast<uint16_t>(x) == x;
410}
411template <> constexpr inline bool isUInt<32>(uint64_t x) {
412 return static_cast<uint32_t>(x) == x;
413}
414
415/// Checks if a unsigned integer is an N bit number shifted left by S.
416template <unsigned N, unsigned S>
417constexpr inline bool isShiftedUInt(uint64_t x) {
418 static_assert(
419 N > 0, "isShiftedUInt<0> doesn't make sense (refers to a 0-bit number)");
420 static_assert(N + S <= 64,
421 "isShiftedUInt<N, S> with N + S > 64 is too wide.");
422 // Per the two static_asserts above, S must be strictly less than 64. So
423 // 1 << S is not undefined behavior.
424 return isUInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
425}
426
427/// Gets the maximum value for a N-bit unsigned integer.
428inline uint64_t maxUIntN(uint64_t N) {
429 assert(N > 0 && N <= 64 && "integer width out of range")(static_cast<void> (0));
430
431 // uint64_t(1) << 64 is undefined behavior, so we can't do
432 // (uint64_t(1) << N) - 1
433 // without checking first that N != 64. But this works and doesn't have a
434 // branch.
435 return UINT64_MAX(18446744073709551615UL) >> (64 - N);
436}
437
438/// Gets the minimum value for a N-bit signed integer.
439inline int64_t minIntN(int64_t N) {
440 assert(N > 0 && N <= 64 && "integer width out of range")(static_cast<void> (0));
441
442 return UINT64_C(1)1UL + ~(UINT64_C(1)1UL << (N - 1));
443}
444
445/// Gets the maximum value for a N-bit signed integer.
446inline int64_t maxIntN(int64_t N) {
447 assert(N > 0 && N <= 64 && "integer width out of range")(static_cast<void> (0));
448
449 // This relies on two's complement wraparound when N == 64, so we convert to
450 // int64_t only at the very end to avoid UB.
451 return (UINT64_C(1)1UL << (N - 1)) - 1;
452}
453
454/// Checks if an unsigned integer fits into the given (dynamic) bit width.
455inline bool isUIntN(unsigned N, uint64_t x) {
456 return N >= 64 || x <= maxUIntN(N);
457}
458
459/// Checks if an signed integer fits into the given (dynamic) bit width.
460inline bool isIntN(unsigned N, int64_t x) {
461 return N >= 64 || (minIntN(N) <= x && x <= maxIntN(N));
462}
463
464/// Return true if the argument is a non-empty sequence of ones starting at the
465/// least significant bit with the remainder zero (32 bit version).
466/// Ex. isMask_32(0x0000FFFFU) == true.
467constexpr inline bool isMask_32(uint32_t Value) {
468 return Value && ((Value + 1) & Value) == 0;
469}
470
471/// Return true if the argument is a non-empty sequence of ones starting at the
472/// least significant bit with the remainder zero (64 bit version).
473constexpr inline bool isMask_64(uint64_t Value) {
474 return Value && ((Value + 1) & Value) == 0;
475}
476
477/// Return true if the argument contains a non-empty sequence of ones with the
478/// remainder zero (32 bit version.) Ex. isShiftedMask_32(0x0000FF00U) == true.
479constexpr inline bool isShiftedMask_32(uint32_t Value) {
480 return Value && isMask_32((Value - 1) | Value);
481}
482
483/// Return true if the argument contains a non-empty sequence of ones with the
484/// remainder zero (64 bit version.)
485constexpr inline bool isShiftedMask_64(uint64_t Value) {
486 return Value && isMask_64((Value - 1) | Value);
487}
488
489/// Return true if the argument is a power of two > 0.
490/// Ex. isPowerOf2_32(0x00100000U) == true (32 bit edition.)
491constexpr inline bool isPowerOf2_32(uint32_t Value) {
492 return Value && !(Value & (Value - 1));
493}
494
495/// Return true if the argument is a power of two > 0 (64 bit edition.)
496constexpr inline bool isPowerOf2_64(uint64_t Value) {
497 return Value && !(Value & (Value - 1));
19
Assuming 'Value' is 0
20
Returning zero, which participates in a condition later
498}
499
500/// Count the number of ones from the most significant bit to the first
501/// zero bit.
502///
503/// Ex. countLeadingOnes(0xFF0FFF00) == 8.
504/// Only unsigned integral types are allowed.
505///
506/// \param ZB the behavior on an input of all ones. Only ZB_Width and
507/// ZB_Undefined are valid arguments.
508template <typename T>
509unsigned countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
510 static_assert(std::numeric_limits<T>::is_integer &&
511 !std::numeric_limits<T>::is_signed,
512 "Only unsigned integral types are allowed.");
513 return countLeadingZeros<T>(~Value, ZB);
514}
515
516/// Count the number of ones from the least significant bit to the first
517/// zero bit.
518///
519/// Ex. countTrailingOnes(0x00FF00FF) == 8.
520/// Only unsigned integral types are allowed.
521///
522/// \param ZB the behavior on an input of all ones. Only ZB_Width and
523/// ZB_Undefined are valid arguments.
524template <typename T>
525unsigned countTrailingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
526 static_assert(std::numeric_limits<T>::is_integer &&
527 !std::numeric_limits<T>::is_signed,
528 "Only unsigned integral types are allowed.");
529 return countTrailingZeros<T>(~Value, ZB);
530}
531
532namespace detail {
533template <typename T, std::size_t SizeOfT> struct PopulationCounter {
534 static unsigned count(T Value) {
535 // Generic version, forward to 32 bits.
536 static_assert(SizeOfT <= 4, "Not implemented!");
537#if defined(__GNUC__4)
538 return __builtin_popcount(Value);
539#else
540 uint32_t v = Value;
541 v = v - ((v >> 1) & 0x55555555);
542 v = (v & 0x33333333) + ((v >> 2) & 0x33333333);
543 return ((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24;
544#endif
545 }
546};
547
548template <typename T> struct PopulationCounter<T, 8> {
549 static unsigned count(T Value) {
550#if defined(__GNUC__4)
551 return __builtin_popcountll(Value);
552#else
553 uint64_t v = Value;
554 v = v - ((v >> 1) & 0x5555555555555555ULL);
555 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
556 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
557 return unsigned((uint64_t)(v * 0x0101010101010101ULL) >> 56);
558#endif
559 }
560};
561} // namespace detail
562
563/// Count the number of set bits in a value.
564/// Ex. countPopulation(0xF000F000) = 8
565/// Returns 0 if the word is zero.
566template <typename T>
567inline unsigned countPopulation(T Value) {
568 static_assert(std::numeric_limits<T>::is_integer &&
569 !std::numeric_limits<T>::is_signed,
570 "Only unsigned integral types are allowed.");
571 return detail::PopulationCounter<T, sizeof(T)>::count(Value);
572}
573
574/// Compile time Log2.
575/// Valid only for positive powers of two.
576template <size_t kValue> constexpr inline size_t CTLog2() {
577 static_assert(kValue > 0 && llvm::isPowerOf2_64(kValue),
578 "Value is not a valid power of 2");
579 return 1 + CTLog2<kValue / 2>();
580}
581
582template <> constexpr inline size_t CTLog2<1>() { return 0; }
583
584/// Return the log base 2 of the specified value.
585inline double Log2(double Value) {
586#if defined(__ANDROID_API__) && __ANDROID_API__ < 18
587 return __builtin_log(Value) / __builtin_log(2.0);
588#else
589 return log2(Value);
590#endif
591}
592
593/// Return the floor log base 2 of the specified value, -1 if the value is zero.
594/// (32 bit edition.)
595/// Ex. Log2_32(32) == 5, Log2_32(1) == 0, Log2_32(0) == -1, Log2_32(6) == 2
596inline unsigned Log2_32(uint32_t Value) {
597 return 31 - countLeadingZeros(Value);
24
Returning the value 4294967295
598}
599
600/// Return the floor log base 2 of the specified value, -1 if the value is zero.
601/// (64 bit edition.)
602inline unsigned Log2_64(uint64_t Value) {
603 return 63 - countLeadingZeros(Value);
604}
605
606/// Return the ceil log base 2 of the specified value, 32 if the value is zero.
607/// (32 bit edition).
608/// Ex. Log2_32_Ceil(32) == 5, Log2_32_Ceil(1) == 0, Log2_32_Ceil(6) == 3
609inline unsigned Log2_32_Ceil(uint32_t Value) {
610 return 32 - countLeadingZeros(Value - 1);
611}
612
613/// Return the ceil log base 2 of the specified value, 64 if the value is zero.
614/// (64 bit edition.)
615inline unsigned Log2_64_Ceil(uint64_t Value) {
616 return 64 - countLeadingZeros(Value - 1);
617}
618
619/// Return the greatest common divisor of the values using Euclid's algorithm.
620template <typename T>
621inline T greatestCommonDivisor(T A, T B) {
622 while (B) {
623 T Tmp = B;
624 B = A % B;
625 A = Tmp;
626 }
627 return A;
628}
629
630inline uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B) {
631 return greatestCommonDivisor<uint64_t>(A, B);
632}
633
634/// This function takes a 64-bit integer and returns the bit equivalent double.
635inline double BitsToDouble(uint64_t Bits) {
636 double D;
637 static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
638 memcpy(&D, &Bits, sizeof(Bits));
639 return D;
640}
641
642/// This function takes a 32-bit integer and returns the bit equivalent float.
643inline float BitsToFloat(uint32_t Bits) {
644 float F;
645 static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
646 memcpy(&F, &Bits, sizeof(Bits));
647 return F;
648}
649
650/// This function takes a double and returns the bit equivalent 64-bit integer.
651/// Note that copying doubles around changes the bits of NaNs on some hosts,
652/// notably x86, so this routine cannot be used if these bits are needed.
653inline uint64_t DoubleToBits(double Double) {
654 uint64_t Bits;
655 static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
656 memcpy(&Bits, &Double, sizeof(Double));
657 return Bits;
658}
659
660/// This function takes a float and returns the bit equivalent 32-bit integer.
661/// Note that copying floats around changes the bits of NaNs on some hosts,
662/// notably x86, so this routine cannot be used if these bits are needed.
663inline uint32_t FloatToBits(float Float) {
664 uint32_t Bits;
665 static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
666 memcpy(&Bits, &Float, sizeof(Float));
667 return Bits;
668}
669
670/// A and B are either alignments or offsets. Return the minimum alignment that
671/// may be assumed after adding the two together.
672constexpr inline uint64_t MinAlign(uint64_t A, uint64_t B) {
673 // The largest power of 2 that divides both A and B.
674 //
675 // Replace "-Value" by "1+~Value" in the following commented code to avoid
676 // MSVC warning C4146
677 // return (A | B) & -(A | B);
678 return (A | B) & (1 + ~(A | B));
679}
680
681/// Returns the next power of two (in 64-bits) that is strictly greater than A.
682/// Returns zero on overflow.
683inline uint64_t NextPowerOf2(uint64_t A) {
684 A |= (A >> 1);
685 A |= (A >> 2);
686 A |= (A >> 4);
687 A |= (A >> 8);
688 A |= (A >> 16);
689 A |= (A >> 32);
690 return A + 1;
691}
692
693/// Returns the power of two which is less than or equal to the given value.
694/// Essentially, it is a floor operation across the domain of powers of two.
695inline uint64_t PowerOf2Floor(uint64_t A) {
696 if (!A) return 0;
697 return 1ull << (63 - countLeadingZeros(A, ZB_Undefined));
698}
699
700/// Returns the power of two which is greater than or equal to the given value.
701/// Essentially, it is a ceil operation across the domain of powers of two.
702inline uint64_t PowerOf2Ceil(uint64_t A) {
703 if (!A)
704 return 0;
705 return NextPowerOf2(A - 1);
706}
707
708/// Returns the next integer (mod 2**64) that is greater than or equal to
709/// \p Value and is a multiple of \p Align. \p Align must be non-zero.
710///
711/// If non-zero \p Skew is specified, the return value will be a minimal
712/// integer that is greater than or equal to \p Value and equal to
713/// \p Align * N + \p Skew for some integer N. If \p Skew is larger than
714/// \p Align, its value is adjusted to '\p Skew mod \p Align'.
715///
716/// Examples:
717/// \code
718/// alignTo(5, 8) = 8
719/// alignTo(17, 8) = 24
720/// alignTo(~0LL, 8) = 0
721/// alignTo(321, 255) = 510
722///
723/// alignTo(5, 8, 7) = 7
724/// alignTo(17, 8, 1) = 17
725/// alignTo(~0LL, 8, 3) = 3
726/// alignTo(321, 255, 42) = 552
727/// \endcode
728inline uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
729 assert(Align != 0u && "Align can't be 0.")(static_cast<void> (0));
730 Skew %= Align;
731 return (Value + Align - 1 - Skew) / Align * Align + Skew;
732}
733
734/// Returns the next integer (mod 2**64) that is greater than or equal to
735/// \p Value and is a multiple of \c Align. \c Align must be non-zero.
736template <uint64_t Align> constexpr inline uint64_t alignTo(uint64_t Value) {
737 static_assert(Align != 0u, "Align must be non-zero");
738 return (Value + Align - 1) / Align * Align;
739}
740
741/// Returns the integer ceil(Numerator / Denominator).
742inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) {
743 return alignTo(Numerator, Denominator) / Denominator;
744}
745
746/// Returns the integer nearest(Numerator / Denominator).
747inline uint64_t divideNearest(uint64_t Numerator, uint64_t Denominator) {
748 return (Numerator + (Denominator / 2)) / Denominator;
749}
750
751/// Returns the largest uint64_t less than or equal to \p Value and is
752/// \p Skew mod \p Align. \p Align must be non-zero
753inline uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
754 assert(Align != 0u && "Align can't be 0.")(static_cast<void> (0));
755 Skew %= Align;
756 return (Value - Skew) / Align * Align + Skew;
757}
758
759/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
760/// Requires 0 < B <= 32.
761template <unsigned B> constexpr inline int32_t SignExtend32(uint32_t X) {
762 static_assert(B > 0, "Bit width can't be 0.");
763 static_assert(B <= 32, "Bit width out of range.");
764 return int32_t(X << (32 - B)) >> (32 - B);
765}
766
767/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
768/// Requires 0 < B <= 32.
769inline int32_t SignExtend32(uint32_t X, unsigned B) {
770 assert(B > 0 && "Bit width can't be 0.")(static_cast<void> (0));
771 assert(B <= 32 && "Bit width out of range.")(static_cast<void> (0));
772 return int32_t(X << (32 - B)) >> (32 - B);
773}
774
775/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
776/// Requires 0 < B <= 64.
777template <unsigned B> constexpr inline int64_t SignExtend64(uint64_t x) {
778 static_assert(B > 0, "Bit width can't be 0.");
779 static_assert(B <= 64, "Bit width out of range.");
780 return int64_t(x << (64 - B)) >> (64 - B);
781}
782
783/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
784/// Requires 0 < B <= 64.
785inline int64_t SignExtend64(uint64_t X, unsigned B) {
786 assert(B > 0 && "Bit width can't be 0.")(static_cast<void> (0));
787 assert(B <= 64 && "Bit width out of range.")(static_cast<void> (0));
788 return int64_t(X << (64 - B)) >> (64 - B);
789}
790
791/// Subtract two unsigned integers, X and Y, of type T and return the absolute
792/// value of the result.
793template <typename T>
794std::enable_if_t<std::is_unsigned<T>::value, T> AbsoluteDifference(T X, T Y) {
795 return X > Y ? (X - Y) : (Y - X);
796}
797
798/// Add two unsigned integers, X and Y, of type T. Clamp the result to the
799/// maximum representable value of T on overflow. ResultOverflowed indicates if
800/// the result is larger than the maximum representable value of type T.
801template <typename T>
802std::enable_if_t<std::is_unsigned<T>::value, T>
803SaturatingAdd(T X, T Y, bool *ResultOverflowed = nullptr) {
804 bool Dummy;
805 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
806 // Hacker's Delight, p. 29
807 T Z = X + Y;
808 Overflowed = (Z < X || Z < Y);
809 if (Overflowed)
810 return std::numeric_limits<T>::max();
811 else
812 return Z;
813}
814
815/// Multiply two unsigned integers, X and Y, of type T. Clamp the result to the
816/// maximum representable value of T on overflow. ResultOverflowed indicates if
817/// the result is larger than the maximum representable value of type T.
818template <typename T>
819std::enable_if_t<std::is_unsigned<T>::value, T>
820SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) {
821 bool Dummy;
822 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
823
824 // Hacker's Delight, p. 30 has a different algorithm, but we don't use that
825 // because it fails for uint16_t (where multiplication can have undefined
826 // behavior due to promotion to int), and requires a division in addition
827 // to the multiplication.
828
829 Overflowed = false;
830
831 // Log2(Z) would be either Log2Z or Log2Z + 1.
832 // Special case: if X or Y is 0, Log2_64 gives -1, and Log2Z
833 // will necessarily be less than Log2Max as desired.
834 int Log2Z = Log2_64(X) + Log2_64(Y);
835 const T Max = std::numeric_limits<T>::max();
836 int Log2Max = Log2_64(Max);
837 if (Log2Z < Log2Max) {
838 return X * Y;
839 }
840 if (Log2Z > Log2Max) {
841 Overflowed = true;
842 return Max;
843 }
844
845 // We're going to use the top bit, and maybe overflow one
846 // bit past it. Multiply all but the bottom bit then add
847 // that on at the end.
848 T Z = (X >> 1) * Y;
849 if (Z & ~(Max >> 1)) {
850 Overflowed = true;
851 return Max;
852 }
853 Z <<= 1;
854 if (X & 1)
855 return SaturatingAdd(Z, Y, ResultOverflowed);
856
857 return Z;
858}
859
860/// Multiply two unsigned integers, X and Y, and add the unsigned integer, A to
861/// the product. Clamp the result to the maximum representable value of T on
862/// overflow. ResultOverflowed indicates if the result is larger than the
863/// maximum representable value of type T.
864template <typename T>
865std::enable_if_t<std::is_unsigned<T>::value, T>
866SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed = nullptr) {
867 bool Dummy;
868 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
869
870 T Product = SaturatingMultiply(X, Y, &Overflowed);
871 if (Overflowed)
872 return Product;
873
874 return SaturatingAdd(A, Product, &Overflowed);
875}
876
877/// Use this rather than HUGE_VALF; the latter causes warnings on MSVC.
878extern const float huge_valf;
879
880
881/// Add two signed integers, computing the two's complement truncated result,
882/// returning true if overflow occured.
883template <typename T>
884std::enable_if_t<std::is_signed<T>::value, T> AddOverflow(T X, T Y, T &Result) {
885#if __has_builtin(__builtin_add_overflow)1
886 return __builtin_add_overflow(X, Y, &Result);
887#else
888 // Perform the unsigned addition.
889 using U = std::make_unsigned_t<T>;
890 const U UX = static_cast<U>(X);
891 const U UY = static_cast<U>(Y);
892 const U UResult = UX + UY;
893
894 // Convert to signed.
895 Result = static_cast<T>(UResult);
896
897 // Adding two positive numbers should result in a positive number.
898 if (X > 0 && Y > 0)
899 return Result <= 0;
900 // Adding two negatives should result in a negative number.
901 if (X < 0 && Y < 0)
902 return Result >= 0;
903 return false;
904#endif
905}
906
907/// Subtract two signed integers, computing the two's complement truncated
908/// result, returning true if an overflow ocurred.
909template <typename T>
910std::enable_if_t<std::is_signed<T>::value, T> SubOverflow(T X, T Y, T &Result) {
911#if __has_builtin(__builtin_sub_overflow)1
912 return __builtin_sub_overflow(X, Y, &Result);
913#else
914 // Perform the unsigned addition.
915 using U = std::make_unsigned_t<T>;
916 const U UX = static_cast<U>(X);
917 const U UY = static_cast<U>(Y);
918 const U UResult = UX - UY;
919
920 // Convert to signed.
921 Result = static_cast<T>(UResult);
922
923 // Subtracting a positive number from a negative results in a negative number.
924 if (X <= 0 && Y > 0)
925 return Result >= 0;
926 // Subtracting a negative number from a positive results in a positive number.
927 if (X >= 0 && Y < 0)
928 return Result <= 0;
929 return false;
930#endif
931}
932
933/// Multiply two signed integers, computing the two's complement truncated
934/// result, returning true if an overflow ocurred.
935template <typename T>
936std::enable_if_t<std::is_signed<T>::value, T> MulOverflow(T X, T Y, T &Result) {
937 // Perform the unsigned multiplication on absolute values.
938 using U = std::make_unsigned_t<T>;
939 const U UX = X < 0 ? (0 - static_cast<U>(X)) : static_cast<U>(X);
940 const U UY = Y < 0 ? (0 - static_cast<U>(Y)) : static_cast<U>(Y);
941 const U UResult = UX * UY;
942
943 // Convert to signed.
944 const bool IsNegative = (X < 0) ^ (Y < 0);
945 Result = IsNegative ? (0 - UResult) : UResult;
946
947 // If any of the args was 0, result is 0 and no overflow occurs.
948 if (UX == 0 || UY == 0)
949 return false;
950
951 // UX and UY are in [1, 2^n], where n is the number of digits.
952 // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
953 // positive) divided by an argument compares to the other.
954 if (IsNegative)
955 return UX > (static_cast<U>(std::numeric_limits<T>::max()) + U(1)) / UY;
956 else
957 return UX > (static_cast<U>(std::numeric_limits<T>::max())) / UY;
958}
959
960} // End llvm namespace
961
962#endif