Line data Source code
1 : //===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This implements the SelectionDAG class.
11 : //
12 : //===----------------------------------------------------------------------===//
13 :
14 : #include "llvm/CodeGen/SelectionDAG.h"
15 : #include "SDNodeDbgValue.h"
16 : #include "llvm/ADT/APFloat.h"
17 : #include "llvm/ADT/APInt.h"
18 : #include "llvm/ADT/APSInt.h"
19 : #include "llvm/ADT/ArrayRef.h"
20 : #include "llvm/ADT/BitVector.h"
21 : #include "llvm/ADT/FoldingSet.h"
22 : #include "llvm/ADT/None.h"
23 : #include "llvm/ADT/STLExtras.h"
24 : #include "llvm/ADT/SmallPtrSet.h"
25 : #include "llvm/ADT/SmallVector.h"
26 : #include "llvm/ADT/Triple.h"
27 : #include "llvm/ADT/Twine.h"
28 : #include "llvm/Analysis/ValueTracking.h"
29 : #include "llvm/CodeGen/ISDOpcodes.h"
30 : #include "llvm/CodeGen/MachineBasicBlock.h"
31 : #include "llvm/CodeGen/MachineConstantPool.h"
32 : #include "llvm/CodeGen/MachineFrameInfo.h"
33 : #include "llvm/CodeGen/MachineFunction.h"
34 : #include "llvm/CodeGen/MachineMemOperand.h"
35 : #include "llvm/CodeGen/RuntimeLibcalls.h"
36 : #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
37 : #include "llvm/CodeGen/SelectionDAGNodes.h"
38 : #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
39 : #include "llvm/CodeGen/TargetLowering.h"
40 : #include "llvm/CodeGen/TargetRegisterInfo.h"
41 : #include "llvm/CodeGen/TargetSubtargetInfo.h"
42 : #include "llvm/CodeGen/ValueTypes.h"
43 : #include "llvm/IR/Constant.h"
44 : #include "llvm/IR/Constants.h"
45 : #include "llvm/IR/DataLayout.h"
46 : #include "llvm/IR/DebugInfoMetadata.h"
47 : #include "llvm/IR/DebugLoc.h"
48 : #include "llvm/IR/DerivedTypes.h"
49 : #include "llvm/IR/Function.h"
50 : #include "llvm/IR/GlobalValue.h"
51 : #include "llvm/IR/Metadata.h"
52 : #include "llvm/IR/Type.h"
53 : #include "llvm/IR/Value.h"
54 : #include "llvm/Support/Casting.h"
55 : #include "llvm/Support/CodeGen.h"
56 : #include "llvm/Support/Compiler.h"
57 : #include "llvm/Support/Debug.h"
58 : #include "llvm/Support/ErrorHandling.h"
59 : #include "llvm/Support/KnownBits.h"
60 : #include "llvm/Support/MachineValueType.h"
61 : #include "llvm/Support/ManagedStatic.h"
62 : #include "llvm/Support/MathExtras.h"
63 : #include "llvm/Support/Mutex.h"
64 : #include "llvm/Support/raw_ostream.h"
65 : #include "llvm/Target/TargetMachine.h"
66 : #include "llvm/Target/TargetOptions.h"
67 : #include <algorithm>
68 : #include <cassert>
69 : #include <cstdint>
70 : #include <cstdlib>
71 : #include <limits>
72 : #include <set>
73 : #include <string>
74 : #include <utility>
75 : #include <vector>
76 :
77 : using namespace llvm;
78 :
79 : /// makeVTList - Return an instance of the SDVTList struct initialized with the
80 : /// specified members.
81 : static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
82 : SDVTList Res = {VTs, NumVTs};
83 : return Res;
84 : }
85 :
86 : // Default null implementations of the callbacks.
87 0 : void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
88 28085090 : void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
89 :
90 : #define DEBUG_TYPE "selectiondag"
91 :
92 : static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
93 : cl::Hidden, cl::init(true),
94 : cl::desc("Gang up loads and stores generated by inlining of memcpy"));
95 :
96 : static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
97 : cl::desc("Number limit for gluing ld/st of memcpy."),
98 : cl::Hidden, cl::init(0));
99 :
100 0 : static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
101 : LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
102 0 : }
103 :
104 : //===----------------------------------------------------------------------===//
105 : // ConstantFPSDNode Class
106 : //===----------------------------------------------------------------------===//
107 :
108 : /// isExactlyValue - We don't rely on operator== working on double values, as
109 : /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
110 : /// As such, this method can be used to do an exact bit-for-bit comparison of
111 : /// two floating point values.
112 3 : bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
113 6 : return getValueAPF().bitwiseIsEqual(V);
114 : }
115 :
116 4228 : bool ConstantFPSDNode::isValueValidForType(EVT VT,
117 : const APFloat& Val) {
118 : assert(VT.isFloatingPoint() && "Can only convert between FP types");
119 :
120 : // convert modifies in place, so make a copy.
121 : APFloat Val2 = APFloat(Val);
122 : bool losesInfo;
123 4228 : (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
124 : APFloat::rmNearestTiesToEven,
125 : &losesInfo);
126 4228 : return !losesInfo;
127 : }
128 :
129 : //===----------------------------------------------------------------------===//
130 : // ISD Namespace
131 : //===----------------------------------------------------------------------===//
132 :
133 99772 : bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
134 : auto *BV = dyn_cast<BuildVectorSDNode>(N);
135 : if (!BV)
136 : return false;
137 :
138 : APInt SplatUndef;
139 : unsigned SplatBitSize;
140 : bool HasUndefs;
141 12746 : unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
142 6373 : return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
143 6373 : EltSize) &&
144 6073 : EltSize == SplatBitSize;
145 : }
146 :
147 : // FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
148 : // specializations of the more general isConstantSplatVector()?
149 :
150 457773 : bool ISD::isBuildVectorAllOnes(const SDNode *N) {
151 : // Look through a bit convert.
152 1097784 : while (N->getOpcode() == ISD::BITCAST)
153 91119 : N = N->getOperand(0).getNode();
154 :
155 457773 : if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
156 :
157 234075 : unsigned i = 0, e = N->getNumOperands();
158 :
159 : // Skip over all of the undef values.
160 235996 : while (i != e && N->getOperand(i).isUndef())
161 1921 : ++i;
162 :
163 : // Do not accept an all-undef vector.
164 234075 : if (i == e) return false;
165 :
166 : // Do not accept build_vectors that aren't all constants or which have non-~0
167 : // elements. We have to be a bit careful here, as the type of the constant
168 : // may not be the same as the type of the vector elements due to type
169 : // legalization (the elements are promoted to a legal type for the target and
170 : // a vector of a type may be legal when the base element type is not).
171 : // We only want to check enough bits to cover the vector elements, because
172 : // we care if the resultant vector is all ones, not whether the individual
173 : // constants are.
174 234034 : SDValue NotZero = N->getOperand(i);
175 468068 : unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
176 : if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
177 437818 : if (CN->getAPIntValue().countTrailingOnes() < EltSize)
178 : return false;
179 : } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
180 16446 : if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
181 : return false;
182 : } else
183 : return false;
184 :
185 : // Okay, we have at least one ~0 value, check to see if the rest match or are
186 : // undefs. Even with the above element type twiddling, this should be OK, as
187 : // the same type legalization should have applied to all the elements.
188 711518 : for (++i; i != e; ++i)
189 538988 : if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())
190 : return false;
191 : return true;
192 : }
193 :
194 2256187 : bool ISD::isBuildVectorAllZeros(const SDNode *N) {
195 : // Look through a bit convert.
196 5309294 : while (N->getOpcode() == ISD::BITCAST)
197 398460 : N = N->getOperand(0).getNode();
198 :
199 2256187 : if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
200 :
201 : bool IsAllUndef = true;
202 5533988 : for (const SDValue &Op : N->op_values()) {
203 9939896 : if (Op.isUndef())
204 : continue;
205 : IsAllUndef = false;
206 : // Do not accept build_vectors that aren't all constants or which have non-0
207 : // elements. We have to be a bit careful here, as the type of the constant
208 : // may not be the same as the type of the vector elements due to type
209 : // legalization (the elements are promoted to a legal type for the target
210 : // and a vector of a type may be legal when the base element type is not).
211 : // We only want to check enough bits to cover the vector elements, because
212 : // we care if the resultant vector is all zeros, not whether the individual
213 : // constants are.
214 9916496 : unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
215 : if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
216 9797308 : if (CN->getAPIntValue().countTrailingZeros() < EltSize)
217 : return false;
218 : } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
219 92078 : if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
220 : return false;
221 : } else
222 : return false;
223 : }
224 :
225 : // Do not accept an all-undef vector.
226 564040 : if (IsAllUndef)
227 44 : return false;
228 : return true;
229 : }
230 :
231 7799003 : bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
232 7799003 : if (N->getOpcode() != ISD::BUILD_VECTOR)
233 : return false;
234 :
235 2901488 : for (const SDValue &Op : N->op_values()) {
236 4525358 : if (Op.isUndef())
237 : continue;
238 : if (!isa<ConstantSDNode>(Op))
239 : return false;
240 : }
241 : return true;
242 : }
243 :
244 1210683 : bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
245 1210683 : if (N->getOpcode() != ISD::BUILD_VECTOR)
246 : return false;
247 :
248 30527 : for (const SDValue &Op : N->op_values()) {
249 53450 : if (Op.isUndef())
250 : continue;
251 : if (!isa<ConstantFPSDNode>(Op))
252 : return false;
253 : }
254 : return true;
255 : }
256 :
257 728763 : bool ISD::allOperandsUndef(const SDNode *N) {
258 : // Return false if the node has no operands.
259 : // This is "logically inconsistent" with the definition of "all" but
260 : // is probably the desired behavior.
261 728763 : if (N->getNumOperands() == 0)
262 : return false;
263 :
264 734817 : for (const SDValue &Op : N->op_values())
265 1469322 : if (!Op.isUndef())
266 : return false;
267 :
268 : return true;
269 : }
270 :
271 323471 : bool ISD::matchUnaryPredicate(SDValue Op,
272 : std::function<bool(ConstantSDNode *)> Match) {
273 : if (auto *Cst = dyn_cast<ConstantSDNode>(Op))
274 278889 : return Match(Cst);
275 :
276 44582 : if (ISD::BUILD_VECTOR != Op.getOpcode())
277 : return false;
278 :
279 17942 : EVT SVT = Op.getValueType().getScalarType();
280 16335 : for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
281 : auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i));
282 45246 : if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
283 8187 : return false;
284 : }
285 : return true;
286 : }
287 :
288 49148 : bool ISD::matchBinaryPredicate(
289 : SDValue LHS, SDValue RHS,
290 : std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match) {
291 49194 : if (LHS.getValueType() != RHS.getValueType())
292 1498 : return false;
293 :
294 : if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
295 : if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
296 16356 : return Match(LHSCst, RHSCst);
297 :
298 31294 : if (ISD::BUILD_VECTOR != LHS.getOpcode() ||
299 : ISD::BUILD_VECTOR != RHS.getOpcode())
300 : return false;
301 :
302 593 : EVT SVT = LHS.getValueType().getScalarType();
303 9120 : for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
304 : auto *LHSCst = dyn_cast<ConstantSDNode>(LHS.getOperand(i));
305 : auto *RHSCst = dyn_cast<ConstantSDNode>(RHS.getOperand(i));
306 8581 : if (!LHSCst || !RHSCst)
307 : return false;
308 17162 : if (LHSCst->getValueType(0) != SVT ||
309 8581 : LHSCst->getValueType(0) != RHSCst->getValueType(0))
310 0 : return false;
311 8581 : if (!Match(LHSCst, RHSCst))
312 : return false;
313 : }
314 : return true;
315 : }
316 :
317 354 : ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
318 354 : switch (ExtType) {
319 12 : case ISD::EXTLOAD:
320 12 : return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
321 : case ISD::SEXTLOAD:
322 : return ISD::SIGN_EXTEND;
323 187 : case ISD::ZEXTLOAD:
324 187 : return ISD::ZERO_EXTEND;
325 : default:
326 : break;
327 : }
328 :
329 0 : llvm_unreachable("Invalid LoadExtType");
330 : }
331 :
332 401136 : ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
333 : // To perform this operation, we just need to swap the L and G bits of the
334 : // operation.
335 401136 : unsigned OldL = (Operation >> 2) & 1;
336 401136 : unsigned OldG = (Operation >> 1) & 1;
337 802272 : return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
338 401136 : (OldL << 1) | // New G bit
339 401136 : (OldG << 2)); // New L bit.
340 : }
341 :
342 56594 : ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
343 : unsigned Operation = Op;
344 56594 : if (isInteger)
345 53770 : Operation ^= 7; // Flip L, G, E bits, but not U.
346 : else
347 2824 : Operation ^= 15; // Flip all of the condition bits.
348 :
349 56594 : if (Operation > ISD::SETTRUE2)
350 302 : Operation &= ~8; // Don't let N and U bits get set.
351 :
352 56594 : return ISD::CondCode(Operation);
353 : }
354 :
355 : /// For an integer comparison, return 1 if the comparison is a signed operation
356 : /// and 2 if the result is an unsigned comparison. Return zero if the operation
357 : /// does not depend on the sign of the input (setne and seteq).
358 : static int isSignedOp(ISD::CondCode Opcode) {
359 : switch (Opcode) {
360 0 : default: llvm_unreachable("Illegal integer setcc operation!");
361 : case ISD::SETEQ:
362 : case ISD::SETNE: return 0;
363 : case ISD::SETLT:
364 : case ISD::SETLE:
365 : case ISD::SETGT:
366 : case ISD::SETGE: return 1;
367 : case ISD::SETULT:
368 : case ISD::SETULE:
369 : case ISD::SETUGT:
370 : case ISD::SETUGE: return 2;
371 : }
372 : }
373 :
374 43 : ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
375 : bool IsInteger) {
376 68 : if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
377 : // Cannot fold a signed integer setcc with an unsigned integer setcc.
378 : return ISD::SETCC_INVALID;
379 :
380 28 : unsigned Op = Op1 | Op2; // Combine all of the condition bits.
381 :
382 : // If the N and U bits get set, then the resultant comparison DOES suddenly
383 : // care about orderedness, and it is true when ordered.
384 28 : if (Op > ISD::SETTRUE2)
385 17 : Op &= ~16; // Clear the U bit if the N bit is set.
386 :
387 : // Canonicalize illegal integer setcc's.
388 28 : if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
389 : Op = ISD::SETNE;
390 :
391 : return ISD::CondCode(Op);
392 : }
393 :
394 43 : ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
395 : bool IsInteger) {
396 48 : if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
397 : // Cannot fold a signed setcc with an unsigned setcc.
398 : return ISD::SETCC_INVALID;
399 :
400 : // Combine all of the condition bits.
401 43 : ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
402 :
403 : // Canonicalize illegal integer setcc's.
404 43 : if (IsInteger) {
405 5 : switch (Result) {
406 : default: break;
407 0 : case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
408 1 : case ISD::SETOEQ: // SETEQ & SETU[LG]E
409 1 : case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
410 2 : case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
411 0 : case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
412 : }
413 : }
414 :
415 : return Result;
416 : }
417 :
418 : //===----------------------------------------------------------------------===//
419 : // SDNode Profile Support
420 : //===----------------------------------------------------------------------===//
421 :
422 : /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
423 : static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
424 99407536 : ID.AddInteger(OpC);
425 : }
426 :
427 : /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
428 : /// solely with their pointer.
429 0 : static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
430 0 : ID.AddPointer(VTList.VTs);
431 0 : }
432 :
433 : /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
434 106954606 : static void AddNodeIDOperands(FoldingSetNodeID &ID,
435 : ArrayRef<SDValue> Ops) {
436 220185331 : for (auto& Op : Ops) {
437 113230725 : ID.AddPointer(Op.getNode());
438 113230725 : ID.AddInteger(Op.getResNo());
439 : }
440 106954606 : }
441 :
442 : /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
443 99407537 : static void AddNodeIDOperands(FoldingSetNodeID &ID,
444 : ArrayRef<SDUse> Ops) {
445 218850729 : for (auto& Op : Ops) {
446 119443192 : ID.AddPointer(Op.getNode());
447 119443192 : ID.AddInteger(Op.getResNo());
448 : }
449 99407537 : }
450 :
451 106954607 : static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
452 : SDVTList VTList, ArrayRef<SDValue> OpList) {
453 106954607 : AddNodeIDOpcode(ID, OpC);
454 106954607 : AddNodeIDValueTypes(ID, VTList);
455 106954606 : AddNodeIDOperands(ID, OpList);
456 106954606 : }
457 :
458 : /// If this is an SDNode with special info, add this info to the NodeID data.
459 99607331 : static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
460 199214662 : switch (N->getOpcode()) {
461 : case ISD::TargetExternalSymbol:
462 : case ISD::ExternalSymbol:
463 : case ISD::MCSymbol:
464 : llvm_unreachable("Should only be used on nodes with operands");
465 : default: break; // Normal nodes don't need extra info.
466 : case ISD::TargetConstant:
467 : case ISD::Constant: {
468 : const ConstantSDNode *C = cast<ConstantSDNode>(N);
469 26856752 : ID.AddPointer(C->getConstantIntValue());
470 26856752 : ID.AddBoolean(C->isOpaque());
471 : break;
472 : }
473 : case ISD::TargetConstantFP:
474 : case ISD::ConstantFP:
475 36841 : ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
476 36841 : break;
477 : case ISD::TargetGlobalAddress:
478 : case ISD::GlobalAddress:
479 : case ISD::TargetGlobalTLSAddress:
480 : case ISD::GlobalTLSAddress: {
481 : const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
482 2622136 : ID.AddPointer(GA->getGlobal());
483 2622136 : ID.AddInteger(GA->getOffset());
484 2622136 : ID.AddInteger(GA->getTargetFlags());
485 2622136 : break;
486 : }
487 : case ISD::BasicBlock:
488 337422 : ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
489 337422 : break;
490 : case ISD::Register:
491 17480441 : ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
492 17480441 : break;
493 : case ISD::RegisterMask:
494 805913 : ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
495 805913 : break;
496 : case ISD::SRCVALUE:
497 533 : ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
498 533 : break;
499 : case ISD::FrameIndex:
500 : case ISD::TargetFrameIndex:
501 6792337 : ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
502 6792337 : break;
503 : case ISD::JumpTable:
504 : case ISD::TargetJumpTable:
505 1169 : ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
506 1169 : ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
507 1169 : break;
508 : case ISD::ConstantPool:
509 : case ISD::TargetConstantPool: {
510 : const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
511 34025 : ID.AddInteger(CP->getAlignment());
512 68050 : ID.AddInteger(CP->getOffset());
513 34025 : if (CP->isMachineConstantPoolEntry())
514 76 : CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
515 : else
516 33949 : ID.AddPointer(CP->getConstVal());
517 34025 : ID.AddInteger(CP->getTargetFlags());
518 34025 : break;
519 : }
520 : case ISD::TargetIndex: {
521 : const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
522 0 : ID.AddInteger(TI->getIndex());
523 0 : ID.AddInteger(TI->getOffset());
524 0 : ID.AddInteger(TI->getTargetFlags());
525 0 : break;
526 : }
527 : case ISD::LOAD: {
528 : const LoadSDNode *LD = cast<LoadSDNode>(N);
529 3980082 : ID.AddInteger(LD->getMemoryVT().getRawBits());
530 3980082 : ID.AddInteger(LD->getRawSubclassData());
531 7960164 : ID.AddInteger(LD->getPointerInfo().getAddrSpace());
532 3980082 : break;
533 : }
534 : case ISD::STORE: {
535 : const StoreSDNode *ST = cast<StoreSDNode>(N);
536 6484597 : ID.AddInteger(ST->getMemoryVT().getRawBits());
537 6484597 : ID.AddInteger(ST->getRawSubclassData());
538 12969194 : ID.AddInteger(ST->getPointerInfo().getAddrSpace());
539 6484597 : break;
540 : }
541 : case ISD::MLOAD: {
542 : const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
543 642 : ID.AddInteger(MLD->getMemoryVT().getRawBits());
544 642 : ID.AddInteger(MLD->getRawSubclassData());
545 1284 : ID.AddInteger(MLD->getPointerInfo().getAddrSpace());
546 642 : break;
547 : }
548 : case ISD::MSTORE: {
549 : const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
550 305 : ID.AddInteger(MST->getMemoryVT().getRawBits());
551 305 : ID.AddInteger(MST->getRawSubclassData());
552 610 : ID.AddInteger(MST->getPointerInfo().getAddrSpace());
553 305 : break;
554 : }
555 : case ISD::MGATHER: {
556 : const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N);
557 759 : ID.AddInteger(MG->getMemoryVT().getRawBits());
558 759 : ID.AddInteger(MG->getRawSubclassData());
559 1518 : ID.AddInteger(MG->getPointerInfo().getAddrSpace());
560 759 : break;
561 : }
562 : case ISD::MSCATTER: {
563 : const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N);
564 152 : ID.AddInteger(MS->getMemoryVT().getRawBits());
565 152 : ID.AddInteger(MS->getRawSubclassData());
566 304 : ID.AddInteger(MS->getPointerInfo().getAddrSpace());
567 152 : break;
568 : }
569 : case ISD::ATOMIC_CMP_SWAP:
570 : case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
571 : case ISD::ATOMIC_SWAP:
572 : case ISD::ATOMIC_LOAD_ADD:
573 : case ISD::ATOMIC_LOAD_SUB:
574 : case ISD::ATOMIC_LOAD_AND:
575 : case ISD::ATOMIC_LOAD_CLR:
576 : case ISD::ATOMIC_LOAD_OR:
577 : case ISD::ATOMIC_LOAD_XOR:
578 : case ISD::ATOMIC_LOAD_NAND:
579 : case ISD::ATOMIC_LOAD_MIN:
580 : case ISD::ATOMIC_LOAD_MAX:
581 : case ISD::ATOMIC_LOAD_UMIN:
582 : case ISD::ATOMIC_LOAD_UMAX:
583 : case ISD::ATOMIC_LOAD:
584 : case ISD::ATOMIC_STORE: {
585 : const AtomicSDNode *AT = cast<AtomicSDNode>(N);
586 20009 : ID.AddInteger(AT->getMemoryVT().getRawBits());
587 20009 : ID.AddInteger(AT->getRawSubclassData());
588 40018 : ID.AddInteger(AT->getPointerInfo().getAddrSpace());
589 20009 : break;
590 : }
591 : case ISD::PREFETCH: {
592 : const MemSDNode *PF = cast<MemSDNode>(N);
593 430 : ID.AddInteger(PF->getPointerInfo().getAddrSpace());
594 215 : break;
595 : }
596 : case ISD::VECTOR_SHUFFLE: {
597 : const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
598 964548 : for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
599 841752 : i != e; ++i)
600 1560708 : ID.AddInteger(SVN->getMaskElt(i));
601 : break;
602 : }
603 : case ISD::TargetBlockAddress:
604 : case ISD::BlockAddress: {
605 : const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
606 148 : ID.AddPointer(BA->getBlockAddress());
607 148 : ID.AddInteger(BA->getOffset());
608 148 : ID.AddInteger(BA->getTargetFlags());
609 148 : break;
610 : }
611 : } // end switch (N->getOpcode())
612 :
613 : // Target specific memory nodes could also have address spaces to check.
614 99607331 : if (N->isTargetMemoryOpcode())
615 20214 : ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
616 99607331 : }
617 :
618 : /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
619 : /// data.
620 99407536 : static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
621 99407536 : AddNodeIDOpcode(ID, N->getOpcode());
622 : // Add the return value info.
623 99407536 : AddNodeIDValueTypes(ID, N->getVTList());
624 : // Add the operand info.
625 99407537 : AddNodeIDOperands(ID, N->ops());
626 :
627 : // Handle SDNode leafs with special info.
628 99407537 : AddNodeIDCustom(ID, N);
629 99407537 : }
630 :
631 : //===----------------------------------------------------------------------===//
632 : // SelectionDAG Class
633 : //===----------------------------------------------------------------------===//
634 :
635 : /// doNotCSE - Return true if CSE should not be performed for this node.
636 12199535 : static bool doNotCSE(SDNode *N) {
637 12199535 : if (N->getValueType(0) == MVT::Glue)
638 856 : return true; // Never CSE anything that produces a flag.
639 :
640 24397358 : switch (N->getOpcode()) {
641 : default: break;
642 : case ISD::HANDLENODE:
643 : case ISD::EH_LABEL:
644 : return true; // Never CSE these nodes.
645 : }
646 :
647 : // Check that remaining values produced are not flags.
648 13879193 : for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
649 : if (N->getValueType(i) == MVT::Glue)
650 2468238 : return true; // Never CSE anything that produces a flag.
651 :
652 : return false;
653 : }
654 :
655 : /// RemoveDeadNodes - This method deletes all unreachable nodes in the
656 : /// SelectionDAG.
657 5620176 : void SelectionDAG::RemoveDeadNodes() {
658 : // Create a dummy node (which is not added to allnodes), that adds a reference
659 : // to the root node, preventing it from being deleted.
660 11240353 : HandleSDNode Dummy(getRoot());
661 :
662 : SmallVector<SDNode*, 128> DeadNodes;
663 :
664 : // Add all obviously-dead nodes to the DeadNodes worklist.
665 172909948 : for (SDNode &Node : allnodes())
666 167289771 : if (Node.use_empty())
667 871555 : DeadNodes.push_back(&Node);
668 :
669 5620177 : RemoveDeadNodes(DeadNodes);
670 :
671 : // If the root changed (e.g. it was a dead load, update the root).
672 5620177 : setRoot(Dummy.getValue());
673 5620177 : }
674 :
675 : /// RemoveDeadNodes - This method deletes the unreachable nodes in the
676 : /// given list, and any nodes that become unreachable as a result.
677 15585629 : void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
678 :
679 : // Process the worklist, deleting the nodes and adding their uses to the
680 : // worklist.
681 25110714 : while (!DeadNodes.empty()) {
682 : SDNode *N = DeadNodes.pop_back_val();
683 : // Skip to next node if we've already managed to delete the node. This could
684 : // happen if replacing a node causes a node previously added to the node to
685 : // be deleted.
686 9525085 : if (N->getOpcode() == ISD::DELETED_NODE)
687 : continue;
688 :
689 22894621 : for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
690 13369539 : DUL->NodeDeleted(N, nullptr);
691 :
692 : // Take the node out of the appropriate CSE map.
693 9525082 : RemoveNodeFromCSEMaps(N);
694 :
695 : // Next, brutally remove the operand list. This is safe to do, as there are
696 : // no cycles in the graph.
697 19667898 : for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
698 10142816 : SDUse &Use = *I++;
699 10142816 : SDNode *Operand = Use.getNode();
700 10142816 : Use.set(SDValue());
701 :
702 : // Now that we removed this operand, see if there are no uses of it left.
703 10142816 : if (Operand->use_empty())
704 3558807 : DeadNodes.push_back(Operand);
705 : }
706 :
707 9525082 : DeallocateNode(N);
708 : }
709 15585629 : }
710 :
711 659691 : void SelectionDAG::RemoveDeadNode(SDNode *N){
712 : SmallVector<SDNode*, 16> DeadNodes(1, N);
713 :
714 : // Create a dummy node that adds a reference to the root node, preventing
715 : // it from being deleted. (This matters if the root is an operand of the
716 : // dead node.)
717 1319382 : HandleSDNode Dummy(getRoot());
718 :
719 659691 : RemoveDeadNodes(DeadNodes);
720 659691 : }
721 :
722 10467238 : void SelectionDAG::DeleteNode(SDNode *N) {
723 : // First take this out of the appropriate CSE map.
724 10467238 : RemoveNodeFromCSEMaps(N);
725 :
726 : // Finally, remove uses due to operands of this node, remove from the
727 : // AllNodes list, and delete the node.
728 10467238 : DeleteNodeNotInCSEMaps(N);
729 10467237 : }
730 :
731 10482788 : void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
732 : assert(N->getIterator() != AllNodes.begin() &&
733 : "Cannot delete the entry node!");
734 : assert(N->use_empty() && "Cannot delete a node that is not dead!");
735 :
736 : // Drop all of the operands and decrement used node's use counts.
737 10482788 : N->DropOperands();
738 :
739 10482789 : DeallocateNode(N);
740 10482788 : }
741 :
742 56715322 : void SDDbgInfo::erase(const SDNode *Node) {
743 56715322 : DbgValMapType::iterator I = DbgValMap.find(Node);
744 56715321 : if (I == DbgValMap.end())
745 56685812 : return;
746 97957 : for (auto &Val: I->second)
747 68448 : Val->setIsInvalidated();
748 : DbgValMap.erase(I);
749 : }
750 :
751 56715321 : void SelectionDAG::DeallocateNode(SDNode *N) {
752 : // If we have operands, deallocate them.
753 56715321 : removeOperands(N);
754 :
755 : NodeAllocator.Deallocate(AllNodes.remove(N));
756 :
757 : // Set the opcode to DELETED_NODE to help catch bugs when node
758 : // memory is reallocated.
759 : // FIXME: There are places in SDag that have grown a dependency on the opcode
760 : // value in the released node.
761 : __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
762 56715322 : N->NodeType = ISD::DELETED_NODE;
763 :
764 : // If any of the SDDbgValue nodes refer to this SDNode, invalidate
765 : // them and forget about that node.
766 56715322 : DbgInfo->erase(N);
767 56715321 : }
768 :
769 : #ifndef NDEBUG
770 : /// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid.
771 : static void VerifySDNode(SDNode *N) {
772 : switch (N->getOpcode()) {
773 : default:
774 : break;
775 : case ISD::BUILD_PAIR: {
776 : EVT VT = N->getValueType(0);
777 : assert(N->getNumValues() == 1 && "Too many results!");
778 : assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
779 : "Wrong return type!");
780 : assert(N->getNumOperands() == 2 && "Wrong number of operands!");
781 : assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
782 : "Mismatched operand types!");
783 : assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
784 : "Wrong operand type!");
785 : assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
786 : "Wrong return type size");
787 : break;
788 : }
789 : case ISD::BUILD_VECTOR: {
790 : assert(N->getNumValues() == 1 && "Too many results!");
791 : assert(N->getValueType(0).isVector() && "Wrong return type!");
792 : assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
793 : "Wrong number of operands!");
794 : EVT EltVT = N->getValueType(0).getVectorElementType();
795 : for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
796 : assert((I->getValueType() == EltVT ||
797 : (EltVT.isInteger() && I->getValueType().isInteger() &&
798 : EltVT.bitsLE(I->getValueType()))) &&
799 : "Wrong operand type!");
800 : assert(I->getValueType() == N->getOperand(0).getValueType() &&
801 : "Operands must all have the same type");
802 : }
803 : break;
804 : }
805 : }
806 : }
807 : #endif // NDEBUG
808 :
809 : /// Insert a newly allocated node into the DAG.
810 : ///
811 : /// Handles insertion into the all nodes list and CSE map, as well as
812 : /// verification and other common operations when a new node is allocated.
813 58014571 : void SelectionDAG::InsertNode(SDNode *N) {
814 : AllNodes.push_back(N);
815 : #ifndef NDEBUG
816 : N->PersistentId = NextPersistentId++;
817 : VerifySDNode(N);
818 : #endif
819 58014571 : }
820 :
821 : /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
822 : /// correspond to it. This is useful when we're about to delete or repurpose
823 : /// the node. We don't want future request for structurally identical nodes
824 : /// to return N anymore.
825 43967227 : bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
826 : bool Erased = false;
827 87934454 : switch (N->getOpcode()) {
828 : case ISD::HANDLENODE: return false; // noop.
829 : case ISD::CONDCODE:
830 : assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
831 : "Cond code doesn't exist!");
832 187884 : Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
833 187884 : CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
834 187884 : break;
835 12147 : case ISD::ExternalSymbol:
836 24258 : Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
837 12147 : break;
838 : case ISD::TargetExternalSymbol: {
839 : ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
840 406 : Erased = TargetExternalSymbols.erase(
841 812 : std::pair<std::string,unsigned char>(ESN->getSymbol(),
842 406 : ESN->getTargetFlags()));
843 406 : break;
844 : }
845 : case ISD::MCSymbol: {
846 : auto *MCSN = cast<MCSymbolSDNode>(N);
847 0 : Erased = MCSymbols.erase(MCSN->getMCSymbol());
848 0 : break;
849 : }
850 : case ISD::VALUETYPE: {
851 92967 : EVT VT = cast<VTSDNode>(N)->getVT();
852 92967 : if (VT.isExtended()) {
853 6171 : Erased = ExtendedValueTypeNodes.erase(VT);
854 : } else {
855 86796 : Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
856 86796 : ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
857 : }
858 : break;
859 : }
860 43611122 : default:
861 : // Remove it from the CSE Map.
862 : assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
863 : assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
864 : Erased = CSEMap.RemoveNode(N);
865 43611123 : break;
866 : }
867 : #ifndef NDEBUG
868 : // Verify that the node was actually in one of the CSE maps, unless it has a
869 : // flag result (which cannot be CSE'd) or is one of the special cases that are
870 : // not subject to CSE.
871 : if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
872 : !N->isMachineOpcode() && !doNotCSE(N)) {
873 : N->dump(this);
874 : dbgs() << "\n";
875 : llvm_unreachable("Node is not in map!");
876 : }
877 : #endif
878 : return Erased;
879 : }
880 :
881 : /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
882 : /// maps and modified in place. Add it back to the CSE maps, unless an identical
883 : /// node already exists, in which case transfer all its users to the existing
884 : /// node. This transfer can potentially trigger recursive merging.
885 : void
886 11977687 : SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
887 : // For node types that aren't CSE'd, just act as if no identical node
888 : // already exists.
889 11977687 : if (!doNotCSE(N)) {
890 : SDNode *Existing = CSEMap.GetOrInsertNode(N);
891 9010866 : if (Existing != N) {
892 : // If there was already an existing matching node, use ReplaceAllUsesWith
893 : // to replace the dead one with the existing one. This can cause
894 : // recursive merging of other unrelated nodes down the line.
895 15551 : ReplaceAllUsesWith(N, Existing);
896 :
897 : // N is now dead. Inform the listeners and delete it.
898 65874 : for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
899 50323 : DUL->NodeDeleted(N, Existing);
900 15551 : DeleteNodeNotInCSEMaps(N);
901 15551 : return;
902 : }
903 : }
904 :
905 : // If the node doesn't already exist, we updated it. Inform listeners.
906 40772063 : for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
907 28809927 : DUL->NodeUpdated(N);
908 : }
909 :
910 : /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
911 : /// were replaced with those specified. If this node is never memoized,
912 : /// return null, otherwise return a pointer to the slot it would take. If a
913 : /// node already exists with these operands, the slot will be non-null.
914 2806 : SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
915 : void *&InsertPos) {
916 2806 : if (doNotCSE(N))
917 : return nullptr;
918 :
919 2806 : SDValue Ops[] = { Op };
920 : FoldingSetNodeID ID;
921 8418 : AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
922 2806 : AddNodeIDCustom(ID, N);
923 2806 : SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
924 2806 : if (Node)
925 0 : Node->intersectFlagsWith(N->getFlags());
926 : return Node;
927 : }
928 :
929 : /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
930 : /// were replaced with those specified. If this node is never memoized,
931 : /// return null, otherwise return a pointer to the slot it would take. If a
932 : /// node already exists with these operands, the slot will be non-null.
933 53434 : SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
934 : SDValue Op1, SDValue Op2,
935 : void *&InsertPos) {
936 53434 : if (doNotCSE(N))
937 : return nullptr;
938 :
939 53434 : SDValue Ops[] = { Op1, Op2 };
940 : FoldingSetNodeID ID;
941 160302 : AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
942 53434 : AddNodeIDCustom(ID, N);
943 53434 : SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
944 53434 : if (Node)
945 698 : Node->intersectFlagsWith(N->getFlags());
946 : return Node;
947 : }
948 :
949 : /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
950 : /// were replaced with those specified. If this node is never memoized,
951 : /// return null, otherwise return a pointer to the slot it would take. If a
952 : /// node already exists with these operands, the slot will be non-null.
953 165608 : SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
954 : void *&InsertPos) {
955 165608 : if (doNotCSE(N))
956 : return nullptr;
957 :
958 : FoldingSetNodeID ID;
959 430662 : AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
960 143554 : AddNodeIDCustom(ID, N);
961 143554 : SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
962 143554 : if (Node)
963 128 : Node->intersectFlagsWith(N->getFlags());
964 : return Node;
965 : }
966 :
967 358526 : unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
968 : Type *Ty = VT == MVT::iPTR ?
969 0 : PointerType::get(Type::getInt8Ty(*getContext()), 0) :
970 358526 : VT.getTypeForEVT(*getContext());
971 :
972 358526 : return getDataLayout().getABITypeAlignment(Ty);
973 : }
974 :
975 : // EntryNode could meaningfully have debug info if we can find it...
976 29267 : SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
977 : : TM(tm), OptLevel(OL),
978 58534 : EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
979 87801 : Root(getEntryNode()) {
980 29267 : InsertNode(&EntryNode);
981 29267 : DbgInfo = new SDDbgInfo();
982 29267 : }
983 :
984 405299 : void SelectionDAG::init(MachineFunction &NewMF,
985 : OptimizationRemarkEmitter &NewORE,
986 : Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
987 : LegacyDivergenceAnalysis * Divergence) {
988 405299 : MF = &NewMF;
989 405299 : SDAGISelPass = PassPtr;
990 405299 : ORE = &NewORE;
991 405299 : TLI = getSubtarget().getTargetLowering();
992 810598 : TSI = getSubtarget().getSelectionDAGInfo();
993 405299 : LibInfo = LibraryInfo;
994 405299 : Context = &MF->getFunction().getContext();
995 405299 : DA = Divergence;
996 405299 : }
997 :
998 87357 : SelectionDAG::~SelectionDAG() {
999 : assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1000 29119 : allnodes_clear();
1001 29119 : OperandRecycler.clear(OperandAllocator);
1002 29119 : delete DbgInfo;
1003 29119 : }
1004 :
1005 1298169 : void SelectionDAG::allnodes_clear() {
1006 : assert(&*AllNodes.begin() == &EntryNode);
1007 : AllNodes.remove(AllNodes.begin());
1008 38005620 : while (!AllNodes.empty())
1009 36707451 : DeallocateNode(&AllNodes.front());
1010 : #ifndef NDEBUG
1011 : NextPersistentId = 0;
1012 : #endif
1013 1298169 : }
1014 :
1015 30537298 : SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1016 : void *&InsertPos) {
1017 : SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1018 : if (N) {
1019 : switch (N->getOpcode()) {
1020 : default: break;
1021 : case ISD::Constant:
1022 : case ISD::ConstantFP:
1023 : llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1024 : "debug location. Use another overload.");
1025 : }
1026 : }
1027 30537299 : return N;
1028 : }
1029 :
1030 76417950 : SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1031 : const SDLoc &DL, void *&InsertPos) {
1032 : SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1033 76417950 : if (N) {
1034 61430194 : switch (N->getOpcode()) {
1035 : case ISD::Constant:
1036 : case ISD::ConstantFP:
1037 : // Erase debug location from the node if the node is used at several
1038 : // different places. Do not propagate one location to all uses as it
1039 : // will cause a worse single stepping debugging experience.
1040 6939737 : if (N->getDebugLoc() != DL.getDebugLoc())
1041 2449122 : N->setDebugLoc(DebugLoc());
1042 : break;
1043 23775360 : default:
1044 : // When the node's point of use is located earlier in the instruction
1045 : // sequence than its prior point of use, update its debug info to the
1046 : // earlier location.
1047 47550720 : if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1048 155598 : N->setDebugLoc(DL.getDebugLoc());
1049 : break;
1050 : }
1051 : }
1052 76417950 : return N;
1053 : }
1054 :
1055 1269050 : void SelectionDAG::clear() {
1056 1269050 : allnodes_clear();
1057 1269050 : OperandRecycler.clear(OperandAllocator);
1058 1269050 : OperandAllocator.Reset();
1059 1269050 : CSEMap.clear();
1060 :
1061 : ExtendedValueTypeNodes.clear();
1062 1269050 : ExternalSymbols.clear();
1063 : TargetExternalSymbols.clear();
1064 1269050 : MCSymbols.clear();
1065 : std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1066 : static_cast<CondCodeSDNode*>(nullptr));
1067 : std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1068 : static_cast<SDNode*>(nullptr));
1069 :
1070 1269050 : EntryNode.UseList = nullptr;
1071 1269050 : InsertNode(&EntryNode);
1072 1269050 : Root = getEntryNode();
1073 1269050 : DbgInfo->clear();
1074 1269050 : }
1075 :
1076 69 : SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1077 69 : return VT.bitsGT(Op.getValueType())
1078 11 : ? getNode(ISD::FP_EXTEND, DL, VT, Op)
1079 69 : : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL));
1080 : }
1081 :
1082 61971 : SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1083 61971 : return VT.bitsGT(Op.getValueType()) ?
1084 14767 : getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1085 61971 : getNode(ISD::TRUNCATE, DL, VT, Op);
1086 : }
1087 :
1088 234806 : SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1089 234806 : return VT.bitsGT(Op.getValueType()) ?
1090 36785 : getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1091 234806 : getNode(ISD::TRUNCATE, DL, VT, Op);
1092 : }
1093 :
1094 823242 : SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1095 823242 : return VT.bitsGT(Op.getValueType()) ?
1096 14094 : getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1097 823242 : getNode(ISD::TRUNCATE, DL, VT, Op);
1098 : }
1099 :
1100 1235 : SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1101 : EVT OpVT) {
1102 2470 : if (VT.bitsLE(Op.getValueType()))
1103 753 : return getNode(ISD::TRUNCATE, SL, VT, Op);
1104 :
1105 482 : TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
1106 482 : return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1107 : }
1108 :
1109 170068 : SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1110 : assert(!VT.isVector() &&
1111 : "getZeroExtendInReg should use the vector element type instead of "
1112 : "the vector type!");
1113 345948 : if (Op.getValueType().getScalarType() == VT) return Op;
1114 164409 : unsigned BitWidth = Op.getScalarValueSizeInBits();
1115 : APInt Imm = APInt::getLowBitsSet(BitWidth,
1116 164409 : VT.getSizeInBits());
1117 : return getNode(ISD::AND, DL, Op.getValueType(), Op,
1118 328818 : getConstant(Imm, DL, Op.getValueType()));
1119 : }
1120 :
1121 300 : SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL,
1122 : EVT VT) {
1123 : assert(VT.isVector() && "This DAG node is restricted to vector types.");
1124 : assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
1125 : "The sizes of the input and result must match in order to perform the "
1126 : "extend in-register.");
1127 : assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
1128 : "The destination vector type must have fewer lanes than the input.");
1129 300 : return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
1130 : }
1131 :
1132 1187 : SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, const SDLoc &DL,
1133 : EVT VT) {
1134 : assert(VT.isVector() && "This DAG node is restricted to vector types.");
1135 : assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
1136 : "The sizes of the input and result must match in order to perform the "
1137 : "extend in-register.");
1138 : assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
1139 : "The destination vector type must have fewer lanes than the input.");
1140 1187 : return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
1141 : }
1142 :
1143 2268 : SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL,
1144 : EVT VT) {
1145 : assert(VT.isVector() && "This DAG node is restricted to vector types.");
1146 : assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
1147 : "The sizes of the input and result must match in order to perform the "
1148 : "extend in-register.");
1149 : assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
1150 : "The destination vector type must have fewer lanes than the input.");
1151 2268 : return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
1152 : }
1153 :
1154 : /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1155 14325 : SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1156 14325 : EVT EltVT = VT.getScalarType();
1157 : SDValue NegOne =
1158 28650 : getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT);
1159 14325 : return getNode(ISD::XOR, DL, VT, Val, NegOne);
1160 : }
1161 :
1162 109 : SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1163 109 : SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1164 109 : return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1165 : }
1166 :
1167 10969 : SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1168 : EVT OpVT) {
1169 10969 : if (!V)
1170 9249 : return getConstant(0, DL, VT);
1171 :
1172 1720 : switch (TLI->getBooleanContents(OpVT)) {
1173 1632 : case TargetLowering::ZeroOrOneBooleanContent:
1174 : case TargetLowering::UndefinedBooleanContent:
1175 1632 : return getConstant(1, DL, VT);
1176 88 : case TargetLowering::ZeroOrNegativeOneBooleanContent:
1177 88 : return getAllOnesConstant(DL, VT);
1178 : }
1179 0 : llvm_unreachable("Unexpected boolean content enum!");
1180 : }
1181 :
1182 26156678 : SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1183 : bool isT, bool isO) {
1184 26156678 : EVT EltVT = VT.getScalarType();
1185 : assert((EltVT.getSizeInBits() >= 64 ||
1186 : (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
1187 : "getConstant with a uint64_t value that doesn't fit in the type!");
1188 52313356 : return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1189 : }
1190 :
1191 28882776 : SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1192 : bool isT, bool isO) {
1193 28882776 : return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1194 : }
1195 :
1196 30636318 : SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1197 : EVT VT, bool isT, bool isO) {
1198 : assert(VT.isInteger() && "Cannot create FP integer constant!");
1199 :
1200 30636318 : EVT EltVT = VT.getScalarType();
1201 : const ConstantInt *Elt = &Val;
1202 :
1203 : // In some cases the vector type is legal but the element type is illegal and
1204 : // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1205 : // inserted value (the type does not need to match the vector element type).
1206 : // Any extra bits introduced will be truncated away.
1207 30636318 : if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1208 : TargetLowering::TypePromoteInteger) {
1209 7488 : EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1210 7488 : APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1211 7488 : Elt = ConstantInt::get(*getContext(), NewVal);
1212 : }
1213 : // In other cases the element type is illegal and needs to be expanded, for
1214 : // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1215 : // the value into n parts and use a vector type with n-times the elements.
1216 : // Then bitcast to the type requested.
1217 : // Legalizing constants too early makes the DAGCombiner's job harder so we
1218 : // only legalize if the DAG tells us we must produce legal types.
1219 47952013 : else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1220 147727 : TLI->getTypeAction(*getContext(), EltVT) ==
1221 : TargetLowering::TypeExpandInteger) {
1222 : const APInt &NewVal = Elt->getValue();
1223 371 : EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1224 371 : unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1225 371 : unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1226 371 : EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1227 :
1228 : // Check the temporary vector is the correct size. If this fails then
1229 : // getTypeToTransformTo() probably returned a type whose size (in bits)
1230 : // isn't a power-of-2 factor of the requested type size.
1231 : assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1232 :
1233 : SmallVector<SDValue, 2> EltParts;
1234 1113 : for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
1235 1484 : EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
1236 1484 : .zextOrTrunc(ViaEltSizeInBits), DL,
1237 1484 : ViaEltVT, isT, isO));
1238 : }
1239 :
1240 : // EltParts is currently in little endian order. If we actually want
1241 : // big-endian order then reverse it now.
1242 371 : if (getDataLayout().isBigEndian())
1243 : std::reverse(EltParts.begin(), EltParts.end());
1244 :
1245 : // The elements must be reversed when the element order is different
1246 : // to the endianness of the elements (because the BITCAST is itself a
1247 : // vector shuffle in this situation). However, we do not need any code to
1248 : // perform this reversal because getConstant() is producing a vector
1249 : // splat.
1250 : // This situation occurs in MIPS MSA.
1251 :
1252 : SmallVector<SDValue, 8> Ops;
1253 1179 : for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1254 808 : Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
1255 :
1256 371 : SDValue V = getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1257 371 : return V;
1258 : }
1259 :
1260 : assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1261 : "APInt size does not match type size!");
1262 30635947 : unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1263 : FoldingSetNodeID ID;
1264 30635947 : AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1265 30635947 : ID.AddPointer(Elt);
1266 : ID.AddBoolean(isO);
1267 30635947 : void *IP = nullptr;
1268 : SDNode *N = nullptr;
1269 30635947 : if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1270 22341689 : if (!VT.isVector())
1271 22164596 : return SDValue(N, 0);
1272 :
1273 8471351 : if (!N) {
1274 8294258 : N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT);
1275 8294258 : CSEMap.InsertNode(N, IP);
1276 8294258 : InsertNode(N);
1277 : NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
1278 : }
1279 :
1280 : SDValue Result(N, 0);
1281 8471351 : if (VT.isVector())
1282 399950 : Result = getSplatBuildVector(VT, DL, Result);
1283 :
1284 8471351 : return Result;
1285 : }
1286 :
1287 4281565 : SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1288 : bool isTarget) {
1289 8563130 : return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1290 : }
1291 :
1292 20660 : SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1293 : bool isTarget) {
1294 20660 : return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1295 : }
1296 :
1297 44024 : SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1298 : EVT VT, bool isTarget) {
1299 : assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1300 :
1301 44024 : EVT EltVT = VT.getScalarType();
1302 :
1303 : // Do the map lookup using the actual bit pattern for the floating point
1304 : // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1305 : // we don't have issues with SNANs.
1306 44024 : unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1307 : FoldingSetNodeID ID;
1308 44024 : AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1309 44024 : ID.AddPointer(&V);
1310 44024 : void *IP = nullptr;
1311 : SDNode *N = nullptr;
1312 44024 : if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1313 7311 : if (!VT.isVector())
1314 5923 : return SDValue(N, 0);
1315 :
1316 38101 : if (!N) {
1317 36713 : N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT);
1318 36713 : CSEMap.InsertNode(N, IP);
1319 36713 : InsertNode(N);
1320 : }
1321 :
1322 : SDValue Result(N, 0);
1323 38101 : if (VT.isVector())
1324 4416 : Result = getSplatBuildVector(VT, DL, Result);
1325 : NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1326 38101 : return Result;
1327 : }
1328 :
1329 7239 : SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1330 : bool isTarget) {
1331 7239 : EVT EltVT = VT.getScalarType();
1332 : if (EltVT == MVT::f32)
1333 9574 : return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1334 : else if (EltVT == MVT::f64)
1335 4536 : return getConstantFP(APFloat(Val), DL, VT, isTarget);
1336 : else if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1337 : EltVT == MVT::f16) {
1338 : bool Ignored;
1339 184 : APFloat APF = APFloat(Val);
1340 184 : APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
1341 : &Ignored);
1342 184 : return getConstantFP(APF, DL, VT, isTarget);
1343 : } else
1344 0 : llvm_unreachable("Unsupported type in getConstantFP");
1345 : }
1346 :
1347 3735543 : SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1348 : EVT VT, int64_t Offset, bool isTargetGA,
1349 : unsigned char TargetFlags) {
1350 : assert((TargetFlags == 0 || isTargetGA) &&
1351 : "Cannot set target flags on target-independent globals");
1352 :
1353 : // Truncate (with sign-extension) the offset value to the pointer size.
1354 3735543 : unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1355 3735543 : if (BitWidth < 64)
1356 800517 : Offset = SignExtend64(Offset, BitWidth);
1357 :
1358 : unsigned Opc;
1359 3735543 : if (GV->isThreadLocal())
1360 2458 : Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1361 : else
1362 3733085 : Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1363 :
1364 : FoldingSetNodeID ID;
1365 3735543 : AddNodeIDNode(ID, Opc, getVTList(VT), None);
1366 3735543 : ID.AddPointer(GV);
1367 3735543 : ID.AddInteger(Offset);
1368 3735543 : ID.AddInteger(TargetFlags);
1369 3735543 : void *IP = nullptr;
1370 3735543 : if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
1371 319022 : return SDValue(E, 0);
1372 :
1373 3416521 : auto *N = newSDNode<GlobalAddressSDNode>(
1374 3416521 : Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
1375 3416521 : CSEMap.InsertNode(N, IP);
1376 3416521 : InsertNode(N);
1377 3416521 : return SDValue(N, 0);
1378 : }
1379 :
1380 6646243 : SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1381 6646243 : unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1382 : FoldingSetNodeID ID;
1383 6646243 : AddNodeIDNode(ID, Opc, getVTList(VT), None);
1384 6646243 : ID.AddInteger(FI);
1385 6646243 : void *IP = nullptr;
1386 6646243 : if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1387 1541782 : return SDValue(E, 0);
1388 :
1389 5104461 : auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1390 5104461 : CSEMap.InsertNode(N, IP);
1391 5104461 : InsertNode(N);
1392 5104461 : return SDValue(N, 0);
1393 : }
1394 :
1395 9548 : SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1396 : unsigned char TargetFlags) {
1397 : assert((TargetFlags == 0 || isTarget) &&
1398 : "Cannot set target flags on target-independent jump tables");
1399 9548 : unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1400 : FoldingSetNodeID ID;
1401 9548 : AddNodeIDNode(ID, Opc, getVTList(VT), None);
1402 9548 : ID.AddInteger(JTI);
1403 9548 : ID.AddInteger(TargetFlags);
1404 9548 : void *IP = nullptr;
1405 9548 : if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1406 51 : return SDValue(E, 0);
1407 :
1408 9497 : auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1409 9497 : CSEMap.InsertNode(N, IP);
1410 9497 : InsertNode(N);
1411 9497 : return SDValue(N, 0);
1412 : }
1413 :
1414 99449 : SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1415 : unsigned Alignment, int Offset,
1416 : bool isTarget,
1417 : unsigned char TargetFlags) {
1418 : assert((TargetFlags == 0 || isTarget) &&
1419 : "Cannot set target flags on target-independent globals");
1420 99449 : if (Alignment == 0)
1421 33606 : Alignment = MF->getFunction().optForSize()
1422 33606 : ? getDataLayout().getABITypeAlignment(C->getType())
1423 33400 : : getDataLayout().getPrefTypeAlignment(C->getType());
1424 99449 : unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1425 : FoldingSetNodeID ID;
1426 99449 : AddNodeIDNode(ID, Opc, getVTList(VT), None);
1427 99449 : ID.AddInteger(Alignment);
1428 99449 : ID.AddInteger(Offset);
1429 99449 : ID.AddPointer(C);
1430 99449 : ID.AddInteger(TargetFlags);
1431 99449 : void *IP = nullptr;
1432 99449 : if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1433 5108 : return SDValue(E, 0);
1434 :
1435 94341 : auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment,
1436 : TargetFlags);
1437 94341 : CSEMap.InsertNode(N, IP);
1438 94341 : InsertNode(N);
1439 94341 : return SDValue(N, 0);
1440 : }
1441 :
1442 265 : SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1443 : unsigned Alignment, int Offset,
1444 : bool isTarget,
1445 : unsigned char TargetFlags) {
1446 : assert((TargetFlags == 0 || isTarget) &&
1447 : "Cannot set target flags on target-independent globals");
1448 265 : if (Alignment == 0)
1449 0 : Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
1450 265 : unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1451 : FoldingSetNodeID ID;
1452 265 : AddNodeIDNode(ID, Opc, getVTList(VT), None);
1453 265 : ID.AddInteger(Alignment);
1454 265 : ID.AddInteger(Offset);
1455 265 : C->addSelectionDAGCSEId(ID);
1456 265 : ID.AddInteger(TargetFlags);
1457 265 : void *IP = nullptr;
1458 265 : if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1459 0 : return SDValue(E, 0);
1460 :
1461 265 : auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment,
1462 : TargetFlags);
1463 265 : CSEMap.InsertNode(N, IP);
1464 265 : InsertNode(N);
1465 265 : return SDValue(N, 0);
1466 : }
1467 :
1468 0 : SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1469 : unsigned char TargetFlags) {
1470 : FoldingSetNodeID ID;
1471 0 : AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
1472 0 : ID.AddInteger(Index);
1473 0 : ID.AddInteger(Offset);
1474 0 : ID.AddInteger(TargetFlags);
1475 0 : void *IP = nullptr;
1476 0 : if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1477 0 : return SDValue(E, 0);
1478 :
1479 0 : auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags);
1480 0 : CSEMap.InsertNode(N, IP);
1481 0 : InsertNode(N);
1482 0 : return SDValue(N, 0);
1483 : }
1484 :
1485 725492 : SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1486 : FoldingSetNodeID ID;
1487 725492 : AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
1488 725492 : ID.AddPointer(MBB);
1489 725492 : void *IP = nullptr;
1490 725492 : if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1491 45 : return SDValue(E, 0);
1492 :
1493 725447 : auto *N = newSDNode<BasicBlockSDNode>(MBB);
1494 725447 : CSEMap.InsertNode(N, IP);
1495 725447 : InsertNode(N);
1496 725447 : return SDValue(N, 0);
1497 : }
1498 :
1499 122129 : SDValue SelectionDAG::getValueType(EVT VT) {
1500 122129 : if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1501 114385 : ValueTypeNodes.size())
1502 5992 : ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1503 :
1504 122129 : SDNode *&N = VT.isExtended() ?
1505 122129 : ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1506 :
1507 122129 : if (N) return SDValue(N, 0);
1508 92967 : N = newSDNode<VTSDNode>(VT);
1509 92967 : InsertNode(N);
1510 92967 : return SDValue(N, 0);
1511 : }
1512 :
1513 14848 : SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1514 14848 : SDNode *&N = ExternalSymbols[Sym];
1515 14848 : if (N) return SDValue(N, 0);
1516 12149 : N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
1517 12149 : InsertNode(N);
1518 12149 : return SDValue(N, 0);
1519 : }
1520 :
1521 75 : SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1522 75 : SDNode *&N = MCSymbols[Sym];
1523 75 : if (N)
1524 18 : return SDValue(N, 0);
1525 57 : N = newSDNode<MCSymbolSDNode>(Sym, VT);
1526 57 : InsertNode(N);
1527 57 : return SDValue(N, 0);
1528 : }
1529 :
1530 34417 : SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1531 : unsigned char TargetFlags) {
1532 : SDNode *&N =
1533 34417 : TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
1534 34417 : TargetFlags)];
1535 34417 : if (N) return SDValue(N, 0);
1536 23258 : N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
1537 23258 : InsertNode(N);
1538 23258 : return SDValue(N, 0);
1539 : }
1540 :
1541 255130 : SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1542 510260 : if ((unsigned)Cond >= CondCodeNodes.size())
1543 12165 : CondCodeNodes.resize(Cond+1);
1544 :
1545 510260 : if (!CondCodeNodes[Cond]) {
1546 187885 : auto *N = newSDNode<CondCodeSDNode>(Cond);
1547 187885 : CondCodeNodes[Cond] = N;
1548 187885 : InsertNode(N);
1549 : }
1550 :
1551 510260 : return SDValue(CondCodeNodes[Cond], 0);
1552 : }
1553 :
1554 : /// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
1555 : /// point at N1 to point at N2 and indices that point at N2 to point at N1.
1556 1228 : static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
1557 : std::swap(N1, N2);
1558 : ShuffleVectorSDNode::commuteMask(M);
1559 1228 : }
1560 :
1561 122631 : SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
1562 : SDValue N2, ArrayRef<int> Mask) {
1563 : assert(VT.getVectorNumElements() == Mask.size() &&
1564 : "Must have the same number of vector elements as mask elements!");
1565 : assert(VT == N1.getValueType() && VT == N2.getValueType() &&
1566 : "Invalid VECTOR_SHUFFLE");
1567 :
1568 : // Canonicalize shuffle undef, undef -> undef
1569 245262 : if (N1.isUndef() && N2.isUndef())
1570 102 : return getUNDEF(VT);
1571 :
1572 : // Validate that all indices in Mask are within the range of the elements
1573 : // input to the shuffle.
1574 122529 : int NElts = Mask.size();
1575 : assert(llvm::all_of(Mask,
1576 : [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
1577 : "Index out of range");
1578 :
1579 : // Copy the mask so we can do any needed cleanup.
1580 : SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end());
1581 :
1582 : // Canonicalize shuffle v, v -> v, undef
1583 122529 : if (N1 == N2) {
1584 7969 : N2 = getUNDEF(VT);
1585 67988 : for (int i = 0; i != NElts; ++i)
1586 120038 : if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
1587 : }
1588 :
1589 : // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
1590 245058 : if (N1.isUndef())
1591 187 : commuteShuffle(N1, N2, MaskVec);
1592 :
1593 122529 : if (TLI->hasVectorBlend()) {
1594 : // If shuffling a splat, try to blend the splat instead. We do this here so
1595 : // that even when this arises during lowering we don't have to re-handle it.
1596 : auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
1597 : BitVector UndefElements;
1598 : SDValue Splat = BV->getSplatValue(&UndefElements);
1599 : if (!Splat)
1600 : return;
1601 :
1602 : for (int i = 0; i < NElts; ++i) {
1603 : if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
1604 : continue;
1605 :
1606 : // If this input comes from undef, mark it as such.
1607 : if (UndefElements[MaskVec[i] - Offset]) {
1608 : MaskVec[i] = -1;
1609 : continue;
1610 : }
1611 :
1612 : // If we can blend a non-undef lane, use that instead.
1613 : if (!UndefElements[i])
1614 : MaskVec[i] = i + Offset;
1615 : }
1616 112865 : };
1617 : if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
1618 4367 : BlendSplat(N1BV, 0);
1619 : if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
1620 5246 : BlendSplat(N2BV, NElts);
1621 : }
1622 :
1623 : // Canonicalize all index into lhs, -> shuffle lhs, undef
1624 : // Canonicalize all index into rhs, -> shuffle rhs, undef
1625 : bool AllLHS = true, AllRHS = true;
1626 122529 : bool N2Undef = N2.isUndef();
1627 1482004 : for (int i = 0; i != NElts; ++i) {
1628 2718950 : if (MaskVec[i] >= NElts) {
1629 174571 : if (N2Undef)
1630 3308 : MaskVec[i] = -1;
1631 : else
1632 : AllLHS = false;
1633 1184904 : } else if (MaskVec[i] >= 0) {
1634 : AllRHS = false;
1635 : }
1636 : }
1637 122529 : if (AllLHS && AllRHS)
1638 212 : return getUNDEF(VT);
1639 122317 : if (AllLHS && !N2Undef)
1640 3732 : N2 = getUNDEF(VT);
1641 122317 : if (AllRHS) {
1642 1041 : N1 = getUNDEF(VT);
1643 1041 : commuteShuffle(N1, N2, MaskVec);
1644 : }
1645 : // Reset our undef status after accounting for the mask.
1646 122317 : N2Undef = N2.isUndef();
1647 : // Re-check whether both sides ended up undef.
1648 244634 : if (N1.isUndef() && N2Undef)
1649 0 : return getUNDEF(VT);
1650 :
1651 : // If Identity shuffle return that node.
1652 : bool Identity = true, AllSame = true;
1653 1479525 : for (int i = 0; i != NElts; ++i) {
1654 2714416 : if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
1655 1357208 : if (MaskVec[i] != MaskVec[0]) AllSame = false;
1656 : }
1657 122317 : if (Identity && NElts)
1658 13480 : return N1;
1659 :
1660 : // Shuffling a constant splat doesn't change the result.
1661 108837 : if (N2Undef) {
1662 64142 : SDValue V = N1;
1663 :
1664 : // Look through any bitcasts. We check that these don't change the number
1665 : // (and size) of elements and just changes their types.
1666 80689 : while (V.getOpcode() == ISD::BITCAST)
1667 16547 : V = V->getOperand(0);
1668 :
1669 : // A splat should always show up as a build vector node.
1670 : if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
1671 : BitVector UndefElements;
1672 1981 : SDValue Splat = BV->getSplatValue(&UndefElements);
1673 : // If this is a splat of an undef, shuffling it is also undef.
1674 1981 : if (Splat && Splat.isUndef())
1675 0 : return getUNDEF(VT);
1676 :
1677 : bool SameNumElts =
1678 1981 : V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
1679 :
1680 : // We only have a splat which can skip shuffles if there is a splatted
1681 : // value and no undef lanes rearranged by the shuffle.
1682 2367 : if (Splat && UndefElements.none()) {
1683 : // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
1684 : // number of elements match or the value splatted is a zero constant.
1685 242 : if (SameNumElts)
1686 215 : return N1;
1687 : if (auto *C = dyn_cast<ConstantSDNode>(Splat))
1688 54 : if (C->isNullValue())
1689 20 : return N1;
1690 : }
1691 :
1692 : // If the shuffle itself creates a splat, build the vector directly.
1693 1746 : if (AllSame && SameNumElts) {
1694 614 : EVT BuildVT = BV->getValueType(0);
1695 614 : const SDValue &Splatted = BV->getOperand(MaskVec[0]);
1696 614 : SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
1697 :
1698 : // We may have jumped through bitcasts, so the type of the
1699 : // BUILD_VECTOR may not match the type of the shuffle.
1700 614 : if (BuildVT != VT)
1701 0 : NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
1702 614 : return NewBV;
1703 : }
1704 : }
1705 : }
1706 :
1707 : FoldingSetNodeID ID;
1708 107988 : SDValue Ops[2] = { N1, N2 };
1709 107988 : AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
1710 1369506 : for (int i = 0; i != NElts; ++i)
1711 2523036 : ID.AddInteger(MaskVec[i]);
1712 :
1713 107988 : void* IP = nullptr;
1714 107988 : if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1715 943 : return SDValue(E, 0);
1716 :
1717 : // Allocate the mask array for the node out of the BumpPtrAllocator, since
1718 : // SDNode doesn't have access to it. This memory will be "leaked" when
1719 : // the node is deallocated, but recovered when the NodeAllocator is released.
1720 107045 : int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
1721 : std::copy(MaskVec.begin(), MaskVec.end(), MaskAlloc);
1722 :
1723 214090 : auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
1724 : dl.getDebugLoc(), MaskAlloc);
1725 107045 : createOperands(N, Ops);
1726 :
1727 107045 : CSEMap.InsertNode(N, IP);
1728 107045 : InsertNode(N);
1729 : SDValue V = SDValue(N, 0);
1730 : NewSDValueDbgMsg(V, "Creating new node: ", this);
1731 107045 : return V;
1732 : }
1733 :
1734 5620 : SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
1735 5620 : EVT VT = SV.getValueType(0);
1736 5620 : SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
1737 : ShuffleVectorSDNode::commuteMask(MaskVec);
1738 :
1739 5620 : SDValue Op0 = SV.getOperand(0);
1740 5620 : SDValue Op1 = SV.getOperand(1);
1741 11240 : return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
1742 : }
1743 :
1744 21039996 : SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
1745 : FoldingSetNodeID ID;
1746 21039996 : AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
1747 21039995 : ID.AddInteger(RegNo);
1748 21039995 : void *IP = nullptr;
1749 21039995 : if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1750 14313049 : return SDValue(E, 0);
1751 :
1752 6726947 : auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
1753 6726947 : N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA);
1754 6726947 : CSEMap.InsertNode(N, IP);
1755 6726947 : InsertNode(N);
1756 6726947 : return SDValue(N, 0);
1757 : }
1758 :
1759 1003767 : SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
1760 : FoldingSetNodeID ID;
1761 1003767 : AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
1762 1003767 : ID.AddPointer(RegMask);
1763 1003767 : void *IP = nullptr;
1764 1003767 : if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1765 411879 : return SDValue(E, 0);
1766 :
1767 591888 : auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
1768 591888 : CSEMap.InsertNode(N, IP);
1769 591888 : InsertNode(N);
1770 591888 : return SDValue(N, 0);
1771 : }
1772 :
1773 993978 : SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
1774 : MCSymbol *Label) {
1775 993978 : return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
1776 : }
1777 :
1778 993979 : SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
1779 : SDValue Root, MCSymbol *Label) {
1780 : FoldingSetNodeID ID;
1781 993979 : SDValue Ops[] = { Root };
1782 993979 : AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
1783 993979 : ID.AddPointer(Label);
1784 993979 : void *IP = nullptr;
1785 993979 : if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1786 0 : return SDValue(E, 0);
1787 :
1788 1987958 : auto *N = newSDNode<LabelSDNode>(dl.getIROrder(), dl.getDebugLoc(), Label);
1789 993979 : createOperands(N, Ops);
1790 :
1791 993979 : CSEMap.InsertNode(N, IP);
1792 993979 : InsertNode(N);
1793 993979 : return SDValue(N, 0);
1794 : }
1795 :
1796 480 : SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
1797 : int64_t Offset,
1798 : bool isTarget,
1799 : unsigned char TargetFlags) {
1800 480 : unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
1801 :
1802 : FoldingSetNodeID ID;
1803 480 : AddNodeIDNode(ID, Opc, getVTList(VT), None);
1804 480 : ID.AddPointer(BA);
1805 480 : ID.AddInteger(Offset);
1806 480 : ID.AddInteger(TargetFlags);
1807 480 : void *IP = nullptr;
1808 480 : if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1809 2 : return SDValue(E, 0);
1810 :
1811 478 : auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
1812 478 : CSEMap.InsertNode(N, IP);
1813 478 : InsertNode(N);
1814 478 : return SDValue(N, 0);
1815 : }
1816 :
1817 690 : SDValue SelectionDAG::getSrcValue(const Value *V) {
1818 : assert((!V || V->getType()->isPointerTy()) &&
1819 : "SrcValue is not a pointer?");
1820 :
1821 : FoldingSetNodeID ID;
1822 690 : AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
1823 690 : ID.AddPointer(V);
1824 :
1825 690 : void *IP = nullptr;
1826 690 : if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1827 179 : return SDValue(E, 0);
1828 :
1829 511 : auto *N = newSDNode<SrcValueSDNode>(V);
1830 511 : CSEMap.InsertNode(N, IP);
1831 511 : InsertNode(N);
1832 511 : return SDValue(N, 0);
1833 : }
1834 :
1835 17390 : SDValue SelectionDAG::getMDNode(const MDNode *MD) {
1836 : FoldingSetNodeID ID;
1837 17390 : AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
1838 17390 : ID.AddPointer(MD);
1839 :
1840 17390 : void *IP = nullptr;
1841 17390 : if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1842 0 : return SDValue(E, 0);
1843 :
1844 17390 : auto *N = newSDNode<MDNodeSDNode>(MD);
1845 17390 : CSEMap.InsertNode(N, IP);
1846 17390 : InsertNode(N);
1847 17390 : return SDValue(N, 0);
1848 : }
1849 :
1850 560630 : SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
1851 50 : if (VT == V.getValueType())
1852 118088 : return V;
1853 :
1854 885084 : return getNode(ISD::BITCAST, SDLoc(V), VT, V);
1855 : }
1856 :
1857 213 : SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
1858 : unsigned SrcAS, unsigned DestAS) {
1859 213 : SDValue Ops[] = {Ptr};
1860 : FoldingSetNodeID ID;
1861 213 : AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
1862 213 : ID.AddInteger(SrcAS);
1863 213 : ID.AddInteger(DestAS);
1864 :
1865 213 : void *IP = nullptr;
1866 213 : if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1867 0 : return SDValue(E, 0);
1868 :
1869 426 : auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
1870 : VT, SrcAS, DestAS);
1871 213 : createOperands(N, Ops);
1872 :
1873 213 : CSEMap.InsertNode(N, IP);
1874 213 : InsertNode(N);
1875 213 : return SDValue(N, 0);
1876 : }
1877 :
1878 : /// getShiftAmountOperand - Return the specified value casted to
1879 : /// the target's desired shift amount type.
1880 282891 : SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
1881 282891 : EVT OpTy = Op.getValueType();
1882 282891 : EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
1883 565782 : if (OpTy == ShTy || OpTy.isVector()) return Op;
1884 :
1885 54060 : return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
1886 : }
1887 :
1888 33 : SDValue SelectionDAG::expandVAArg(SDNode *Node) {
1889 : SDLoc dl(Node);
1890 33 : const TargetLowering &TLI = getTargetLoweringInfo();
1891 33 : const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
1892 33 : EVT VT = Node->getValueType(0);
1893 33 : SDValue Tmp1 = Node->getOperand(0);
1894 33 : SDValue Tmp2 = Node->getOperand(1);
1895 33 : unsigned Align = Node->getConstantOperandVal(3);
1896 :
1897 : SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
1898 66 : Tmp2, MachinePointerInfo(V));
1899 33 : SDValue VAList = VAListLoad;
1900 :
1901 33 : if (Align > TLI.getMinStackArgumentAlignment()) {
1902 : assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2");
1903 :
1904 13 : VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
1905 13 : getConstant(Align - 1, dl, VAList.getValueType()));
1906 :
1907 13 : VAList = getNode(ISD::AND, dl, VAList.getValueType(), VAList,
1908 13 : getConstant(-(int64_t)Align, dl, VAList.getValueType()));
1909 : }
1910 :
1911 : // Increment the pointer, VAList, to the next vaarg
1912 33 : Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
1913 33 : getConstant(getDataLayout().getTypeAllocSize(
1914 33 : VT.getTypeForEVT(*getContext())),
1915 33 : dl, VAList.getValueType()));
1916 : // Store the incremented VAList to the legalized pointer
1917 33 : Tmp1 =
1918 33 : getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
1919 : // Load the actual argument out of the pointer VAList
1920 33 : return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
1921 : }
1922 :
1923 7 : SDValue SelectionDAG::expandVACopy(SDNode *Node) {
1924 : SDLoc dl(Node);
1925 : const TargetLowering &TLI = getTargetLoweringInfo();
1926 : // This defaults to loading a pointer from the input and storing it to the
1927 : // output, returning the chain.
1928 14 : const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
1929 7 : const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
1930 : SDValue Tmp1 =
1931 : getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
1932 14 : Node->getOperand(2), MachinePointerInfo(VS));
1933 7 : return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
1934 7 : MachinePointerInfo(VD));
1935 : }
1936 :
1937 2765 : SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
1938 2765 : MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
1939 : unsigned ByteSize = VT.getStoreSize();
1940 2765 : Type *Ty = VT.getTypeForEVT(*getContext());
1941 : unsigned StackAlign =
1942 2765 : std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign);
1943 :
1944 2765 : int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);
1945 2765 : return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
1946 : }
1947 :
1948 3221 : SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
1949 6442 : unsigned Bytes = std::max(VT1.getStoreSize(), VT2.getStoreSize());
1950 3221 : Type *Ty1 = VT1.getTypeForEVT(*getContext());
1951 3221 : Type *Ty2 = VT2.getTypeForEVT(*getContext());
1952 3221 : const DataLayout &DL = getDataLayout();
1953 : unsigned Align =
1954 3221 : std::max(DL.getPrefTypeAlignment(Ty1), DL.getPrefTypeAlignment(Ty2));
1955 :
1956 3221 : MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
1957 3221 : int FrameIdx = MFI.CreateStackObject(Bytes, Align, false);
1958 3221 : return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
1959 : }
1960 :
1961 354680 : SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
1962 : ISD::CondCode Cond, const SDLoc &dl) {
1963 354680 : EVT OpVT = N1.getValueType();
1964 :
1965 : // These setcc operations always fold.
1966 354680 : switch (Cond) {
1967 : default: break;
1968 35 : case ISD::SETFALSE:
1969 35 : case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
1970 40 : case ISD::SETTRUE:
1971 40 : case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
1972 :
1973 : case ISD::SETOEQ:
1974 : case ISD::SETOGT:
1975 : case ISD::SETOGE:
1976 : case ISD::SETOLT:
1977 : case ISD::SETOLE:
1978 : case ISD::SETONE:
1979 : case ISD::SETO:
1980 : case ISD::SETUO:
1981 : case ISD::SETUEQ:
1982 : case ISD::SETUNE:
1983 : assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
1984 : break;
1985 : }
1986 :
1987 : if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
1988 199628 : const APInt &C2 = N2C->getAPIntValue();
1989 : if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
1990 10279 : const APInt &C1 = N1C->getAPIntValue();
1991 :
1992 10279 : switch (Cond) {
1993 0 : default: llvm_unreachable("Unknown integer setcc!");
1994 8016 : case ISD::SETEQ: return getBoolConstant(C1 == C2, dl, VT, OpVT);
1995 148 : case ISD::SETNE: return getBoolConstant(C1 != C2, dl, VT, OpVT);
1996 23 : case ISD::SETULT: return getBoolConstant(C1.ult(C2), dl, VT, OpVT);
1997 19 : case ISD::SETUGT: return getBoolConstant(C1.ugt(C2), dl, VT, OpVT);
1998 37 : case ISD::SETULE: return getBoolConstant(C1.ule(C2), dl, VT, OpVT);
1999 16 : case ISD::SETUGE: return getBoolConstant(C1.uge(C2), dl, VT, OpVT);
2000 1946 : case ISD::SETLT: return getBoolConstant(C1.slt(C2), dl, VT, OpVT);
2001 57 : case ISD::SETGT: return getBoolConstant(C1.sgt(C2), dl, VT, OpVT);
2002 4 : case ISD::SETLE: return getBoolConstant(C1.sle(C2), dl, VT, OpVT);
2003 13 : case ISD::SETGE: return getBoolConstant(C1.sge(C2), dl, VT, OpVT);
2004 : }
2005 : }
2006 : }
2007 : if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1)) {
2008 : if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2)) {
2009 672 : APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
2010 224 : switch (Cond) {
2011 : default: break;
2012 0 : case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2013 0 : return getUNDEF(VT);
2014 : LLVM_FALLTHROUGH;
2015 : case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
2016 88 : OpVT);
2017 0 : case ISD::SETNE: if (R==APFloat::cmpUnordered)
2018 0 : return getUNDEF(VT);
2019 : LLVM_FALLTHROUGH;
2020 0 : case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2021 0 : R==APFloat::cmpLessThan, dl, VT,
2022 0 : OpVT);
2023 0 : case ISD::SETLT: if (R==APFloat::cmpUnordered)
2024 0 : return getUNDEF(VT);
2025 : LLVM_FALLTHROUGH;
2026 : case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
2027 23 : OpVT);
2028 0 : case ISD::SETGT: if (R==APFloat::cmpUnordered)
2029 0 : return getUNDEF(VT);
2030 : LLVM_FALLTHROUGH;
2031 : case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl,
2032 10 : VT, OpVT);
2033 0 : case ISD::SETLE: if (R==APFloat::cmpUnordered)
2034 0 : return getUNDEF(VT);
2035 : LLVM_FALLTHROUGH;
2036 : case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan ||
2037 : R==APFloat::cmpEqual, dl, VT,
2038 4 : OpVT);
2039 0 : case ISD::SETGE: if (R==APFloat::cmpUnordered)
2040 0 : return getUNDEF(VT);
2041 : LLVM_FALLTHROUGH;
2042 6 : case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2043 6 : R==APFloat::cmpEqual, dl, VT, OpVT);
2044 8 : case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
2045 8 : OpVT);
2046 23 : case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
2047 23 : OpVT);
2048 0 : case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered ||
2049 0 : R==APFloat::cmpEqual, dl, VT,
2050 0 : OpVT);
2051 16 : case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
2052 16 : OpVT);
2053 14 : case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered ||
2054 7 : R==APFloat::cmpLessThan, dl, VT,
2055 7 : OpVT);
2056 36 : case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2057 : R==APFloat::cmpUnordered, dl, VT,
2058 36 : OpVT);
2059 1 : case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl,
2060 1 : VT, OpVT);
2061 2 : case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
2062 2 : OpVT);
2063 : }
2064 : } else {
2065 : // Ensure that the constant occurs on the RHS.
2066 508 : ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
2067 : MVT CompVT = N1.getValueType().getSimpleVT();
2068 1016 : if (!TLI->isCondCodeLegal(SwappedCond, CompVT))
2069 290 : return SDValue();
2070 :
2071 218 : return getSetCC(dl, VT, N2, N1, SwappedCond);
2072 : }
2073 : }
2074 :
2075 : // Could not fold it.
2076 343594 : return SDValue();
2077 : }
2078 :
2079 : /// See if the specified operand can be simplified with the knowledge that only
2080 : /// the bits specified by Mask are used.
2081 926531 : SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &Mask) {
2082 926531 : switch (V.getOpcode()) {
2083 : default:
2084 : break;
2085 8050 : case ISD::Constant: {
2086 : const ConstantSDNode *CV = cast<ConstantSDNode>(V.getNode());
2087 : assert(CV && "Const value should be ConstSDNode.");
2088 8050 : const APInt &CVal = CV->getAPIntValue();
2089 8050 : APInt NewVal = CVal & Mask;
2090 8050 : if (NewVal != CVal)
2091 492 : return getConstant(NewVal, SDLoc(V), V.getValueType());
2092 : break;
2093 : }
2094 6215 : case ISD::OR:
2095 : case ISD::XOR:
2096 : // If the LHS or RHS don't contribute bits to the or, drop them.
2097 6215 : if (MaskedValueIsZero(V.getOperand(0), Mask))
2098 702 : return V.getOperand(1);
2099 5513 : if (MaskedValueIsZero(V.getOperand(1), Mask))
2100 780 : return V.getOperand(0);
2101 : break;
2102 49529 : case ISD::SRL:
2103 : // Only look at single-use SRLs.
2104 : if (!V.getNode()->hasOneUse())
2105 : break;
2106 : if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
2107 : // See if we can recursively simplify the LHS.
2108 45304 : unsigned Amt = RHSC->getZExtValue();
2109 :
2110 : // Watch out for shift count overflow though.
2111 45304 : if (Amt >= Mask.getBitWidth())
2112 : break;
2113 : APInt NewMask = Mask << Amt;
2114 45303 : if (SDValue SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask))
2115 1198 : return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS,
2116 2396 : V.getOperand(1));
2117 : }
2118 : break;
2119 11591 : case ISD::AND: {
2120 : // X & -1 -> X (ignoring bits which aren't demanded).
2121 11591 : ConstantSDNode *AndVal = isConstOrConstSplat(V.getOperand(1));
2122 22484 : if (AndVal && Mask.isSubsetOf(AndVal->getAPIntValue()))
2123 1417 : return V.getOperand(0);
2124 : break;
2125 : }
2126 537 : case ISD::ANY_EXTEND: {
2127 537 : SDValue Src = V.getOperand(0);
2128 537 : unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
2129 : // Being conservative here - only peek through if we only demand bits in the
2130 : // non-extended source (even though the extended bits are technically undef).
2131 537 : if (Mask.getActiveBits() > SrcBitWidth)
2132 : break;
2133 530 : APInt SrcMask = Mask.trunc(SrcBitWidth);
2134 530 : if (SDValue DemandedSrc = GetDemandedBits(Src, SrcMask))
2135 116 : return getNode(ISD::ANY_EXTEND, SDLoc(V), V.getValueType(), DemandedSrc);
2136 : break;
2137 : }
2138 : }
2139 922130 : return SDValue();
2140 : }
2141 :
2142 : /// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2143 : /// use this predicate to simplify operations downstream.
2144 93302 : bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2145 93302 : unsigned BitWidth = Op.getScalarValueSizeInBits();
2146 93302 : return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);
2147 : }
2148 :
2149 : /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2150 : /// this predicate to simplify operations downstream. Mask is known to be zero
2151 : /// for bits that V cannot have.
2152 1831295 : bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
2153 : unsigned Depth) const {
2154 3662590 : return Mask.isSubsetOf(computeKnownBits(Op, Depth).Zero);
2155 : }
2156 :
2157 : /// Helper function that checks to see if a node is a constant or a
2158 : /// build vector of splat constants at least within the demanded elts.
2159 0 : static ConstantSDNode *isConstOrDemandedConstSplat(SDValue N,
2160 : const APInt &DemandedElts) {
2161 : if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
2162 0 : return CN;
2163 0 : if (N.getOpcode() != ISD::BUILD_VECTOR)
2164 0 : return nullptr;
2165 0 : EVT VT = N.getValueType();
2166 : ConstantSDNode *Cst = nullptr;
2167 : unsigned NumElts = VT.getVectorNumElements();
2168 : assert(DemandedElts.getBitWidth() == NumElts && "Unexpected vector size");
2169 0 : for (unsigned i = 0; i != NumElts; ++i) {
2170 0 : if (!DemandedElts[i])
2171 0 : continue;
2172 : ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(i));
2173 0 : if (!C || (Cst && Cst->getAPIntValue() != C->getAPIntValue()) ||
2174 0 : C->getValueType(0) != VT.getScalarType())
2175 0 : return nullptr;
2176 : Cst = C;
2177 : }
2178 : return Cst;
2179 : }
2180 :
2181 : /// If a SHL/SRA/SRL node has a constant or splat constant shift amount that
2182 : /// is less than the element bit-width of the shift node, return it.
2183 1245154 : static const APInt *getValidShiftAmountConstant(SDValue V) {
2184 2490308 : if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1))) {
2185 : // Shifting more than the bitwidth is not valid.
2186 1198817 : const APInt &ShAmt = SA->getAPIntValue();
2187 1198817 : if (ShAmt.ult(V.getScalarValueSizeInBits()))
2188 1198774 : return &ShAmt;
2189 : }
2190 : return nullptr;
2191 : }
2192 :
2193 : /// Determine which bits of Op are known to be either zero or one and return
2194 : /// them in Known. For vectors, the known bits are those that are shared by
2195 : /// every vector element.
2196 20256543 : KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
2197 40513086 : EVT VT = Op.getValueType();
2198 : APInt DemandedElts = VT.isVector()
2199 : ? APInt::getAllOnesValue(VT.getVectorNumElements())
2200 21280072 : : APInt(1, 1);
2201 20256543 : return computeKnownBits(Op, DemandedElts, Depth);
2202 : }
2203 :
2204 : /// Determine which bits of Op are known to be either zero or one and return
2205 : /// them in Known. The DemandedElts argument allows us to only collect the known
2206 : /// bits that are shared by the requested vector elements.
2207 38967734 : KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
2208 : unsigned Depth) const {
2209 38967734 : unsigned BitWidth = Op.getScalarValueSizeInBits();
2210 :
2211 38967734 : KnownBits Known(BitWidth); // Don't know anything.
2212 :
2213 : if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2214 : // We know all of the bits for a constant!
2215 16413928 : Known.One = C->getAPIntValue();
2216 8206964 : Known.Zero = ~Known.One;
2217 8206964 : return Known;
2218 : }
2219 : if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) {
2220 : // We know all of the bits for a constant fp!
2221 11789 : Known.One = C->getValueAPF().bitcastToAPInt();
2222 5877 : Known.Zero = ~Known.One;
2223 5877 : return Known;
2224 : }
2225 :
2226 30754893 : if (Depth == 6)
2227 : return Known; // Limit search depth.
2228 :
2229 29501615 : KnownBits Known2;
2230 29501615 : unsigned NumElts = DemandedElts.getBitWidth();
2231 :
2232 29501615 : if (!DemandedElts)
2233 : return Known; // No demanded elts, better to assume we don't know anything.
2234 :
2235 : unsigned Opcode = Op.getOpcode();
2236 29501615 : switch (Opcode) {
2237 704995 : case ISD::BUILD_VECTOR:
2238 : // Collect the known bits that are shared by every demanded vector element.
2239 : assert(NumElts == Op.getValueType().getVectorNumElements() &&
2240 : "Unexpected vector size");
2241 704995 : Known.Zero.setAllBits(); Known.One.setAllBits();
2242 4438538 : for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
2243 3946802 : if (!DemandedElts[i])
2244 2258333 : continue;
2245 :
2246 1688469 : SDValue SrcOp = Op.getOperand(i);
2247 1688469 : Known2 = computeKnownBits(SrcOp, Depth + 1);
2248 :
2249 : // BUILD_VECTOR can implicitly truncate sources, we must handle this.
2250 1688469 : if (SrcOp.getValueSizeInBits() != BitWidth) {
2251 : assert(SrcOp.getValueSizeInBits() > BitWidth &&
2252 : "Expected BUILD_VECTOR implicit truncation");
2253 24841 : Known2 = Known2.trunc(BitWidth);
2254 : }
2255 :
2256 : // Known bits are the values that are shared by every demanded element.
2257 : Known.One &= Known2.One;
2258 : Known.Zero &= Known2.Zero;
2259 :
2260 : // If we don't know any bits, early out.
2261 1688469 : if (Known.isUnknown())
2262 : break;
2263 491736 : }
2264 : break;
2265 : case ISD::VECTOR_SHUFFLE: {
2266 : // Collect the known bits that are shared by every vector element referenced
2267 : // by the shuffle.
2268 : APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
2269 201908 : Known.Zero.setAllBits(); Known.One.setAllBits();
2270 : const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
2271 : assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
2272 2172394 : for (unsigned i = 0; i != NumElts; ++i) {
2273 2028983 : if (!DemandedElts[i])
2274 : continue;
2275 :
2276 701062 : int M = SVN->getMaskElt(i);
2277 701062 : if (M < 0) {
2278 : // For UNDEF elements, we don't know anything about the common state of
2279 : // the shuffle result.
2280 : Known.resetAll();
2281 58497 : DemandedLHS.clearAllBits();
2282 58497 : DemandedRHS.clearAllBits();
2283 58497 : break;
2284 : }
2285 :
2286 642565 : if ((unsigned)M < NumElts)
2287 538474 : DemandedLHS.setBit((unsigned)M % NumElts);
2288 : else
2289 104091 : DemandedRHS.setBit((unsigned)M % NumElts);
2290 : }
2291 : // Known bits are the values that are shared by every demanded element.
2292 201908 : if (!!DemandedLHS) {
2293 116728 : SDValue LHS = Op.getOperand(0);
2294 116728 : Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);
2295 : Known.One &= Known2.One;
2296 : Known.Zero &= Known2.Zero;
2297 : }
2298 : // If we don't know any bits, early out.
2299 201908 : if (Known.isUnknown())
2300 : break;
2301 29076 : if (!!DemandedRHS) {
2302 27412 : SDValue RHS = Op.getOperand(1);
2303 27412 : Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);
2304 : Known.One &= Known2.One;
2305 : Known.Zero &= Known2.Zero;
2306 : }
2307 : break;
2308 : }
2309 38456 : case ISD::CONCAT_VECTORS: {
2310 : // Split DemandedElts and test each of the demanded subvectors.
2311 38456 : Known.Zero.setAllBits(); Known.One.setAllBits();
2312 115368 : EVT SubVectorVT = Op.getOperand(0).getValueType();
2313 : unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
2314 : unsigned NumSubVectors = Op.getNumOperands();
2315 70144 : for (unsigned i = 0; i != NumSubVectors; ++i) {
2316 62278 : APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
2317 124556 : DemandedSub = DemandedSub.trunc(NumSubVectorElts);
2318 62278 : if (!!DemandedSub) {
2319 38717 : SDValue Sub = Op.getOperand(i);
2320 38717 : Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
2321 : Known.One &= Known2.One;
2322 : Known.Zero &= Known2.Zero;
2323 : }
2324 : // If we don't know any bits, early out.
2325 62278 : if (Known.isUnknown())
2326 : break;
2327 : }
2328 : break;
2329 : }
2330 42644 : case ISD::INSERT_SUBVECTOR: {
2331 : // If we know the element index, demand any elements from the subvector and
2332 : // the remainder from the src its inserted into, otherwise demand them all.
2333 42644 : SDValue Src = Op.getOperand(0);
2334 42644 : SDValue Sub = Op.getOperand(1);
2335 : ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2336 42644 : unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2337 42644 : if (SubIdx && SubIdx->getAPIntValue().ule(NumElts - NumSubElts)) {
2338 42644 : Known.One.setAllBits();
2339 42644 : Known.Zero.setAllBits();
2340 42644 : uint64_t Idx = SubIdx->getZExtValue();
2341 42644 : APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2342 42644 : if (!!DemandedSubElts) {
2343 32683 : Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);
2344 32683 : if (Known.isUnknown())
2345 : break; // early-out.
2346 : }
2347 11391 : APInt SubMask = APInt::getBitsSet(NumElts, Idx, Idx + NumSubElts);
2348 11391 : APInt DemandedSrcElts = DemandedElts & ~SubMask;
2349 11391 : if (!!DemandedSrcElts) {
2350 10802 : Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
2351 : Known.One &= Known2.One;
2352 : Known.Zero &= Known2.Zero;
2353 : }
2354 : } else {
2355 0 : Known = computeKnownBits(Sub, Depth + 1);
2356 0 : if (Known.isUnknown())
2357 : break; // early-out.
2358 0 : Known2 = computeKnownBits(Src, Depth + 1);
2359 0 : Known.One &= Known2.One;
2360 0 : Known.Zero &= Known2.Zero;
2361 : }
2362 : break;
2363 : }
2364 173499 : case ISD::EXTRACT_SUBVECTOR: {
2365 : // If we know the element index, just demand that subvector elements,
2366 : // otherwise demand them all.
2367 173499 : SDValue Src = Op.getOperand(0);
2368 : ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
2369 173499 : unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2370 173499 : if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
2371 : // Offset the demanded elts by the subvector index.
2372 : uint64_t Idx = SubIdx->getZExtValue();
2373 173499 : APInt DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2374 173499 : Known = computeKnownBits(Src, DemandedSrc, Depth + 1);
2375 : } else {
2376 0 : Known = computeKnownBits(Src, Depth + 1);
2377 : }
2378 : break;
2379 : }
2380 1074811 : case ISD::BITCAST: {
2381 1074811 : SDValue N0 = Op.getOperand(0);
2382 1074811 : EVT SubVT = N0.getValueType();
2383 : unsigned SubBitWidth = SubVT.getScalarSizeInBits();
2384 :
2385 : // Ignore bitcasts from unsupported types.
2386 1074811 : if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
2387 : break;
2388 :
2389 : // Fast handling of 'identity' bitcasts.
2390 1074571 : if (BitWidth == SubBitWidth) {
2391 41720 : Known = computeKnownBits(N0, DemandedElts, Depth + 1);
2392 41720 : break;
2393 : }
2394 :
2395 1032851 : bool IsLE = getDataLayout().isLittleEndian();
2396 :
2397 : // Bitcast 'small element' vector to 'large element' scalar/vector.
2398 1032851 : if ((BitWidth % SubBitWidth) == 0) {
2399 : assert(N0.getValueType().isVector() && "Expected bitcast from vector");
2400 :
2401 : // Collect known bits for the (larger) output by collecting the known
2402 : // bits from each set of sub elements and shift these into place.
2403 : // We need to separately call computeKnownBits for each set of
2404 : // sub elements as the knownbits for each is likely to be different.
2405 548134 : unsigned SubScale = BitWidth / SubBitWidth;
2406 548134 : APInt SubDemandedElts(NumElts * SubScale, 0);
2407 2390846 : for (unsigned i = 0; i != NumElts; ++i)
2408 1842712 : if (DemandedElts[i])
2409 1311571 : SubDemandedElts.setBit(i * SubScale);
2410 :
2411 2313736 : for (unsigned i = 0; i != SubScale; ++i) {
2412 3531204 : Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
2413 1765602 : Depth + 1);
2414 1765602 : unsigned Shifts = IsLE ? i : SubScale - 1 - i;
2415 3531204 : Known.One |= Known2.One.zext(BitWidth).shl(SubBitWidth * Shifts);
2416 3532640 : Known.Zero |= Known2.Zero.zext(BitWidth).shl(SubBitWidth * Shifts);
2417 : }
2418 : }
2419 :
2420 : // Bitcast 'large element' scalar/vector to 'small element' vector.
2421 1032851 : if ((SubBitWidth % BitWidth) == 0) {
2422 : assert(Op.getValueType().isVector() && "Expected bitcast to vector");
2423 :
2424 : // Collect known bits for the (smaller) output by collecting the known
2425 : // bits from the overlapping larger input elements and extracting the
2426 : // sub sections we actually care about.
2427 484717 : unsigned SubScale = SubBitWidth / BitWidth;
2428 484717 : APInt SubDemandedElts(NumElts / SubScale, 0);
2429 9464804 : for (unsigned i = 0; i != NumElts; ++i)
2430 8980087 : if (DemandedElts[i])
2431 1831260 : SubDemandedElts.setBit(i / SubScale);
2432 :
2433 484717 : Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);
2434 :
2435 484717 : Known.Zero.setAllBits(); Known.One.setAllBits();
2436 3536640 : for (unsigned i = 0; i != NumElts; ++i)
2437 3509718 : if (DemandedElts[i]) {
2438 633683 : unsigned Shifts = IsLE ? i : NumElts - 1 - i;
2439 633683 : unsigned Offset = (Shifts % SubScale) * BitWidth;
2440 1267366 : Known.One &= Known2.One.lshr(Offset).trunc(BitWidth);
2441 1267366 : Known.Zero &= Known2.Zero.lshr(Offset).trunc(BitWidth);
2442 : // If we don't know any bits, early out.
2443 633683 : if (Known.isUnknown())
2444 : break;
2445 : }
2446 : }
2447 : break;
2448 : }
2449 951000 : case ISD::AND:
2450 : // If either the LHS or the RHS are Zero, the result is zero.
2451 1902000 : Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2452 1902000 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2453 :
2454 : // Output known-1 bits are only known if set in both the LHS & RHS.
2455 951000 : Known.One &= Known2.One;
2456 : // Output known-0 are known to be clear if zero in either the LHS | RHS.
2457 951000 : Known.Zero |= Known2.Zero;
2458 : break;
2459 239609 : case ISD::OR:
2460 479218 : Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2461 479218 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2462 :
2463 : // Output known-0 bits are only known if clear in both the LHS & RHS.
2464 239609 : Known.Zero &= Known2.Zero;
2465 : // Output known-1 are known to be set if set in either the LHS | RHS.
2466 239609 : Known.One |= Known2.One;
2467 : break;
2468 305202 : case ISD::XOR: {
2469 610404 : Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2470 610404 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2471 :
2472 : // Output known-0 bits are known if clear or set in both the LHS & RHS.
2473 915606 : APInt KnownZeroOut = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
2474 : // Output known-1 are known to be set if set in only one of the LHS, RHS.
2475 305202 : Known.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);
2476 305202 : Known.Zero = KnownZeroOut;
2477 : break;
2478 : }
2479 186398 : case ISD::MUL: {
2480 372796 : Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2481 372796 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2482 :
2483 : // If low bits are zero in either operand, output low known-0 bits.
2484 : // Also compute a conservative estimate for high known-0 bits.
2485 : // More trickiness is possible, but this is sufficient for the
2486 : // interesting case of alignment computation.
2487 186398 : unsigned TrailZ = Known.countMinTrailingZeros() +
2488 186398 : Known2.countMinTrailingZeros();
2489 186398 : unsigned LeadZ = std::max(Known.countMinLeadingZeros() +
2490 : Known2.countMinLeadingZeros(),
2491 186398 : BitWidth) - BitWidth;
2492 :
2493 : Known.resetAll();
2494 186398 : Known.Zero.setLowBits(std::min(TrailZ, BitWidth));
2495 186398 : Known.Zero.setHighBits(std::min(LeadZ, BitWidth));
2496 : break;
2497 : }
2498 5742 : case ISD::UDIV: {
2499 : // For the purposes of computing leading zeros we can conservatively
2500 : // treat a udiv as a logical right shift by the power of 2 known to
2501 : // be less than the denominator.
2502 11484 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2503 : unsigned LeadZ = Known2.countMinLeadingZeros();
2504 :
2505 11484 : Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2506 : unsigned RHSMaxLeadingZeros = Known2.countMaxLeadingZeros();
2507 5742 : if (RHSMaxLeadingZeros != BitWidth)
2508 10322 : LeadZ = std::min(BitWidth, LeadZ + BitWidth - RHSMaxLeadingZeros - 1);
2509 :
2510 5742 : Known.Zero.setHighBits(LeadZ);
2511 : break;
2512 : }
2513 78259 : case ISD::SELECT:
2514 : case ISD::VSELECT:
2515 156518 : Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
2516 : // If we don't know any bits, early out.
2517 78259 : if (Known.isUnknown())
2518 : break;
2519 38874 : Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);
2520 :
2521 : // Only known if known in both the LHS and RHS.
2522 19437 : Known.One &= Known2.One;
2523 19437 : Known.Zero &= Known2.Zero;
2524 : break;
2525 35752 : case ISD::SELECT_CC:
2526 71504 : Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);
2527 : // If we don't know any bits, early out.
2528 35752 : if (Known.isUnknown())
2529 : break;
2530 13022 : Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
2531 :
2532 : // Only known if known in both the LHS and RHS.
2533 6511 : Known.One &= Known2.One;
2534 6511 : Known.Zero &= Known2.Zero;
2535 : break;
2536 2643 : case ISD::SMULO:
2537 : case ISD::UMULO:
2538 : case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
2539 2643 : if (Op.getResNo() != 1)
2540 : break;
2541 : // The boolean result conforms to getBooleanContents.
2542 : // If we know the result of a setcc has the top bits zero, use this info.
2543 : // We know that we have an integer-based boolean since these operations
2544 : // are only available for integer.
2545 4328 : if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
2546 1922 : TargetLowering::ZeroOrOneBooleanContent &&
2547 : BitWidth > 1)
2548 1438 : Known.Zero.setBitsFrom(1);
2549 : break;
2550 424430 : case ISD::SETCC:
2551 : // If we know the result of a setcc has the top bits zero, use this info.
2552 848860 : if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2553 424430 : TargetLowering::ZeroOrOneBooleanContent &&
2554 : BitWidth > 1)
2555 196311 : Known.Zero.setBitsFrom(1);
2556 : break;
2557 487975 : case ISD::SHL:
2558 487975 : if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
2559 914762 : Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2560 457381 : unsigned Shift = ShAmt->getZExtValue();
2561 457381 : Known.Zero <<= Shift;
2562 457381 : Known.One <<= Shift;
2563 : // Low bits are known zero.
2564 : Known.Zero.setLowBits(Shift);
2565 : }
2566 : break;
2567 695418 : case ISD::SRL:
2568 695418 : if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
2569 1362528 : Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2570 681264 : unsigned Shift = ShAmt->getZExtValue();
2571 681264 : Known.Zero.lshrInPlace(Shift);
2572 681264 : Known.One.lshrInPlace(Shift);
2573 : // High bits are known zero.
2574 : Known.Zero.setHighBits(Shift);
2575 14154 : } else if (auto *BV = dyn_cast<BuildVectorSDNode>(Op.getOperand(1))) {
2576 : // If the shift amount is a vector of constants see if we can bound
2577 : // the number of upper zero bits.
2578 784 : unsigned ShiftAmountMin = BitWidth;
2579 5368 : for (unsigned i = 0; i != BV->getNumOperands(); ++i) {
2580 2483 : if (auto *C = dyn_cast<ConstantSDNode>(BV->getOperand(i))) {
2581 2452 : const APInt &ShAmt = C->getAPIntValue();
2582 2452 : if (ShAmt.ult(BitWidth)) {
2583 1900 : ShiftAmountMin = std::min<unsigned>(ShiftAmountMin,
2584 2502 : ShAmt.getZExtValue());
2585 : continue;
2586 : }
2587 : }
2588 : // Don't know anything.
2589 583 : ShiftAmountMin = 0;
2590 583 : break;
2591 : }
2592 :
2593 784 : Known.Zero.setHighBits(ShiftAmountMin);
2594 : }
2595 : break;
2596 61761 : case ISD::SRA:
2597 61761 : if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
2598 120258 : Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2599 60129 : unsigned Shift = ShAmt->getZExtValue();
2600 : // Sign extend known zero/one bit (else is unknown).
2601 60129 : Known.Zero.ashrInPlace(Shift);
2602 60129 : Known.One.ashrInPlace(Shift);
2603 : }
2604 : break;
2605 46106 : case ISD::SIGN_EXTEND_INREG: {
2606 46106 : EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2607 : unsigned EBits = EVT.getScalarSizeInBits();
2608 :
2609 : // Sign extension. Compute the demanded bits in the result that are not
2610 : // present in the input.
2611 46106 : APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
2612 :
2613 : APInt InSignMask = APInt::getSignMask(EBits);
2614 46106 : APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
2615 :
2616 : // If the sign extended bits are demanded, we know that the sign
2617 : // bit is demanded.
2618 92212 : InSignMask = InSignMask.zext(BitWidth);
2619 46106 : if (NewBits.getBoolValue())
2620 : InputDemandedBits |= InSignMask;
2621 :
2622 92212 : Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2623 46106 : Known.One &= InputDemandedBits;
2624 46106 : Known.Zero &= InputDemandedBits;
2625 :
2626 : // If the sign bit of the input is known set or clear, then we know the
2627 : // top bits of the result.
2628 46106 : if (Known.Zero.intersects(InSignMask)) { // Input sign bit known clear
2629 : Known.Zero |= NewBits;
2630 2 : Known.One &= ~NewBits;
2631 46104 : } else if (Known.One.intersects(InSignMask)) { // Input sign bit known set
2632 : Known.One |= NewBits;
2633 3 : Known.Zero &= ~NewBits;
2634 : } else { // Input sign bit unknown
2635 46101 : Known.Zero &= ~NewBits;
2636 46101 : Known.One &= ~NewBits;
2637 : }
2638 : break;
2639 : }
2640 786 : case ISD::CTTZ:
2641 : case ISD::CTTZ_ZERO_UNDEF: {
2642 1572 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2643 : // If we have a known 1, its position is our upper bound.
2644 : unsigned PossibleTZ = Known2.countMaxTrailingZeros();
2645 786 : unsigned LowBits = Log2_32(PossibleTZ) + 1;
2646 786 : Known.Zero.setBitsFrom(LowBits);
2647 : break;
2648 : }
2649 3805 : case ISD::CTLZ:
2650 : case ISD::CTLZ_ZERO_UNDEF: {
2651 7610 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2652 : // If we have a known 1, its position is our upper bound.
2653 : unsigned PossibleLZ = Known2.countMaxLeadingZeros();
2654 3805 : unsigned LowBits = Log2_32(PossibleLZ) + 1;
2655 3805 : Known.Zero.setBitsFrom(LowBits);
2656 : break;
2657 : }
2658 3196 : case ISD::CTPOP: {
2659 6392 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2660 : // If we know some of the bits are zero, they can't be one.
2661 : unsigned PossibleOnes = Known2.countMaxPopulation();
2662 6392 : Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1);
2663 : break;
2664 : }
2665 : case ISD::LOAD: {
2666 : LoadSDNode *LD = cast<LoadSDNode>(Op);
2667 : // If this is a ZEXTLoad and we are looking at the loaded value.
2668 248259 : if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
2669 248172 : EVT VT = LD->getMemoryVT();
2670 : unsigned MemBits = VT.getScalarSizeInBits();
2671 248172 : Known.Zero.setBitsFrom(MemBits);
2672 11193446 : } else if (const MDNode *Ranges = LD->getRanges()) {
2673 2367 : if (LD->getExtensionType() == ISD::NON_EXTLOAD)
2674 2361 : computeKnownBitsFromRangeMetadata(*Ranges, Known);
2675 : }
2676 : break;
2677 : }
2678 7853 : case ISD::ZERO_EXTEND_VECTOR_INREG: {
2679 15706 : EVT InVT = Op.getOperand(0).getValueType();
2680 7853 : APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
2681 15706 : Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
2682 7853 : Known = Known.zext(BitWidth);
2683 7853 : Known.Zero.setBitsFrom(InVT.getScalarSizeInBits());
2684 : break;
2685 : }
2686 194122 : case ISD::ZERO_EXTEND: {
2687 194122 : EVT InVT = Op.getOperand(0).getValueType();
2688 388244 : Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2689 194122 : Known = Known.zext(BitWidth);
2690 194122 : Known.Zero.setBitsFrom(InVT.getScalarSizeInBits());
2691 : break;
2692 : }
2693 : // TODO ISD::SIGN_EXTEND_VECTOR_INREG
2694 88793 : case ISD::SIGN_EXTEND: {
2695 177586 : Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2696 : // If the sign bit is known to be zero or one, then sext will extend
2697 : // it to the top bits, else it will just zext.
2698 88793 : Known = Known.sext(BitWidth);
2699 88793 : break;
2700 : }
2701 310057 : case ISD::ANY_EXTEND: {
2702 620114 : Known = computeKnownBits(Op.getOperand(0), Depth+1);
2703 310057 : Known = Known.zext(BitWidth);
2704 310057 : break;
2705 : }
2706 428348 : case ISD::TRUNCATE: {
2707 856696 : Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2708 428348 : Known = Known.trunc(BitWidth);
2709 428348 : break;
2710 : }
2711 198778 : case ISD::AssertZext: {
2712 198778 : EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2713 198778 : APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
2714 397556 : Known = computeKnownBits(Op.getOperand(0), Depth+1);
2715 397556 : Known.Zero |= (~InMask);
2716 397556 : Known.One &= (~Known.Zero);
2717 : break;
2718 : }
2719 26 : case ISD::FGETSIGN:
2720 : // All bits are zero except the low bit.
2721 26 : Known.Zero.setBitsFrom(1);
2722 : break;
2723 6970 : case ISD::USUBO:
2724 : case ISD::SSUBO:
2725 6970 : if (Op.getResNo() == 1) {
2726 : // If we know the result of a setcc has the top bits zero, use this info.
2727 13114 : if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2728 6557 : TargetLowering::ZeroOrOneBooleanContent &&
2729 : BitWidth > 1)
2730 1464 : Known.Zero.setBitsFrom(1);
2731 : break;
2732 : }
2733 : LLVM_FALLTHROUGH;
2734 : case ISD::SUB:
2735 : case ISD::SUBC: {
2736 178939 : if (ConstantSDNode *CLHS = isConstOrConstSplat(Op.getOperand(0))) {
2737 : // We know that the top bits of C-X are clear if X contains less bits
2738 : // than C (i.e. no wrap-around can happen). For example, 20-X is
2739 : // positive if we can prove that X is >= 0 and < 16.
2740 15270 : if (CLHS->getAPIntValue().isNonNegative()) {
2741 14170 : unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
2742 : // NLZ can't be BitWidth with no sign bit
2743 14142 : APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
2744 42426 : Known2 = computeKnownBits(Op.getOperand(1), DemandedElts,
2745 14142 : Depth + 1);
2746 :
2747 : // If all of the MaskV bits are known to be zero, then we know the
2748 : // output top bits are zero, because we now know that the output is
2749 : // from [0-C].
2750 14142 : if ((Known2.Zero & MaskV) == MaskV) {
2751 114 : unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
2752 : // Top bits known zero.
2753 114 : Known.Zero.setHighBits(NLZ2);
2754 : }
2755 : }
2756 : }
2757 :
2758 : // If low bits are know to be zero in both operands, then we know they are
2759 : // going to be 0 in the result. Both addition and complement operations
2760 : // preserve the low zero bits.
2761 357878 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2762 178939 : unsigned KnownZeroLow = Known2.countMinTrailingZeros();
2763 178939 : if (KnownZeroLow == 0)
2764 : break;
2765 :
2766 29620 : Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2767 14810 : KnownZeroLow = std::min(KnownZeroLow, Known2.countMinTrailingZeros());
2768 14810 : Known.Zero.setLowBits(KnownZeroLow);
2769 : break;
2770 : }
2771 23524 : case ISD::UADDO:
2772 : case ISD::SADDO:
2773 : case ISD::ADDCARRY:
2774 23524 : if (Op.getResNo() == 1) {
2775 : // If we know the result of a setcc has the top bits zero, use this info.
2776 8370 : if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2777 4185 : TargetLowering::ZeroOrOneBooleanContent &&
2778 : BitWidth > 1)
2779 3166 : Known.Zero.setBitsFrom(1);
2780 : break;
2781 : }
2782 : LLVM_FALLTHROUGH;
2783 : case ISD::ADD:
2784 : case ISD::ADDC:
2785 : case ISD::ADDE: {
2786 : // Output known-0 bits are known if clear or set in both the low clear bits
2787 : // common to both LHS & RHS. For example, 8+(X<<3) is known to have the
2788 : // low 3 bits clear.
2789 : // Output known-0 bits are also known if the top bits of each input are
2790 : // known to be clear. For example, if one input has the top 10 bits clear
2791 : // and the other has the top 8 bits clear, we know the top 7 bits of the
2792 : // output must be clear.
2793 8986514 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2794 4493257 : unsigned KnownZeroHigh = Known2.countMinLeadingZeros();
2795 4493257 : unsigned KnownZeroLow = Known2.countMinTrailingZeros();
2796 :
2797 8986514 : Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2798 4534543 : KnownZeroHigh = std::min(KnownZeroHigh, Known2.countMinLeadingZeros());
2799 4493257 : KnownZeroLow = std::min(KnownZeroLow, Known2.countMinTrailingZeros());
2800 :
2801 4493257 : if (Opcode == ISD::ADDE || Opcode == ISD::ADDCARRY) {
2802 : // With ADDE and ADDCARRY, a carry bit may be added in, so we can only
2803 : // use this information if we know (at least) that the low two bits are
2804 : // clear. We then return to the caller that the low bit is unknown but
2805 : // that other bits are known zero.
2806 13193 : if (KnownZeroLow >= 2)
2807 372 : Known.Zero.setBits(1, KnownZeroLow);
2808 : break;
2809 : }
2810 :
2811 4480064 : Known.Zero.setLowBits(KnownZeroLow);
2812 4480064 : if (KnownZeroHigh > 1)
2813 62020 : Known.Zero.setHighBits(KnownZeroHigh - 1);
2814 : break;
2815 : }
2816 1462 : case ISD::SREM:
2817 1462 : if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) {
2818 902 : const APInt &RA = Rem->getAPIntValue().abs();
2819 451 : if (RA.isPowerOf2()) {
2820 103 : APInt LowBits = RA - 1;
2821 206 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2822 :
2823 : // The low bits of the first operand are unchanged by the srem.
2824 206 : Known.Zero = Known2.Zero & LowBits;
2825 103 : Known.One = Known2.One & LowBits;
2826 :
2827 : // If the first operand is non-negative or has all low bits zero, then
2828 : // the upper bits are all zero.
2829 511 : if (Known2.Zero[BitWidth-1] || ((Known2.Zero & LowBits) == LowBits))
2830 4 : Known.Zero |= ~LowBits;
2831 :
2832 : // If the first operand is negative and not all low bits are zero, then
2833 : // the upper bits are all one.
2834 206 : if (Known2.One[BitWidth-1] && ((Known2.One & LowBits) != 0))
2835 0 : Known.One |= ~LowBits;
2836 : assert((Known.Zero & Known.One) == 0&&"Bits known to be one AND zero?");
2837 : }
2838 : }
2839 : break;
2840 3566 : case ISD::UREM: {
2841 3566 : if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) {
2842 2529 : const APInt &RA = Rem->getAPIntValue();
2843 2529 : if (RA.isPowerOf2()) {
2844 1012 : APInt LowBits = (RA - 1);
2845 2024 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2846 :
2847 : // The upper bits are all zero, the lower ones are unchanged.
2848 1012 : Known.Zero = Known2.Zero | ~LowBits;
2849 1012 : Known.One = Known2.One & LowBits;
2850 : break;
2851 : }
2852 : }
2853 :
2854 : // Since the result is less than or equal to either operand, any leading
2855 : // zero bits in either operand must also exist in the result.
2856 5108 : Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2857 5108 : Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2858 :
2859 : uint32_t Leaders =
2860 6616 : std::max(Known.countMinLeadingZeros(), Known2.countMinLeadingZeros());
2861 : Known.resetAll();
2862 : Known.Zero.setHighBits(Leaders);
2863 : break;
2864 : }
2865 128 : case ISD::EXTRACT_ELEMENT: {
2866 256 : Known = computeKnownBits(Op.getOperand(0), Depth+1);
2867 128 : const unsigned Index = Op.getConstantOperandVal(1);
2868 128 : const unsigned BitWidth = Op.getValueSizeInBits();
2869 :
2870 : // Remove low part of known bits mask
2871 128 : Known.Zero = Known.Zero.getHiBits(Known.Zero.getBitWidth() - Index * BitWidth);
2872 128 : Known.One = Known.One.getHiBits(Known.One.getBitWidth() - Index * BitWidth);
2873 :
2874 : // Remove high part of known bit mask
2875 128 : Known = Known.trunc(BitWidth);
2876 128 : break;
2877 : }
2878 599317 : case ISD::EXTRACT_VECTOR_ELT: {
2879 599317 : SDValue InVec = Op.getOperand(0);
2880 599317 : SDValue EltNo = Op.getOperand(1);
2881 599317 : EVT VecVT = InVec.getValueType();
2882 599317 : const unsigned BitWidth = Op.getValueSizeInBits();
2883 : const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
2884 : const unsigned NumSrcElts = VecVT.getVectorNumElements();
2885 : // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2886 : // anything about the extended bits.
2887 599317 : if (BitWidth > EltBitWidth)
2888 12245 : Known = Known.trunc(EltBitWidth);
2889 : ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
2890 1197088 : if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) {
2891 : // If we know the element index, just demand that vector element.
2892 598544 : unsigned Idx = ConstEltNo->getZExtValue();
2893 598544 : APInt DemandedElt = APInt::getOneBitSet(NumSrcElts, Idx);
2894 598544 : Known = computeKnownBits(InVec, DemandedElt, Depth + 1);
2895 : } else {
2896 : // Unknown element index, so ignore DemandedElts and demand them all.
2897 773 : Known = computeKnownBits(InVec, Depth + 1);
2898 : }
2899 599317 : if (BitWidth > EltBitWidth)
2900 12245 : Known = Known.zext(BitWidth);
2901 : break;
2902 : }
2903 7902 : case ISD::INSERT_VECTOR_ELT: {
2904 7902 : SDValue InVec = Op.getOperand(0);
2905 7902 : SDValue InVal = Op.getOperand(1);
2906 7902 : SDValue EltNo = Op.getOperand(2);
2907 :
2908 : ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
2909 15792 : if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
2910 : // If we know the element index, split the demand between the
2911 : // source vector and the inserted element.
2912 7906 : Known.Zero = Known.One = APInt::getAllOnesValue(BitWidth);
2913 15792 : unsigned EltIdx = CEltNo->getZExtValue();
2914 :
2915 : // If we demand the inserted element then add its common known bits.
2916 7896 : if (DemandedElts[EltIdx]) {
2917 6619 : Known2 = computeKnownBits(InVal, Depth + 1);
2918 6619 : Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth());
2919 13238 : Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth());
2920 : }
2921 :
2922 : // If we demand the source vector then add its common known bits, ensuring
2923 : // that we don't demand the inserted element.
2924 15792 : APInt VectorElts = DemandedElts & ~(APInt::getOneBitSet(NumElts, EltIdx));
2925 7896 : if (!!VectorElts) {
2926 5080 : Known2 = computeKnownBits(InVec, VectorElts, Depth + 1);
2927 : Known.One &= Known2.One;
2928 : Known.Zero &= Known2.Zero;
2929 : }
2930 : } else {
2931 : // Unknown element index, so ignore DemandedElts and demand them all.
2932 6 : Known = computeKnownBits(InVec, Depth + 1);
2933 6 : Known2 = computeKnownBits(InVal, Depth + 1);
2934 6 : Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth());
2935 12 : Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth());
2936 : }
2937 : break;
2938 : }
2939 295 : case ISD::BITREVERSE: {
2940 590 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2941 295 : Known.Zero = Known2.Zero.reverseBits();
2942 295 : Known.One = Known2.One.reverseBits();
2943 295 : break;
2944 : }
2945 4253 : case ISD::BSWAP: {
2946 8506 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2947 4253 : Known.Zero = Known2.Zero.byteSwap();
2948 4253 : Known.One = Known2.One.byteSwap();
2949 4253 : break;
2950 : }
2951 2687 : case ISD::ABS: {
2952 5374 : Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2953 :
2954 : // If the source's MSB is zero then we know the rest of the bits already.
2955 2687 : if (Known2.isNonNegative()) {
2956 0 : Known.Zero = Known2.Zero;
2957 0 : Known.One = Known2.One;
2958 0 : break;
2959 : }
2960 :
2961 : // We only know that the absolute values's MSB will be zero iff there is
2962 : // a set bit that isn't the sign bit (otherwise it could be INT_MIN).
2963 2687 : Known2.One.clearSignBit();
2964 2687 : if (Known2.One.getBoolValue()) {
2965 2 : Known.Zero = APInt::getSignMask(BitWidth);
2966 2 : break;
2967 : }
2968 : break;
2969 : }
2970 3478 : case ISD::UMIN: {
2971 6956 : Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2972 6956 : Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2973 :
2974 : // UMIN - we know that the result will have the maximum of the
2975 : // known zero leading bits of the inputs.
2976 3478 : unsigned LeadZero = Known.countMinLeadingZeros();
2977 4901 : LeadZero = std::max(LeadZero, Known2.countMinLeadingZeros());
2978 :
2979 : Known.Zero &= Known2.Zero;
2980 3478 : Known.One &= Known2.One;
2981 3478 : Known.Zero.setHighBits(LeadZero);
2982 : break;
2983 : }
2984 3243 : case ISD::UMAX: {
2985 6486 : Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2986 6486 : Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2987 :
2988 : // UMAX - we know that the result will have the maximum of the
2989 : // known one leading bits of the inputs.
2990 3243 : unsigned LeadOne = Known.countMinLeadingOnes();
2991 3243 : LeadOne = std::max(LeadOne, Known2.countMinLeadingOnes());
2992 :
2993 3243 : Known.Zero &= Known2.Zero;
2994 : Known.One &= Known2.One;
2995 3243 : Known.One.setHighBits(LeadOne);
2996 : break;
2997 : }
2998 12085 : case ISD::SMIN:
2999 : case ISD::SMAX: {
3000 : // If we have a clamp pattern, we know that the number of sign bits will be
3001 : // the minimum of the clamp min/max range.
3002 : bool IsMax = (Opcode == ISD::SMAX);
3003 : ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3004 12085 : if ((CstLow = isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts)))
3005 6083 : if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3006 903 : CstHigh = isConstOrDemandedConstSplat(Op.getOperand(0).getOperand(1),
3007 : DemandedElts);
3008 12085 : if (CstLow && CstHigh) {
3009 895 : if (!IsMax)
3010 : std::swap(CstLow, CstHigh);
3011 :
3012 895 : const APInt &ValueLow = CstLow->getAPIntValue();
3013 895 : const APInt &ValueHigh = CstHigh->getAPIntValue();
3014 895 : if (ValueLow.sle(ValueHigh)) {
3015 894 : unsigned LowSignBits = ValueLow.getNumSignBits();
3016 894 : unsigned HighSignBits = ValueHigh.getNumSignBits();
3017 894 : unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
3018 894 : if (ValueLow.isNegative() && ValueHigh.isNegative()) {
3019 1 : Known.One.setHighBits(MinSignBits);
3020 668 : break;
3021 : }
3022 1560 : if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
3023 667 : Known.Zero.setHighBits(MinSignBits);
3024 : break;
3025 : }
3026 : }
3027 : }
3028 :
3029 : // Fallback - just get the shared known bits of the operands.
3030 22834 : Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3031 11417 : if (Known.isUnknown()) break; // Early-out
3032 0 : Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3033 0 : Known.Zero &= Known2.Zero;
3034 0 : Known.One &= Known2.One;
3035 : break;
3036 : }
3037 1396852 : case ISD::FrameIndex:
3038 : case ISD::TargetFrameIndex:
3039 1396852 : TLI->computeKnownBitsForFrameIndex(Op, Known, DemandedElts, *this, Depth);
3040 1396852 : break;
3041 :
3042 9766870 : default:
3043 9766870 : if (Opcode < ISD::BUILTIN_OP_END)
3044 : break;
3045 : LLVM_FALLTHROUGH;
3046 : case ISD::INTRINSIC_WO_CHAIN:
3047 : case ISD::INTRINSIC_W_CHAIN:
3048 : case ISD::INTRINSIC_VOID:
3049 : // Allow the target to implement this method for its nodes.
3050 4173022 : TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
3051 4173022 : break;
3052 : }
3053 :
3054 : assert(!Known.hasConflict() && "Bits known to be one AND zero?");
3055 : return Known;
3056 : }
3057 :
3058 130947 : SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0,
3059 : SDValue N1) const {
3060 : // X + 0 never overflow
3061 130947 : if (isNullConstant(N1))
3062 : return OFK_Never;
3063 :
3064 130947 : KnownBits N1Known;
3065 130947 : computeKnownBits(N1, N1Known);
3066 130947 : if (N1Known.Zero.getBoolValue()) {
3067 125956 : KnownBits N0Known;
3068 126087 : computeKnownBits(N0, N0Known);
3069 :
3070 : bool overflow;
3071 252174 : (void)(~N0Known.Zero).uadd_ov(~N1Known.Zero, overflow);
3072 126087 : if (!overflow)
3073 131 : return OFK_Never;
3074 : }
3075 :
3076 : // mulhi + 1 never overflow
3077 261907 : if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
3078 132191 : (~N1Known.Zero & 0x01) == ~N1Known.Zero)
3079 : return OFK_Never;
3080 :
3081 261186 : if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) {
3082 233 : KnownBits N0Known;
3083 233 : computeKnownBits(N0, N0Known);
3084 :
3085 233 : if ((~N0Known.Zero & 0x01) == ~N0Known.Zero)
3086 0 : return OFK_Never;
3087 : }
3088 :
3089 : return OFK_Sometime;
3090 : }
3091 :
3092 26286 : bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
3093 52572 : EVT OpVT = Val.getValueType();
3094 : unsigned BitWidth = OpVT.getScalarSizeInBits();
3095 :
3096 : // Is the constant a known power of 2?
3097 : if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val))
3098 42903 : return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3099 :
3100 : // A left-shift of a constant one will have exactly one bit set because
3101 : // shifting the bit off the end is undefined.
3102 4854 : if (Val.getOpcode() == ISD::SHL) {
3103 57 : auto *C = isConstOrConstSplat(Val.getOperand(0));
3104 57 : if (C && C->getAPIntValue() == 1)
3105 : return true;
3106 : }
3107 :
3108 : // Similarly, a logical right-shift of a constant sign-bit will have exactly
3109 : // one bit set.
3110 4805 : if (Val.getOpcode() == ISD::SRL) {
3111 23 : auto *C = isConstOrConstSplat(Val.getOperand(0));
3112 23 : if (C && C->getAPIntValue().isSignMask())
3113 : return true;
3114 : }
3115 :
3116 : // Are all operands of a build vector constant powers of two?
3117 4800 : if (Val.getOpcode() == ISD::BUILD_VECTOR)
3118 3366 : if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {
3119 : if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
3120 : return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3121 : return false;
3122 : }))
3123 : return true;
3124 :
3125 : // More could be done here, though the above checks are enough
3126 : // to handle some common cases.
3127 :
3128 : // Fall back to computeKnownBits to catch other known cases.
3129 8270 : KnownBits Known = computeKnownBits(Val);
3130 4302 : return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
3131 : }
3132 :
3133 617146 : unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
3134 1234292 : EVT VT = Op.getValueType();
3135 : APInt DemandedElts = VT.isVector()
3136 : ? APInt::getAllOnesValue(VT.getVectorNumElements())
3137 665483 : : APInt(1, 1);
3138 617146 : return ComputeNumSignBits(Op, DemandedElts, Depth);
3139 : }
3140 :
3141 720913 : unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
3142 : unsigned Depth) const {
3143 1441826 : EVT VT = Op.getValueType();
3144 : assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
3145 720913 : unsigned VTBits = VT.getScalarSizeInBits();
3146 720913 : unsigned NumElts = DemandedElts.getBitWidth();
3147 : unsigned Tmp, Tmp2;
3148 720913 : unsigned FirstAnswer = 1;
3149 :
3150 : if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3151 29191 : const APInt &Val = C->getAPIntValue();
3152 29191 : return Val.getNumSignBits();
3153 : }
3154 :
3155 691722 : if (Depth == 6)
3156 : return 1; // Limit search depth.
3157 :
3158 691404 : if (!DemandedElts)
3159 : return 1; // No demanded elts, better to assume we don't know anything.
3160 :
3161 : unsigned Opcode = Op.getOpcode();
3162 691404 : switch (Opcode) {
3163 441828 : default: break;
3164 4314 : case ISD::AssertSext:
3165 4314 : Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3166 249576 : return VTBits-Tmp+1;
3167 9356 : case ISD::AssertZext:
3168 9356 : Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3169 9356 : return VTBits-Tmp;
3170 :
3171 4121 : case ISD::BUILD_VECTOR:
3172 4121 : Tmp = VTBits;
3173 20722 : for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
3174 16601 : if (!DemandedElts[i])
3175 6767 : continue;
3176 :
3177 9834 : SDValue SrcOp = Op.getOperand(i);
3178 19668 : Tmp2 = ComputeNumSignBits(Op.getOperand(i), Depth + 1);
3179 :
3180 : // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3181 9834 : if (SrcOp.getValueSizeInBits() != VTBits) {
3182 : assert(SrcOp.getValueSizeInBits() > VTBits &&
3183 : "Expected BUILD_VECTOR implicit truncation");
3184 24 : unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
3185 24 : Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
3186 : }
3187 9834 : Tmp = std::min(Tmp, Tmp2);
3188 4121 : }
3189 4121 : return Tmp;
3190 :
3191 : case ISD::VECTOR_SHUFFLE: {
3192 : // Collect the minimum number of sign bits that are shared by every vector
3193 : // element referenced by the shuffle.
3194 : APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
3195 : const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
3196 : assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3197 15420 : for (unsigned i = 0; i != NumElts; ++i) {
3198 13124 : int M = SVN->getMaskElt(i);
3199 13124 : if (!DemandedElts[i])
3200 : continue;
3201 : // For UNDEF elements, we don't know anything about the common state of
3202 : // the shuffle result.
3203 11193 : if (M < 0)
3204 : return 1;
3205 11119 : if ((unsigned)M < NumElts)
3206 10304 : DemandedLHS.setBit((unsigned)M % NumElts);
3207 : else
3208 815 : DemandedRHS.setBit((unsigned)M % NumElts);
3209 : }
3210 2296 : Tmp = std::numeric_limits<unsigned>::max();
3211 2296 : if (!!DemandedLHS)
3212 4586 : Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
3213 2296 : if (!!DemandedRHS) {
3214 542 : Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
3215 271 : Tmp = std::min(Tmp, Tmp2);
3216 : }
3217 : // If we don't know anything, early out and try computeKnownBits fall-back.
3218 2296 : if (Tmp == 1)
3219 : break;
3220 : assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3221 : return Tmp;
3222 : }
3223 :
3224 23994 : case ISD::BITCAST: {
3225 23994 : SDValue N0 = Op.getOperand(0);
3226 23994 : EVT SrcVT = N0.getValueType();
3227 : unsigned SrcBits = SrcVT.getScalarSizeInBits();
3228 :
3229 : // Ignore bitcasts from unsupported types..
3230 23994 : if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
3231 : break;
3232 :
3233 : // Fast handling of 'identity' bitcasts.
3234 23985 : if (VTBits == SrcBits)
3235 11174 : return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
3236 :
3237 22102 : bool IsLE = getDataLayout().isLittleEndian();
3238 :
3239 : // Bitcast 'large element' scalar/vector to 'small element' vector.
3240 22102 : if ((SrcBits % VTBits) == 0) {
3241 : assert(VT.isVector() && "Expected bitcast to vector");
3242 :
3243 9291 : unsigned Scale = SrcBits / VTBits;
3244 9291 : APInt SrcDemandedElts(NumElts / Scale, 0);
3245 79671 : for (unsigned i = 0; i != NumElts; ++i)
3246 70380 : if (DemandedElts[i])
3247 64893 : SrcDemandedElts.setBit(i / Scale);
3248 :
3249 : // Fast case - sign splat can be simply split across the small elements.
3250 9291 : Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);
3251 9291 : if (Tmp == SrcBits)
3252 : return VTBits;
3253 :
3254 : // Slow case - determine how far the sign extends into each sub-element.
3255 6376 : Tmp2 = VTBits;
3256 12788 : for (unsigned i = 0; i != NumElts; ++i)
3257 10204 : if (DemandedElts[i]) {
3258 7571 : unsigned SubOffset = i % Scale;
3259 7571 : SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
3260 7571 : SubOffset = SubOffset * VTBits;
3261 7571 : if (Tmp <= SubOffset)
3262 : return 1;
3263 6358 : Tmp2 = std::min(Tmp2, Tmp - SubOffset);
3264 : }
3265 2584 : return Tmp2;
3266 : }
3267 : break;
3268 : }
3269 :
3270 2548 : case ISD::SIGN_EXTEND:
3271 2548 : Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
3272 5096 : return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
3273 3315 : case ISD::SIGN_EXTEND_INREG:
3274 : // Max of the input and what this extends.
3275 3315 : Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
3276 3315 : Tmp = VTBits-Tmp+1;
3277 6630 : Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3278 3315 : return std::max(Tmp, Tmp2);
3279 86 : case ISD::SIGN_EXTEND_VECTOR_INREG: {
3280 86 : SDValue Src = Op.getOperand(0);
3281 86 : EVT SrcVT = Src.getValueType();
3282 86 : APInt DemandedSrcElts = DemandedElts.zextOrSelf(SrcVT.getVectorNumElements());
3283 86 : Tmp = VTBits - SrcVT.getScalarSizeInBits();
3284 86 : return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
3285 : }
3286 :
3287 2950 : case ISD::SRA:
3288 5900 : Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3289 : // SRA X, C -> adds C sign bits.
3290 2950 : if (ConstantSDNode *C =
3291 5900 : isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts)) {
3292 2798 : APInt ShiftVal = C->getAPIntValue();
3293 2798 : ShiftVal += Tmp;
3294 8172 : Tmp = ShiftVal.uge(VTBits) ? VTBits : ShiftVal.getZExtValue();
3295 : }
3296 2950 : return Tmp;
3297 8412 : case ISD::SHL:
3298 8412 : if (ConstantSDNode *C =
3299 8412 : isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts)) {
3300 : // shl destroys sign bits.
3301 15848 : Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3302 23772 : if (C->getAPIntValue().uge(VTBits) || // Bad shift.
3303 7924 : C->getAPIntValue().uge(Tmp)) break; // Shifted all sign bits out.
3304 506 : return Tmp - C->getZExtValue();
3305 488 : }
3306 : break;
3307 27901 : case ISD::AND:
3308 : case ISD::OR:
3309 : case ISD::XOR: // NOT is handled here.
3310 : // Logical binary ops preserve the number of sign bits at the worst.
3311 55802 : Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3312 27901 : if (Tmp != 1) {
3313 24124 : Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3314 12062 : FirstAnswer = std::min(Tmp, Tmp2);
3315 : // We computed what we know about the sign bits as our first
3316 : // answer. Now proceed to the generic code that uses
3317 : // computeKnownBits, and pick whichever answer is better.
3318 : }
3319 : break;
3320 :
3321 809 : case ISD::SELECT:
3322 : case ISD::VSELECT:
3323 1618 : Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3324 809 : if (Tmp == 1) return 1; // Early out.
3325 360 : Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3326 180 : return std::min(Tmp, Tmp2);
3327 126 : case ISD::SELECT_CC:
3328 252 : Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3329 126 : if (Tmp == 1) return 1; // Early out.
3330 158 : Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
3331 79 : return std::min(Tmp, Tmp2);
3332 :
3333 602 : case ISD::SMIN:
3334 : case ISD::SMAX: {
3335 : // If we have a clamp pattern, we know that the number of sign bits will be
3336 : // the minimum of the clamp min/max range.
3337 : bool IsMax = (Opcode == ISD::SMAX);
3338 : ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3339 602 : if ((CstLow = isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts)))
3340 1495 : if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3341 489 : CstHigh = isConstOrDemandedConstSplat(Op.getOperand(0).getOperand(1),
3342 : DemandedElts);
3343 602 : if (CstLow && CstHigh) {
3344 489 : if (!IsMax)
3345 : std::swap(CstLow, CstHigh);
3346 1467 : if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
3347 489 : Tmp = CstLow->getAPIntValue().getNumSignBits();
3348 489 : Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
3349 489 : return std::min(Tmp, Tmp2);
3350 : }
3351 : }
3352 :
3353 : // Fallback - just get the minimum number of sign bits of the operands.
3354 226 : Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3355 113 : if (Tmp == 1)
3356 : return 1; // Early out.
3357 110 : Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
3358 55 : return std::min(Tmp, Tmp2);
3359 : }
3360 138 : case ISD::UMIN:
3361 : case ISD::UMAX:
3362 276 : Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3363 138 : if (Tmp == 1)
3364 : return 1; // Early out.
3365 34 : Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
3366 17 : return std::min(Tmp, Tmp2);
3367 144 : case ISD::SADDO:
3368 : case ISD::UADDO:
3369 : case ISD::SSUBO:
3370 : case ISD::USUBO:
3371 : case ISD::SMULO:
3372 : case ISD::UMULO:
3373 144 : if (Op.getResNo() != 1)
3374 : break;
3375 : // The boolean result conforms to getBooleanContents. Fall through.
3376 : // If setcc returns 0/-1, all bits are sign bits.
3377 : // We know that we have an integer-based boolean since these operations
3378 : // are only available for integer.
3379 212 : if (TLI->getBooleanContents(VT.isVector(), false) ==
3380 : TargetLowering::ZeroOrNegativeOneBooleanContent)
3381 : return VTBits;
3382 : break;
3383 7529 : case ISD::SETCC:
3384 : // If setcc returns 0/-1, all bits are sign bits.
3385 22587 : if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3386 : TargetLowering::ZeroOrNegativeOneBooleanContent)
3387 : return VTBits;
3388 : break;
3389 61 : case ISD::ROTL:
3390 : case ISD::ROTR:
3391 : if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
3392 122 : unsigned RotAmt = C->getAPIntValue().urem(VTBits);
3393 :
3394 : // Handle rotate right by N like a rotate left by 32-N.
3395 61 : if (Opcode == ISD::ROTR)
3396 28 : RotAmt = (VTBits - RotAmt) % VTBits;
3397 :
3398 : // If we aren't rotating out all of the known-in sign bits, return the
3399 : // number that are left. This handles rotl(sext(x), 1) for example.
3400 122 : Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3401 61 : if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
3402 : }
3403 : break;
3404 155279 : case ISD::ADD:
3405 : case ISD::ADDC:
3406 : // Add can have at most one carry bit. Thus we know that the output
3407 : // is, at worst, one more bit than the inputs.
3408 310558 : Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3409 155279 : if (Tmp == 1) return 1; // Early out.
3410 :
3411 : // Special case decrementing a value (ADD X, -1):
3412 2329 : if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
3413 2332 : if (CRHS->isAllOnesValue()) {
3414 370 : KnownBits Known = computeKnownBits(Op.getOperand(0), Depth+1);
3415 :
3416 : // If the input is known to be 0 or 1, the output is 0/-1, which is all
3417 : // sign bits set.
3418 476 : if ((Known.Zero | 1).isAllOnesValue())
3419 106 : return VTBits;
3420 :
3421 : // If we are subtracting one from a positive number, there is no carry
3422 : // out of the result.
3423 229 : if (Known.isNonNegative())
3424 : return Tmp;
3425 : }
3426 :
3427 4446 : Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
3428 2223 : if (Tmp2 == 1) return 1;
3429 2007 : return std::min(Tmp, Tmp2)-1;
3430 :
3431 10303 : case ISD::SUB:
3432 20606 : Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
3433 10303 : if (Tmp2 == 1) return 1;
3434 :
3435 : // Handle NEG.
3436 1794 : if (ConstantSDNode *CLHS = isConstOrConstSplat(Op.getOperand(0)))
3437 484 : if (CLHS->isNullValue()) {
3438 292 : KnownBits Known = computeKnownBits(Op.getOperand(1), Depth+1);
3439 : // If the input is known to be 0 or 1, the output is 0/-1, which is all
3440 : // sign bits set.
3441 278 : if ((Known.Zero | 1).isAllOnesValue())
3442 125 : return VTBits;
3443 :
3444 : // If the input is known to be positive (the sign bit is known clear),
3445 : // the output of the NEG has the same number of sign bits as the input.
3446 134 : if (Known.isNonNegative())
3447 : return Tmp2;
3448 :
3449 : // Otherwise, we treat this like a SUB.
3450 : }
3451 :
3452 : // Sub can have at most one carry bit. Thus we know that the output
3453 : // is, at worst, one more bit than the inputs.
3454 1544 : Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3455 772 : if (Tmp == 1) return 1; // Early out.
3456 507 : return std::min(Tmp, Tmp2)-1;
3457 34981 : case ISD::TRUNCATE: {
3458 : // Check if the sign bits of source go down as far as the truncated value.
3459 34981 : unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
3460 69962 : unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3461 34981 : if (NumSrcSignBits > (NumSrcBits - VTBits))
3462 4934 : return NumSrcSignBits - (NumSrcBits - VTBits);
3463 : break;
3464 : }
3465 30 : case ISD::EXTRACT_ELEMENT: {
3466 60 : const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3467 30 : const int BitWidth = Op.getValueSizeInBits();
3468 60 : const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
3469 :
3470 : // Get reverse index (starting from 1), Op1 value indexes elements from
3471 : // little end. Sign starts at big end.
3472 30 : const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
3473 :
3474 : // If the sign portion ends in our element the subtraction gives correct
3475 : // result. Otherwise it gives either negative or > bitwidth result
3476 60 : return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
3477 : }
3478 164 : case ISD::INSERT_VECTOR_ELT: {
3479 164 : SDValue InVec = Op.getOperand(0);
3480 164 : SDValue InVal = Op.getOperand(1);
3481 164 : SDValue EltNo = Op.getOperand(2);
3482 328 : unsigned NumElts = InVec.getValueType().getVectorNumElements();
3483 :
3484 : ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3485 312 : if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3486 : // If we know the element index, split the demand between the
3487 : // source vector and the inserted element.
3488 156 : unsigned EltIdx = CEltNo->getZExtValue();
3489 :
3490 : // If we demand the inserted element then get its sign bits.
3491 156 : Tmp = std::numeric_limits<unsigned>::max();
3492 156 : if (DemandedElts[EltIdx]) {
3493 : // TODO - handle implicit truncation of inserted elements.
3494 154 : if (InVal.getScalarValueSizeInBits() != VTBits)
3495 : break;
3496 154 : Tmp = ComputeNumSignBits(InVal, Depth + 1);
3497 : }
3498 :
3499 : // If we demand the source vector then get its sign bits, and determine
3500 : // the minimum.
3501 : APInt VectorElts = DemandedElts;
3502 : VectorElts.clearBit(EltIdx);
3503 156 : if (!!VectorElts) {
3504 89 : Tmp2 = ComputeNumSignBits(InVec, VectorElts, Depth + 1);
3505 89 : Tmp = std::min(Tmp, Tmp2);
3506 : }
3507 : } else {
3508 : // Unknown element index, so ignore DemandedElts and demand them all.
3509 8 : Tmp = ComputeNumSignBits(InVec, Depth + 1);
3510 8 : Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
3511 8 : Tmp = std::min(Tmp, Tmp2);
3512 : }
3513 : assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3514 164 : return Tmp;
3515 : }
3516 16985 : case ISD::EXTRACT_VECTOR_ELT: {
3517 16985 : SDValue InVec = Op.getOperand(0);
3518 16985 : SDValue EltNo = Op.getOperand(1);
3519 16985 : EVT VecVT = InVec.getValueType();
3520 16985 : const unsigned BitWidth = Op.getValueSizeInBits();
3521 33970 : const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
3522 : const unsigned NumSrcElts = VecVT.getVectorNumElements();
3523 :
3524 : // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
3525 : // anything about sign bits. But if the sizes match we can derive knowledge
3526 : // about sign bits from the vector operand.
3527 16985 : if (BitWidth != EltBitWidth)
3528 : break;
3529 :
3530 : // If we know the element index, just demand that vector element, else for
3531 : // an unknown element index, ignore DemandedElts and demand them all.
3532 15890 : APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
3533 : ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3534 31780 : if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3535 : DemandedSrcElts =
3536 31780 : APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
3537 :
3538 15890 : return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
3539 : }
3540 11110 : case ISD::EXTRACT_SUBVECTOR: {
3541 : // If we know the element index, just demand that subvector elements,
3542 : // otherwise demand them all.
3543 11110 : SDValue Src = Op.getOperand(0);
3544 : ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
3545 11110 : unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3546 11110 : if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
3547 : // Offset the demanded elts by the subvector index.
3548 : uint64_t Idx = SubIdx->getZExtValue();
3549 11110 : APInt DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
3550 11110 : return ComputeNumSignBits(Src, DemandedSrc, Depth + 1);
3551 : }
3552 0 : return ComputeNumSignBits(Src, Depth + 1);
3553 : }
3554 2617 : case ISD::CONCAT_VECTORS:
3555 : // Determine the minimum number of sign bits across all demanded
3556 : // elts of the input vectors. Early out if the result is already 1.
3557 2617 : Tmp = std::numeric_limits<unsigned>::max();
3558 5234 : EVT SubVectorVT = Op.getOperand(0).getValueType();
3559 : unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3560 : unsigned NumSubVectors = Op.getNumOperands();
3561 6729 : for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
3562 4112 : APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
3563 8224 : DemandedSub = DemandedSub.trunc(NumSubVectorElts);
3564 4112 : if (!DemandedSub)
3565 : continue;
3566 5926 : Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
3567 2963 : Tmp = std::min(Tmp, Tmp2);
3568 2617 : }
3569 : assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3570 2617 : return Tmp;
3571 : }
3572 :
3573 : // If we are looking at the loaded value of the SDNode.
3574 441828 : if (Op.getResNo() == 0) {
3575 : // Handle LOADX separately here. EXTLOAD case will fallthrough.
3576 : if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
3577 : unsigned ExtType = LD->getExtensionType();
3578 77947 : switch (ExtType) {
3579 : default: break;
3580 1622 : case ISD::SEXTLOAD: // '17' bits known
3581 1622 : Tmp = LD->getMemoryVT().getScalarSizeInBits();
3582 1622 : return VTBits-Tmp+1;
3583 2975 : case ISD::ZEXTLOAD: // '16' bits known
3584 2975 : Tmp = LD->getMemoryVT().getScalarSizeInBits();
3585 2975 : return VTBits-Tmp;
3586 : }
3587 : }
3588 : }
3589 :
3590 : // Allow the target to implement this method for its nodes.
3591 874462 : if (Opcode >= ISD::BUILTIN_OP_END ||
3592 437231 : Opcode == ISD::INTRINSIC_WO_CHAIN ||
3593 324097 : Opcode == ISD::INTRINSIC_W_CHAIN ||
3594 : Opcode == ISD::INTRINSIC_VOID) {
3595 : unsigned NumBits =
3596 113893 : TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
3597 113893 : if (NumBits > 1)
3598 14058 : FirstAnswer = std::max(FirstAnswer, NumBits);
3599 : }
3600 :
3601 : // Finally, if we can prove that the top bits of the result are 0's or 1's,
3602 : // use this information.
3603 874462 : KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
3604 :
3605 : APInt Mask;
3606 437231 : if (Known.isNonNegative()) { // sign bit is 0
3607 65305 : Mask = Known.Zero;
3608 371926 : } else if (Known.isNegative()) { // sign bit is 1;
3609 110 : Mask = Known.One;
3610 : } else {
3611 : // Nothing known.
3612 371816 : return FirstAnswer;
3613 : }
3614 :
3615 : // Okay, we know that the sign bit in Mask is set. Use CLZ to determine
3616 : // the number of identical bits in the top of the input value.
3617 65415 : Mask = ~Mask;
3618 65415 : Mask <<= Mask.getBitWidth()-VTBits;
3619 : // Return # leading zeros. We use 'min' here in case Val was zero before
3620 : // shifting. We don't want to return '64' as for an i32 "0".
3621 130830 : return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
3622 : }
3623 :
3624 3326674 : bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
3625 3326674 : if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
3626 : !isa<ConstantSDNode>(Op.getOperand(1)))
3627 : return false;
3628 :
3629 1977189 : if (Op.getOpcode() == ISD::OR &&
3630 353738 : !MaskedValueIsZero(Op.getOperand(0),
3631 : cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
3632 196 : return false;
3633 :
3634 : return true;
3635 : }
3636 :
3637 1273 : bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
3638 : // If we're told that NaNs won't happen, assume they won't.
3639 1273 : if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
3640 347 : return true;
3641 :
3642 926 : if (Depth == 6)
3643 : return false; // Limit search depth.
3644 :
3645 : // TODO: Handle vectors.
3646 : // If the value is a constant, we can obviously see if it is a NaN or not.
3647 : if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
3648 188 : return !C->getValueAPF().isNaN() ||
3649 0 : (SNaN && !C->getValueAPF().isSignaling());
3650 : }
3651 :
3652 : unsigned Opcode = Op.getOpcode();
3653 832 : switch (Opcode) {
3654 36 : case ISD::FADD:
3655 : case ISD::FSUB:
3656 : case ISD::FMUL:
3657 : case ISD::FDIV:
3658 : case ISD::FREM:
3659 : case ISD::FSIN:
3660 : case ISD::FCOS: {
3661 36 : if (SNaN)
3662 5 : return true;
3663 : // TODO: Need isKnownNeverInfinity
3664 : return false;
3665 : }
3666 7 : case ISD::FCANONICALIZE:
3667 : case ISD::FEXP:
3668 : case ISD::FEXP2:
3669 : case ISD::FTRUNC:
3670 : case ISD::FFLOOR:
3671 : case ISD::FCEIL:
3672 : case ISD::FROUND:
3673 : case ISD::FRINT:
3674 : case ISD::FNEARBYINT: {
3675 7 : if (SNaN)
3676 : return true;
3677 0 : return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
3678 : }
3679 130 : case ISD::FABS:
3680 : case ISD::FNEG:
3681 : case ISD::FCOPYSIGN: {
3682 260 : return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
3683 : }
3684 3 : case ISD::SELECT:
3685 8 : return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
3686 2 : isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
3687 2 : case ISD::FP_EXTEND:
3688 : case ISD::FP_ROUND: {
3689 2 : if (SNaN)
3690 : return true;
3691 0 : return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
3692 : }
3693 : case ISD::SINT_TO_FP:
3694 : case ISD::UINT_TO_FP:
3695 : return true;
3696 9 : case ISD::FMA:
3697 : case ISD::FMAD: {
3698 9 : if (SNaN)
3699 : return true;
3700 16 : return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
3701 8 : isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
3702 0 : isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
3703 : }
3704 0 : case ISD::FSQRT: // Need is known positive
3705 : case ISD::FLOG:
3706 : case ISD::FLOG2:
3707 : case ISD::FLOG10:
3708 : case ISD::FPOWI:
3709 : case ISD::FPOW: {
3710 0 : if (SNaN)
3711 0 : return true;
3712 : // TODO: Refine on operand
3713 : return false;
3714 : }
3715 :
3716 : // TODO: Handle FMINNUM/FMAXNUM/FMINNAN/FMAXNAN when there is an agreement on
3717 : // what they should do.
3718 0 : case ISD::EXTRACT_VECTOR_ELT: {
3719 0 : return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
3720 : }
3721 643 : default:
3722 1286 : if (Opcode >= ISD::BUILTIN_OP_END ||
3723 643 : Opcode == ISD::INTRINSIC_WO_CHAIN ||
3724 616 : Opcode == ISD::INTRINSIC_W_CHAIN ||
3725 : Opcode == ISD::INTRINSIC_VOID) {
3726 27 : return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
3727 : }
3728 :
3729 : return false;
3730 : }
3731 : }
3732 :
3733 164 : bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
3734 : assert(Op.getValueType().isFloatingPoint() &&
3735 : "Floating point type expected");
3736 :
3737 : // If the value is a constant, we can obviously see if it is a zero or not.
3738 : // TODO: Add BuildVector support.
3739 : if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
3740 52 : return !C->isZero();
3741 : return false;
3742 : }
3743 :
3744 4136 : bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
3745 : assert(!Op.getValueType().isFloatingPoint() &&
3746 : "Floating point types unsupported - use isKnownNeverZeroFloat");
3747 :
3748 : // If the value is a constant, we can obviously see if it is a zero or not.
3749 8272 : if (ISD::matchUnaryPredicate(
3750 : Op, [](ConstantSDNode *C) { return !C->isNullValue(); }))
3751 : return true;
3752 :
3753 : // TODO: Recognize more cases here.
3754 3133 : switch (Op.getOpcode()) {
3755 : default: break;
3756 43 : case ISD::OR:
3757 58 : if (isKnownNeverZero(Op.getOperand(1)) ||
3758 15 : isKnownNeverZero(Op.getOperand(0)))
3759 28 : return true;
3760 : break;
3761 : }
3762 :
3763 : return false;
3764 : }
3765 :
3766 18881 : bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
3767 : // Check the obvious case.
3768 : if (A == B) return true;
3769 :
3770 : // For for negative and positive zero.
3771 : if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
3772 : if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
3773 33 : if (CA->isZero() && CB->isZero()) return true;
3774 :
3775 : // Otherwise they may not be equal.
3776 : return false;
3777 : }
3778 :
3779 : // FIXME: unify with llvm::haveNoCommonBitsSet.
3780 : // FIXME: could also handle masked merge pattern (X & ~M) op (Y & M)
3781 3130653 : bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
3782 : assert(A.getValueType() == B.getValueType() &&
3783 : "Values must have the same type");
3784 6261306 : return (computeKnownBits(A).Zero | computeKnownBits(B).Zero).isAllOnesValue();
3785 : }
3786 :
3787 52276 : static SDValue FoldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
3788 : ArrayRef<SDValue> Ops,
3789 : SelectionDAG &DAG) {
3790 : assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
3791 : assert(llvm::all_of(Ops,
3792 : [Ops](SDValue Op) {
3793 : return Ops[0].getValueType() == Op.getValueType();
3794 : }) &&
3795 : "Concatenation of vectors with inconsistent value types!");
3796 : assert((Ops.size() * Ops[0].getValueType().getVectorNumElements()) ==
3797 : VT.getVectorNumElements() &&
3798 : "Incorrect element count in vector concatenation!");
3799 :
3800 52276 : if (Ops.size() == 1)
3801 0 : return Ops[0];
3802 :
3803 : // Concat of UNDEFs is UNDEF.
3804 52276 : if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
3805 501 : return DAG.getUNDEF(VT);
3806 :
3807 : // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
3808 : // simplified to one big BUILD_VECTOR.
3809 : // FIXME: Add support for SCALAR_TO_VECTOR as well.
3810 51775 : EVT SVT = VT.getScalarType();
3811 : SmallVector<SDValue, 16> Elts;
3812 55647 : for (SDValue Op : Ops) {
3813 54653 : EVT OpVT = Op.getValueType();
3814 54653 : if (Op.isUndef())
3815 1250 : Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
3816 54028 : else if (Op.getOpcode() == ISD::BUILD_VECTOR)
3817 6494 : Elts.append(Op->op_begin(), Op->op_end());
3818 : else
3819 50781 : return SDValue();
3820 : }
3821 :
3822 : // BUILD_VECTOR requires all inputs to be of the same type, find the
3823 : // maximum type and extend them all.
3824 18788 : for (SDValue Op : Elts)
3825 17794 : SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
3826 :
3827 994 : if (SVT.bitsGT(VT.getScalarType()))
3828 13342 : for (SDValue &Op : Elts)
3829 25528 : Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
3830 25528 : ? DAG.getZExtOrTrunc(Op, DL, SVT)
3831 12764 : : DAG.getSExtOrTrunc(Op, DL, SVT);
3832 :
3833 994 : SDValue V = DAG.getBuildVector(VT, DL, Elts);
3834 : NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
3835 994 : return V;
3836 : }
3837 :
3838 : /// Gets or creates the specified node.
3839 7492797 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
3840 : FoldingSetNodeID ID;
3841 7492797 : AddNodeIDNode(ID, Opcode, getVTList(VT), None);
3842 7492797 : void *IP = nullptr;
3843 7492797 : if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
3844 6548325 : return SDValue(E, 0);
3845 :
3846 1888944 : auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
3847 944472 : getVTList(VT));
3848 944472 : CSEMap.InsertNode(N, IP);
3849 :
3850 944472 : InsertNode(N);
3851 : SDValue V = SDValue(N, 0);
3852 : NewSDValueDbgMsg(V, "Creating new node: ", this);
3853 944472 : return V;
3854 : }
3855 :
3856 12862124 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
3857 : SDValue Operand, const SDNodeFlags Flags) {
3858 : // Constant fold unary operations with an integer constant operand. Even
3859 : // opaque constant will be folded, because the folding of unary operations
3860 : // doesn't create new constants with different values. Nevertheless, the
3861 : // opaque flag is preserved during folding to prevent future folding with
3862 : // other constants.
3863 : if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) {
3864 330866 : const APInt &Val = C->getAPIntValue();
3865 330866 : switch (Opcode) {
3866 : default: break;
3867 53687 : case ISD::SIGN_EXTEND:
3868 53687 : return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
3869 107374 : C->isTargetOpcode(), C->isOpaque());
3870 194344 : case ISD::ANY_EXTEND:
3871 : case ISD::ZERO_EXTEND:
3872 : case ISD::TRUNCATE:
3873 194344 : return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
3874 388688 : C->isTargetOpcode(), C->isOpaque());
3875 489 : case ISD::UINT_TO_FP:
3876 : case ISD::SINT_TO_FP: {
3877 : APFloat apf(EVTToAPFloatSemantics(VT),
3878 489 : APInt::getNullValue(VT.getSizeInBits()));
3879 489 : (void)apf.convertFromAPInt(Val,
3880 : Opcode==ISD::SINT_TO_FP,
3881 : APFloat::rmNearestTiesToEven);
3882 489 : return getConstantFP(apf, DL, VT);
3883 : }
3884 : case ISD::BITCAST:
3885 9 : if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
3886 18 : return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
3887 385 : if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
3888 770 : return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
3889 115 : if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
3890 230 : return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
3891 0 : if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
3892 0 : return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
3893 : break;
3894 14 : case ISD::ABS:
3895 14 : return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
3896 28 : C->isOpaque());
3897 452 : case ISD::BITREVERSE:
3898 452 : return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
3899 904 : C->isOpaque());
3900 170 : case ISD::BSWAP:
3901 170 : return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
3902 340 : C->isOpaque());
3903 660 : case ISD::CTPOP:
3904 : return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
3905 1980 : C->isOpaque());
3906 11913 : case ISD::CTLZ:
3907 : case ISD::CTLZ_ZERO_UNDEF:
3908 11913 : return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
3909 23826 : C->isOpaque());
3910 3739 : case ISD::CTTZ:
3911 : case ISD::CTTZ_ZERO_UNDEF:
3912 3739 : return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
3913 7478 : C->isOpaque());
3914 743 : case ISD::FP16_TO_FP: {
3915 : bool Ignored;
3916 : APFloat FPV(APFloat::IEEEhalf(),
3917 1486 : (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
3918 :
3919 : // This can return overflow, underflow, or inexact; we don't care.
3920 : // FIXME need to be more flexible about rounding mode.
3921 743 : (void)FPV.convert(EVTToAPFloatSemantics(VT),
3922 : APFloat::rmNearestTiesToEven, &Ignored);
3923 743 : return getConstantFP(FPV, DL, VT);
3924 : }
3925 : }
3926 : }
3927 :
3928 : // Constant fold unary operations with a floating point constant operand.
3929 : if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) {
3930 5588 : APFloat V = C->getValueAPF(); // make copy
3931 5588 : switch (Opcode) {
3932 264 : case ISD::FNEG:
3933 264 : V.changeSign();
3934 264 : return getConstantFP(V, DL, VT);
3935 21 : case ISD::FABS:
3936 21 : V.clearSign();
3937 21 : return getConstantFP(V, DL, VT);
3938 28 : case ISD::FCEIL: {
3939 28 : APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
3940 28 : if (fs == APFloat::opOK || fs == APFloat::opInexact)
3941 28 : return getConstantFP(V, DL, VT);
3942 : break;
3943 : }
3944 25 : case ISD::FTRUNC: {
3945 25 : APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
3946 25 : if (fs == APFloat::opOK || fs == APFloat::opInexact)
3947 25 : return getConstantFP(V, DL, VT);
3948 : break;
3949 : }
3950 25 : case ISD::FFLOOR: {
3951 25 : APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
3952 25 : if (fs == APFloat::opOK || fs == APFloat::opInexact)
3953 25 : return getConstantFP(V, DL, VT);
3954 : break;
3955 : }
3956 122 : case ISD::FP_EXTEND: {
3957 : bool ignored;
3958 : // This can return overflow, underflow, or inexact; we don't care.
3959 : // FIXME need to be more flexible about rounding mode.
3960 122 : (void)V.convert(EVTToAPFloatSemantics(VT),
3961 : APFloat::rmNearestTiesToEven, &ignored);
3962 122 : return getConstantFP(V, DL, VT);
3963 : }
3964 759 : case ISD::FP_TO_SINT:
3965 : case ISD::FP_TO_UINT: {
3966 : bool ignored;
3967 759 : APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
3968 : // FIXME need to be more flexible about rounding mode.
3969 : APFloat::opStatus s =
3970 759 : V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
3971 759 : if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
3972 : break;
3973 721 : return getConstant(IntVal, DL, VT);
3974 : }
3975 : case ISD::BITCAST:
3976 280 : if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
3977 840 : return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
3978 2599 : else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
3979 7797 : return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
3980 355 : else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
3981 1065 : return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
3982 : break;
3983 65 : case ISD::FP_TO_FP16: {
3984 : bool Ignored;
3985 : // This can return overflow, underflow, or inexact; we don't care.
3986 : // FIXME need to be more flexible about rounding mode.
3987 65 : (void)V.convert(APFloat::IEEEhalf(),
3988 : APFloat::rmNearestTiesToEven, &Ignored);
3989 130 : return getConstant(V.bitcastToAPInt(), DL, VT);
3990 : }
3991 : }
3992 : }
3993 :
3994 : // Constant fold unary operations with a vector integer or float operand.
3995 : if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand)) {
3996 309348 : if (BV->isConstant()) {
3997 256998 : switch (Opcode) {
3998 : default:
3999 : // FIXME: Entirely reasonable to perform folding of other unary
4000 : // operations here as the need arises.
4001 : break;
4002 2763 : case ISD::FNEG:
4003 : case ISD::FABS:
4004 : case ISD::FCEIL:
4005 : case ISD::FTRUNC:
4006 : case ISD::FFLOOR:
4007 : case ISD::FP_EXTEND:
4008 : case ISD::FP_TO_SINT:
4009 : case ISD::FP_TO_UINT:
4010 : case ISD::TRUNCATE:
4011 : case ISD::ANY_EXTEND:
4012 : case ISD::ZERO_EXTEND:
4013 : case ISD::SIGN_EXTEND:
4014 : case ISD::UINT_TO_FP:
4015 : case ISD::SINT_TO_FP:
4016 : case ISD::ABS:
4017 : case ISD::BITREVERSE:
4018 : case ISD::BSWAP:
4019 : case ISD::CTLZ:
4020 : case ISD::CTLZ_ZERO_UNDEF:
4021 : case ISD::CTTZ:
4022 : case ISD::CTTZ_ZERO_UNDEF:
4023 : case ISD::CTPOP: {
4024 2763 : SDValue Ops = { Operand };
4025 2763 : if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
4026 2763 : return Fold;
4027 : }
4028 : }
4029 : }
4030 : }
4031 :
4032 12588156 : unsigned OpOpcode = Operand.getNode()->getOpcode();
4033 12588156 : switch (Opcode) {
4034 8457449 : case ISD::TokenFactor:
4035 : case ISD::MERGE_VALUES:
4036 : case ISD::CONCAT_VECTORS:
4037 8457449 : return Operand; // Factor, merge or concat of one node? No need.
4038 : case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
4039 10471 : case ISD::FP_EXTEND:
4040 : assert(VT.isFloatingPoint() &&
4041 : Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
4042 12913 : if (Operand.getValueType() == VT) return Operand; // noop conversion.
4043 : assert((!VT.isVector() ||
4044 : VT.getVectorNumElements() ==
4045 : Operand.getValueType().getVectorNumElements()) &&
4046 : "Vector element count mismatch!");
4047 : assert(Operand.getValueType().bitsLT(VT) &&
4048 : "Invalid fpext node, dst < src!");
4049 8070 : if (Operand.isUndef())
4050 19 : return getUNDEF(VT);
4051 : break;
4052 72262 : case ISD::SIGN_EXTEND:
4053 : assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4054 : "Invalid SIGN_EXTEND!");
4055 75380 : if (Operand.getValueType() == VT) return Operand; // noop extension
4056 : assert((!VT.isVector() ||
4057 : VT.getVectorNumElements() ==
4058 : Operand.getValueType().getVectorNumElements()) &&
4059 : "Vector element count mismatch!");
4060 : assert(Operand.getValueType().bitsLT(VT) &&
4061 : "Invalid sext node, dst < src!");
4062 69229 : if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
4063 466 : return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4064 68763 : else if (OpOpcode == ISD::UNDEF)
4065 : // sext(undef) = 0, because the top bits will all be the same.
4066 212 : return getConstant(0, DL, VT);
4067 : break;
4068 305427 : case ISD::ZERO_EXTEND:
4069 : assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4070 : "Invalid ZERO_EXTEND!");
4071 311463 : if (Operand.getValueType() == VT) return Operand; // noop extension
4072 : assert((!VT.isVector() ||
4073 : VT.getVectorNumElements() ==
4074 : Operand.getValueType().getVectorNumElements()) &&
4075 : "Vector element count mismatch!");
4076 : assert(Operand.getValueType().bitsLT(VT) &&
4077 : "Invalid zext node, dst < src!");
4078 300438 : if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
4079 32858 : return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0));
4080 267580 : else if (OpOpcode == ISD::UNDEF)
4081 : // zext(undef) = 0, because the top bits will be zero.
4082 883 : return getConstant(0, DL, VT);
4083 : break;
4084 261344 : case ISD::ANY_EXTEND:
4085 : assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4086 : "Invalid ANY_EXTEND!");
4087 378738 : if (Operand.getValueType() == VT) return Operand; // noop extension
4088 : assert((!VT.isVector() ||
4089 : VT.getVectorNumElements() ==
4090 : Operand.getValueType().getVectorNumElements()) &&
4091 : "Vector element count mismatch!");
4092 : assert(Operand.getValueType().bitsLT(VT) &&
4093 : "Invalid anyext node, dst < src!");
4094 :
4095 144856 : if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
4096 : OpOpcode == ISD::ANY_EXTEND)
4097 : // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
4098 2671 : return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4099 142185 : else if (OpOpcode == ISD::UNDEF)
4100 340 : return getUNDEF(VT);
4101 :
4102 : // (ext (trunc x)) -> x
4103 141845 : if (OpOpcode == ISD::TRUNCATE) {
4104 45565 : SDValue OpOp = Operand.getOperand(0);
4105 7 : if (OpOp.getValueType() == VT) {
4106 43640 : transferDbgValues(Operand, OpOp);
4107 43640 : return OpOp;
4108 : }
4109 : }
4110 : break;
4111 1569425 : case ISD::TRUNCATE:
4112 : assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4113 : "Invalid TRUNCATE!");
4114 2308811 : if (Operand.getValueType() == VT) return Operand; // noop truncate
4115 : assert((!VT.isVector() ||
4116 : VT.getVectorNumElements() ==
4117 : Operand.getValueType().getVectorNumElements()) &&
4118 : "Vector element count mismatch!");
4119 : assert(Operand.getValueType().bitsGT(VT) &&
4120 : "Invalid truncate node, src < dst!");
4121 834760 : if (OpOpcode == ISD::TRUNCATE)
4122 45106 : return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
4123 789654 : if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
4124 : OpOpcode == ISD::ANY_EXTEND) {
4125 : // If the source is smaller than the dest, we still need an extend.
4126 28510 : if (Operand.getOperand(0).getValueType().getScalarType()
4127 14255 : .bitsLT(VT.getScalarType()))
4128 4203 : return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4129 20104 : if (Operand.getOperand(0).getValueType().bitsGT(VT))
4130 328 : return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
4131 9724 : return Operand.getOperand(0);
4132 : }
4133 775399 : if (OpOpcode == ISD::UNDEF)
4134 15413 : return getUNDEF(VT);
4135 : break;
4136 542 : case ISD::ABS:
4137 : assert(VT.isInteger() && VT == Operand.getValueType() &&
4138 : "Invalid ABS!");
4139 542 : if (OpOpcode == ISD::UNDEF)
4140 1 : return getUNDEF(VT);
4141 : break;
4142 1049 : case ISD::BSWAP:
4143 : assert(VT.isInteger() && VT == Operand.getValueType() &&
4144 : "Invalid BSWAP!");
4145 : assert((VT.getScalarSizeInBits() % 16 == 0) &&
4146 : "BSWAP types must be a multiple of 16 bits!");
4147 1049 : if (OpOpcode == ISD::UNDEF)
4148 11 : return getUNDEF(VT);
4149 : break;
4150 764 : case ISD::BITREVERSE:
4151 : assert(VT.isInteger() && VT == Operand.getValueType() &&
4152 : "Invalid BITREVERSE!");
4153 764 : if (OpOpcode == ISD::UNDEF)
4154 11 : return getUNDEF(VT);
4155 : break;
4156 995588 : case ISD::BITCAST:
4157 : // Basic sanity checking.
4158 : assert(VT.getSizeInBits() == Operand.getValueSizeInBits() &&
4159 : "Cannot BITCAST between types of different sizes!");
4160 200670 : if (VT == Operand.getValueType()) return Operand; // noop conversion.
4161 798519 : if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
4162 45442 : return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
4163 753077 : if (OpOpcode == ISD::UNDEF)
4164 12743 : return getUNDEF(VT);
4165 : break;
4166 28433 : case ISD::SCALAR_TO_VECTOR:
4167 : assert(VT.isVector() && !Operand.getValueType().isVector() &&
4168 : (VT.getVectorElementType() == Operand.getValueType() ||
4169 : (VT.getVectorElementType().isInteger() &&
4170 : Operand.getValueType().isInteger() &&
4171 : VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
4172 : "Illegal SCALAR_TO_VECTOR node!");
4173 28433 : if (OpOpcode == ISD::UNDEF)
4174 1 : return getUNDEF(VT);
4175 : // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
4176 : if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
4177 1613 : isa<ConstantSDNode>(Operand.getOperand(1)) &&
4178 28432 : Operand.getConstantOperandVal(1) == 0 &&
4179 2250 : Operand.getOperand(0).getValueType() == VT)
4180 840 : return Operand.getOperand(0);
4181 : break;
4182 7478 : case ISD::FNEG:
4183 : // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
4184 7478 : if ((getTarget().Options.UnsafeFPMath || Flags.hasNoSignedZeros()) &&
4185 : OpOpcode == ISD::FSUB)
4186 : return getNode(ISD::FSUB, DL, VT, Operand.getOperand(1),
4187 58 : Operand.getOperand(0), Flags);
4188 7420 : if (OpOpcode == ISD::FNEG) // --X -> X
4189 9 : return Operand.getOperand(0);
4190 : break;
4191 3407 : case ISD::FABS:
4192 3407 : if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
4193 18 : return getNode(ISD::FABS, DL, VT, Operand.getOperand(0));
4194 : break;
4195 : }
4196 :
4197 : SDNode *N;
4198 2857065 : SDVTList VTs = getVTList(VT);
4199 2857065 : SDValue Ops[] = {Operand};
4200 2857065 : if (VT != MVT::Glue) { // Don't CSE flag producing nodes
4201 : FoldingSetNodeID ID;
4202 2856460 : AddNodeIDNode(ID, Opcode, VTs, Ops);
4203 2856460 : void *IP = nullptr;
4204 2856460 : if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
4205 92301 : E->intersectFlagsWith(Flags);
4206 92301 : return SDValue(E, 0);
4207 : }
4208 :
4209 5528318 : N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4210 2764159 : N->setFlags(Flags);
4211 2764159 : createOperands(N, Ops);
4212 2764159 : CSEMap.InsertNode(N, IP);
4213 : } else {
4214 1210 : N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4215 605 : createOperands(N, Ops);
4216 : }
4217 :
4218 2764764 : InsertNode(N);
4219 : SDValue V = SDValue(N, 0);
4220 : NewSDValueDbgMsg(V, "Creating new node: ", this);
4221 2764764 : return V;
4222 : }
4223 :
4224 549292 : static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1,
4225 : const APInt &C2) {
4226 549292 : switch (Opcode) {
4227 594082 : case ISD::ADD: return std::make_pair(C1 + C2, true);
4228 33898 : case ISD::SUB: return std::make_pair(C1 - C2, true);
4229 12568 : case ISD::MUL: return std::make_pair(C1 * C2, true);
4230 31160 : case ISD::AND: return std::make_pair(C1 & C2, true);
4231 6158 : case ISD::OR: return std::make_pair(C1 | C2, true);
4232 8026 : case ISD::XOR: return std::make_pair(C1 ^ C2, true);
4233 29185 : case ISD::SHL: return std::make_pair(C1 << C2, true);
4234 18155 : case ISD::SRL: return std::make_pair(C1.lshr(C2), true);
4235 100 : case ISD::SRA: return std::make_pair(C1.ashr(C2), true);
4236 16 : case ISD::ROTL: return std::make_pair(C1.rotl(C2), true);
4237 0 : case ISD::ROTR: return std::make_pair(C1.rotr(C2), true);
4238 1624 : case ISD::SMIN: return std::make_pair(C1.sle(C2) ? C1 : C2, true);
4239 1632 : case ISD::SMAX: return std::make_pair(C1.sge(C2) ? C1 : C2, true);
4240 1624 : case ISD::UMIN: return std::make_pair(C1.ule(C2) ? C1 : C2, true);
4241 1626 : case ISD::UMAX: return std::make_pair(C1.uge(C2) ? C1 : C2, true);
4242 : case ISD::UDIV:
4243 38 : if (!C2.getBoolValue())
4244 : break;
4245 38 : return std::make_pair(C1.udiv(C2), true);
4246 : case ISD::UREM:
4247 257 : if (!C2.getBoolValue())
4248 : break;
4249 257 : return std::make_pair(C1.urem(C2), true);
4250 : case ISD::SDIV:
4251 110 : if (!C2.getBoolValue())
4252 : break;
4253 110 : return std::make_pair(C1.sdiv(C2), true);
4254 : case ISD::SREM:
4255 91 : if (!C2.getBoolValue())
4256 : break;
4257 91 : return std::make_pair(C1.srem(C2), true);
4258 : }
4259 148857 : return std::make_pair(APInt(1, 0), false);
4260 : }
4261 :
4262 549456 : SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
4263 : EVT VT, const ConstantSDNode *Cst1,
4264 : const ConstantSDNode *Cst2) {
4265 549456 : if (Cst1->isOpaque() || Cst2->isOpaque())
4266 164 : return SDValue();
4267 :
4268 : std::pair<APInt, bool> Folded = FoldValue(Opcode, Cst1->getAPIntValue(),
4269 1647876 : Cst2->getAPIntValue());
4270 549292 : if (!Folded.second)
4271 148857 : return SDValue();
4272 400435 : return getConstant(Folded.first, DL, VT);
4273 : }
4274 :
4275 416494 : SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
4276 : const GlobalAddressSDNode *GA,
4277 : const SDNode *N2) {
4278 416494 : if (GA->getOpcode() != ISD::GlobalAddress)
4279 12 : return SDValue();
4280 416482 : if (!TLI->isOffsetFoldingLegal(GA))
4281 390719 : return SDValue();
4282 : const ConstantSDNode *Cst2 = dyn_cast<ConstantSDNode>(N2);
4283 : if (!Cst2)
4284 2325 : return SDValue();
4285 23438 : int64_t Offset = Cst2->getSExtValue();
4286 23438 : switch (Opcode) {
4287 : case ISD::ADD: break;
4288 1 : case ISD::SUB: Offset = -uint64_t(Offset); break;
4289 16 : default: return SDValue();
4290 : }
4291 23422 : return getGlobalAddress(GA->getGlobal(), SDLoc(Cst2), VT,
4292 23422 : GA->getOffset() + uint64_t(Offset));
4293 : }
4294 :
4295 8824389 : bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
4296 8824389 : switch (Opcode) {
4297 30437 : case ISD::SDIV:
4298 : case ISD::UDIV:
4299 : case ISD::SREM:
4300 : case ISD::UREM: {
4301 : // If a divisor is zero/undef or any element of a divisor vector is
4302 : // zero/undef, the whole op is undef.
4303 : assert(Ops.size() == 2 && "Div/rem should have 2 operands");
4304 30437 : SDValue Divisor = Ops[1];
4305 30437 : if (Divisor.isUndef() || isNullConstant(Divisor))
4306 33 : return true;
4307 :
4308 32408 : return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
4309 : llvm::any_of(Divisor->op_values(),
4310 0 : [](SDValue V) { return V.isUndef() ||
4311 0 : isNullConstant(V); });
4312 : // TODO: Handle signed overflow.
4313 : }
4314 : // TODO: Handle oversized shifts.
4315 : default:
4316 : return false;
4317 : }
4318 : }
4319 :
4320 8185662 : SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
4321 : EVT VT, SDNode *Cst1,
4322 : SDNode *Cst2) {
4323 : // If the opcode is a target-specific ISD node, there's nothing we can
4324 : // do here and the operand rules may not line up with the below, so
4325 : // bail early.
4326 8185662 : if (Opcode >= ISD::BUILTIN_OP_END)
4327 521970 : return SDValue();
4328 :
4329 7663692 : if (isUndef(Opcode, {SDValue(Cst1, 0), SDValue(Cst2, 0)}))
4330 105 : return getUNDEF(VT);
4331 :
4332 : // Handle the case of two scalars.
4333 : if (const ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1)) {
4334 : if (const ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2)) {
4335 542037 : SDValue Folded = FoldConstantArithmetic(Opcode, DL, VT, Scalar1, Scalar2);
4336 : assert((!Folded || !VT.isVector()) &&
4337 : "Can't fold vectors ops with scalar operands");
4338 542037 : return Folded;
4339 : }
4340 : }
4341 :
4342 : // fold (add Sym, c) -> Sym+c
4343 : if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Cst1))
4344 414905 : return FoldSymbolOffset(Opcode, VT, GA, Cst2);
4345 6706645 : if (TLI->isCommutativeBinOp(Opcode))
4346 : if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Cst2))
4347 1589 : return FoldSymbolOffset(Opcode, VT, GA, Cst1);
4348 :
4349 : // For vectors extract each constant element into Inputs so we can constant
4350 : // fold them individually.
4351 : BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
4352 : BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
4353 6705056 : if (!BV1 || !BV2)
4354 6697555 : return SDValue();
4355 :
4356 : assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
4357 :
4358 7501 : EVT SVT = VT.getScalarType();
4359 7501 : EVT LegalSVT = SVT;
4360 7501 : if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
4361 1741 : LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
4362 1741 : if (LegalSVT.bitsLT(SVT))
4363 0 : return SDValue();
4364 : }
4365 : SmallVector<SDValue, 4> Outputs;
4366 35352 : for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
4367 31981 : SDValue V1 = BV1->getOperand(I);
4368 31981 : SDValue V2 = BV2->getOperand(I);
4369 :
4370 31981 : if (SVT.isInteger()) {
4371 61190 : if (V1->getValueType(0).bitsGT(SVT))
4372 380 : V1 = getNode(ISD::TRUNCATE, DL, SVT, V1);
4373 61190 : if (V2->getValueType(0).bitsGT(SVT))
4374 337 : V2 = getNode(ISD::TRUNCATE, DL, SVT, V2);
4375 : }
4376 :
4377 95993 : if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
4378 0 : return SDValue();
4379 :
4380 : // Fold one vector element.
4381 31981 : SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2);
4382 32031 : if (LegalSVT != SVT)
4383 298 : ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
4384 :
4385 : // Scalar folding only succeeded if the result is a constant or UNDEF.
4386 63962 : if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
4387 : ScalarResult.getOpcode() != ISD::ConstantFP)
4388 4130 : return SDValue();
4389 27851 : Outputs.push_back(ScalarResult);
4390 : }
4391 :
4392 : assert(VT.getVectorNumElements() == Outputs.size() &&
4393 : "Vector size mismatch!");
4394 :
4395 : // We may have a vector type but a scalar result. Create a splat.
4396 3371 : Outputs.resize(VT.getVectorNumElements(), Outputs.back());
4397 :
4398 : // Build a big vector out of the scalar elements we generated.
4399 6742 : return getBuildVector(VT, SDLoc(), Outputs);
4400 : }
4401 :
4402 : // TODO: Merge with FoldConstantArithmetic
4403 1145533 : SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
4404 : const SDLoc &DL, EVT VT,
4405 : ArrayRef<SDValue> Ops,
4406 : const SDNodeFlags Flags) {
4407 : // If the opcode is a target-specific ISD node, there's nothing we can
4408 : // do here and the operand rules may not line up with the below, so
4409 : // bail early.
4410 1145533 : if (Opcode >= ISD::BUILTIN_OP_END)
4411 0 : return SDValue();
4412 :
4413 1145533 : if (isUndef(Opcode, Ops))
4414 1 : return getUNDEF(VT);
4415 :
4416 : // We can only fold vectors - maybe merge with FoldConstantArithmetic someday?
4417 1145532 : if (!VT.isVector())
4418 279292 : return SDValue();
4419 :
4420 866240 : unsigned NumElts = VT.getVectorNumElements();
4421 :
4422 : auto IsScalarOrSameVectorSize = [&](const SDValue &Op) {
4423 : return !Op.getValueType().isVector() ||
4424 : Op.getValueType().getVectorNumElements() == NumElts;
4425 : };
4426 :
4427 : auto IsConstantBuildVectorOrUndef = [&](const SDValue &Op) {
4428 : BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op);
4429 : return (Op.isUndef()) || (Op.getOpcode() == ISD::CONDCODE) ||
4430 : (BV && BV->isConstant());
4431 : };
4432 :
4433 : // All operands must be vector types with the same number of elements as
4434 : // the result type and must be either UNDEF or a build vector of constant
4435 : // or UNDEF scalars.
4436 870425 : if (!llvm::all_of(Ops, IsConstantBuildVectorOrUndef) ||
4437 : !llvm::all_of(Ops, IsScalarOrSameVectorSize))
4438 862055 : return SDValue();
4439 :
4440 : // If we are comparing vectors, then the result needs to be a i1 boolean
4441 : // that is then sign-extended back to the legal result type.
4442 4185 : EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
4443 :
4444 : // Find legal integer scalar type for constant promotion and
4445 : // ensure that its scalar size is at least as large as source.
4446 4185 : EVT LegalSVT = VT.getScalarType();
4447 4185 : if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
4448 883 : LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
4449 883 : if (LegalSVT.bitsLT(VT.getScalarType()))
4450 0 : return SDValue();
4451 : }
4452 :
4453 : // Constant fold each scalar lane separately.
4454 : SmallVector<SDValue, 4> ScalarResults;
4455 41876 : for (unsigned i = 0; i != NumElts; i++) {
4456 : SmallVector<SDValue, 4> ScalarOps;
4457 94921 : for (SDValue Op : Ops) {
4458 114240 : EVT InSVT = Op.getValueType().getScalarType();
4459 : BuildVectorSDNode *InBV = dyn_cast<BuildVectorSDNode>(Op);
4460 : if (!InBV) {
4461 : // We've checked that this is UNDEF or a constant of some kind.
4462 8043 : if (Op.isUndef())
4463 138 : ScalarOps.push_back(getUNDEF(InSVT));
4464 : else
4465 7905 : ScalarOps.push_back(Op);
4466 8043 : continue;
4467 : }
4468 :
4469 49077 : SDValue ScalarOp = InBV->getOperand(i);
4470 49077 : EVT ScalarVT = ScalarOp.getValueType();
4471 :
4472 : // Build vector (integer) scalar operands may need implicit
4473 : // truncation - do this before constant folding.
4474 49077 : if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT))
4475 354 : ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
4476 :
4477 49077 : ScalarOps.push_back(ScalarOp);
4478 : }
4479 :
4480 : // Constant fold the scalar operands.
4481 37801 : SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
4482 :
4483 : // Legalize the (integer) scalar constant if necessary.
4484 37817 : if (LegalSVT != SVT)
4485 7055 : ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
4486 :
4487 : // Scalar folding only succeeded if the result is a constant or UNDEF.
4488 75602 : if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
4489 : ScalarResult.getOpcode() != ISD::ConstantFP)
4490 110 : return SDValue();
4491 37691 : ScalarResults.push_back(ScalarResult);
4492 : }
4493 :
4494 4075 : SDValue V = getBuildVector(VT, DL, ScalarResults);
4495 : NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
4496 4075 : return V;
4497 : }
4498 :
4499 13125771 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4500 : SDValue N1, SDValue N2, const SDNodeFlags Flags) {
4501 : ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
4502 : ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
4503 : ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
4504 : ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
4505 :
4506 : // Canonicalize constant to RHS if commutative.
4507 13125771 : if (TLI->isCommutativeBinOp(Opcode)) {
4508 7563095 : if (N1C && !N2C) {
4509 : std::swap(N1C, N2C);
4510 : std::swap(N1, N2);
4511 7546583 : } else if (N1CFP && !N2CFP) {
4512 : std::swap(N1CFP, N2CFP);
4513 : std::swap(N1, N2);
4514 : }
4515 : }
4516 :
4517 13125771 : switch (Opcode) {
4518 : default: break;
4519 2751616 : case ISD::TokenFactor:
4520 : assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
4521 : N2.getValueType() == MVT::Other && "Invalid token factor!");
4522 : // Fold trivial token factors.
4523 2751616 : if (N1.getOpcode() == ISD::EntryToken) return N2;
4524 2751258 : if (N2.getOpcode() == ISD::EntryToken) return N1;
4525 62209 : if (N1 == N2) return N1;
4526 : break;
4527 43930 : case ISD::CONCAT_VECTORS: {
4528 : // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
4529 43930 : SDValue Ops[] = {N1, N2};
4530 43930 : if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
4531 930 : return V;
4532 43000 : break;
4533 : }
4534 286003 : case ISD::AND:
4535 : assert(VT.isInteger() && "This operator does not apply to FP types!");
4536 : assert(N1.getValueType() == N2.getValueType() &&
4537 : N1.getValueType() == VT && "Binary operator types must match!");
4538 : // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
4539 : // worth handling here.
4540 526960 : if (N2C && N2C->isNullValue())
4541 3356 : return N2;
4542 520248 : if (N2C && N2C->isAllOnesValue()) // X & -1 -> X
4543 2624 : return N1;
4544 : break;
4545 7304232 : case ISD::OR:
4546 : case ISD::XOR:
4547 : case ISD::ADD:
4548 : case ISD::SUB:
4549 : assert(VT.isInteger() && "This operator does not apply to FP types!");
4550 : assert(N1.getValueType() == N2.getValueType() &&
4551 : N1.getValueType() == VT && "Binary operator types must match!");
4552 : // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
4553 : // it's worth handling here.
4554 14046648 : if (N2C && N2C->isNullValue())
4555 4867617 : return N1;
4556 : break;
4557 : case ISD::UDIV:
4558 : case ISD::UREM:
4559 : case ISD::MULHU:
4560 : case ISD::MULHS:
4561 : case ISD::MUL:
4562 : case ISD::SDIV:
4563 : case ISD::SREM:
4564 : case ISD::SMIN:
4565 : case ISD::SMAX:
4566 : case ISD::UMIN:
4567 : case ISD::UMAX:
4568 : assert(VT.isInteger() && "This operator does not apply to FP types!");
4569 : assert(N1.getValueType() == N2.getValueType() &&
4570 : N1.getValueType() == VT && "Binary operator types must match!");
4571 : break;
4572 : case ISD::FADD:
4573 : case ISD::FSUB:
4574 : case ISD::FMUL:
4575 : case ISD::FDIV:
4576 : case ISD::FREM:
4577 : assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
4578 : assert(N1.getValueType() == N2.getValueType() &&
4579 : N1.getValueType() == VT && "Binary operator types must match!");
4580 : break;
4581 : case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
4582 : assert(N1.getValueType() == VT &&
4583 : N1.getValueType().isFloatingPoint() &&
4584 : N2.getValueType().isFloatingPoint() &&
4585 : "Invalid FCOPYSIGN!");
4586 : break;
4587 : case ISD::SHL:
4588 : case ISD::SRA:
4589 : case ISD::SRL:
4590 : case ISD::ROTL:
4591 : case ISD::ROTR:
4592 : assert(VT == N1.getValueType() &&
4593 : "Shift operators return type must be the same as their first arg");
4594 : assert(VT.isInteger() && N2.getValueType().isInteger() &&
4595 : "Shifts only work on integers");
4596 : assert((!VT.isVector() || VT == N2.getValueType()) &&
4597 : "Vector shift amounts must be in the same as their first arg");
4598 : // Verify that the shift amount VT is bit enough to hold valid shift
4599 : // amounts. This catches things like trying to shift an i1024 value by an
4600 : // i8, which is easy to fall into in generic code that uses
4601 : // TLI.getShiftAmount().
4602 : assert(N2.getValueSizeInBits() >= Log2_32_Ceil(N1.getValueSizeInBits()) &&
4603 : "Invalid use of small shift amount with oversized value!");
4604 :
4605 : // Always fold shifts of i1 values so the code generator doesn't need to
4606 : // handle them. Since we know the size of the shift has to be less than the
4607 : // size of the value, the shift/rotate count is guaranteed to be zero.
4608 : if (VT == MVT::i1)
4609 45 : return N1;
4610 749899 : if (N2C && N2C->isNullValue())
4611 16745 : return N1;
4612 : break;
4613 : case ISD::FP_ROUND_INREG: {
4614 : EVT EVT = cast<VTSDNode>(N2)->getVT();
4615 : assert(VT == N1.getValueType() && "Not an inreg round!");
4616 : assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
4617 : "Cannot FP_ROUND_INREG integer types");
4618 : assert(EVT.isVector() == VT.isVector() &&
4619 : "FP_ROUND_INREG type should be vector iff the operand "
4620 : "type is vector!");
4621 : assert((!EVT.isVector() ||
4622 : EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
4623 : "Vector element counts must match in FP_ROUND_INREG");
4624 : assert(EVT.bitsLE(VT) && "Not rounding down!");
4625 : (void)EVT;
4626 0 : if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding.
4627 : break;
4628 : }
4629 6253 : case ISD::FP_ROUND:
4630 : assert(VT.isFloatingPoint() &&
4631 : N1.getValueType().isFloatingPoint() &&
4632 : VT.bitsLE(N1.getValueType()) &&
4633 : N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
4634 : "Invalid FP_ROUND!");
4635 6386 : if (N1.getValueType() == VT) return N1; // noop conversion.
4636 : break;
4637 : case ISD::AssertSext:
4638 : case ISD::AssertZext: {
4639 : EVT EVT = cast<VTSDNode>(N2)->getVT();
4640 : assert(VT == N1.getValueType() && "Not an inreg extend!");
4641 : assert(VT.isInteger() && EVT.isInteger() &&
4642 : "Cannot *_EXTEND_INREG FP types");
4643 : assert(!EVT.isVector() &&
4644 : "AssertSExt/AssertZExt type should be the vector element type "
4645 : "rather than the vector type!");
4646 : assert(EVT.bitsLE(VT) && "Not extending!");
4647 6026 : if (VT == EVT) return N1; // noop assertion.
4648 : break;
4649 : }
4650 : case ISD::SIGN_EXTEND_INREG: {
4651 21048 : EVT EVT = cast<VTSDNode>(N2)->getVT();
4652 : assert(VT == N1.getValueType() && "Not an inreg extend!");
4653 : assert(VT.isInteger() && EVT.isInteger() &&
4654 : "Cannot *_EXTEND_INREG FP types");
4655 : assert(EVT.isVector() == VT.isVector() &&
4656 : "SIGN_EXTEND_INREG type should be vector iff the operand "
4657 : "type is vector!");
4658 : assert((!EVT.isVector() ||
4659 : EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
4660 : "Vector element counts must match in SIGN_EXTEND_INREG");
4661 : assert(EVT.bitsLE(VT) && "Not extending!");
4662 21548 : if (EVT == VT) return N1; // Not actually extending
4663 :
4664 : auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
4665 : unsigned FromBits = EVT.getScalarSizeInBits();
4666 : Val <<= Val.getBitWidth() - FromBits;
4667 : Val.ashrInPlace(Val.getBitWidth() - FromBits);
4668 : return getConstant(Val, DL, ConstantVT);
4669 20930 : };
4670 :
4671 20930 : if (N1C) {
4672 286 : const APInt &Val = N1C->getAPIntValue();
4673 572 : return SignExtendInReg(Val, VT);
4674 : }
4675 20644 : if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
4676 : SmallVector<SDValue, 8> Ops;
4677 18 : llvm::EVT OpVT = N1.getOperand(0).getValueType();
4678 47 : for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
4679 76 : SDValue Op = N1.getOperand(i);
4680 38 : if (Op.isUndef()) {
4681 0 : Ops.push_back(getUNDEF(OpVT));
4682 0 : continue;
4683 : }
4684 : ConstantSDNode *C = cast<ConstantSDNode>(Op);
4685 38 : APInt Val = C->getAPIntValue();
4686 38 : Ops.push_back(SignExtendInReg(Val, OpVT));
4687 : }
4688 9 : return getBuildVector(VT, DL, Ops);
4689 : }
4690 20635 : break;
4691 : }
4692 437111 : case ISD::EXTRACT_VECTOR_ELT:
4693 : assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
4694 : "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
4695 : element type of the vector.");
4696 :
4697 : // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
4698 437111 : if (N1.isUndef())
4699 19910 : return getUNDEF(VT);
4700 :
4701 : // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF
4702 1243469 : if (N2C && N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements()))
4703 84 : return getUNDEF(VT);
4704 :
4705 : // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
4706 : // expanding copies of large vectors from registers.
4707 413050 : if (N2C &&
4708 434055 : N1.getOpcode() == ISD::CONCAT_VECTORS &&
4709 : N1.getNumOperands() > 0) {
4710 : unsigned Factor =
4711 50814 : N1.getOperand(0).getValueType().getVectorNumElements();
4712 : return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
4713 33876 : N1.getOperand(N2C->getZExtValue() / Factor),
4714 16938 : getConstant(N2C->getZExtValue() % Factor, DL,
4715 33876 : N2.getValueType()));
4716 : }
4717 :
4718 : // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
4719 : // expanding large vector constants.
4720 400179 : if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
4721 235160 : SDValue Elt = N1.getOperand(N2C->getZExtValue());
4722 :
4723 117620 : if (VT != Elt.getValueType())
4724 : // If the vector element type is not legal, the BUILD_VECTOR operands
4725 : // are promoted and implicitly truncated, and the result implicitly
4726 : // extended. Make that explicit here.
4727 543 : Elt = getAnyExtOrTrunc(Elt, DL, VT);
4728 :
4729 117580 : return Elt;
4730 : }
4731 :
4732 : // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
4733 : // operations are lowered to scalars.
4734 282599 : if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
4735 : // If the indices are the same, return the inserted element else
4736 : // if the indices are known different, extract the element from
4737 : // the original vector.
4738 5982 : SDValue N1Op2 = N1.getOperand(2);
4739 : ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
4740 :
4741 4827 : if (N1Op2C && N2C) {
4742 14445 : if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
4743 1135 : if (VT == N1.getOperand(1).getValueType())
4744 1110 : return N1.getOperand(1);
4745 : else
4746 25 : return getSExtOrTrunc(N1.getOperand(1), DL, VT);
4747 : }
4748 :
4749 3680 : return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
4750 : }
4751 : }
4752 :
4753 : // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
4754 : // when vector types are scalarized and v1iX is legal.
4755 : // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx)
4756 296267 : if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
4757 296267 : N1.getValueType().getVectorNumElements() == 1) {
4758 : return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
4759 223 : N1.getOperand(1));
4760 : }
4761 : break;
4762 35927 : case ISD::EXTRACT_ELEMENT:
4763 : assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
4764 : assert(!N1.getValueType().isVector() && !VT.isVector() &&
4765 : (N1.getValueType().isInteger() == VT.isInteger()) &&
4766 : N1.getValueType() != VT &&
4767 : "Wrong types for EXTRACT_ELEMENT!");
4768 :
4769 : // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
4770 : // 64-bit integers into 32-bit parts. Instead of building the extract of
4771 : // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
4772 35927 : if (N1.getOpcode() == ISD::BUILD_PAIR)
4773 13280 : return N1.getOperand(N2C->getZExtValue());
4774 :
4775 : // EXTRACT_ELEMENT of a constant int is also very common.
4776 29287 : if (N1C) {
4777 5968 : unsigned ElementSize = VT.getSizeInBits();
4778 5968 : unsigned Shift = ElementSize * N2C->getZExtValue();
4779 11936 : APInt ShiftedVal = N1C->getAPIntValue().lshr(Shift);
4780 11936 : return getConstant(ShiftedVal.trunc(ElementSize), DL, VT);
4781 23319 : }
4782 : break;
4783 119113 : case ISD::EXTRACT_SUBVECTOR:
4784 119113 : if (VT.isSimple() && N1.getValueType().isSimple()) {
4785 : assert(VT.isVector() && N1.getValueType().isVector() &&
4786 : "Extract subvector VTs must be a vectors!");
4787 : assert(VT.getVectorElementType() ==
4788 : N1.getValueType().getVectorElementType() &&
4789 : "Extract subvector VTs must have the same element type!");
4790 : assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
4791 : "Extract subvector must be from larger vector to smaller vector!");
4792 :
4793 : if (N2C) {
4794 : assert((VT.getVectorNumElements() + N2C->getZExtValue()
4795 : <= N1.getValueType().getVectorNumElements())
4796 : && "Extract subvector overflow!");
4797 : }
4798 :
4799 : // Trivial extraction.
4800 113355 : if (VT.getSimpleVT() == N1.getSimpleValueType())
4801 18169 : return N1;
4802 :
4803 : // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
4804 95186 : if (N1.isUndef())
4805 768 : return getUNDEF(VT);
4806 :
4807 : // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
4808 : // the concat have the same type as the extract.
4809 94418 : if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
4810 94418 : N1.getNumOperands() > 0 &&
4811 7113 : VT == N1.getOperand(0).getValueType()) {
4812 : unsigned Factor = VT.getVectorNumElements();
4813 17565 : return N1.getOperand(N2C->getZExtValue() / Factor);
4814 : }
4815 :
4816 : // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
4817 : // during shuffle legalization.
4818 88563 : if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
4819 817 : VT == N1.getOperand(1).getValueType())
4820 325 : return N1.getOperand(1);
4821 : }
4822 : break;
4823 : }
4824 :
4825 : // Perform trivial constant folding.
4826 7961505 : if (SDValue SV =
4827 7961505 : FoldConstantArithmetic(Opcode, DL, VT, N1.getNode(), N2.getNode()))
4828 195889 : return SV;
4829 :
4830 : // Constant fold FP operations.
4831 7765616 : bool HasFPExceptions = TLI->hasFloatingPointExceptions();
4832 7765616 : if (N1CFP) {
4833 6962 : if (N2CFP) {
4834 4974 : APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
4835 : APFloat::opStatus s;
4836 4974 : switch (Opcode) {
4837 487 : case ISD::FADD:
4838 487 : s = V1.add(V2, APFloat::rmNearestTiesToEven);
4839 487 : if (!HasFPExceptions || s != APFloat::opInvalidOp)
4840 487 : return getConstantFP(V1, DL, VT);
4841 : break;
4842 75 : case ISD::FSUB:
4843 75 : s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
4844 75 : if (!HasFPExceptions || s!=APFloat::opInvalidOp)
4845 75 : return getConstantFP(V1, DL, VT);
4846 : break;
4847 590 : case ISD::FMUL:
4848 590 : s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
4849 590 : if (!HasFPExceptions || s!=APFloat::opInvalidOp)
4850 590 : return getConstantFP(V1, DL, VT);
4851 : break;
4852 61 : case ISD::FDIV:
4853 61 : s = V1.divide(V2, APFloat::rmNearestTiesToEven);
4854 61 : if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
4855 : s!=APFloat::opDivByZero)) {
4856 38 : return getConstantFP(V1, DL, VT);
4857 : }
4858 : break;
4859 35 : case ISD::FREM :
4860 35 : s = V1.mod(V2);
4861 35 : if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
4862 : s!=APFloat::opDivByZero)) {
4863 21 : return getConstantFP(V1, DL, VT);
4864 : }
4865 : break;
4866 4 : case ISD::FCOPYSIGN:
4867 4 : V1.copySign(V2);
4868 4 : return getConstantFP(V1, DL, VT);
4869 : default: break;
4870 : }
4871 : }
4872 :
4873 5747 : if (Opcode == ISD::FP_ROUND) {
4874 4 : APFloat V = N1CFP->getValueAPF(); // make copy
4875 : bool ignored;
4876 : // This can return overflow, underflow, or inexact; we don't care.
4877 : // FIXME need to be more flexible about rounding mode.
4878 4 : (void)V.convert(EVTToAPFloatSemantics(VT),
4879 : APFloat::rmNearestTiesToEven, &ignored);
4880 4 : return getConstantFP(V, DL, VT);
4881 : }
4882 : }
4883 :
4884 : // Any FP binop with an undef operand is folded to NaN. This matches the
4885 : // behavior of the IR optimizer.
4886 7764397 : switch (Opcode) {
4887 57994 : case ISD::FADD:
4888 : case ISD::FSUB:
4889 : case ISD::FMUL:
4890 : case ISD::FDIV:
4891 : case ISD::FREM:
4892 57994 : if (N1.isUndef() || N2.isUndef())
4893 2544 : return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT);
4894 : }
4895 :
4896 : // Canonicalize an UNDEF to the RHS, even over a constant.
4897 7763125 : if (N1.isUndef()) {
4898 5601 : if (TLI->isCommutativeBinOp(Opcode)) {
4899 : std::swap(N1, N2);
4900 : } else {
4901 2774 : switch (Opcode) {
4902 30 : case ISD::FP_ROUND_INREG:
4903 : case ISD::SIGN_EXTEND_INREG:
4904 : case ISD::SUB:
4905 30 : return getUNDEF(VT); // fold op(undef, arg2) -> undef
4906 307 : case ISD::UDIV:
4907 : case ISD::SDIV:
4908 : case ISD::UREM:
4909 : case ISD::SREM:
4910 : case ISD::SRA:
4911 : case ISD::SRL:
4912 : case ISD::SHL:
4913 307 : return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0
4914 : }
4915 : }
4916 : }
4917 :
4918 : // Fold a bunch of operators when the RHS is undef.
4919 7762788 : if (N2.isUndef()) {
4920 19836 : switch (Opcode) {
4921 897 : case ISD::XOR:
4922 897 : if (N1.isUndef())
4923 : // Handle undef ^ undef -> 0 special case. This is a common
4924 : // idiom (misuse).
4925 166 : return getConstant(0, DL, VT);
4926 : LLVM_FALLTHROUGH;
4927 : case ISD::ADD:
4928 : case ISD::ADDC:
4929 : case ISD::ADDE:
4930 : case ISD::SUB:
4931 : case ISD::UDIV:
4932 : case ISD::SDIV:
4933 : case ISD::UREM:
4934 : case ISD::SREM:
4935 : case ISD::SRA:
4936 : case ISD::SRL:
4937 : case ISD::SHL:
4938 2969 : return getUNDEF(VT); // fold op(arg1, undef) -> undef
4939 987 : case ISD::MUL:
4940 : case ISD::AND:
4941 987 : return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0
4942 454 : case ISD::OR:
4943 454 : return getAllOnesConstant(DL, VT);
4944 : }
4945 : }
4946 :
4947 : // Memoize this node if possible.
4948 : SDNode *N;
4949 7758212 : SDVTList VTs = getVTList(VT);
4950 7758212 : SDValue Ops[] = {N1, N2};
4951 7758212 : if (VT != MVT::Glue) {
4952 : FoldingSetNodeID ID;
4953 7754455 : AddNodeIDNode(ID, Opcode, VTs, Ops);
4954 7754455 : void *IP = nullptr;
4955 7754455 : if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
4956 637129 : E->intersectFlagsWith(Flags);
4957 637129 : return SDValue(E, 0);
4958 : }
4959 :
4960 14234652 : N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4961 7117326 : N->setFlags(Flags);
4962 7117326 : createOperands(N, Ops);
4963 7117326 : CSEMap.InsertNode(N, IP);
4964 : } else {
4965 7514 : N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4966 3757 : createOperands(N, Ops);
4967 : }
4968 :
4969 7121083 : InsertNode(N);
4970 : SDValue V = SDValue(N, 0);
4971 : NewSDValueDbgMsg(V, "Creating new node: ", this);
4972 7121083 : return V;
4973 : }
4974 :
4975 2293847 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4976 : SDValue N1, SDValue N2, SDValue N3,
4977 : const SDNodeFlags Flags) {
4978 : // Perform various simplifications.
4979 2293847 : switch (Opcode) {
4980 : case ISD::FMA: {
4981 : assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
4982 : assert(N1.getValueType() == VT && N2.getValueType() == VT &&
4983 : N3.getValueType() == VT && "FMA types must match!");
4984 : ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
4985 : ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
4986 : ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
4987 6427 : if (N1CFP && N2CFP && N3CFP) {
4988 8 : APFloat V1 = N1CFP->getValueAPF();
4989 8 : const APFloat &V2 = N2CFP->getValueAPF();
4990 8 : const APFloat &V3 = N3CFP->getValueAPF();
4991 : APFloat::opStatus s =
4992 8 : V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
4993 8 : if (!TLI->hasFloatingPointExceptions() || s != APFloat::opInvalidOp)
4994 8 : return getConstantFP(V1, DL, VT);
4995 : }
4996 : break;
4997 : }
4998 13 : case ISD::CONCAT_VECTORS: {
4999 : // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
5000 13 : SDValue Ops[] = {N1, N2, N3};
5001 13 : if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
5002 2 : return V;
5003 11 : break;
5004 : }
5005 : case ISD::SETCC: {
5006 : // Use FoldSetCC to simplify SETCC's.
5007 354537 : if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
5008 10795 : return V;
5009 : // Vector constant folding.
5010 343742 : SDValue Ops[] = {N1, N2, N3};
5011 343742 : if (SDValue V = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) {
5012 : NewSDValueDbgMsg(V, "New node vector constant folding: ", this);
5013 895 : return V;
5014 : }
5015 342847 : break;
5016 : }
5017 : case ISD::SELECT:
5018 : if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
5019 3742 : if (N1C->getZExtValue())
5020 160 : return N2; // select true, X, Y -> X
5021 1711 : return N3; // select false, X, Y -> Y
5022 : }
5023 :
5024 83 : if (N2 == N3) return N2; // select C, X, X -> X
5025 : break;
5026 : case ISD::VECTOR_SHUFFLE:
5027 : llvm_unreachable("should use getVectorShuffle constructor!");
5028 : case ISD::INSERT_VECTOR_ELT: {
5029 : ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
5030 : // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF
5031 121217 : if (N3C && N3C->getZExtValue() >= N1.getValueType().getVectorNumElements())
5032 8 : return getUNDEF(VT);
5033 : break;
5034 : }
5035 23034 : case ISD::INSERT_SUBVECTOR: {
5036 : SDValue Index = N3;
5037 23034 : if (VT.isSimple() && N1.getValueType().isSimple()
5038 46068 : && N2.getValueType().isSimple()) {
5039 : assert(VT.isVector() && N1.getValueType().isVector() &&
5040 : N2.getValueType().isVector() &&
5041 : "Insert subvector VTs must be a vectors");
5042 : assert(VT == N1.getValueType() &&
5043 : "Dest and insert subvector source types must match!");
5044 : assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
5045 : "Insert subvector must be from smaller vector to larger vector!");
5046 : if (isa<ConstantSDNode>(Index)) {
5047 : assert((N2.getValueType().getVectorNumElements() +
5048 : cast<ConstantSDNode>(Index)->getZExtValue()
5049 : <= VT.getVectorNumElements())
5050 : && "Insert subvector overflow!");
5051 : }
5052 :
5053 : // Trivial insertion.
5054 23034 : if (VT.getSimpleVT() == N2.getSimpleValueType())
5055 83 : return N2;
5056 : }
5057 : break;
5058 : }
5059 0 : case ISD::BITCAST:
5060 : // Fold bit_convert nodes from a type to themselves.
5061 0 : if (N1.getValueType() == VT)
5062 0 : return N1;
5063 : break;
5064 : }
5065 :
5066 : // Memoize node if it doesn't produce a flag.
5067 : SDNode *N;
5068 2280102 : SDVTList VTs = getVTList(VT);
5069 2280102 : SDValue Ops[] = {N1, N2, N3};
5070 : if (VT != MVT::Glue) {
5071 : FoldingSetNodeID ID;
5072 2279013 : AddNodeIDNode(ID, Opcode, VTs, Ops);
5073 2279013 : void *IP = nullptr;
5074 2279013 : if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
5075 120708 : E->intersectFlagsWith(Flags);
5076 120708 : return SDValue(E, 0);
5077 : }
5078 :
5079 4316610 : N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5080 2158305 : N->setFlags(Flags);
5081 2158305 : createOperands(N, Ops);
5082 2158305 : CSEMap.InsertNode(N, IP);
5083 : } else {
5084 2178 : N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5085 1089 : createOperands(N, Ops);
5086 : }
5087 :
5088 2159394 : InsertNode(N);
5089 : SDValue V = SDValue(N, 0);
5090 : NewSDValueDbgMsg(V, "Creating new node: ", this);
5091 2159394 : return V;
5092 : }
5093 :
5094 85911 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5095 : SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
5096 85911 : SDValue Ops[] = { N1, N2, N3, N4 };
5097 85911 : return getNode(Opcode, DL, VT, Ops);
5098 : }
5099 :
5100 39265 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5101 : SDValue N1, SDValue N2, SDValue N3, SDValue N4,
5102 : SDValue N5) {
5103 39265 : SDValue Ops[] = { N1, N2, N3, N4, N5 };
5104 39265 : return getNode(Opcode, DL, VT, Ops);
5105 : }
5106 :
5107 : /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
5108 : /// the incoming stack arguments to be loaded from the stack.
5109 155 : SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
5110 : SmallVector<SDValue, 8> ArgChains;
5111 :
5112 : // Include the original chain at the beginning of the list. When this is
5113 : // used by target LowerCall hooks, this helps legalize find the
5114 : // CALLSEQ_BEGIN node.
5115 155 : ArgChains.push_back(Chain);
5116 :
5117 : // Add a chain value for each stack argument.
5118 155 : for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
5119 741 : UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
5120 : if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
5121 : if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
5122 102 : if (FI->getIndex() < 0)
5123 102 : ArgChains.push_back(SDValue(L, 1));
5124 :
5125 : // Build a tokenfactor for all the chains.
5126 310 : return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
5127 : }
5128 :
5129 : /// getMemsetValue - Vectorized representation of the memset value
5130 : /// operand.
5131 282862 : static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
5132 : const SDLoc &dl) {
5133 : assert(!Value.isUndef());
5134 :
5135 : unsigned NumBits = VT.getScalarSizeInBits();
5136 : if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
5137 : assert(C->getAPIntValue().getBitWidth() == 8);
5138 565656 : APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
5139 282828 : if (VT.isInteger())
5140 280273 : return DAG.getConstant(Val, dl, VT);
5141 5110 : return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
5142 2555 : VT);
5143 : }
5144 :
5145 : assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
5146 34 : EVT IntVT = VT.getScalarType();
5147 34 : if (!IntVT.isInteger())
5148 1 : IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
5149 :
5150 34 : Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
5151 34 : if (NumBits > 8) {
5152 : // Use a multiplication with 0x010101... to extend the input to the
5153 : // required length.
5154 36 : APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
5155 18 : Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
5156 18 : DAG.getConstant(Magic, dl, IntVT));
5157 : }
5158 :
5159 34 : if (VT != Value.getValueType() && !VT.isInteger())
5160 1 : Value = DAG.getBitcast(VT.getScalarType(), Value);
5161 34 : if (VT != Value.getValueType())
5162 18 : Value = DAG.getSplatBuildVector(VT, dl, Value);
5163 :
5164 34 : return Value;
5165 : }
5166 :
5167 : /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
5168 : /// used when a memcpy is turned into a memset when the source is a constant
5169 : /// string ptr.
5170 1251 : static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
5171 : const TargetLowering &TLI,
5172 : const ConstantDataArraySlice &Slice) {
5173 : // Handle vector with all elements zero.
5174 1251 : if (Slice.Array == nullptr) {
5175 45 : if (VT.isInteger())
5176 42 : return DAG.getConstant(0, dl, VT);
5177 : else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
5178 3 : return DAG.getConstantFP(0.0, dl, VT);
5179 0 : else if (VT.isVector()) {
5180 : unsigned NumElts = VT.getVectorNumElements();
5181 0 : MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
5182 : return DAG.getNode(ISD::BITCAST, dl, VT,
5183 : DAG.getConstant(0, dl,
5184 0 : EVT::getVectorVT(*DAG.getContext(),
5185 0 : EltVT, NumElts)));
5186 : } else
5187 0 : llvm_unreachable("Expected type!");
5188 : }
5189 :
5190 : assert(!VT.isVector() && "Can't handle vector type here!");
5191 1206 : unsigned NumVTBits = VT.getSizeInBits();
5192 1206 : unsigned NumVTBytes = NumVTBits / 8;
5193 1208 : unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));
5194 :
5195 : APInt Val(NumVTBits, 0);
5196 1206 : if (DAG.getDataLayout().isLittleEndian()) {
5197 5498 : for (unsigned i = 0; i != NumBytes; ++i)
5198 8584 : Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
5199 : } else {
5200 0 : for (unsigned i = 0; i != NumBytes; ++i)
5201 0 : Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
5202 : }
5203 :
5204 : // If the "cost" of materializing the integer immediate is less than the cost
5205 : // of a load, then it is cost effective to turn the load into the immediate.
5206 1206 : Type *Ty = VT.getTypeForEVT(*DAG.getContext());
5207 1206 : if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
5208 1191 : return DAG.getConstant(Val, dl, VT);
5209 15 : return SDValue(nullptr, 0);
5210 : }
5211 :
5212 529852 : SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, unsigned Offset,
5213 : const SDLoc &DL) {
5214 1059704 : EVT VT = Base.getValueType();
5215 529852 : return getNode(ISD::ADD, DL, VT, Base, getConstant(Offset, DL, VT));
5216 : }
5217 :
5218 : /// Returns true if memcpy source is constant data.
5219 0 : static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
5220 : uint64_t SrcDelta = 0;
5221 : GlobalAddressSDNode *G = nullptr;
5222 0 : if (Src.getOpcode() == ISD::GlobalAddress)
5223 : G = cast<GlobalAddressSDNode>(Src);
5224 0 : else if (Src.getOpcode() == ISD::ADD &&
5225 0 : Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
5226 0 : Src.getOperand(1).getOpcode() == ISD::Constant) {
5227 : G = cast<GlobalAddressSDNode>(Src.getOperand(0));
5228 0 : SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
5229 : }
5230 0 : if (!G)
5231 0 : return false;
5232 :
5233 0 : return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,
5234 0 : SrcDelta + G->getOffset());
5235 : }
5236 :
5237 : /// Determines the optimal series of memory ops to replace the memset / memcpy.
5238 : /// Return true if the number of memory ops is below the threshold (Limit).
5239 : /// It returns the types of the sequence of memory ops to perform
5240 : /// memset / memcpy by reference.
5241 257329 : static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
5242 : unsigned Limit, uint64_t Size,
5243 : unsigned DstAlign, unsigned SrcAlign,
5244 : bool IsMemset,
5245 : bool ZeroMemset,
5246 : bool MemcpyStrSrc,
5247 : bool AllowOverlap,
5248 : unsigned DstAS, unsigned SrcAS,
5249 : SelectionDAG &DAG,
5250 : const TargetLowering &TLI) {
5251 : assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
5252 : "Expecting memcpy / memset source to meet alignment requirement!");
5253 : // If 'SrcAlign' is zero, that means the memory operation does not need to
5254 : // load the value, i.e. memset or memcpy from constant string. Otherwise,
5255 : // it's the inferred alignment of the source. 'DstAlign', on the other hand,
5256 : // is the specified alignment of the memory operation. If it is zero, that
5257 : // means it's possible to change the alignment of the destination.
5258 : // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
5259 : // not need to be loaded.
5260 : EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
5261 : IsMemset, ZeroMemset, MemcpyStrSrc,
5262 257329 : DAG.getMachineFunction());
5263 :
5264 : if (VT == MVT::Other) {
5265 : // Use the largest integer type whose alignment constraints are satisfied.
5266 : // We only need to check DstAlign here as SrcAlign is always greater or
5267 : // equal to DstAlign (or zero).
5268 490 : VT = MVT::i64;
5269 1426 : while (DstAlign && DstAlign < VT.getSizeInBits() / 8 &&
5270 601 : !TLI.allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign))
5271 335 : VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
5272 : assert(VT.isInteger());
5273 :
5274 : // Find the largest legal integer type.
5275 : MVT LVT = MVT::i64;
5276 338 : while (!TLI.isTypeLegal(LVT))
5277 338 : LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
5278 : assert(LVT.isInteger());
5279 :
5280 : // If the type we've chosen is larger than the largest legal integer type
5281 : // then use that instead.
5282 490 : if (VT.bitsGT(LVT))
5283 67 : VT = LVT;
5284 : }
5285 :
5286 : unsigned NumMemOps = 0;
5287 690452 : while (Size != 0) {
5288 434574 : unsigned VTSize = VT.getSizeInBits() / 8;
5289 566382 : while (VTSize > Size) {
5290 : // For now, only use non-vector load / store's for the left-over pieces.
5291 131808 : EVT NewVT = VT;
5292 : unsigned NewVTSize;
5293 :
5294 : bool Found = false;
5295 131808 : if (VT.isVector() || VT.isFloatingPoint()) {
5296 129755 : NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
5297 128934 : if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
5298 128934 : TLI.isSafeMemOpType(NewVT.getSimpleVT()))
5299 : Found = true;
5300 : else if (NewVT == MVT::i64 &&
5301 702 : TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
5302 1463 : TLI.isSafeMemOpType(MVT::f64)) {
5303 : // i64 is usually not legal on 32-bit targets, but f64 may be.
5304 701 : NewVT = MVT::f64;
5305 : Found = true;
5306 : }
5307 : }
5308 :
5309 : if (!Found) {
5310 : do {
5311 2173 : NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
5312 : if (NewVT == MVT::i8)
5313 : break;
5314 1632 : } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
5315 : }
5316 131808 : NewVTSize = NewVT.getSizeInBits() / 8;
5317 :
5318 : // If the new VT cannot cover all of the remaining bits, then consider
5319 : // issuing a (or a pair of) unaligned and overlapping load / store.
5320 : // FIXME: Only does this for 64-bit or more since we don't have proper
5321 : // cost model for unaligned load / store.
5322 : bool Fast;
5323 131117 : if (NumMemOps && AllowOverlap &&
5324 130536 : VTSize >= 8 && NewVTSize < Size &&
5325 132254 : TLI.allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign, &Fast) && Fast)
5326 214 : VTSize = Size;
5327 : else {
5328 131594 : VT = NewVT;
5329 : VTSize = NewVTSize;
5330 : }
5331 : }
5332 :
5333 434574 : if (++NumMemOps > Limit)
5334 : return false;
5335 :
5336 433123 : MemOps.push_back(VT);
5337 433123 : Size -= VTSize;
5338 : }
5339 :
5340 : return true;
5341 : }
5342 :
5343 257329 : static bool shouldLowerMemFuncForSize(const MachineFunction &MF) {
5344 : // On Darwin, -Os means optimize for size without hurting performance, so
5345 : // only really optimize for size when -Oz (MinSize) is used.
5346 257329 : if (MF.getTarget().getTargetTriple().isOSDarwin())
5347 166 : return MF.getFunction().optForMinSize();
5348 257163 : return MF.getFunction().optForSize();
5349 : }
5350 :
5351 38 : static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
5352 : SmallVector<SDValue, 32> &OutChains, unsigned From,
5353 : unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
5354 : SmallVector<SDValue, 16> &OutStoreChains) {
5355 : assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
5356 : assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
5357 : SmallVector<SDValue, 16> GluedLoadChains;
5358 113 : for (unsigned i = From; i < To; ++i) {
5359 150 : OutChains.push_back(OutLoadChains[i]);
5360 75 : GluedLoadChains.push_back(OutLoadChains[i]);
5361 : }
5362 :
5363 : // Chain for all loads.
5364 : SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
5365 38 : GluedLoadChains);
5366 :
5367 113 : for (unsigned i = From; i < To; ++i) {
5368 75 : StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);
5369 : SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(),
5370 : ST->getBasePtr(), ST->getMemoryVT(),
5371 150 : ST->getMemOperand());
5372 75 : OutChains.push_back(NewStore);
5373 : }
5374 38 : }
5375 :
5376 101560 : static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
5377 : SDValue Chain, SDValue Dst, SDValue Src,
5378 : uint64_t Size, unsigned Align,
5379 : bool isVol, bool AlwaysInline,
5380 : MachinePointerInfo DstPtrInfo,
5381 : MachinePointerInfo SrcPtrInfo) {
5382 : // Turn a memcpy of undef to nop.
5383 101560 : if (Src.isUndef())
5384 5 : return Chain;
5385 :
5386 : // Expand memcpy to a series of load and store ops if the size operand falls
5387 : // below a certain threshold.
5388 : // TODO: In the AlwaysInline case, if the size is big then generate a loop
5389 : // rather than maybe a humongous number of loads and stores.
5390 101555 : const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5391 101555 : const DataLayout &DL = DAG.getDataLayout();
5392 101555 : LLVMContext &C = *DAG.getContext();
5393 : std::vector<EVT> MemOps;
5394 : bool DstAlignCanChange = false;
5395 101555 : MachineFunction &MF = DAG.getMachineFunction();
5396 101555 : MachineFrameInfo &MFI = MF.getFrameInfo();
5397 101555 : bool OptSize = shouldLowerMemFuncForSize(MF);
5398 : FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
5399 81687 : if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
5400 : DstAlignCanChange = true;
5401 101555 : unsigned SrcAlign = DAG.InferPtrAlignment(Src);
5402 101555 : if (Align > SrcAlign)
5403 : SrcAlign = Align;
5404 : ConstantDataArraySlice Slice;
5405 101555 : bool CopyFromConstant = isMemSrcFromConstant(Src, Slice);
5406 101555 : bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
5407 101555 : unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
5408 :
5409 222990 : if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
5410 : (DstAlignCanChange ? 0 : Align),
5411 : (isZeroConstant ? 0 : SrcAlign),
5412 : false, false, CopyFromConstant, true,
5413 : DstPtrInfo.getAddrSpace(),
5414 : SrcPtrInfo.getAddrSpace(),
5415 : DAG, TLI))
5416 370 : return SDValue();
5417 :
5418 101185 : if (DstAlignCanChange) {
5419 81576 : Type *Ty = MemOps[0].getTypeForEVT(C);
5420 81576 : unsigned NewAlign = (unsigned)DL.getABITypeAlignment(Ty);
5421 :
5422 : // Don't promote to an alignment that would require dynamic stack
5423 : // realignment.
5424 81576 : const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
5425 81576 : if (!TRI->needsStackRealignment(MF))
5426 81574 : while (NewAlign > Align &&
5427 6042 : DL.exceedsNaturalStackAlignment(NewAlign))
5428 3 : NewAlign /= 2;
5429 :
5430 81576 : if (NewAlign > Align) {
5431 : // Give the stack frame object a larger alignment if needed.
5432 12084 : if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
5433 : MFI.setObjectAlignment(FI->getIndex(), NewAlign);
5434 : Align = NewAlign;
5435 : }
5436 : }
5437 :
5438 : MachineMemOperand::Flags MMOFlags =
5439 101185 : isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
5440 : SmallVector<SDValue, 16> OutLoadChains;
5441 : SmallVector<SDValue, 16> OutStoreChains;
5442 : SmallVector<SDValue, 32> OutChains;
5443 202370 : unsigned NumMemOps = MemOps.size();
5444 : uint64_t SrcOff = 0, DstOff = 0;
5445 213158 : for (unsigned i = 0; i != NumMemOps; ++i) {
5446 111973 : EVT VT = MemOps[i];
5447 111973 : unsigned VTSize = VT.getSizeInBits() / 8;
5448 111973 : SDValue Value, Store;
5449 :
5450 111973 : if (VTSize > Size) {
5451 : // Issuing an unaligned load / store pair that overlaps with the previous
5452 : // pair. Adjust the offset accordingly.
5453 : assert(i == NumMemOps-1 && i != 0);
5454 203 : SrcOff -= VTSize - Size;
5455 : DstOff -= VTSize - Size;
5456 : }
5457 :
5458 111973 : if (CopyFromConstant &&
5459 2867 : (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
5460 : // It's unlikely a store of a vector immediate can be done in a single
5461 : // instruction. It would require a load from a constantpool first.
5462 : // We only handle zero vectors here.
5463 : // FIXME: Handle other cases where store of vector immediate is done in
5464 : // a single instruction.
5465 : ConstantDataArraySlice SubSlice;
5466 1251 : if (SrcOff < Slice.Length) {
5467 1207 : SubSlice = Slice;
5468 : SubSlice.move(SrcOff);
5469 : } else {
5470 : // This is an out-of-bounds access and hence UB. Pretend we read zero.
5471 44 : SubSlice.Array = nullptr;
5472 44 : SubSlice.Offset = 0;
5473 44 : SubSlice.Length = VTSize;
5474 : }
5475 1251 : Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
5476 1251 : if (Value.getNode()) {
5477 1236 : Store = DAG.getStore(Chain, dl, Value,
5478 : DAG.getMemBasePlusOffset(Dst, DstOff, dl),
5479 : DstPtrInfo.getWithOffset(DstOff), Align,
5480 1236 : MMOFlags);
5481 1236 : OutChains.push_back(Store);
5482 : }
5483 : }
5484 :
5485 111973 : if (!Store.getNode()) {
5486 : // The type might not be legal for the target. This should only happen
5487 : // if the type is smaller than a legal type, as on PPC, so the right
5488 : // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
5489 : // to Load/Store if NVT==VT.
5490 : // FIXME does the case above also need this?
5491 110737 : EVT NVT = TLI.getTypeToTransformTo(C, VT);
5492 : assert(NVT.bitsGE(VT));
5493 :
5494 : bool isDereferenceable =
5495 110737 : SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
5496 : MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
5497 110737 : if (isDereferenceable)
5498 : SrcMMOFlags |= MachineMemOperand::MODereferenceable;
5499 :
5500 110737 : Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
5501 : DAG.getMemBasePlusOffset(Src, SrcOff, dl),
5502 : SrcPtrInfo.getWithOffset(SrcOff), VT,
5503 221474 : MinAlign(SrcAlign, SrcOff), SrcMMOFlags);
5504 110737 : OutLoadChains.push_back(Value.getValue(1));
5505 :
5506 110737 : Store = DAG.getTruncStore(
5507 : Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl),
5508 110737 : DstPtrInfo.getWithOffset(DstOff), VT, Align, MMOFlags);
5509 110737 : OutStoreChains.push_back(Store);
5510 : }
5511 111973 : SrcOff += VTSize;
5512 : DstOff += VTSize;
5513 111973 : Size -= VTSize;
5514 : }
5515 :
5516 101185 : unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
5517 101185 : TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
5518 101185 : unsigned NumLdStInMemcpy = OutStoreChains.size();
5519 :
5520 101185 : if (NumLdStInMemcpy) {
5521 : // It may be that memcpy might be converted to memset if it's memcpy
5522 : // of constants. In such a case, we won't have loads and stores, but
5523 : // just stores. In the absence of loads, there is nothing to gang up.
5524 100601 : if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
5525 : // If target does not care, just leave as it.
5526 211225 : for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
5527 221324 : OutChains.push_back(OutLoadChains[i]);
5528 110662 : OutChains.push_back(OutStoreChains[i]);
5529 : }
5530 : } else {
5531 : // Ld/St less than/equal limit set by target.
5532 38 : if (NumLdStInMemcpy <= GluedLdStLimit) {
5533 38 : chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
5534 : NumLdStInMemcpy, OutLoadChains,
5535 : OutStoreChains);
5536 : } else {
5537 0 : unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
5538 0 : unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
5539 : unsigned GlueIter = 0;
5540 :
5541 0 : for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
5542 0 : unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
5543 : unsigned IndexTo = NumLdStInMemcpy - GlueIter;
5544 :
5545 0 : chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo,
5546 : OutLoadChains, OutStoreChains);
5547 0 : GlueIter += GluedLdStLimit;
5548 : }
5549 :
5550 : // Residual ld/st.
5551 0 : if (RemainingLdStInMemcpy) {
5552 0 : chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
5553 : RemainingLdStInMemcpy, OutLoadChains,
5554 : OutStoreChains);
5555 : }
5556 : }
5557 : }
5558 : }
5559 101185 : return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
5560 : }
5561 :
5562 84 : static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
5563 : SDValue Chain, SDValue Dst, SDValue Src,
5564 : uint64_t Size, unsigned Align,
5565 : bool isVol, bool AlwaysInline,
5566 : MachinePointerInfo DstPtrInfo,
5567 : MachinePointerInfo SrcPtrInfo) {
5568 : // Turn a memmove of undef to nop.
5569 168 : if (Src.isUndef())
5570 0 : return Chain;
5571 :
5572 : // Expand memmove to a series of load and store ops if the size operand falls
5573 : // below a certain threshold.
5574 84 : const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5575 84 : const DataLayout &DL = DAG.getDataLayout();
5576 84 : LLVMContext &C = *DAG.getContext();
5577 : std::vector<EVT> MemOps;
5578 : bool DstAlignCanChange = false;
5579 84 : MachineFunction &MF = DAG.getMachineFunction();
5580 84 : MachineFrameInfo &MFI = MF.getFrameInfo();
5581 84 : bool OptSize = shouldLowerMemFuncForSize(MF);
5582 : FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
5583 1 : if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
5584 : DstAlignCanChange = true;
5585 84 : unsigned SrcAlign = DAG.InferPtrAlignment(Src);
5586 84 : if (Align > SrcAlign)
5587 : SrcAlign = Align;
5588 84 : unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
5589 :
5590 167 : if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
5591 : (DstAlignCanChange ? 0 : Align), SrcAlign,
5592 : false, false, false, false,
5593 : DstPtrInfo.getAddrSpace(),
5594 : SrcPtrInfo.getAddrSpace(),
5595 : DAG, TLI))
5596 57 : return SDValue();
5597 :
5598 27 : if (DstAlignCanChange) {
5599 1 : Type *Ty = MemOps[0].getTypeForEVT(C);
5600 1 : unsigned NewAlign = (unsigned)DL.getABITypeAlignment(Ty);
5601 1 : if (NewAlign > Align) {
5602 : // Give the stack frame object a larger alignment if needed.
5603 2 : if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
5604 : MFI.setObjectAlignment(FI->getIndex(), NewAlign);
5605 : Align = NewAlign;
5606 : }
5607 : }
5608 :
5609 : MachineMemOperand::Flags MMOFlags =
5610 27 : isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
5611 : uint64_t SrcOff = 0, DstOff = 0;
5612 : SmallVector<SDValue, 8> LoadValues;
5613 : SmallVector<SDValue, 8> LoadChains;
5614 : SmallVector<SDValue, 8> OutChains;
5615 54 : unsigned NumMemOps = MemOps.size();
5616 56 : for (unsigned i = 0; i < NumMemOps; i++) {
5617 29 : EVT VT = MemOps[i];
5618 29 : unsigned VTSize = VT.getSizeInBits() / 8;
5619 29 : SDValue Value;
5620 :
5621 : bool isDereferenceable =
5622 29 : SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
5623 : MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
5624 29 : if (isDereferenceable)
5625 : SrcMMOFlags |= MachineMemOperand::MODereferenceable;
5626 :
5627 29 : Value =
5628 29 : DAG.getLoad(VT, dl, Chain, DAG.getMemBasePlusOffset(Src, SrcOff, dl),
5629 29 : SrcPtrInfo.getWithOffset(SrcOff), SrcAlign, SrcMMOFlags);
5630 29 : LoadValues.push_back(Value);
5631 29 : LoadChains.push_back(Value.getValue(1));
5632 29 : SrcOff += VTSize;
5633 : }
5634 27 : Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
5635 : OutChains.clear();
5636 56 : for (unsigned i = 0; i < NumMemOps; i++) {
5637 29 : EVT VT = MemOps[i];
5638 29 : unsigned VTSize = VT.getSizeInBits() / 8;
5639 29 : SDValue Store;
5640 :
5641 29 : Store = DAG.getStore(Chain, dl, LoadValues[i],
5642 : DAG.getMemBasePlusOffset(Dst, DstOff, dl),
5643 29 : DstPtrInfo.getWithOffset(DstOff), Align, MMOFlags);
5644 29 : OutChains.push_back(Store);
5645 29 : DstOff += VTSize;
5646 : }
5647 :
5648 27 : return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
5649 : }
5650 :
5651 : /// Lower the call to 'memset' intrinsic function into a series of store
5652 : /// operations.
5653 : ///
5654 : /// \param DAG Selection DAG where lowered code is placed.
5655 : /// \param dl Link to corresponding IR location.
5656 : /// \param Chain Control flow dependency.
5657 : /// \param Dst Pointer to destination memory location.
5658 : /// \param Src Value of byte to write into the memory.
5659 : /// \param Size Number of bytes to write.
5660 : /// \param Align Alignment of the destination in bytes.
5661 : /// \param isVol True if destination is volatile.
5662 : /// \param DstPtrInfo IR information on the memory pointer.
5663 : /// \returns New head in the control flow, if lowering was successful, empty
5664 : /// SDValue otherwise.
5665 : ///
5666 : /// The function tries to replace 'llvm.memset' intrinsic with several store
5667 : /// operations and value calculation code. This is usually profitable for small
5668 : /// memory size.
5669 155691 : static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
5670 : SDValue Chain, SDValue Dst, SDValue Src,
5671 : uint64_t Size, unsigned Align, bool isVol,
5672 : MachinePointerInfo DstPtrInfo) {
5673 : // Turn a memset of undef to nop.
5674 155691 : if (Src.isUndef())
5675 1 : return Chain;
5676 :
5677 : // Expand memset to a series of load/store ops if the size operand
5678 : // falls below a certain threshold.
5679 155690 : const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5680 : std::vector<EVT> MemOps;
5681 : bool DstAlignCanChange = false;
5682 155690 : MachineFunction &MF = DAG.getMachineFunction();
5683 155690 : MachineFrameInfo &MFI = MF.getFrameInfo();
5684 155690 : bool OptSize = shouldLowerMemFuncForSize(MF);
5685 : FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
5686 4127 : if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
5687 : DstAlignCanChange = true;
5688 : bool IsZeroVal =
5689 311282 : isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
5690 311380 : if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
5691 : Size, (DstAlignCanChange ? 0 : Align), 0,
5692 : true, IsZeroVal, false, true,
5693 : DstPtrInfo.getAddrSpace(), ~0u,
5694 : DAG, TLI))
5695 1024 : return SDValue();
5696 :
5697 154666 : if (DstAlignCanChange) {
5698 4047 : Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
5699 4047 : unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
5700 4047 : if (NewAlign > Align) {
5701 : // Give the stack frame object a larger alignment if needed.
5702 1154 : if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
5703 : MFI.setObjectAlignment(FI->getIndex(), NewAlign);
5704 : Align = NewAlign;
5705 : }
5706 : }
5707 :
5708 : SmallVector<SDValue, 8> OutChains;
5709 : uint64_t DstOff = 0;
5710 154666 : unsigned NumMemOps = MemOps.size();
5711 :
5712 : // Find the largest store and generate the bit pattern for it.
5713 154666 : EVT LargestVT = MemOps[0];
5714 304965 : for (unsigned i = 1; i < NumMemOps; i++)
5715 300598 : if (MemOps[i].bitsGT(LargestVT))
5716 0 : LargestVT = MemOps[i];
5717 154666 : SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
5718 :
5719 459631 : for (unsigned i = 0; i < NumMemOps; i++) {
5720 304965 : EVT VT = MemOps[i];
5721 304965 : unsigned VTSize = VT.getSizeInBits() / 8;
5722 304965 : if (VTSize > Size) {
5723 : // Issuing an unaligned load / store pair that overlaps with the previous
5724 : // pair. Adjust the offset accordingly.
5725 : assert(i == NumMemOps-1 && i != 0);
5726 11 : DstOff -= VTSize - Size;
5727 : }
5728 :
5729 : // If this store is smaller than the largest store see whether we can get
5730 : // the smaller value for free with a truncate.
5731 304965 : SDValue Value = MemSetValue;
5732 304965 : if (VT.bitsLT(LargestVT)) {
5733 128384 : if (!LargestVT.isVector() && !VT.isVector() &&
5734 69 : TLI.isTruncateFree(LargestVT, VT))
5735 50 : Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
5736 : else
5737 128196 : Value = getMemsetValue(Src, VT, DAG, dl);
5738 : }
5739 : assert(Value.getValueType() == VT && "Value with wrong type.");
5740 : SDValue Store = DAG.getStore(
5741 : Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl),
5742 : DstPtrInfo.getWithOffset(DstOff), Align,
5743 609915 : isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone);
5744 304965 : OutChains.push_back(Store);
5745 304965 : DstOff += VT.getSizeInBits() / 8;
5746 304965 : Size -= VTSize;
5747 : }
5748 :
5749 154666 : return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
5750 : }
5751 :
5752 8892 : static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
5753 : unsigned AS) {
5754 : // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
5755 : // pointer operands can be losslessly bitcasted to pointers of address space 0
5756 8892 : if (AS != 0 && !TLI->isNoopAddrSpaceCast(AS, 0)) {
5757 0 : report_fatal_error("cannot lower memory intrinsic in address space " +
5758 : Twine(AS));
5759 : }
5760 8892 : }
5761 :
5762 101934 : SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
5763 : SDValue Src, SDValue Size, unsigned Align,
5764 : bool isVol, bool AlwaysInline, bool isTailCall,
5765 : MachinePointerInfo DstPtrInfo,
5766 : MachinePointerInfo SrcPtrInfo) {
5767 : assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
5768 :
5769 : // Check to see if we should lower the memcpy to loads and stores first.
5770 : // For cases within the target-specified limits, this is the best choice.
5771 : ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
5772 : if (ConstantSize) {
5773 : // Memcpy with size zero? Just return the original chain.
5774 203124 : if (ConstantSize->isNullValue())
5775 101192 : return Chain;
5776 :
5777 : SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
5778 : ConstantSize->getZExtValue(),Align,
5779 203114 : isVol, false, DstPtrInfo, SrcPtrInfo);
5780 101557 : if (Result.getNode())
5781 101187 : return Result;
5782 : }
5783 :
5784 : // Then check to see if we should lower the memcpy with target-specific
5785 : // code. If the target chooses to do this, this is the next best.
5786 742 : if (TSI) {
5787 : SDValue Result = TSI->EmitTargetCodeForMemcpy(
5788 : *this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline,
5789 742 : DstPtrInfo, SrcPtrInfo);
5790 742 : if (Result.getNode())
5791 157 : return Result;
5792 : }
5793 :
5794 : // If we really need inline code and the target declined to provide it,
5795 : // use a (potentially long) sequence of loads and stores.
5796 585 : if (AlwaysInline) {
5797 : assert(ConstantSize && "AlwaysInline requires a constant size!");
5798 : return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
5799 : ConstantSize->getZExtValue(), Align, isVol,
5800 6 : true, DstPtrInfo, SrcPtrInfo);
5801 : }
5802 :
5803 582 : checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
5804 582 : checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
5805 :
5806 : // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
5807 : // memcpy is not guaranteed to be safe. libc memcpys aren't required to
5808 : // respect volatile, so they may do things like read or write memory
5809 : // beyond the given memory regions. But fixing this isn't easy, and most
5810 : // people don't care.
5811 :
5812 : // Emit a library call.
5813 : TargetLowering::ArgListTy Args;
5814 : TargetLowering::ArgListEntry Entry;
5815 582 : Entry.Ty = getDataLayout().getIntPtrType(*getContext());
5816 582 : Entry.Node = Dst; Args.push_back(Entry);
5817 582 : Entry.Node = Src; Args.push_back(Entry);
5818 582 : Entry.Node = Size; Args.push_back(Entry);
5819 : // FIXME: pass in SDLoc
5820 582 : TargetLowering::CallLoweringInfo CLI(*this);
5821 : CLI.setDebugLoc(dl)
5822 582 : .setChain(Chain)
5823 582 : .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
5824 582 : Dst.getValueType().getTypeForEVT(*getContext()),
5825 582 : getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
5826 : TLI->getPointerTy(getDataLayout())),
5827 1164 : std::move(Args))
5828 : .setDiscardResult()
5829 : .setTailCall(isTailCall);
5830 :
5831 582 : std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
5832 582 : return CallResult.second;
5833 : }
5834 :
5835 6 : SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
5836 : SDValue Dst, unsigned DstAlign,
5837 : SDValue Src, unsigned SrcAlign,
5838 : SDValue Size, Type *SizeTy,
5839 : unsigned ElemSz, bool isTailCall,
5840 : MachinePointerInfo DstPtrInfo,
5841 : MachinePointerInfo SrcPtrInfo) {
5842 : // Emit a library call.
5843 : TargetLowering::ArgListTy Args;
5844 : TargetLowering::ArgListEntry Entry;
5845 6 : Entry.Ty = getDataLayout().getIntPtrType(*getContext());
5846 6 : Entry.Node = Dst;
5847 6 : Args.push_back(Entry);
5848 :
5849 6 : Entry.Node = Src;
5850 6 : Args.push_back(Entry);
5851 :
5852 6 : Entry.Ty = SizeTy;
5853 6 : Entry.Node = Size;
5854 6 : Args.push_back(Entry);
5855 :
5856 : RTLIB::Libcall LibraryCall =
5857 6 : RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElemSz);
5858 6 : if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
5859 0 : report_fatal_error("Unsupported element size");
5860 :
5861 6 : TargetLowering::CallLoweringInfo CLI(*this);
5862 : CLI.setDebugLoc(dl)
5863 6 : .setChain(Chain)
5864 6 : .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
5865 6 : Type::getVoidTy(*getContext()),
5866 6 : getExternalSymbol(TLI->getLibcallName(LibraryCall),
5867 : TLI->getPointerTy(getDataLayout())),
5868 12 : std::move(Args))
5869 : .setDiscardResult()
5870 : .setTailCall(isTailCall);
5871 :
5872 6 : std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
5873 6 : return CallResult.second;
5874 : }
5875 :
5876 3440 : SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
5877 : SDValue Src, SDValue Size, unsigned Align,
5878 : bool isVol, bool isTailCall,
5879 : MachinePointerInfo DstPtrInfo,
5880 : MachinePointerInfo SrcPtrInfo) {
5881 : assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
5882 :
5883 : // Check to see if we should lower the memmove to loads and stores first.
5884 : // For cases within the target-specified limits, this is the best choice.
5885 : ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
5886 : if (ConstantSize) {
5887 : // Memmove with size zero? Just return the original chain.
5888 168 : if (ConstantSize->isNullValue())
5889 27 : return Chain;
5890 :
5891 : SDValue Result =
5892 : getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
5893 : ConstantSize->getZExtValue(), Align, isVol,
5894 168 : false, DstPtrInfo, SrcPtrInfo);
5895 84 : if (Result.getNode())
5896 27 : return Result;
5897 : }
5898 :
5899 : // Then check to see if we should lower the memmove with target-specific
5900 : // code. If the target chooses to do this, this is the next best.
5901 3413 : if (TSI) {
5902 : SDValue Result = TSI->EmitTargetCodeForMemmove(
5903 3413 : *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
5904 3413 : if (Result.getNode())
5905 48 : return Result;
5906 : }
5907 :
5908 3365 : checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
5909 3365 : checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
5910 :
5911 : // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
5912 : // not be safe. See memcpy above for more details.
5913 :
5914 : // Emit a library call.
5915 : TargetLowering::ArgListTy Args;
5916 : TargetLowering::ArgListEntry Entry;
5917 3365 : Entry.Ty = getDataLayout().getIntPtrType(*getContext());
5918 3365 : Entry.Node = Dst; Args.push_back(Entry);
5919 3365 : Entry.Node = Src; Args.push_back(Entry);
5920 3365 : Entry.Node = Size; Args.push_back(Entry);
5921 : // FIXME: pass in SDLoc
5922 3365 : TargetLowering::CallLoweringInfo CLI(*this);
5923 : CLI.setDebugLoc(dl)
5924 3365 : .setChain(Chain)
5925 3365 : .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
5926 3365 : Dst.getValueType().getTypeForEVT(*getContext()),
5927 3365 : getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
5928 : TLI->getPointerTy(getDataLayout())),
5929 6730 : std::move(Args))
5930 : .setDiscardResult()
5931 : .setTailCall(isTailCall);
5932 :
5933 3365 : std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
5934 3365 : return CallResult.second;
5935 : }
5936 :
5937 6 : SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
5938 : SDValue Dst, unsigned DstAlign,
5939 : SDValue Src, unsigned SrcAlign,
5940 : SDValue Size, Type *SizeTy,
5941 : unsigned ElemSz, bool isTailCall,
5942 : MachinePointerInfo DstPtrInfo,
5943 : MachinePointerInfo SrcPtrInfo) {
5944 : // Emit a library call.
5945 : TargetLowering::ArgListTy Args;
5946 : TargetLowering::ArgListEntry Entry;
5947 6 : Entry.Ty = getDataLayout().getIntPtrType(*getContext());
5948 6 : Entry.Node = Dst;
5949 6 : Args.push_back(Entry);
5950 :
5951 6 : Entry.Node = Src;
5952 6 : Args.push_back(Entry);
5953 :
5954 6 : Entry.Ty = SizeTy;
5955 6 : Entry.Node = Size;
5956 6 : Args.push_back(Entry);
5957 :
5958 : RTLIB::Libcall LibraryCall =
5959 6 : RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElemSz);
5960 6 : if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
5961 0 : report_fatal_error("Unsupported element size");
5962 :
5963 6 : TargetLowering::CallLoweringInfo CLI(*this);
5964 : CLI.setDebugLoc(dl)
5965 6 : .setChain(Chain)
5966 6 : .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
5967 6 : Type::getVoidTy(*getContext()),
5968 6 : getExternalSymbol(TLI->getLibcallName(LibraryCall),
5969 : TLI->getPointerTy(getDataLayout())),
5970 12 : std::move(Args))
5971 : .setDiscardResult()
5972 : .setTailCall(isTailCall);
5973 :
5974 6 : std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
5975 6 : return CallResult.second;
5976 : }
5977 :
5978 155847 : SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
5979 : SDValue Src, SDValue Size, unsigned Align,
5980 : bool isVol, bool isTailCall,
5981 : MachinePointerInfo DstPtrInfo) {
5982 : assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
5983 :
5984 : // Check to see if we should lower the memset to stores first.
5985 : // For cases within the target-specified limits, this is the best choice.
5986 : ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
5987 : if (ConstantSize) {
5988 : // Memset with size zero? Just return the original chain.
5989 311398 : if (ConstantSize->isNullValue())
5990 154675 : return Chain;
5991 :
5992 : SDValue Result =
5993 : getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
5994 311382 : Align, isVol, DstPtrInfo);
5995 :
5996 155691 : if (Result.getNode())
5997 154667 : return Result;
5998 : }
5999 :
6000 : // Then check to see if we should lower the memset with target-specific
6001 : // code. If the target chooses to do this, this is the next best.
6002 1172 : if (TSI) {
6003 : SDValue Result = TSI->EmitTargetCodeForMemset(
6004 1172 : *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo);
6005 1172 : if (Result.getNode())
6006 174 : return Result;
6007 : }
6008 :
6009 998 : checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
6010 :
6011 : // Emit a library call.
6012 998 : Type *IntPtrTy = getDataLayout().getIntPtrType(*getContext());
6013 : TargetLowering::ArgListTy Args;
6014 : TargetLowering::ArgListEntry Entry;
6015 998 : Entry.Node = Dst; Entry.Ty = IntPtrTy;
6016 998 : Args.push_back(Entry);
6017 998 : Entry.Node = Src;
6018 1996 : Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
6019 998 : Args.push_back(Entry);
6020 998 : Entry.Node = Size;
6021 998 : Entry.Ty = IntPtrTy;
6022 998 : Args.push_back(Entry);
6023 :
6024 : // FIXME: pass in SDLoc
6025 998 : TargetLowering::CallLoweringInfo CLI(*this);
6026 : CLI.setDebugLoc(dl)
6027 998 : .setChain(Chain)
6028 998 : .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
6029 998 : Dst.getValueType().getTypeForEVT(*getContext()),
6030 998 : getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
6031 : TLI->getPointerTy(getDataLayout())),
6032 1996 : std::move(Args))
6033 : .setDiscardResult()
6034 : .setTailCall(isTailCall);
6035 :
6036 998 : std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6037 998 : return CallResult.second;
6038 : }
6039 :
6040 6 : SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
6041 : SDValue Dst, unsigned DstAlign,
6042 : SDValue Value, SDValue Size, Type *SizeTy,
6043 : unsigned ElemSz, bool isTailCall,
6044 : MachinePointerInfo DstPtrInfo) {
6045 : // Emit a library call.
6046 : TargetLowering::ArgListTy Args;
6047 : TargetLowering::ArgListEntry Entry;
6048 6 : Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6049 6 : Entry.Node = Dst;
6050 6 : Args.push_back(Entry);
6051 :
6052 6 : Entry.Ty = Type::getInt8Ty(*getContext());
6053 6 : Entry.Node = Value;
6054 6 : Args.push_back(Entry);
6055 :
6056 6 : Entry.Ty = SizeTy;
6057 6 : Entry.Node = Size;
6058 6 : Args.push_back(Entry);
6059 :
6060 : RTLIB::Libcall LibraryCall =
6061 6 : RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElemSz);
6062 6 : if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
6063 0 : report_fatal_error("Unsupported element size");
6064 :
6065 6 : TargetLowering::CallLoweringInfo CLI(*this);
6066 : CLI.setDebugLoc(dl)
6067 6 : .setChain(Chain)
6068 6 : .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
6069 6 : Type::getVoidTy(*getContext()),
6070 6 : getExternalSymbol(TLI->getLibcallName(LibraryCall),
6071 : TLI->getPointerTy(getDataLayout())),
6072 12 : std::move(Args))
6073 : .setDiscardResult()
6074 : .setTailCall(isTailCall);
6075 :
6076 6 : std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6077 6 : return CallResult.second;
6078 : }
6079 :
6080 37405 : SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
6081 : SDVTList VTList, ArrayRef<SDValue> Ops,
6082 : MachineMemOperand *MMO) {
6083 : FoldingSetNodeID ID;
6084 37405 : ID.AddInteger(MemVT.getRawBits());
6085 37405 : AddNodeIDNode(ID, Opcode, VTList, Ops);
6086 37405 : ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6087 37405 : void* IP = nullptr;
6088 37405 : if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6089 0 : cast<AtomicSDNode>(E)->refineAlignment(MMO);
6090 0 : return SDValue(E, 0);
6091 : }
6092 :
6093 74810 : auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
6094 : VTList, MemVT, MMO);
6095 37405 : createOperands(N, Ops);
6096 :
6097 37405 : CSEMap.InsertNode(N, IP);
6098 37405 : InsertNode(N);
6099 37405 : return SDValue(N, 0);
6100 : }
6101 :
6102 6128 : SDValue SelectionDAG::getAtomicCmpSwap(
6103 : unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain,
6104 : SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
6105 : unsigned Alignment, AtomicOrdering SuccessOrdering,
6106 : AtomicOrdering FailureOrdering, SyncScope::ID SSID) {
6107 : assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
6108 : Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
6109 : assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
6110 :
6111 6128 : if (Alignment == 0) // Ensure that codegen never sees alignment 0
6112 6128 : Alignment = getEVTAlignment(MemVT);
6113 :
6114 6128 : MachineFunction &MF = getMachineFunction();
6115 :
6116 : // FIXME: Volatile isn't really correct; we should keep track of atomic
6117 : // orderings in the memoperand.
6118 : auto Flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad |
6119 : MachineMemOperand::MOStore;
6120 : MachineMemOperand *MMO =
6121 12256 : MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
6122 6128 : AAMDNodes(), nullptr, SSID, SuccessOrdering,
6123 : FailureOrdering);
6124 :
6125 6128 : return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO);
6126 : }
6127 :
6128 17827 : SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
6129 : EVT MemVT, SDVTList VTs, SDValue Chain,
6130 : SDValue Ptr, SDValue Cmp, SDValue Swp,
6131 : MachineMemOperand *MMO) {
6132 : assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
6133 : Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
6134 : assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
6135 :
6136 17827 : SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
6137 17827 : return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
6138 : }
6139 :
6140 10294 : SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
6141 : SDValue Chain, SDValue Ptr, SDValue Val,
6142 : const Value *PtrVal, unsigned Alignment,
6143 : AtomicOrdering Ordering,
6144 : SyncScope::ID SSID) {
6145 10294 : if (Alignment == 0) // Ensure that codegen never sees alignment 0
6146 8674 : Alignment = getEVTAlignment(MemVT);
6147 :
6148 10294 : MachineFunction &MF = getMachineFunction();
6149 : // An atomic store does not load. An atomic load does not store.
6150 : // (An atomicrmw obviously both loads and stores.)
6151 : // For now, atomics are considered to be volatile always, and they are
6152 : // chained as such.
6153 : // FIXME: Volatile isn't really correct; we should keep track of atomic
6154 : // orderings in the memoperand.
6155 : auto Flags = MachineMemOperand::MOVolatile;
6156 10294 : if (Opcode != ISD::ATOMIC_STORE)
6157 : Flags |= MachineMemOperand::MOLoad;
6158 10294 : if (Opcode != ISD::ATOMIC_LOAD)
6159 : Flags |= MachineMemOperand::MOStore;
6160 :
6161 : MachineMemOperand *MMO =
6162 30882 : MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
6163 10294 : MemVT.getStoreSize(), Alignment, AAMDNodes(),
6164 : nullptr, SSID, Ordering);
6165 :
6166 10294 : return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO);
6167 : }
6168 :
6169 13206 : SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
6170 : SDValue Chain, SDValue Ptr, SDValue Val,
6171 : MachineMemOperand *MMO) {
6172 : assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
6173 : Opcode == ISD::ATOMIC_LOAD_SUB ||
6174 : Opcode == ISD::ATOMIC_LOAD_AND ||
6175 : Opcode == ISD::ATOMIC_LOAD_CLR ||
6176 : Opcode == ISD::ATOMIC_LOAD_OR ||
6177 : Opcode == ISD::ATOMIC_LOAD_XOR ||
6178 : Opcode == ISD::ATOMIC_LOAD_NAND ||
6179 : Opcode == ISD::ATOMIC_LOAD_MIN ||
6180 : Opcode == ISD::ATOMIC_LOAD_MAX ||
6181 : Opcode == ISD::ATOMIC_LOAD_UMIN ||
6182 : Opcode == ISD::ATOMIC_LOAD_UMAX ||
6183 : Opcode == ISD::ATOMIC_SWAP ||
6184 : Opcode == ISD::ATOMIC_STORE) &&
6185 : "Invalid Atomic Op");
6186 :
6187 13206 : EVT VT = Val.getValueType();
6188 :
6189 1742 : SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
6190 13206 : getVTList(VT, MVT::Other);
6191 13206 : SDValue Ops[] = {Chain, Ptr, Val};
6192 13206 : return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
6193 : }
6194 :
6195 6372 : SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
6196 : EVT VT, SDValue Chain, SDValue Ptr,
6197 : MachineMemOperand *MMO) {
6198 : assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
6199 :
6200 6372 : SDVTList VTs = getVTList(VT, MVT::Other);
6201 6372 : SDValue Ops[] = {Chain, Ptr};
6202 6372 : return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
6203 : }
6204 :
6205 : /// getMergeValues - Create a MERGE_VALUES node from the given operands.
6206 421494 : SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
6207 421494 : if (Ops.size() == 1)
6208 350819 : return Ops[0];
6209 :
6210 : SmallVector<EVT, 4> VTs;
6211 : VTs.reserve(Ops.size());
6212 214115 : for (unsigned i = 0; i < Ops.size(); ++i)
6213 286880 : VTs.push_back(Ops[i].getValueType());
6214 70675 : return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
6215 : }
6216 :
6217 9795 : SDValue SelectionDAG::getMemIntrinsicNode(
6218 : unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
6219 : EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align,
6220 : MachineMemOperand::Flags Flags, unsigned Size) {
6221 9795 : if (Align == 0) // Ensure that codegen never sees alignment 0
6222 4746 : Align = getEVTAlignment(MemVT);
6223 :
6224 9795 : if (!Size)
6225 : Size = MemVT.getStoreSize();
6226 :
6227 9795 : MachineFunction &MF = getMachineFunction();
6228 : MachineMemOperand *MMO =
6229 9795 : MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
6230 :
6231 9795 : return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
6232 : }
6233 :
6234 22667 : SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
6235 : SDVTList VTList,
6236 : ArrayRef<SDValue> Ops, EVT MemVT,
6237 : MachineMemOperand *MMO) {
6238 : assert((Opcode == ISD::INTRINSIC_VOID ||
6239 : Opcode == ISD::INTRINSIC_W_CHAIN ||
6240 : Opcode == ISD::PREFETCH ||
6241 : Opcode == ISD::LIFETIME_START ||
6242 : Opcode == ISD::LIFETIME_END ||
6243 : ((int)Opcode <= std::numeric_limits<int>::max() &&
6244 : (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
6245 : "Opcode is not a memory-accessing opcode!");
6246 :
6247 : // Memoize the node unless it returns a flag.
6248 : MemIntrinsicSDNode *N;
6249 22667 : if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
6250 : FoldingSetNodeID ID;
6251 16718 : AddNodeIDNode(ID, Opcode, VTList, Ops);
6252 33436 : ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
6253 : Opcode, dl.getIROrder(), VTList, MemVT, MMO));
6254 16718 : ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6255 16718 : void *IP = nullptr;
6256 16718 : if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6257 0 : cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
6258 0 : return SDValue(E, 0);
6259 : }
6260 :
6261 33436 : N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
6262 : VTList, MemVT, MMO);
6263 16718 : createOperands(N, Ops);
6264 :
6265 16718 : CSEMap.InsertNode(N, IP);
6266 : } else {
6267 11898 : N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
6268 : VTList, MemVT, MMO);
6269 5949 : createOperands(N, Ops);
6270 : }
6271 22667 : InsertNode(N);
6272 22667 : return SDValue(N, 0);
6273 : }
6274 :
6275 : /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
6276 : /// MachinePointerInfo record from it. This is particularly useful because the
6277 : /// code generator has many cases where it doesn't bother passing in a
6278 : /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
6279 0 : static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
6280 : SelectionDAG &DAG, SDValue Ptr,
6281 : int64_t Offset = 0) {
6282 : // If this is FI+Offset, we can model it.
6283 : if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
6284 : return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
6285 0 : FI->getIndex(), Offset);
6286 :
6287 : // If this is (FI+Offset1)+Offset2, we can model it.
6288 : if (Ptr.getOpcode() != ISD::ADD ||
6289 0 : !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
6290 : !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
6291 0 : return Info;
6292 :
6293 0 : int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
6294 : return MachinePointerInfo::getFixedStack(
6295 : DAG.getMachineFunction(), FI,
6296 0 : Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
6297 : }
6298 :
6299 : /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
6300 : /// MachinePointerInfo record from it. This is particularly useful because the
6301 : /// code generator has many cases where it doesn't bother passing in a
6302 : /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
6303 0 : static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
6304 : SelectionDAG &DAG, SDValue Ptr,
6305 : SDValue OffsetOp) {
6306 : // If the 'Offset' value isn't a constant, we can't handle this.
6307 : if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
6308 0 : return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue());
6309 0 : if (OffsetOp.isUndef())
6310 0 : return InferPointerInfo(Info, DAG, Ptr);
6311 0 : return Info;
6312 : }
6313 :
6314 2978631 : SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
6315 : EVT VT, const SDLoc &dl, SDValue Chain,
6316 : SDValue Ptr, SDValue Offset,
6317 : MachinePointerInfo PtrInfo, EVT MemVT,
6318 : unsigned Alignment,
6319 : MachineMemOperand::Flags MMOFlags,
6320 : const AAMDNodes &AAInfo, const MDNode *Ranges) {
6321 : assert(Chain.getValueType() == MVT::Other &&
6322 : "Invalid chain type");
6323 2978631 : if (Alignment == 0) // Ensure that codegen never sees alignment 0
6324 126295 : Alignment = getEVTAlignment(MemVT);
6325 :
6326 : MMOFlags |= MachineMemOperand::MOLoad;
6327 : assert((MMOFlags & MachineMemOperand::MOStore) == 0);
6328 : // If we don't have a PtrInfo, infer the trivial frame index case to simplify
6329 : // clients.
6330 2978631 : if (PtrInfo.V.isNull())
6331 17458 : PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
6332 :
6333 2978631 : MachineFunction &MF = getMachineFunction();
6334 2978631 : MachineMemOperand *MMO = MF.getMachineMemOperand(
6335 : PtrInfo, MMOFlags, MemVT.getStoreSize(), Alignment, AAInfo, Ranges);
6336 2978631 : return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
6337 : }
6338 :
6339 3316856 : SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
6340 : EVT VT, const SDLoc &dl, SDValue Chain,
6341 : SDValue Ptr, SDValue Offset, EVT MemVT,
6342 : MachineMemOperand *MMO) {
6343 3324912 : if (VT == MemVT) {
6344 3240583 : ExtType = ISD::NON_EXTLOAD;
6345 : } else if (ExtType == ISD::NON_EXTLOAD) {
6346 : assert(VT == MemVT && "Non-extending load from different memory type!");
6347 : } else {
6348 : // Extending load.
6349 : assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
6350 : "Should only be an extending load, not truncating!");
6351 : assert(VT.isInteger() == MemVT.isInteger() &&
6352 : "Cannot convert from FP to Int or Int -> FP!");
6353 : assert(VT.isVector() == MemVT.isVector() &&
6354 : "Cannot use an ext load to convert to or from a vector!");
6355 : assert((!VT.isVector() ||
6356 : VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
6357 : "Cannot use an ext load to change the number of vector elements!");
6358 : }
6359 :
6360 3316856 : bool Indexed = AM != ISD::UNINDEXED;
6361 : assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
6362 :
6363 : SDVTList VTs = Indexed ?
6364 3316856 : getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
6365 3316856 : SDValue Ops[] = { Chain, Ptr, Offset };
6366 : FoldingSetNodeID ID;
6367 3316856 : AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
6368 3316856 : ID.AddInteger(MemVT.getRawBits());
6369 6633712 : ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
6370 : dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
6371 6633712 : ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6372 3316856 : void *IP = nullptr;
6373 3316856 : if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6374 158363 : cast<LoadSDNode>(E)->refineAlignment(MMO);
6375 158363 : return SDValue(E, 0);
6376 : }
6377 6316986 : auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
6378 : ExtType, MemVT, MMO);
6379 3158493 : createOperands(N, Ops);
6380 :
6381 3158493 : CSEMap.InsertNode(N, IP);
6382 3158493 : InsertNode(N);
6383 : SDValue V(N, 0);
6384 : NewSDValueDbgMsg(V, "Creating new node: ", this);
6385 3158493 : return V;
6386 : }
6387 :
6388 2791084 : SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
6389 : SDValue Ptr, MachinePointerInfo PtrInfo,
6390 : unsigned Alignment,
6391 : MachineMemOperand::Flags MMOFlags,
6392 : const AAMDNodes &AAInfo, const MDNode *Ranges) {
6393 5582168 : SDValue Undef = getUNDEF(Ptr.getValueType());
6394 : return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
6395 2791084 : PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
6396 : }
6397 :
6398 275484 : SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
6399 : SDValue Ptr, MachineMemOperand *MMO) {
6400 550968 : SDValue Undef = getUNDEF(Ptr.getValueType());
6401 : return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
6402 275484 : VT, MMO);
6403 : }
6404 :
6405 165219 : SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
6406 : EVT VT, SDValue Chain, SDValue Ptr,
6407 : MachinePointerInfo PtrInfo, EVT MemVT,
6408 : unsigned Alignment,
6409 : MachineMemOperand::Flags MMOFlags,
6410 : const AAMDNodes &AAInfo) {
6411 330438 : SDValue Undef = getUNDEF(Ptr.getValueType());
6412 : return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
6413 165219 : MemVT, Alignment, MMOFlags, AAInfo);
6414 : }
6415 :
6416 61473 : SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
6417 : EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
6418 : MachineMemOperand *MMO) {
6419 122946 : SDValue Undef = getUNDEF(Ptr.getValueType());
6420 : return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
6421 61473 : MemVT, MMO);
6422 : }
6423 :
6424 563 : SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
6425 : SDValue Base, SDValue Offset,
6426 : ISD::MemIndexedMode AM) {
6427 : LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
6428 : assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
6429 : // Don't propagate the invariant or dereferenceable flags.
6430 : auto MMOFlags =
6431 563 : LD->getMemOperand()->getFlags() &
6432 : ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
6433 : return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
6434 563 : LD->getChain(), Base, Offset, LD->getPointerInfo(),
6435 : LD->getMemoryVT(), LD->getAlignment(), MMOFlags,
6436 563 : LD->getAAInfo());
6437 : }
6438 :
6439 2898479 : SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6440 : SDValue Ptr, MachinePointerInfo PtrInfo,
6441 : unsigned Alignment,
6442 : MachineMemOperand::Flags MMOFlags,
6443 : const AAMDNodes &AAInfo) {
6444 : assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
6445 2898479 : if (Alignment == 0) // Ensure that codegen never sees alignment 0
6446 211170 : Alignment = getEVTAlignment(Val.getValueType());
6447 :
6448 : MMOFlags |= MachineMemOperand::MOStore;
6449 : assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
6450 :
6451 2898479 : if (PtrInfo.V.isNull())
6452 22472 : PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
6453 :
6454 2898479 : MachineFunction &MF = getMachineFunction();
6455 2898479 : MachineMemOperand *MMO = MF.getMachineMemOperand(
6456 2898479 : PtrInfo, MMOFlags, Val.getValueType().getStoreSize(), Alignment, AAInfo);
6457 2898479 : return getStore(Chain, dl, Val, Ptr, MMO);
6458 : }
6459 :
6460 3475944 : SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6461 : SDValue Ptr, MachineMemOperand *MMO) {
6462 : assert(Chain.getValueType() == MVT::Other &&
6463 : "Invalid chain type");
6464 6951888 : EVT VT = Val.getValueType();
6465 3475944 : SDVTList VTs = getVTList(MVT::Other);
6466 6951888 : SDValue Undef = getUNDEF(Ptr.getValueType());
6467 3475944 : SDValue Ops[] = { Chain, Val, Ptr, Undef };
6468 : FoldingSetNodeID ID;
6469 3475944 : AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
6470 3475944 : ID.AddInteger(VT.getRawBits());
6471 3475944 : ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
6472 3475944 : dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO));
6473 6951888 : ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6474 3475944 : void *IP = nullptr;
6475 3475944 : if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6476 8751 : cast<StoreSDNode>(E)->refineAlignment(MMO);
6477 8751 : return SDValue(E, 0);
6478 : }
6479 6934386 : auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
6480 3467193 : ISD::UNINDEXED, false, VT, MMO);
6481 3467193 : createOperands(N, Ops);
6482 :
6483 3467193 : CSEMap.InsertNode(N, IP);
6484 3467193 : InsertNode(N);
6485 : SDValue V(N, 0);
6486 : NewSDValueDbgMsg(V, "Creating new node: ", this);
6487 3467193 : return V;
6488 : }
6489 :
6490 155768 : SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6491 : SDValue Ptr, MachinePointerInfo PtrInfo,
6492 : EVT SVT, unsigned Alignment,
6493 : MachineMemOperand::Flags MMOFlags,
6494 : const AAMDNodes &AAInfo) {
6495 : assert(Chain.getValueType() == MVT::Other &&
6496 : "Invalid chain type");
6497 155768 : if (Alignment == 0) // Ensure that codegen never sees alignment 0
6498 728 : Alignment = getEVTAlignment(SVT);
6499 :
6500 : MMOFlags |= MachineMemOperand::MOStore;
6501 : assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
6502 :
6503 155768 : if (PtrInfo.V.isNull())
6504 7334 : PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
6505 :
6506 155768 : MachineFunction &MF = getMachineFunction();
6507 155768 : MachineMemOperand *MMO = MF.getMachineMemOperand(
6508 : PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
6509 155768 : return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
6510 : }
6511 :
6512 173372 : SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6513 : SDValue Ptr, EVT SVT,
6514 : MachineMemOperand *MMO) {
6515 173372 : EVT VT = Val.getValueType();
6516 :
6517 : assert(Chain.getValueType() == MVT::Other &&
6518 : "Invalid chain type");
6519 173906 : if (VT == SVT)
6520 146896 : return getStore(Chain, dl, Val, Ptr, MMO);
6521 :
6522 : assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
6523 : "Should only be a truncating store, not extending!");
6524 : assert(VT.isInteger() == SVT.isInteger() &&
6525 : "Can't do FP-INT conversion!");
6526 : assert(VT.isVector() == SVT.isVector() &&
6527 : "Cannot use trunc store to convert to or from a vector!");
6528 : assert((!VT.isVector() ||
6529 : VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
6530 : "Cannot use trunc store to change the number of vector elements!");
6531 :
6532 26476 : SDVTList VTs = getVTList(MVT::Other);
6533 52952 : SDValue Undef = getUNDEF(Ptr.getValueType());
6534 26476 : SDValue Ops[] = { Chain, Val, Ptr, Undef };
6535 : FoldingSetNodeID ID;
6536 26476 : AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
6537 26476 : ID.AddInteger(SVT.getRawBits());
6538 26476 : ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
6539 26476 : dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO));
6540 52952 : ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6541 26476 : void *IP = nullptr;
6542 26476 : if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6543 617 : cast<StoreSDNode>(E)->refineAlignment(MMO);
6544 617 : return SDValue(E, 0);
6545 : }
6546 51718 : auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
6547 25859 : ISD::UNINDEXED, true, SVT, MMO);
6548 25859 : createOperands(N, Ops);
6549 :
6550 25859 : CSEMap.InsertNode(N, IP);
6551 25859 : InsertNode(N);
6552 : SDValue V(N, 0);
6553 : NewSDValueDbgMsg(V, "Creating new node: ", this);
6554 25859 : return V;
6555 : }
6556 :
6557 310 : SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
6558 : SDValue Base, SDValue Offset,
6559 : ISD::MemIndexedMode AM) {
6560 : StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
6561 : assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
6562 620 : SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
6563 310 : SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
6564 : FoldingSetNodeID ID;
6565 310 : AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
6566 310 : ID.AddInteger(ST->getMemoryVT().getRawBits());
6567 310 : ID.AddInteger(ST->getRawSubclassData());
6568 620 : ID.AddInteger(ST->getPointerInfo().getAddrSpace());
6569 310 : void *IP = nullptr;
6570 310 : if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
6571 0 : return SDValue(E, 0);
6572 :
6573 620 : auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
6574 310 : ST->isTruncatingStore(), ST->getMemoryVT(),
6575 310 : ST->getMemOperand());
6576 310 : createOperands(N, Ops);
6577 :
6578 310 : CSEMap.InsertNode(N, IP);
6579 310 : InsertNode(N);
6580 : SDValue V(N, 0);
6581 : NewSDValueDbgMsg(V, "Creating new node: ", this);
6582 310 : return V;
6583 : }
6584 :
6585 612 : SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
6586 : SDValue Ptr, SDValue Mask, SDValue PassThru,
6587 : EVT MemVT, MachineMemOperand *MMO,
6588 : ISD::LoadExtType ExtTy, bool isExpanding) {
6589 612 : SDVTList VTs = getVTList(VT, MVT::Other);
6590 612 : SDValue Ops[] = { Chain, Ptr, Mask, PassThru };
6591 : FoldingSetNodeID ID;
6592 612 : AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
6593 612 : ID.AddInteger(VT.getRawBits());
6594 1224 : ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
6595 : dl.getIROrder(), VTs, ExtTy, isExpanding, MemVT, MMO));
6596 1224 : ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6597 612 : void *IP = nullptr;
6598 612 : if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6599 0 : cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
6600 0 : return SDValue(E, 0);
6601 : }
6602 1224 : auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
6603 : ExtTy, isExpanding, MemVT, MMO);
6604 612 : createOperands(N, Ops);
6605 :
6606 612 : CSEMap.InsertNode(N, IP);
6607 612 : InsertNode(N);
6608 : SDValue V(N, 0);
6609 : NewSDValueDbgMsg(V, "Creating new node: ", this);
6610 612 : return V;
6611 : }
6612 :
6613 374 : SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
6614 : SDValue Val, SDValue Ptr, SDValue Mask,
6615 : EVT MemVT, MachineMemOperand *MMO,
6616 : bool IsTruncating, bool IsCompressing) {
6617 : assert(Chain.getValueType() == MVT::Other &&
6618 : "Invalid chain type");
6619 374 : EVT VT = Val.getValueType();
6620 374 : SDVTList VTs = getVTList(MVT::Other);
6621 374 : SDValue Ops[] = { Chain, Val, Ptr, Mask };
6622 : FoldingSetNodeID ID;
6623 374 : AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
6624 374 : ID.AddInteger(VT.getRawBits());
6625 748 : ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
6626 : dl.getIROrder(), VTs, IsTruncating, IsCompressing, MemVT, MMO));
6627 748 : ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6628 374 : void *IP = nullptr;
6629 374 : if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6630 0 : cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
6631 0 : return SDValue(E, 0);
6632 : }
6633 748 : auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
6634 : IsTruncating, IsCompressing, MemVT, MMO);
6635 374 : createOperands(N, Ops);
6636 :
6637 374 : CSEMap.InsertNode(N, IP);
6638 374 : InsertNode(N);
6639 : SDValue V(N, 0);
6640 : NewSDValueDbgMsg(V, "Creating new node: ", this);
6641 374 : return V;
6642 : }
6643 :
6644 473 : SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
6645 : ArrayRef<SDValue> Ops,
6646 : MachineMemOperand *MMO) {
6647 : assert(Ops.size() == 6 && "Incompatible number of operands");
6648 :
6649 : FoldingSetNodeID ID;
6650 473 : AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
6651 473 : ID.AddInteger(VT.getRawBits());
6652 946 : ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
6653 : dl.getIROrder(), VTs, VT, MMO));
6654 946 : ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6655 473 : void *IP = nullptr;
6656 473 : if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6657 10 : cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
6658 10 : return SDValue(E, 0);
6659 : }
6660 :
6661 926 : auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
6662 : VTs, VT, MMO);
6663 463 : createOperands(N, Ops);
6664 :
6665 : assert(N->getPassThru().getValueType() == N->getValueType(0) &&
6666 : "Incompatible type of the PassThru value in MaskedGatherSDNode");
6667 : assert(N->getMask().getValueType().getVectorNumElements() ==
6668 : N->getValueType(0).getVectorNumElements() &&
6669 : "Vector width mismatch between mask and data");
6670 : assert(N->getIndex().getValueType().getVectorNumElements() >=
6671 : N->getValueType(0).getVectorNumElements() &&
6672 : "Vector width mismatch between index and data");
6673 : assert(isa<ConstantSDNode>(N->getScale()) &&
6674 : cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&
6675 : "Scale should be a constant power of 2");
6676 :
6677 463 : CSEMap.InsertNode(N, IP);
6678 463 : InsertNode(N);
6679 : SDValue V(N, 0);
6680 : NewSDValueDbgMsg(V, "Creating new node: ", this);
6681 463 : return V;
6682 : }
6683 :
6684 168 : SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
6685 : ArrayRef<SDValue> Ops,
6686 : MachineMemOperand *MMO) {
6687 : assert(Ops.size() == 6 && "Incompatible number of operands");
6688 :
6689 : FoldingSetNodeID ID;
6690 168 : AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
6691 168 : ID.AddInteger(VT.getRawBits());
6692 336 : ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
6693 : dl.getIROrder(), VTs, VT, MMO));
6694 336 : ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6695 168 : void *IP = nullptr;
6696 168 : if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6697 0 : cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
6698 0 : return SDValue(E, 0);
6699 : }
6700 336 : auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
6701 : VTs, VT, MMO);
6702 168 : createOperands(N, Ops);
6703 :
6704 : assert(N->getMask().getValueType().getVectorNumElements() ==
6705 : N->getValue().getValueType().getVectorNumElements() &&
6706 : "Vector width mismatch between mask and data");
6707 : assert(N->getIndex().getValueType().getVectorNumElements() >=
6708 : N->getValue().getValueType().getVectorNumElements() &&
6709 : "Vector width mismatch between index and data");
6710 : assert(isa<ConstantSDNode>(N->getScale()) &&
6711 : cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&
6712 : "Scale should be a constant power of 2");
6713 :
6714 168 : CSEMap.InsertNode(N, IP);
6715 168 : InsertNode(N);
6716 : SDValue V(N, 0);
6717 : NewSDValueDbgMsg(V, "Creating new node: ", this);
6718 168 : return V;
6719 : }
6720 :
6721 257 : SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
6722 : SDValue Ptr, SDValue SV, unsigned Align) {
6723 257 : SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
6724 257 : return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
6725 : }
6726 :
6727 6874 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6728 : ArrayRef<SDUse> Ops) {
6729 6874 : switch (Ops.size()) {
6730 0 : case 0: return getNode(Opcode, DL, VT);
6731 30 : case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
6732 3759 : case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
6733 56 : case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
6734 : default: break;
6735 : }
6736 :
6737 : // Copy from an SDUse array into an SDValue array for use with
6738 : // the regular getNode logic.
6739 : SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
6740 3029 : return getNode(Opcode, DL, VT, NewOps);
6741 : }
6742 :
6743 12460271 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6744 : ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
6745 12460271 : unsigned NumOps = Ops.size();
6746 12460271 : switch (NumOps) {
6747 0 : case 0: return getNode(Opcode, DL, VT);
6748 8540167 : case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);
6749 4431550 : case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
6750 1010818 : case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags);
6751 : default: break;
6752 : }
6753 :
6754 1198920 : switch (Opcode) {
6755 : default: break;
6756 8333 : case ISD::CONCAT_VECTORS:
6757 : // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
6758 8333 : if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
6759 563 : return V;
6760 7770 : break;
6761 : case ISD::SELECT_CC:
6762 : assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
6763 : assert(Ops[0].getValueType() == Ops[1].getValueType() &&
6764 : "LHS and RHS of condition must have same type!");
6765 : assert(Ops[2].getValueType() == Ops[3].getValueType() &&
6766 : "True and False arms of SelectCC must have same type!");
6767 : assert(Ops[2].getValueType() == VT &&
6768 : "select_cc node must be of same type as true and false value!");
6769 : break;
6770 : case ISD::BR_CC:
6771 : assert(NumOps == 5 && "BR_CC takes 5 operands!");
6772 : assert(Ops[2].getValueType() == Ops[3].getValueType() &&
6773 : "LHS/RHS of comparison should match types!");
6774 : break;
6775 : }
6776 :
6777 : // Memoize nodes.
6778 : SDNode *N;
6779 1198357 : SDVTList VTs = getVTList(VT);
6780 :
6781 1198357 : if (VT != MVT::Glue) {
6782 : FoldingSetNodeID ID;
6783 1198357 : AddNodeIDNode(ID, Opcode, VTs, Ops);
6784 1198357 : void *IP = nullptr;
6785 :
6786 1198357 : if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
6787 222527 : return SDValue(E, 0);
6788 :
6789 1951660 : N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6790 975830 : createOperands(N, Ops);
6791 :
6792 975830 : CSEMap.InsertNode(N, IP);
6793 : } else {
6794 0 : N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6795 0 : createOperands(N, Ops);
6796 : }
6797 :
6798 975830 : InsertNode(N);
6799 : SDValue V(N, 0);
6800 : NewSDValueDbgMsg(V, "Creating new node: ", this);
6801 975830 : return V;
6802 : }
6803 :
6804 19003 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
6805 : ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
6806 19003 : return getNode(Opcode, DL, getVTList(ResultTys), Ops);
6807 : }
6808 :
6809 12723310 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
6810 : ArrayRef<SDValue> Ops) {
6811 12723310 : if (VTList.NumVTs == 1)
6812 3541929 : return getNode(Opcode, DL, VTList.VTs[0], Ops);
6813 :
6814 : #if 0
6815 : switch (Opcode) {
6816 : // FIXME: figure out how to safely handle things like
6817 : // int foo(int x) { return 1 << (x & 255); }
6818 : // int bar() { return foo(256); }
6819 : case ISD::SRA_PARTS:
6820 : case ISD::SRL_PARTS:
6821 : case ISD::SHL_PARTS:
6822 : if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
6823 : cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
6824 : return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
6825 : else if (N3.getOpcode() == ISD::AND)
6826 : if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
6827 : // If the and is only masking out bits that cannot effect the shift,
6828 : // eliminate the and.
6829 : unsigned NumBits = VT.getScalarSizeInBits()*2;
6830 : if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
6831 : return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
6832 : }
6833 : break;
6834 : }
6835 : #endif
6836 :
6837 : // Memoize the node unless it returns a flag.
6838 : SDNode *N;
6839 9181381 : if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
6840 : FoldingSetNodeID ID;
6841 3233224 : AddNodeIDNode(ID, Opcode, VTList, Ops);
6842 3233224 : void *IP = nullptr;
6843 3233224 : if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
6844 145163 : return SDValue(E, 0);
6845 :
6846 6176122 : N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
6847 3088061 : createOperands(N, Ops);
6848 3088061 : CSEMap.InsertNode(N, IP);
6849 : } else {
6850 11896314 : N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
6851 5948157 : createOperands(N, Ops);
6852 : }
6853 9036218 : InsertNode(N);
6854 : SDValue V(N, 0);
6855 : NewSDValueDbgMsg(V, "Creating new node: ", this);
6856 9036218 : return V;
6857 : }
6858 :
6859 0 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
6860 : SDVTList VTList) {
6861 0 : return getNode(Opcode, DL, VTList, None);
6862 : }
6863 :
6864 4770 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
6865 : SDValue N1) {
6866 4770 : SDValue Ops[] = { N1 };
6867 4770 : return getNode(Opcode, DL, VTList, Ops);
6868 : }
6869 :
6870 499875 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
6871 : SDValue N1, SDValue N2) {
6872 499875 : SDValue Ops[] = { N1, N2 };
6873 499875 : return getNode(Opcode, DL, VTList, Ops);
6874 : }
6875 :
6876 221023 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
6877 : SDValue N1, SDValue N2, SDValue N3) {
6878 221023 : SDValue Ops[] = { N1, N2, N3 };
6879 221023 : return getNode(Opcode, DL, VTList, Ops);
6880 : }
6881 :
6882 822 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
6883 : SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
6884 822 : SDValue Ops[] = { N1, N2, N3, N4 };
6885 822 : return getNode(Opcode, DL, VTList, Ops);
6886 : }
6887 :
6888 358 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
6889 : SDValue N1, SDValue N2, SDValue N3, SDValue N4,
6890 : SDValue N5) {
6891 358 : SDValue Ops[] = { N1, N2, N3, N4, N5 };
6892 358 : return getNode(Opcode, DL, VTList, Ops);
6893 : }
6894 :
6895 97341570 : SDVTList SelectionDAG::getVTList(EVT VT) {
6896 97341570 : return makeVTList(SDNode::getValueTypeList(VT), 1);
6897 : }
6898 :
6899 15266062 : SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
6900 : FoldingSetNodeID ID;
6901 15266062 : ID.AddInteger(2U);
6902 15266062 : ID.AddInteger(VT1.getRawBits());
6903 15266063 : ID.AddInteger(VT2.getRawBits());
6904 :
6905 15266063 : void *IP = nullptr;
6906 : SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
6907 15266063 : if (!Result) {
6908 102451 : EVT *Array = Allocator.Allocate<EVT>(2);
6909 102451 : Array[0] = VT1;
6910 102451 : Array[1] = VT2;
6911 102451 : Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
6912 102451 : VTListMap.InsertNode(Result, IP);
6913 : }
6914 15266063 : return Result->getSDVTList();
6915 : }
6916 :
6917 443360 : SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
6918 : FoldingSetNodeID ID;
6919 443360 : ID.AddInteger(3U);
6920 443360 : ID.AddInteger(VT1.getRawBits());
6921 443360 : ID.AddInteger(VT2.getRawBits());
6922 443360 : ID.AddInteger(VT3.getRawBits());
6923 :
6924 443360 : void *IP = nullptr;
6925 : SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
6926 443360 : if (!Result) {
6927 9955 : EVT *Array = Allocator.Allocate<EVT>(3);
6928 9955 : Array[0] = VT1;
6929 9955 : Array[1] = VT2;
6930 9955 : Array[2] = VT3;
6931 9955 : Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
6932 9955 : VTListMap.InsertNode(Result, IP);
6933 : }
6934 443360 : return Result->getSDVTList();
6935 : }
6936 :
6937 124 : SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
6938 : FoldingSetNodeID ID;
6939 124 : ID.AddInteger(4U);
6940 124 : ID.AddInteger(VT1.getRawBits());
6941 124 : ID.AddInteger(VT2.getRawBits());
6942 124 : ID.AddInteger(VT3.getRawBits());
6943 124 : ID.AddInteger(VT4.getRawBits());
6944 :
6945 124 : void *IP = nullptr;
6946 : SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
6947 124 : if (!Result) {
6948 74 : EVT *Array = Allocator.Allocate<EVT>(4);
6949 74 : Array[0] = VT1;
6950 74 : Array[1] = VT2;
6951 74 : Array[2] = VT3;
6952 74 : Array[3] = VT4;
6953 74 : Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
6954 74 : VTListMap.InsertNode(Result, IP);
6955 : }
6956 124 : return Result->getSDVTList();
6957 : }
6958 :
6959 6242288 : SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
6960 6242288 : unsigned NumVTs = VTs.size();
6961 : FoldingSetNodeID ID;
6962 6242288 : ID.AddInteger(NumVTs);
6963 17305016 : for (unsigned index = 0; index < NumVTs; index++) {
6964 22125456 : ID.AddInteger(VTs[index].getRawBits());
6965 : }
6966 :
6967 6242288 : void *IP = nullptr;
6968 : SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
6969 6242288 : if (!Result) {
6970 59134 : EVT *Array = Allocator.Allocate<EVT>(NumVTs);
6971 : std::copy(VTs.begin(), VTs.end(), Array);
6972 59134 : Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
6973 59134 : VTListMap.InsertNode(Result, IP);
6974 : }
6975 6242288 : return Result->getSDVTList();
6976 : }
6977 :
6978 :
6979 : /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
6980 : /// specified operands. If the resultant node already exists in the DAG,
6981 : /// this does not modify the specified node, instead it returns the node that
6982 : /// already exists. If the resultant node does not exist in the DAG, the
6983 : /// input node is returned. As a degenerate case, if you specify the same
6984 : /// input operands as the node already has, the input node is returned.
6985 2806 : SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
6986 : assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
6987 :
6988 : // Check to see if there is no change.
6989 2806 : if (Op == N->getOperand(0)) return N;
6990 :
6991 : // See if the modified node already exists.
6992 2806 : void *InsertPos = nullptr;
6993 2806 : if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
6994 : return Existing;
6995 :
6996 : // Nope it doesn't. Remove the node from its current place in the maps.
6997 2806 : if (InsertPos)
6998 2806 : if (!RemoveNodeFromCSEMaps(N))
6999 0 : InsertPos = nullptr;
7000 :
7001 : // Now we update the operands.
7002 2806 : N->OperandList[0].set(Op);
7003 :
7004 2806 : updateDivergence(N);
7005 : // If this gets put into a CSE map, add it.
7006 2806 : if (InsertPos) CSEMap.InsertNode(N, InsertPos);
7007 : return N;
7008 : }
7009 :
7010 53434 : SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
7011 : assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
7012 :
7013 : // Check to see if there is no change.
7014 83977 : if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
7015 : return N; // No operands changed, just return the input node.
7016 :
7017 : // See if the modified node already exists.
7018 53434 : void *InsertPos = nullptr;
7019 53434 : if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
7020 : return Existing;
7021 :
7022 : // Nope it doesn't. Remove the node from its current place in the maps.
7023 52736 : if (InsertPos)
7024 52736 : if (!RemoveNodeFromCSEMaps(N))
7025 0 : InsertPos = nullptr;
7026 :
7027 : // Now we update the operands.
7028 52736 : if (N->OperandList[0] != Op1)
7029 : N->OperandList[0].set(Op1);
7030 52736 : if (N->OperandList[1] != Op2)
7031 39633 : N->OperandList[1].set(Op2);
7032 :
7033 52736 : updateDivergence(N);
7034 : // If this gets put into a CSE map, add it.
7035 52736 : if (InsertPos) CSEMap.InsertNode(N, InsertPos);
7036 : return N;
7037 : }
7038 :
7039 96652 : SDNode *SelectionDAG::
7040 : UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
7041 96652 : SDValue Ops[] = { Op1, Op2, Op3 };
7042 96652 : return UpdateNodeOperands(N, Ops);
7043 : }
7044 :
7045 753 : SDNode *SelectionDAG::
7046 : UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
7047 : SDValue Op3, SDValue Op4) {
7048 753 : SDValue Ops[] = { Op1, Op2, Op3, Op4 };
7049 753 : return UpdateNodeOperands(N, Ops);
7050 : }
7051 :
7052 2018 : SDNode *SelectionDAG::
7053 : UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
7054 : SDValue Op3, SDValue Op4, SDValue Op5) {
7055 2018 : SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
7056 2018 : return UpdateNodeOperands(N, Ops);
7057 : }
7058 :
7059 12091054 : SDNode *SelectionDAG::
7060 : UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
7061 12091054 : unsigned NumOps = Ops.size();
7062 : assert(N->getNumOperands() == NumOps &&
7063 : "Update with wrong number of operands");
7064 :
7065 : // If no operands changed just return the input node.
7066 24182108 : if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
7067 : return N;
7068 :
7069 : // See if the modified node already exists.
7070 165608 : void *InsertPos = nullptr;
7071 165608 : if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
7072 : return Existing;
7073 :
7074 : // Nope it doesn't. Remove the node from its current place in the maps.
7075 165480 : if (InsertPos)
7076 143426 : if (!RemoveNodeFromCSEMaps(N))
7077 0 : InsertPos = nullptr;
7078 :
7079 : // Now we update the operands.
7080 679288 : for (unsigned i = 0; i != NumOps; ++i)
7081 1027616 : if (N->OperandList[i] != Ops[i])
7082 : N->OperandList[i].set(Ops[i]);
7083 :
7084 165480 : updateDivergence(N);
7085 : // If this gets put into a CSE map, add it.
7086 165480 : if (InsertPos) CSEMap.InsertNode(N, InsertPos);
7087 : return N;
7088 : }
7089 :
7090 : /// DropOperands - Release the operands and set this node to have
7091 : /// zero operands.
7092 23450611 : void SDNode::DropOperands() {
7093 : // Unlike the code in MorphNodeTo that does this, we don't need to
7094 : // watch for dead nodes here.
7095 55303994 : for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
7096 31853383 : SDUse &Use = *I++;
7097 31853383 : Use.set(SDValue());
7098 : }
7099 23450611 : }
7100 :
7101 5092181 : void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
7102 : ArrayRef<MachineMemOperand *> NewMemRefs) {
7103 5092181 : if (NewMemRefs.empty()) {
7104 : N->clearMemRefs();
7105 535 : return;
7106 : }
7107 :
7108 : // Check if we can avoid allocating by storing a single reference directly.
7109 5091646 : if (NewMemRefs.size() == 1) {
7110 4758454 : N->MemRefs = NewMemRefs[0];
7111 4758454 : N->NumMemRefs = 1;
7112 4758454 : return;
7113 : }
7114 :
7115 : MachineMemOperand **MemRefsBuffer =
7116 333192 : Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size());
7117 : std::copy(NewMemRefs.begin(), NewMemRefs.end(), MemRefsBuffer);
7118 : N->MemRefs = MemRefsBuffer;
7119 333192 : N->NumMemRefs = static_cast<int>(NewMemRefs.size());
7120 : }
7121 :
7122 : /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
7123 : /// machine opcode.
7124 : ///
7125 6893 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7126 : EVT VT) {
7127 6893 : SDVTList VTs = getVTList(VT);
7128 6893 : return SelectNodeTo(N, MachineOpc, VTs, None);
7129 : }
7130 :
7131 245 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7132 : EVT VT, SDValue Op1) {
7133 245 : SDVTList VTs = getVTList(VT);
7134 245 : SDValue Ops[] = { Op1 };
7135 245 : return SelectNodeTo(N, MachineOpc, VTs, Ops);
7136 : }
7137 :
7138 2155 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7139 : EVT VT, SDValue Op1,
7140 : SDValue Op2) {
7141 2155 : SDVTList VTs = getVTList(VT);
7142 2155 : SDValue Ops[] = { Op1, Op2 };
7143 2155 : return SelectNodeTo(N, MachineOpc, VTs, Ops);
7144 : }
7145 :
7146 52 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7147 : EVT VT, SDValue Op1,
7148 : SDValue Op2, SDValue Op3) {
7149 52 : SDVTList VTs = getVTList(VT);
7150 52 : SDValue Ops[] = { Op1, Op2, Op3 };
7151 52 : return SelectNodeTo(N, MachineOpc, VTs, Ops);
7152 : }
7153 :
7154 4443 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7155 : EVT VT, ArrayRef<SDValue> Ops) {
7156 4443 : SDVTList VTs = getVTList(VT);
7157 4443 : return SelectNodeTo(N, MachineOpc, VTs, Ops);
7158 : }
7159 :
7160 10 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7161 : EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
7162 10 : SDVTList VTs = getVTList(VT1, VT2);
7163 10 : return SelectNodeTo(N, MachineOpc, VTs, Ops);
7164 : }
7165 :
7166 7 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7167 : EVT VT1, EVT VT2) {
7168 7 : SDVTList VTs = getVTList(VT1, VT2);
7169 7 : return SelectNodeTo(N, MachineOpc, VTs, None);
7170 : }
7171 :
7172 5 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7173 : EVT VT1, EVT VT2, EVT VT3,
7174 : ArrayRef<SDValue> Ops) {
7175 5 : SDVTList VTs = getVTList(VT1, VT2, VT3);
7176 5 : return SelectNodeTo(N, MachineOpc, VTs, Ops);
7177 : }
7178 :
7179 0 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7180 : EVT VT1, EVT VT2,
7181 : SDValue Op1, SDValue Op2) {
7182 0 : SDVTList VTs = getVTList(VT1, VT2);
7183 0 : SDValue Ops[] = { Op1, Op2 };
7184 0 : return SelectNodeTo(N, MachineOpc, VTs, Ops);
7185 : }
7186 :
7187 43013 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7188 : SDVTList VTs,ArrayRef<SDValue> Ops) {
7189 43013 : SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
7190 : // Reset the NodeID to -1.
7191 : New->setNodeId(-1);
7192 43013 : if (New != N) {
7193 82 : ReplaceAllUsesWith(N, New);
7194 82 : RemoveDeadNode(N);
7195 : }
7196 43013 : return New;
7197 : }
7198 :
7199 : /// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
7200 : /// the line number information on the merged node since it is not possible to
7201 : /// preserve the information that operation is associated with multiple lines.
7202 : /// This will make the debugger working better at -O0, were there is a higher
7203 : /// probability having other instructions associated with that line.
7204 : ///
7205 : /// For IROrder, we keep the smaller of the two
7206 111392 : SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
7207 : DebugLoc NLoc = N->getDebugLoc();
7208 111392 : if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) {
7209 0 : N->setDebugLoc(DebugLoc());
7210 : }
7211 236417 : unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
7212 : N->setIROrder(Order);
7213 111392 : return N;
7214 : }
7215 :
7216 : /// MorphNodeTo - This *mutates* the specified node to have the specified
7217 : /// return type, opcode, and operands.
7218 : ///
7219 : /// Note that MorphNodeTo returns the resultant node. If there is already a
7220 : /// node of the specified opcode and operands, it returns that node instead of
7221 : /// the current one. Note that the SDLoc need not be the same.
7222 : ///
7223 : /// Using MorphNodeTo is faster than creating a new node and swapping it in
7224 : /// with ReplaceAllUsesWith both because it often avoids allocating a new
7225 : /// node, and because it doesn't require CSE recalculation for any of
7226 : /// the node's users.
7227 : ///
7228 : /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
7229 : /// As a consequence it isn't appropriate to use from within the DAG combiner or
7230 : /// the legalizer which maintain worklists that would need to be updated when
7231 : /// deleting things.
7232 11825958 : SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
7233 : SDVTList VTs, ArrayRef<SDValue> Ops) {
7234 : // If an identical node already exists, use it.
7235 11825958 : void *IP = nullptr;
7236 11825958 : if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
7237 : FoldingSetNodeID ID;
7238 8736240 : AddNodeIDNode(ID, Opc, VTs, Ops);
7239 17472479 : if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))
7240 55410 : return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
7241 : }
7242 :
7243 11798252 : if (!RemoveNodeFromCSEMaps(N))
7244 3016491 : IP = nullptr;
7245 :
7246 : // Start the morphing.
7247 11798253 : N->NodeType = Opc;
7248 11798253 : N->ValueList = VTs.VTs;
7249 11798253 : N->NumValues = VTs.NumVTs;
7250 :
7251 : // Clear the operands list, updating used nodes to remove this from their
7252 : // use list. Keep track of any operands that become dead as a result.
7253 : SmallPtrSet<SDNode*, 16> DeadNodeSet;
7254 48091576 : for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
7255 36293323 : SDUse &Use = *I++;
7256 : SDNode *Used = Use.getNode();
7257 36293323 : Use.set(SDValue());
7258 36293323 : if (Used->use_empty())
7259 14398824 : DeadNodeSet.insert(Used);
7260 : }
7261 :
7262 : // For MachineNode, initialize the memory references information.
7263 : if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
7264 : MN->clearMemRefs();
7265 :
7266 : // Swap for an appropriately sized array from the recycler.
7267 11798253 : removeOperands(N);
7268 11798253 : createOperands(N, Ops);
7269 :
7270 : // Delete any nodes that are still dead after adding the uses for the
7271 : // new operands.
7272 11798253 : if (!DeadNodeSet.empty()) {
7273 : SmallVector<SDNode *, 16> DeadNodes;
7274 23688761 : for (SDNode *N : DeadNodeSet)
7275 14398823 : if (N->use_empty())
7276 4419209 : DeadNodes.push_back(N);
7277 9289938 : RemoveDeadNodes(DeadNodes);
7278 : }
7279 :
7280 11798253 : if (IP)
7281 : CSEMap.InsertNode(N, IP); // Memoize the new node.
7282 : return N;
7283 : }
7284 :
7285 464 : SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
7286 464 : unsigned OrigOpc = Node->getOpcode();
7287 : unsigned NewOpc;
7288 : bool IsUnary = false;
7289 : bool IsTernary = false;
7290 : switch (OrigOpc) {
7291 0 : default:
7292 0 : llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
7293 : case ISD::STRICT_FADD: NewOpc = ISD::FADD; break;
7294 : case ISD::STRICT_FSUB: NewOpc = ISD::FSUB; break;
7295 : case ISD::STRICT_FMUL: NewOpc = ISD::FMUL; break;
7296 : case ISD::STRICT_FDIV: NewOpc = ISD::FDIV; break;
7297 : case ISD::STRICT_FREM: NewOpc = ISD::FREM; break;
7298 : case ISD::STRICT_FMA: NewOpc = ISD::FMA; IsTernary = true; break;
7299 : case ISD::STRICT_FSQRT: NewOpc = ISD::FSQRT; IsUnary = true; break;
7300 : case ISD::STRICT_FPOW: NewOpc = ISD::FPOW; break;
7301 : case ISD::STRICT_FPOWI: NewOpc = ISD::FPOWI; break;
7302 : case ISD::STRICT_FSIN: NewOpc = ISD::FSIN; IsUnary = true; break;
7303 : case ISD::STRICT_FCOS: NewOpc = ISD::FCOS; IsUnary = true; break;
7304 : case ISD::STRICT_FEXP: NewOpc = ISD::FEXP; IsUnary = true; break;
7305 : case ISD::STRICT_FEXP2: NewOpc = ISD::FEXP2; IsUnary = true; break;
7306 : case ISD::STRICT_FLOG: NewOpc = ISD::FLOG; IsUnary = true; break;
7307 : case ISD::STRICT_FLOG10: NewOpc = ISD::FLOG10; IsUnary = true; break;
7308 : case ISD::STRICT_FLOG2: NewOpc = ISD::FLOG2; IsUnary = true; break;
7309 : case ISD::STRICT_FRINT: NewOpc = ISD::FRINT; IsUnary = true; break;
7310 : case ISD::STRICT_FNEARBYINT:
7311 : NewOpc = ISD::FNEARBYINT;
7312 : IsUnary = true;
7313 : break;
7314 : }
7315 :
7316 : // We're taking this node out of the chain, so we need to re-link things.
7317 464 : SDValue InputChain = Node->getOperand(0);
7318 : SDValue OutputChain = SDValue(Node, 1);
7319 464 : ReplaceAllUsesOfValueWith(OutputChain, InputChain);
7320 :
7321 928 : SDVTList VTs = getVTList(Node->getOperand(1).getValueType());
7322 : SDNode *Res = nullptr;
7323 464 : if (IsUnary)
7324 522 : Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1) });
7325 203 : else if (IsTernary)
7326 78 : Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1),
7327 : Node->getOperand(2),
7328 : Node->getOperand(3)});
7329 : else
7330 328 : Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1),
7331 : Node->getOperand(2) });
7332 :
7333 : // MorphNodeTo can operate in two ways: if an existing node with the
7334 : // specified operands exists, it can just return it. Otherwise, it
7335 : // updates the node in place to have the requested operands.
7336 464 : if (Res == Node) {
7337 : // If we updated the node in place, reset the node ID. To the isel,
7338 : // this should be just like a newly allocated machine node.
7339 : Res->setNodeId(-1);
7340 : } else {
7341 0 : ReplaceAllUsesWith(Node, Res);
7342 0 : RemoveDeadNode(Node);
7343 : }
7344 :
7345 464 : return Res;
7346 : }
7347 :
7348 : /// getMachineNode - These are used for target selectors to create a new node
7349 : /// with specified return type(s), MachineInstr opcode, and operands.
7350 : ///
7351 : /// Note that getMachineNode returns the resultant node. If there is already a
7352 : /// node of the specified opcode and operands, it returns that node instead of
7353 : /// the current one.
7354 3939 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
7355 : EVT VT) {
7356 3939 : SDVTList VTs = getVTList(VT);
7357 3939 : return getMachineNode(Opcode, dl, VTs, None);
7358 : }
7359 :
7360 50759 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
7361 : EVT VT, SDValue Op1) {
7362 50759 : SDVTList VTs = getVTList(VT);
7363 50759 : SDValue Ops[] = { Op1 };
7364 50759 : return getMachineNode(Opcode, dl, VTs, Ops);
7365 : }
7366 :
7367 59968 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
7368 : EVT VT, SDValue Op1, SDValue Op2) {
7369 59968 : SDVTList VTs = getVTList(VT);
7370 59968 : SDValue Ops[] = { Op1, Op2 };
7371 59968 : return getMachineNode(Opcode, dl, VTs, Ops);
7372 : }
7373 :
7374 5706 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
7375 : EVT VT, SDValue Op1, SDValue Op2,
7376 : SDValue Op3) {
7377 5706 : SDVTList VTs = getVTList(VT);
7378 5706 : SDValue Ops[] = { Op1, Op2, Op3 };
7379 5706 : return getMachineNode(Opcode, dl, VTs, Ops);
7380 : }
7381 :
7382 45701 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
7383 : EVT VT, ArrayRef<SDValue> Ops) {
7384 45701 : SDVTList VTs = getVTList(VT);
7385 45701 : return getMachineNode(Opcode, dl, VTs, Ops);
7386 : }
7387 :
7388 9498 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
7389 : EVT VT1, EVT VT2, SDValue Op1,
7390 : SDValue Op2) {
7391 9498 : SDVTList VTs = getVTList(VT1, VT2);
7392 9498 : SDValue Ops[] = { Op1, Op2 };
7393 9498 : return getMachineNode(Opcode, dl, VTs, Ops);
7394 : }
7395 :
7396 96 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
7397 : EVT VT1, EVT VT2, SDValue Op1,
7398 : SDValue Op2, SDValue Op3) {
7399 96 : SDVTList VTs = getVTList(VT1, VT2);
7400 96 : SDValue Ops[] = { Op1, Op2, Op3 };
7401 96 : return getMachineNode(Opcode, dl, VTs, Ops);
7402 : }
7403 :
7404 215698 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
7405 : EVT VT1, EVT VT2,
7406 : ArrayRef<SDValue> Ops) {
7407 215698 : SDVTList VTs = getVTList(VT1, VT2);
7408 215698 : return getMachineNode(Opcode, dl, VTs, Ops);
7409 : }
7410 :
7411 27 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
7412 : EVT VT1, EVT VT2, EVT VT3,
7413 : SDValue Op1, SDValue Op2) {
7414 27 : SDVTList VTs = getVTList(VT1, VT2, VT3);
7415 27 : SDValue Ops[] = { Op1, Op2 };
7416 27 : return getMachineNode(Opcode, dl, VTs, Ops);
7417 : }
7418 :
7419 178 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
7420 : EVT VT1, EVT VT2, EVT VT3,
7421 : SDValue Op1, SDValue Op2,
7422 : SDValue Op3) {
7423 178 : SDVTList VTs = getVTList(VT1, VT2, VT3);
7424 178 : SDValue Ops[] = { Op1, Op2, Op3 };
7425 178 : return getMachineNode(Opcode, dl, VTs, Ops);
7426 : }
7427 :
7428 396 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
7429 : EVT VT1, EVT VT2, EVT VT3,
7430 : ArrayRef<SDValue> Ops) {
7431 396 : SDVTList VTs = getVTList(VT1, VT2, VT3);
7432 396 : return getMachineNode(Opcode, dl, VTs, Ops);
7433 : }
7434 :
7435 2605 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
7436 : ArrayRef<EVT> ResultTys,
7437 : ArrayRef<SDValue> Ops) {
7438 2605 : SDVTList VTs = getVTList(ResultTys);
7439 2605 : return getMachineNode(Opcode, dl, VTs, Ops);
7440 : }
7441 :
7442 647724 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
7443 : SDVTList VTs,
7444 : ArrayRef<SDValue> Ops) {
7445 647724 : bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
7446 : MachineSDNode *N;
7447 647724 : void *IP = nullptr;
7448 :
7449 : if (DoCSE) {
7450 : FoldingSetNodeID ID;
7451 632191 : AddNodeIDNode(ID, ~Opcode, VTs, Ops);
7452 632191 : IP = nullptr;
7453 632191 : if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
7454 83687 : return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
7455 : }
7456 : }
7457 :
7458 : // Allocate a new MachineSDNode.
7459 1128074 : N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7460 564037 : createOperands(N, Ops);
7461 :
7462 564037 : if (DoCSE)
7463 548504 : CSEMap.InsertNode(N, IP);
7464 :
7465 564037 : InsertNode(N);
7466 564037 : return N;
7467 : }
7468 :
7469 : /// getTargetExtractSubreg - A convenience function for creating
7470 : /// TargetOpcode::EXTRACT_SUBREG nodes.
7471 42118 : SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
7472 : SDValue Operand) {
7473 42118 : SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
7474 42118 : SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
7475 : VT, Operand, SRIdxVal);
7476 42118 : return SDValue(Subreg, 0);
7477 : }
7478 :
7479 : /// getTargetInsertSubreg - A convenience function for creating
7480 : /// TargetOpcode::INSERT_SUBREG nodes.
7481 1711 : SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
7482 : SDValue Operand, SDValue Subreg) {
7483 1711 : SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
7484 1711 : SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
7485 : VT, Operand, Subreg, SRIdxVal);
7486 1711 : return SDValue(Result, 0);
7487 : }
7488 :
7489 : /// getNodeIfExists - Get the specified node if it's already available, or
7490 : /// else return NULL.
7491 635726 : SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
7492 : ArrayRef<SDValue> Ops,
7493 : const SDNodeFlags Flags) {
7494 635726 : if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
7495 : FoldingSetNodeID ID;
7496 635726 : AddNodeIDNode(ID, Opcode, VTList, Ops);
7497 635726 : void *IP = nullptr;
7498 1271452 : if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) {
7499 20 : E->intersectFlagsWith(Flags);
7500 : return E;
7501 : }
7502 : }
7503 : return nullptr;
7504 : }
7505 :
7506 : /// getDbgValue - Creates a SDDbgValue node.
7507 : ///
7508 : /// SDNode
7509 51798 : SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
7510 : SDNode *N, unsigned R, bool IsIndirect,
7511 : const DebugLoc &DL, unsigned O) {
7512 : assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
7513 : "Expected inlined-at fields to agree");
7514 51798 : return new (DbgInfo->getAlloc())
7515 51798 : SDDbgValue(Var, Expr, N, R, IsIndirect, DL, O);
7516 : }
7517 :
7518 : /// Constant
7519 25834 : SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
7520 : DIExpression *Expr,
7521 : const Value *C,
7522 : const DebugLoc &DL, unsigned O) {
7523 : assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
7524 : "Expected inlined-at fields to agree");
7525 25834 : return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, C, DL, O);
7526 : }
7527 :
7528 : /// FrameIndex
7529 16651 : SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
7530 : DIExpression *Expr, unsigned FI,
7531 : bool IsIndirect,
7532 : const DebugLoc &DL,
7533 : unsigned O) {
7534 : assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
7535 : "Expected inlined-at fields to agree");
7536 16651 : return new (DbgInfo->getAlloc())
7537 16651 : SDDbgValue(Var, Expr, FI, IsIndirect, DL, O, SDDbgValue::FRAMEIX);
7538 : }
7539 :
7540 : /// VReg
7541 10696 : SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var,
7542 : DIExpression *Expr,
7543 : unsigned VReg, bool IsIndirect,
7544 : const DebugLoc &DL, unsigned O) {
7545 : assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
7546 : "Expected inlined-at fields to agree");
7547 10696 : return new (DbgInfo->getAlloc())
7548 10696 : SDDbgValue(Var, Expr, VReg, IsIndirect, DL, O, SDDbgValue::VREG);
7549 : }
7550 :
7551 13786791 : void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
7552 : unsigned OffsetInBits, unsigned SizeInBits,
7553 : bool InvalidateDbg) {
7554 : SDNode *FromNode = From.getNode();
7555 : SDNode *ToNode = To.getNode();
7556 : assert(FromNode && ToNode && "Can't modify dbg values");
7557 :
7558 : // PR35338
7559 : // TODO: assert(From != To && "Redundant dbg value transfer");
7560 : // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
7561 13786102 : if (From == To || FromNode == ToNode)
7562 13781568 : return;
7563 :
7564 9617177 : if (!FromNode->getHasDebugValue())
7565 : return;
7566 :
7567 : SmallVector<SDDbgValue *, 2> ClonedDVs;
7568 19741 : for (SDDbgValue *Dbg : GetDbgValues(FromNode)) {
7569 9295 : if (Dbg->getKind() != SDDbgValue::SDNODE || Dbg->isInvalidated())
7570 1323 : continue;
7571 :
7572 : // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
7573 :
7574 : // Just transfer the dbg value attached to From.
7575 8345 : if (Dbg->getResNo() != From.getResNo())
7576 : continue;
7577 :
7578 7977 : DIVariable *Var = Dbg->getVariable();
7579 7977 : auto *Expr = Dbg->getExpression();
7580 : // If a fragment is requested, update the expression.
7581 7977 : if (SizeInBits) {
7582 : // When splitting a larger (e.g., sign-extended) value whose
7583 : // lower bits are described with an SDDbgValue, do not attempt
7584 : // to transfer the SDDbgValue to the upper bits.
7585 60 : if (auto FI = Expr->getFragmentInfo())
7586 10 : if (OffsetInBits + SizeInBits > FI->SizeInBits)
7587 : continue;
7588 : auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
7589 59 : SizeInBits);
7590 59 : if (!Fragment)
7591 : continue;
7592 55 : Expr = *Fragment;
7593 : }
7594 : // Clone the SDDbgValue and move it to To.
7595 : SDDbgValue *Clone =
7596 15944 : getDbgValue(Var, Expr, ToNode, To.getResNo(), Dbg->isIndirect(),
7597 7972 : Dbg->getDebugLoc(), Dbg->getOrder());
7598 7972 : ClonedDVs.push_back(Clone);
7599 :
7600 7972 : if (InvalidateDbg)
7601 : Dbg->setIsInvalidated();
7602 : }
7603 :
7604 13195 : for (SDDbgValue *Dbg : ClonedDVs)
7605 7972 : AddDbgValue(Dbg, ToNode, false);
7606 : }
7607 :
7608 5739205 : void SelectionDAG::salvageDebugInfo(SDNode &N) {
7609 5739205 : if (!N.getHasDebugValue())
7610 5733106 : return;
7611 :
7612 : SmallVector<SDDbgValue *, 2> ClonedDVs;
7613 29794 : for (auto DV : GetDbgValues(&N)) {
7614 17596 : if (DV->isInvalidated())
7615 : continue;
7616 17596 : switch (N.getOpcode()) {
7617 : default:
7618 10938 : break;
7619 6658 : case ISD::ADD:
7620 6658 : SDValue N0 = N.getOperand(0);
7621 6658 : SDValue N1 = N.getOperand(1);
7622 13316 : if (!isConstantIntBuildVectorOrConstantInt(N0) &&
7623 6658 : isConstantIntBuildVectorOrConstantInt(N1)) {
7624 : uint64_t Offset = N.getConstantOperandVal(1);
7625 : // Rewrite an ADD constant node into a DIExpression. Since we are
7626 : // performing arithmetic to compute the variable's *value* in the
7627 : // DIExpression, we need to mark the expression with a
7628 : // DW_OP_stack_value.
7629 5707 : auto *DIExpr = DV->getExpression();
7630 5707 : DIExpr = DIExpression::prepend(DIExpr, DIExpression::NoDeref, Offset,
7631 : DIExpression::NoDeref,
7632 : DIExpression::WithStackValue);
7633 : SDDbgValue *Clone =
7634 5707 : getDbgValue(DV->getVariable(), DIExpr, N0.getNode(), N0.getResNo(),
7635 5707 : DV->isIndirect(), DV->getDebugLoc(), DV->getOrder());
7636 5707 : ClonedDVs.push_back(Clone);
7637 : DV->setIsInvalidated();
7638 : LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
7639 : N0.getNode()->dumprFull(this);
7640 : dbgs() << " into " << *DIExpr << '\n');
7641 : }
7642 : }
7643 : }
7644 :
7645 11806 : for (SDDbgValue *Dbg : ClonedDVs)
7646 5707 : AddDbgValue(Dbg, Dbg->getSDNode(), false);
7647 : }
7648 :
7649 : /// Creates a SDDbgLabel node.
7650 1 : SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
7651 : const DebugLoc &DL, unsigned O) {
7652 : assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
7653 : "Expected inlined-at fields to agree");
7654 1 : return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
7655 : }
7656 :
7657 : namespace {
7658 :
7659 : /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
7660 : /// pointed to by a use iterator is deleted, increment the use iterator
7661 : /// so that it doesn't dangle.
7662 : ///
7663 : class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
7664 : SDNode::use_iterator &UI;
7665 : SDNode::use_iterator &UE;
7666 :
7667 24710 : void NodeDeleted(SDNode *N, SDNode *E) override {
7668 : // Increment the iterator as needed.
7669 49420 : while (UI != UE && N == *UI)
7670 : ++UI;
7671 24710 : }
7672 :
7673 : public:
7674 : RAUWUpdateListener(SelectionDAG &d,
7675 : SDNode::use_iterator &ui,
7676 : SDNode::use_iterator &ue)
7677 22633592 : : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
7678 : };
7679 :
7680 : } // end anonymous namespace
7681 :
7682 : /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
7683 : /// This can cause recursive merging of nodes in the DAG.
7684 : ///
7685 : /// This version assumes From has a single result value.
7686 : ///
7687 3158805 : void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
7688 : SDNode *From = FromN.getNode();
7689 : assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
7690 : "Cannot replace with this method!");
7691 : assert(From != To.getNode() && "Cannot replace uses of with self");
7692 :
7693 : // Preserve Debug Values
7694 3158805 : transferDbgValues(FromN, To);
7695 :
7696 : // Iterate over all the existing uses of From. New uses will be added
7697 : // to the beginning of the use list, which we avoid visiting.
7698 : // This specifically avoids visiting uses of From that arise while the
7699 : // replacement is happening, because any such uses would be the result
7700 : // of CSE: If an existing node looks like From after one of its operands
7701 : // is replaced by To, we don't want to replace of all its users with To
7702 : // too. See PR3018 for more info.
7703 3158805 : SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
7704 : RAUWUpdateListener Listener(*this, UI, UE);
7705 6995949 : while (UI != UE) {
7706 : SDNode *User = *UI;
7707 :
7708 : // This node is about to morph, remove its old self from the CSE maps.
7709 3837144 : RemoveNodeFromCSEMaps(User);
7710 :
7711 : // A user can appear in a use list multiple times, and when this
7712 : // happens the uses are usually next to each other in the list.
7713 : // To help reduce the number of CSE recomputations, process all
7714 : // the uses of this user that we can find this way.
7715 : do {
7716 3849291 : SDUse &Use = UI.getUse();
7717 : ++UI;
7718 : Use.set(To);
7719 7698582 : if (To->isDivergent() != From->isDivergent())
7720 7176 : updateDivergence(User);
7721 3849291 : } while (UI != UE && *UI == User);
7722 : // Now that we have modified User, add it back to the CSE maps. If it
7723 : // already exists there, recursively merge the results together.
7724 3837144 : AddModifiedNodeToCSEMaps(User);
7725 : }
7726 :
7727 : // If we just RAUW'd the root, take note.
7728 3158805 : if (FromN == getRoot())
7729 7548 : setRoot(To);
7730 3158805 : }
7731 :
7732 : /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
7733 : /// This can cause recursive merging of nodes in the DAG.
7734 : ///
7735 : /// This version assumes that for each value of From, there is a
7736 : /// corresponding value in To in the same position with the same type.
7737 : ///
7738 1896978 : void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
7739 : #ifndef NDEBUG
7740 : for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
7741 : assert((!From->hasAnyUseOfValue(i) ||
7742 : From->getValueType(i) == To->getValueType(i)) &&
7743 : "Cannot use this version of ReplaceAllUsesWith!");
7744 : #endif
7745 :
7746 : // Handle the trivial case.
7747 1896978 : if (From == To)
7748 0 : return;
7749 :
7750 : // Preserve Debug Info. Only do this if there's a use.
7751 3938345 : for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
7752 2041367 : if (From->hasAnyUseOfValue(i)) {
7753 : assert((i < To->getNumValues()) && "Invalid To location");
7754 1905253 : transferDbgValues(SDValue(From, i), SDValue(To, i));
7755 : }
7756 :
7757 : // Iterate over just the existing users of From. See the comments in
7758 : // the ReplaceAllUsesWith above.
7759 1896978 : SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
7760 : RAUWUpdateListener Listener(*this, UI, UE);
7761 4413185 : while (UI != UE) {
7762 : SDNode *User = *UI;
7763 :
7764 : // This node is about to morph, remove its old self from the CSE maps.
7765 2516207 : RemoveNodeFromCSEMaps(User);
7766 :
7767 : // A user can appear in a use list multiple times, and when this
7768 : // happens the uses are usually next to each other in the list.
7769 : // To help reduce the number of CSE recomputations, process all
7770 : // the uses of this user that we can find this way.
7771 : do {
7772 2523274 : SDUse &Use = UI.getUse();
7773 : ++UI;
7774 : Use.setNode(To);
7775 5046548 : if (To->isDivergent() != From->isDivergent())
7776 1685 : updateDivergence(User);
7777 2523274 : } while (UI != UE && *UI == User);
7778 :
7779 : // Now that we have modified User, add it back to the CSE maps. If it
7780 : // already exists there, recursively merge the results together.
7781 2516207 : AddModifiedNodeToCSEMaps(User);
7782 : }
7783 :
7784 : // If we just RAUW'd the root, take note.
7785 1896978 : if (From == getRoot().getNode())
7786 52190 : setRoot(SDValue(To, getRoot().getResNo()));
7787 : }
7788 :
7789 : /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
7790 : /// This can cause recursive merging of nodes in the DAG.
7791 : ///
7792 : /// This version can replace From with any result values. To must match the
7793 : /// number and types of values returned by From.
7794 1901177 : void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
7795 3802354 : if (From->getNumValues() == 1) // Handle the simple case efficiently.
7796 764144 : return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
7797 :
7798 : // Preserve Debug Info.
7799 3423700 : for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
7800 4573334 : transferDbgValues(SDValue(From, i), To[i]);
7801 :
7802 : // Iterate over just the existing users of From. See the comments in
7803 : // the ReplaceAllUsesWith above.
7804 1137033 : SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
7805 : RAUWUpdateListener Listener(*this, UI, UE);
7806 3136516 : while (UI != UE) {
7807 : SDNode *User = *UI;
7808 :
7809 : // This node is about to morph, remove its old self from the CSE maps.
7810 1999483 : RemoveNodeFromCSEMaps(User);
7811 :
7812 : // A user can appear in a use list multiple times, and when this happens the
7813 : // uses are usually next to each other in the list. To help reduce the
7814 : // number of CSE and divergence recomputations, process all the uses of this
7815 : // user that we can find this way.
7816 : bool To_IsDivergent = false;
7817 : do {
7818 2004711 : SDUse &Use = UI.getUse();
7819 2004711 : const SDValue &ToOp = To[Use.getResNo()];
7820 : ++UI;
7821 : Use.set(ToOp);
7822 2004711 : To_IsDivergent |= ToOp->isDivergent();
7823 2004711 : } while (UI != UE && *UI == User);
7824 :
7825 1999483 : if (To_IsDivergent != From->isDivergent())
7826 4008 : updateDivergence(User);
7827 :
7828 : // Now that we have modified User, add it back to the CSE maps. If it
7829 : // already exists there, recursively merge the results together.
7830 1999483 : AddModifiedNodeToCSEMaps(User);
7831 : }
7832 :
7833 : // If we just RAUW'd the root, take note.
7834 1137033 : if (From == getRoot().getNode())
7835 12 : setRoot(SDValue(To[getRoot().getResNo()]));
7836 : }
7837 :
7838 : /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
7839 : /// uses of other values produced by From.getNode() alone. The Deleted
7840 : /// vector is handled the same way as for ReplaceAllUsesWith.
7841 6366992 : void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
7842 : // Handle the really simple, really trivial case efficiently.
7843 1243012 : if (From == To) return;
7844 :
7845 : // Handle the simple, trivial, case efficiently.
7846 6366056 : if (From.getNode()->getNumValues() == 1) {
7847 1242076 : ReplaceAllUsesWith(From, To);
7848 1242076 : return;
7849 : }
7850 :
7851 : // Preserve Debug Info.
7852 5123980 : transferDbgValues(From, To);
7853 :
7854 : // Iterate over just the existing users of From. See the comments in
7855 : // the ReplaceAllUsesWith above.
7856 5123980 : SDNode::use_iterator UI = From.getNode()->use_begin(),
7857 : UE = From.getNode()->use_end();
7858 : RAUWUpdateListener Listener(*this, UI, UE);
7859 11237568 : while (UI != UE) {
7860 : SDNode *User = *UI;
7861 : bool UserRemovedFromCSEMaps = false;
7862 :
7863 : // A user can appear in a use list multiple times, and when this
7864 : // happens the uses are usually next to each other in the list.
7865 : // To help reduce the number of CSE recomputations, process all
7866 : // the uses of this user that we can find this way.
7867 : do {
7868 6944727 : SDUse &Use = UI.getUse();
7869 :
7870 : // Skip uses of different values from the same node.
7871 6944727 : if (Use.getResNo() != From.getResNo()) {
7872 : ++UI;
7873 3320022 : continue;
7874 : }
7875 :
7876 : // If this node hasn't been modified yet, it's still in the CSE maps,
7877 : // so remove its old self from the CSE maps.
7878 3624705 : if (!UserRemovedFromCSEMaps) {
7879 3622879 : RemoveNodeFromCSEMaps(User);
7880 : UserRemovedFromCSEMaps = true;
7881 : }
7882 :
7883 : ++UI;
7884 : Use.set(To);
7885 7249410 : if (To->isDivergent() != From->isDivergent())
7886 647 : updateDivergence(User);
7887 6944727 : } while (UI != UE && *UI == User);
7888 : // We are iterating over all uses of the From node, so if a use
7889 : // doesn't use the specific value, no changes are made.
7890 6113588 : if (!UserRemovedFromCSEMaps)
7891 : continue;
7892 :
7893 : // Now that we have modified User, add it back to the CSE maps. If it
7894 : // already exists there, recursively merge the results together.
7895 3622879 : AddModifiedNodeToCSEMaps(User);
7896 : }
7897 :
7898 : // If we just RAUW'd the root, take note.
7899 5123980 : if (From == getRoot())
7900 36106 : setRoot(To);
7901 : }
7902 :
7903 : namespace {
7904 :
7905 : /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
7906 : /// to record information about a use.
7907 : struct UseMemo {
7908 : SDNode *User;
7909 : unsigned Index;
7910 : SDUse *Use;
7911 : };
7912 :
7913 : /// operator< - Sort Memos by User.
7914 0 : bool operator<(const UseMemo &L, const UseMemo &R) {
7915 0 : return (intptr_t)L.User < (intptr_t)R.User;
7916 : }
7917 :
7918 : } // end anonymous namespace
7919 :
7920 248750 : void SelectionDAG::updateDivergence(SDNode * N)
7921 : {
7922 248750 : if (TLI->isSDNodeAlwaysUniform(N))
7923 : return;
7924 242537 : bool IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA);
7925 917324 : for (auto &Op : N->ops()) {
7926 674787 : if (Op.Val.getValueType() != MVT::Other)
7927 460890 : IsDivergent |= Op.getNode()->isDivergent();
7928 : }
7929 242537 : if (N->SDNodeBits.IsDivergent != IsDivergent) {
7930 12125 : N->SDNodeBits.IsDivergent = IsDivergent;
7931 26337 : for (auto U : N->uses()) {
7932 14212 : updateDivergence(U);
7933 : }
7934 : }
7935 : }
7936 :
7937 :
7938 0 : void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode*>& Order) {
7939 : DenseMap<SDNode *, unsigned> Degree;
7940 0 : Order.reserve(AllNodes.size());
7941 0 : for (auto & N : allnodes()) {
7942 0 : unsigned NOps = N.getNumOperands();
7943 0 : Degree[&N] = NOps;
7944 0 : if (0 == NOps)
7945 0 : Order.push_back(&N);
7946 : }
7947 : for (std::vector<SDNode *>::iterator I = Order.begin();
7948 0 : I!=Order.end();++I) {
7949 0 : SDNode * N = *I;
7950 0 : for (auto U : N->uses()) {
7951 : unsigned &UnsortedOps = Degree[U];
7952 0 : if (0 == --UnsortedOps)
7953 0 : Order.push_back(U);
7954 : }
7955 : }
7956 0 : }
7957 :
7958 : #ifndef NDEBUG
7959 : void SelectionDAG::VerifyDAGDiverence()
7960 : {
7961 : std::vector<SDNode*> TopoOrder;
7962 : CreateTopologicalOrder(TopoOrder);
7963 : const TargetLowering &TLI = getTargetLoweringInfo();
7964 : DenseMap<const SDNode *, bool> DivergenceMap;
7965 : for (auto &N : allnodes()) {
7966 : DivergenceMap[&N] = false;
7967 : }
7968 : for (auto N : TopoOrder) {
7969 : bool IsDivergent = DivergenceMap[N];
7970 : bool IsSDNodeDivergent = TLI.isSDNodeSourceOfDivergence(N, FLI, DA);
7971 : for (auto &Op : N->ops()) {
7972 : if (Op.Val.getValueType() != MVT::Other)
7973 : IsSDNodeDivergent |= DivergenceMap[Op.getNode()];
7974 : }
7975 : if (!IsDivergent && IsSDNodeDivergent && !TLI.isSDNodeAlwaysUniform(N)) {
7976 : DivergenceMap[N] = true;
7977 : }
7978 : }
7979 : for (auto &N : allnodes()) {
7980 : (void)N;
7981 : assert(DivergenceMap[&N] == N.isDivergent() &&
7982 : "Divergence bit inconsistency detected\n");
7983 : }
7984 : }
7985 : #endif
7986 :
7987 :
7988 : /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
7989 : /// uses of other values produced by From.getNode() alone. The same value
7990 : /// may appear in both the From and To list. The Deleted vector is
7991 : /// handled the same way as for ReplaceAllUsesWith.
7992 1087 : void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
7993 : const SDValue *To,
7994 : unsigned Num){
7995 : // Handle the simple, trivial case efficiently.
7996 1087 : if (Num == 1)
7997 0 : return ReplaceAllUsesOfValueWith(*From, *To);
7998 :
7999 1087 : transferDbgValues(*From, *To);
8000 :
8001 : // Read up all the uses and make records of them. This helps
8002 : // processing new uses that are introduced during the
8003 : // replacement process.
8004 1087 : SmallVector<UseMemo, 4> Uses;
8005 3491 : for (unsigned i = 0; i != Num; ++i) {
8006 2404 : unsigned FromResNo = From[i].getResNo();
8007 2404 : SDNode *FromNode = From[i].getNode();
8008 2404 : for (SDNode::use_iterator UI = FromNode->use_begin(),
8009 6677 : E = FromNode->use_end(); UI != E; ++UI) {
8010 : SDUse &Use = UI.getUse();
8011 4273 : if (Use.getResNo() == FromResNo) {
8012 2107 : UseMemo Memo = { *UI, i, &Use };
8013 2107 : Uses.push_back(Memo);
8014 : }
8015 : }
8016 : }
8017 :
8018 : // Sort the uses, so that all the uses from a given User are together.
8019 1087 : llvm::sort(Uses);
8020 :
8021 3061 : for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
8022 3061 : UseIndex != UseIndexEnd; ) {
8023 : // We know that this user uses some value of From. If it is the right
8024 : // value, update it.
8025 1974 : SDNode *User = Uses[UseIndex].User;
8026 :
8027 : // This node is about to morph, remove its old self from the CSE maps.
8028 1974 : RemoveNodeFromCSEMaps(User);
8029 :
8030 : // The Uses array is sorted, so all the uses for a given User
8031 : // are next to each other in the list.
8032 : // To help reduce the number of CSE recomputations, process all
8033 : // the uses of this user that we can find this way.
8034 : do {
8035 2107 : unsigned i = Uses[UseIndex].Index;
8036 2107 : SDUse &Use = *Uses[UseIndex].Use;
8037 2107 : ++UseIndex;
8038 :
8039 2107 : Use.set(To[i]);
8040 2107 : } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
8041 :
8042 : // Now that we have modified User, add it back to the CSE maps. If it
8043 : // already exists there, recursively merge the results together.
8044 1974 : AddModifiedNodeToCSEMaps(User);
8045 : }
8046 : }
8047 :
8048 : /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
8049 : /// based on their topological order. It returns the maximum id and a vector
8050 : /// of the SDNodes* in assigned order by reference.
8051 2796839 : unsigned SelectionDAG::AssignTopologicalOrder() {
8052 : unsigned DAGSize = 0;
8053 :
8054 : // SortedPos tracks the progress of the algorithm. Nodes before it are
8055 : // sorted, nodes after it are unsorted. When the algorithm completes
8056 : // it is at the end of the list.
8057 : allnodes_iterator SortedPos = allnodes_begin();
8058 :
8059 : // Visit all the nodes. Move nodes with no operands to the front of
8060 : // the list immediately. Annotate nodes that do have operands with their
8061 : // operand count. Before we do this, the Node Id fields of the nodes
8062 : // may contain arbitrary values. After, the Node Id fields for nodes
8063 : // before SortedPos will contain the topological sort index, and the
8064 : // Node Id fields for nodes At SortedPos and after will contain the
8065 : // count of outstanding operands.
8066 86636216 : for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
8067 : SDNode *N = &*I++;
8068 83839377 : checkForCycles(N, this);
8069 83839377 : unsigned Degree = N->getNumOperands();
8070 83839377 : if (Degree == 0) {
8071 : // A node with no uses, add it to the result array immediately.
8072 35277928 : N->setNodeId(DAGSize++);
8073 : allnodes_iterator Q(N);
8074 35277928 : if (Q != SortedPos)
8075 : SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
8076 : assert(SortedPos != AllNodes.end() && "Overran node list");
8077 : ++SortedPos;
8078 : } else {
8079 : // Temporarily use the Node Id as scratch space for the degree count.
8080 48561449 : N->setNodeId(Degree);
8081 : }
8082 : }
8083 :
8084 : // Visit all the nodes. As we iterate, move nodes into sorted order,
8085 : // such that by the time the end is reached all nodes will be sorted.
8086 86636216 : for (SDNode &Node : allnodes()) {
8087 : SDNode *N = &Node;
8088 83839377 : checkForCycles(N, this);
8089 : // N is in sorted position, so all its uses have one less operand
8090 : // that needs to be sorted.
8091 83839377 : for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
8092 231533005 : UI != UE; ++UI) {
8093 : SDNode *P = *UI;
8094 147693628 : unsigned Degree = P->getNodeId();
8095 : assert(Degree != 0 && "Invalid node degree");
8096 147693628 : --Degree;
8097 147693628 : if (Degree == 0) {
8098 : // All of P's operands are sorted, so P may sorted now.
8099 48561449 : P->setNodeId(DAGSize++);
8100 48561449 : if (P->getIterator() != SortedPos)
8101 : SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
8102 : assert(SortedPos != AllNodes.end() && "Overran node list");
8103 : ++SortedPos;
8104 : } else {
8105 : // Update P's outstanding operand count.
8106 99132179 : P->setNodeId(Degree);
8107 : }
8108 : }
8109 83839377 : if (Node.getIterator() == SortedPos) {
8110 : #ifndef NDEBUG
8111 : allnodes_iterator I(N);
8112 : SDNode *S = &*++I;
8113 : dbgs() << "Overran sorted position:\n";
8114 : S->dumprFull(this); dbgs() << "\n";
8115 : dbgs() << "Checking if this is due to cycles\n";
8116 : checkForCycles(this, true);
8117 : #endif
8118 0 : llvm_unreachable(nullptr);
8119 : }
8120 : }
8121 :
8122 : assert(SortedPos == AllNodes.end() &&
8123 : "Topological sort incomplete!");
8124 : assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
8125 : "First node in topological sort is not the entry token!");
8126 : assert(AllNodes.front().getNodeId() == 0 &&
8127 : "First node in topological sort has non-zero id!");
8128 : assert(AllNodes.front().getNumOperands() == 0 &&
8129 : "First node in topological sort has operands!");
8130 : assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
8131 : "Last node in topologic sort has unexpected id!");
8132 : assert(AllNodes.back().use_empty() &&
8133 : "Last node in topologic sort has users!");
8134 : assert(DAGSize == allnodes_size() && "Node count mismatch!");
8135 2796839 : return DAGSize;
8136 : }
8137 :
8138 : /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
8139 : /// value is produced by SD.
8140 104979 : void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
8141 104979 : if (SD) {
8142 : assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
8143 : SD->setHasDebugValue(true);
8144 : }
8145 104979 : DbgInfo->add(DB, SD, isParameter);
8146 104979 : }
8147 :
8148 1 : void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) {
8149 1 : DbgInfo->add(DB);
8150 1 : }
8151 :
8152 3209 : SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
8153 : SDValue NewMemOp) {
8154 : assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
8155 : // The new memory operation must have the same position as the old load in
8156 : // terms of memory dependency. Create a TokenFactor for the old load and new
8157 : // memory operation and update uses of the old load's output chain to use that
8158 : // TokenFactor.
8159 : SDValue OldChain = SDValue(OldLoad, 1);
8160 3209 : SDValue NewChain = SDValue(NewMemOp.getNode(), 1);
8161 3209 : if (!OldLoad->hasAnyUseOfValue(1))
8162 2933 : return NewChain;
8163 :
8164 : SDValue TokenFactor =
8165 276 : getNode(ISD::TokenFactor, SDLoc(OldLoad), MVT::Other, OldChain, NewChain);
8166 276 : ReplaceAllUsesOfValueWith(OldChain, TokenFactor);
8167 276 : UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewChain);
8168 276 : return TokenFactor;
8169 : }
8170 :
8171 : //===----------------------------------------------------------------------===//
8172 : // SDNode Class
8173 : //===----------------------------------------------------------------------===//
8174 :
8175 7797899 : bool llvm::isNullConstant(SDValue V) {
8176 : ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
8177 12200984 : return Const != nullptr && Const->isNullValue();
8178 : }
8179 :
8180 1163661 : bool llvm::isNullFPConstant(SDValue V) {
8181 : ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
8182 171128 : return Const != nullptr && Const->isZero() && !Const->isNegative();
8183 : }
8184 :
8185 1028890 : bool llvm::isAllOnesConstant(SDValue V) {
8186 : ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
8187 1579348 : return Const != nullptr && Const->isAllOnesValue();
8188 : }
8189 :
8190 329655 : bool llvm::isOneConstant(SDValue V) {
8191 : ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
8192 411344 : return Const != nullptr && Const->isOne();
8193 : }
8194 :
8195 9609070 : SDValue llvm::peekThroughBitcasts(SDValue V) {
8196 11099790 : while (V.getOpcode() == ISD::BITCAST)
8197 1490720 : V = V.getOperand(0);
8198 9609070 : return V;
8199 : }
8200 :
8201 478443 : SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
8202 631623 : while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse())
8203 153180 : V = V.getOperand(0);
8204 478443 : return V;
8205 : }
8206 :
8207 3686362 : bool llvm::isBitwiseNot(SDValue V) {
8208 3686362 : if (V.getOpcode() != ISD::XOR)
8209 : return false;
8210 3199 : ConstantSDNode *C = isConstOrConstSplat(peekThroughBitcasts(V.getOperand(1)));
8211 5112 : return C && C->isAllOnesValue();
8212 : }
8213 :
8214 12598492 : ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs) {
8215 : if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
8216 : return CN;
8217 :
8218 : if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
8219 : BitVector UndefElements;
8220 309159 : ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements);
8221 :
8222 : // BuildVectors can truncate their operands. Ignore that case here.
8223 596579 : if (CN && (UndefElements.none() || AllowUndefs) &&
8224 880158 : CN->getValueType(0) == N.getValueType().getScalarType())
8225 : return CN;
8226 : }
8227 :
8228 : return nullptr;
8229 : }
8230 :
8231 232893 : ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
8232 : if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
8233 : return CN;
8234 :
8235 : if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
8236 : BitVector UndefElements;
8237 6833 : ConstantFPSDNode *CN = BV->getConstantFPSplatNode(&UndefElements);
8238 9466 : if (CN && (UndefElements.none() || AllowUndefs))
8239 : return CN;
8240 : }
8241 :
8242 : return nullptr;
8243 : }
8244 :
8245 25935645 : HandleSDNode::~HandleSDNode() {
8246 12967822 : DropOperands();
8247 12967823 : }
8248 :
8249 3416521 : GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
8250 : const DebugLoc &DL,
8251 : const GlobalValue *GA, EVT VT,
8252 3416521 : int64_t o, unsigned char TF)
8253 4182596 : : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
8254 3416521 : TheGlobal = GA;
8255 3416521 : }
8256 :
8257 213 : AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl,
8258 : EVT VT, unsigned SrcAS,
8259 213 : unsigned DestAS)
8260 : : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
8261 215 : SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
8262 :
8263 13553091 : MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
8264 13553091 : SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
8265 15371418 : : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
8266 27106182 : MemSDNodeBits.IsVolatile = MMO->isVolatile();
8267 27106182 : MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
8268 27106182 : MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
8269 27106182 : MemSDNodeBits.IsInvariant = MMO->isInvariant();
8270 :
8271 : // We check here that the size of the memory operand fits within the size of
8272 : // the MMO. This is because the MMO might indicate only a possible address
8273 : // range instead of specifying the affected memory addresses precisely.
8274 : assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
8275 13553091 : }
8276 :
8277 : /// Profile - Gather unique data for the node.
8278 : ///
8279 99407536 : void SDNode::Profile(FoldingSetNodeID &ID) const {
8280 99407536 : AddNodeIDNode(ID, this);
8281 99407537 : }
8282 :
8283 : namespace {
8284 :
8285 26640 : struct EVTArray {
8286 : std::vector<EVT> VTs;
8287 :
8288 26999 : EVTArray() {
8289 26999 : VTs.reserve(MVT::LAST_VALUETYPE);
8290 3104885 : for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
8291 3077886 : VTs.push_back(MVT((MVT::SimpleValueType)i));
8292 26999 : }
8293 : };
8294 :
8295 : } // end anonymous namespace
8296 :
8297 : static ManagedStatic<std::set<EVT, EVT::compareRawBits>> EVTs;
8298 : static ManagedStatic<EVTArray> SimpleVTArray;
8299 : static ManagedStatic<sys::SmartMutex<true>> VTMutex;
8300 :
8301 : /// getValueTypeList - Return a pointer to the specified value type.
8302 : ///
8303 136745722 : const EVT *SDNode::getValueTypeList(EVT VT) {
8304 136745722 : if (VT.isExtended()) {
8305 105074 : sys::SmartScopedLock<true> Lock(*VTMutex);
8306 : return &(*EVTs->insert(VT).first);
8307 : } else {
8308 : assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
8309 : "Value type out of range!");
8310 136640648 : return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
8311 : }
8312 : }
8313 :
8314 : /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
8315 : /// indicated value. This method ignores uses of other values defined by this
8316 : /// operation.
8317 21373567 : bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
8318 : assert(Value < getNumValues() && "Bad value!");
8319 :
8320 : // TODO: Only iterate over uses of a given value of the node
8321 49555235 : for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
8322 34009579 : if (UI.getUse().getResNo() == Value) {
8323 27201255 : if (NUses == 0)
8324 : return false;
8325 21373344 : --NUses;
8326 : }
8327 : }
8328 :
8329 : // Found exactly the right number of uses?
8330 15545656 : return NUses == 0;
8331 : }
8332 :
8333 : /// hasAnyUseOfValue - Return true if there are any use of the indicated
8334 : /// value. This method ignores uses of other values defined by this operation.
8335 23816815 : bool SDNode::hasAnyUseOfValue(unsigned Value) const {
8336 : assert(Value < getNumValues() && "Bad value!");
8337 :
8338 34719265 : for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
8339 28285355 : if (UI.getUse().getResNo() == Value)
8340 : return true;
8341 :
8342 : return false;
8343 : }
8344 :
8345 : /// isOnlyUserOf - Return true if this node is the only use of N.
8346 372966 : bool SDNode::isOnlyUserOf(const SDNode *N) const {
8347 : bool Seen = false;
8348 569042 : for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
8349 : SDNode *User = *I;
8350 446365 : if (User == this)
8351 : Seen = true;
8352 : else
8353 : return false;
8354 : }
8355 :
8356 : return Seen;
8357 : }
8358 :
8359 : /// Return true if the only users of N are contained in Nodes.
8360 51087 : bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
8361 : bool Seen = false;
8362 89071 : for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
8363 84162 : SDNode *User = *I;
8364 84162 : if (llvm::any_of(Nodes,
8365 0 : [&User](const SDNode *Node) { return User == Node; }))
8366 : Seen = true;
8367 : else
8368 46178 : return false;
8369 : }
8370 :
8371 : return Seen;
8372 : }
8373 :
8374 : /// isOperand - Return true if this node is an operand of N.
8375 2655514 : bool SDValue::isOperandOf(const SDNode *N) const {
8376 11683639 : for (const SDValue &Op : N->op_values())
8377 9064607 : if (*this == Op)
8378 : return true;
8379 : return false;
8380 : }
8381 :
8382 201 : bool SDNode::isOperandOf(const SDNode *N) const {
8383 633 : for (const SDValue &Op : N->op_values())
8384 572 : if (this == Op.getNode())
8385 : return true;
8386 : return false;
8387 : }
8388 :
8389 : /// reachesChainWithoutSideEffects - Return true if this operand (which must
8390 : /// be a chain) reaches the specified operand without crossing any
8391 : /// side-effecting instructions on any chain path. In practice, this looks
8392 : /// through token factors and non-volatile loads. In order to remain efficient,
8393 : /// this only looks a couple of nodes in, it does not do an exhaustive search.
8394 : ///
8395 : /// Note that we only need to examine chains when we're searching for
8396 : /// side-effects; SelectionDAG requires that all side-effects are represented
8397 : /// by chains, even if another operand would force a specific ordering. This
8398 : /// constraint is necessary to allow transformations like splitting loads.
8399 2452 : bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
8400 : unsigned Depth) const {
8401 2452 : if (*this == Dest) return true;
8402 :
8403 : // Don't search too deeply, we just want to be able to see through
8404 : // TokenFactor's etc.
8405 1012 : if (Depth == 0) return false;
8406 :
8407 : // If this is a token factor, all inputs to the TF happen in parallel.
8408 977 : if (getOpcode() == ISD::TokenFactor) {
8409 : // First, try a shallow search.
8410 78 : if (is_contained((*this)->ops(), Dest)) {
8411 : // We found the chain we want as an operand of this TokenFactor.
8412 : // Essentially, we reach the chain without side-effects if we could
8413 : // serialize the TokenFactor into a simple chain of operations with
8414 : // Dest as the last operation. This is automatically true if the
8415 : // chain has one use: there are no other ordering constraints.
8416 : // If the chain has more than one use, we give up: some other
8417 : // use of Dest might force a side-effect between Dest and the current
8418 : // node.
8419 44 : if (Dest.hasOneUse())
8420 : return true;
8421 : }
8422 : // Next, try a deep search: check whether every operand of the TokenFactor
8423 : // reaches Dest.
8424 124 : return llvm::all_of((*this)->ops(), [=](SDValue Op) {
8425 0 : return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
8426 : });
8427 : }
8428 :
8429 : // Loads don't have side effects, look through them.
8430 : if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
8431 139 : if (!Ld->isVolatile())
8432 139 : return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
8433 : }
8434 : return false;
8435 : }
8436 :
8437 3675 : bool SDNode::hasPredecessor(const SDNode *N) const {
8438 : SmallPtrSet<const SDNode *, 32> Visited;
8439 : SmallVector<const SDNode *, 16> Worklist;
8440 3675 : Worklist.push_back(this);
8441 3675 : return hasPredecessorHelper(N, Visited, Worklist);
8442 : }
8443 :
8444 852880 : void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
8445 852880 : this->Flags.intersectWith(Flags);
8446 852880 : }
8447 :
8448 : SDValue
8449 87947 : SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
8450 : ArrayRef<ISD::NodeType> CandidateBinOps) {
8451 : // The pattern must end in an extract from index 0.
8452 175894 : if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
8453 175894 : !isNullConstant(Extract->getOperand(1)))
8454 43654 : return SDValue();
8455 :
8456 44293 : SDValue Op = Extract->getOperand(0);
8457 88586 : unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements());
8458 :
8459 : // Match against one of the candidate binary ops.
8460 44293 : if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) {
8461 0 : return Op.getOpcode() == unsigned(BinOp);
8462 : }))
8463 42679 : return SDValue();
8464 :
8465 : // At each stage, we're looking for something that looks like:
8466 : // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
8467 : // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
8468 : // i32 undef, i32 undef, i32 undef, i32 undef>
8469 : // %a = binop <8 x i32> %op, %s
8470 : // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
8471 : // we expect something like:
8472 : // <4,5,6,7,u,u,u,u>
8473 : // <2,3,u,u,u,u,u,u>
8474 : // <1,u,u,u,u,u,u,u>
8475 : unsigned CandidateBinOp = Op.getOpcode();
8476 5285 : for (unsigned i = 0; i < Stages; ++i) {
8477 4277 : if (Op.getOpcode() != CandidateBinOp)
8478 2 : return SDValue();
8479 :
8480 4275 : SDValue Op0 = Op.getOperand(0);
8481 4275 : SDValue Op1 = Op.getOperand(1);
8482 :
8483 : ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Op0);
8484 : if (Shuffle) {
8485 57 : Op = Op1;
8486 : } else {
8487 : Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1);
8488 4218 : Op = Op0;
8489 : }
8490 :
8491 : // The first operand of the shuffle should be the same as the other operand
8492 : // of the binop.
8493 4275 : if (!Shuffle || Shuffle->getOperand(0) != Op)
8494 604 : return SDValue();
8495 :
8496 : // Verify the shuffle has the expected (at this stage of the pyramid) mask.
8497 24300 : for (int Index = 0, MaskEnd = 1 << i; Index < MaskEnd; ++Index)
8498 41258 : if (Shuffle->getMaskElt(Index) != MaskEnd + Index)
8499 0 : return SDValue();
8500 : }
8501 :
8502 1008 : BinOp = (ISD::NodeType)CandidateBinOp;
8503 1008 : return Op;
8504 : }
8505 :
8506 5670 : SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
8507 : assert(N->getNumValues() == 1 &&
8508 : "Can't unroll a vector with multiple results!");
8509 :
8510 11340 : EVT VT = N->getValueType(0);
8511 : unsigned NE = VT.getVectorNumElements();
8512 5670 : EVT EltVT = VT.getVectorElementType();
8513 : SDLoc dl(N);
8514 :
8515 : SmallVector<SDValue, 8> Scalars;
8516 5670 : SmallVector<SDValue, 4> Operands(N->getNumOperands());
8517 :
8518 : // If ResNE is 0, fully unroll the vector op.
8519 5670 : if (ResNE == 0)
8520 : ResNE = NE;
8521 37 : else if (NE > ResNE)
8522 : NE = ResNE;
8523 :
8524 : unsigned i;
8525 22381 : for (i= 0; i != NE; ++i) {
8526 41166 : for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
8527 48910 : SDValue Operand = N->getOperand(j);
8528 24455 : EVT OperandVT = Operand.getValueType();
8529 24455 : if (OperandVT.isVector()) {
8530 : // A vector operand; extract a single element.
8531 23923 : EVT OperandEltVT = OperandVT.getVectorElementType();
8532 23923 : Operands[j] =
8533 23923 : getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand,
8534 23923 : getConstant(i, dl, TLI->getVectorIdxTy(getDataLayout())));
8535 : } else {
8536 : // A scalar operand; just use it as is.
8537 532 : Operands[j] = Operand;
8538 : }
8539 : }
8540 :
8541 33422 : switch (N->getOpcode()) {
8542 16159 : default: {
8543 16159 : Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
8544 32318 : N->getFlags()));
8545 16159 : break;
8546 : }
8547 : case ISD::VSELECT:
8548 52 : Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
8549 52 : break;
8550 : case ISD::SHL:
8551 : case ISD::SRA:
8552 : case ISD::SRL:
8553 : case ISD::ROTL:
8554 : case ISD::ROTR:
8555 412 : Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
8556 : getShiftAmountOperand(Operands[0].getValueType(),
8557 824 : Operands[1])));
8558 412 : break;
8559 : case ISD::SIGN_EXTEND_INREG:
8560 : case ISD::FP_ROUND_INREG: {
8561 88 : EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
8562 88 : Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
8563 : Operands[0],
8564 176 : getValueType(ExtVT)));
8565 : }
8566 : }
8567 : }
8568 :
8569 5719 : for (; i < ResNE; ++i)
8570 49 : Scalars.push_back(getUNDEF(EltVT));
8571 :
8572 5670 : EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
8573 5670 : return getBuildVector(VecVT, dl, Scalars);
8574 : }
8575 :
8576 5803 : bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
8577 : LoadSDNode *Base,
8578 : unsigned Bytes,
8579 : int Dist) const {
8580 5803 : if (LD->isVolatile() || Base->isVolatile())
8581 91 : return false;
8582 5712 : if (LD->isIndexed() || Base->isIndexed())
8583 : return false;
8584 : if (LD->getChain() != Base->getChain())
8585 : return false;
8586 5364 : EVT VT = LD->getValueType(0);
8587 5364 : if (VT.getSizeInBits() / 8 != Bytes)
8588 : return false;
8589 :
8590 5364 : auto BaseLocDecomp = BaseIndexOffset::match(Base, *this);
8591 5364 : auto LocDecomp = BaseIndexOffset::match(LD, *this);
8592 :
8593 5364 : int64_t Offset = 0;
8594 5364 : if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
8595 5102 : return (Dist * Bytes == Offset);
8596 : return false;
8597 : }
8598 :
8599 : /// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if
8600 : /// it cannot be inferred.
8601 7623693 : unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
8602 : // If this is a GlobalAddress + cst, return the alignment.
8603 : const GlobalValue *GV;
8604 7623693 : int64_t GVOffset = 0;
8605 7623693 : if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
8606 2814552 : unsigned IdxWidth = getDataLayout().getIndexTypeSizeInBits(GV->getType());
8607 2819717 : KnownBits Known(IdxWidth);
8608 2814552 : llvm::computeKnownBits(GV, Known, getDataLayout());
8609 2814552 : unsigned AlignBits = Known.countMinTrailingZeros();
8610 2814552 : unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0;
8611 2809387 : if (Align)
8612 5618774 : return MinAlign(Align, GVOffset);
8613 : }
8614 :
8615 : // If this is a direct reference to a stack slot, use information about the
8616 : // stack slot's alignment.
8617 : int FrameIdx = 1 << 31;
8618 : int64_t FrameOffset = 0;
8619 : if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
8620 1734115 : FrameIdx = FI->getIndex();
8621 3080191 : } else if (isBaseWithConstantOffset(Ptr) &&
8622 : isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
8623 : // Handle FI+Cst
8624 245236 : FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
8625 245236 : FrameOffset = Ptr.getConstantOperandVal(1);
8626 : }
8627 :
8628 1979351 : if (FrameIdx != (1 << 31)) {
8629 1979351 : const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
8630 3958702 : unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx),
8631 1979351 : FrameOffset);
8632 1979351 : return FIInfoAlign;
8633 : }
8634 :
8635 : return 0;
8636 : }
8637 :
8638 : /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
8639 : /// which is split (or expanded) into two not necessarily identical pieces.
8640 116767 : std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
8641 : // Currently all types are split in half.
8642 : EVT LoVT, HiVT;
8643 116767 : if (!VT.isVector())
8644 1838 : LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
8645 : else
8646 114929 : LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());
8647 :
8648 116767 : return std::make_pair(LoVT, HiVT);
8649 : }
8650 :
8651 : /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
8652 : /// low/high part.
8653 : std::pair<SDValue, SDValue>
8654 17633 : SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
8655 : const EVT &HiVT) {
8656 : assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <=
8657 : N.getValueType().getVectorNumElements() &&
8658 : "More vector elements requested than available!");
8659 : SDValue Lo, Hi;
8660 17633 : Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
8661 17633 : getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout())));
8662 17633 : Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
8663 : getConstant(LoVT.getVectorNumElements(), DL,
8664 35266 : TLI->getVectorIdxTy(getDataLayout())));
8665 17633 : return std::make_pair(Lo, Hi);
8666 : }
8667 :
8668 28057 : void SelectionDAG::ExtractVectorElements(SDValue Op,
8669 : SmallVectorImpl<SDValue> &Args,
8670 : unsigned Start, unsigned Count) {
8671 28057 : EVT VT = Op.getValueType();
8672 28057 : if (Count == 0)
8673 : Count = VT.getVectorNumElements();
8674 :
8675 28057 : EVT EltVT = VT.getVectorElementType();
8676 28057 : EVT IdxTy = TLI->getVectorIdxTy(getDataLayout());
8677 : SDLoc SL(Op);
8678 135426 : for (unsigned i = Start, e = Start + Count; i != e; ++i) {
8679 214738 : Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
8680 214738 : Op, getConstant(i, SL, IdxTy)));
8681 : }
8682 28057 : }
8683 :
8684 : // getAddressSpace - Return the address space this GlobalAddress belongs to.
8685 8718 : unsigned GlobalAddressSDNode::getAddressSpace() const {
8686 17436 : return getGlobal()->getType()->getAddressSpace();
8687 : }
8688 :
8689 34263 : Type *ConstantPoolSDNode::getType() const {
8690 34263 : if (isMachineConstantPoolEntry())
8691 252 : return Val.MachineCPVal->getType();
8692 34011 : return Val.ConstVal->getType();
8693 : }
8694 :
8695 32900 : bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
8696 : unsigned &SplatBitSize,
8697 : bool &HasAnyUndefs,
8698 : unsigned MinSplatBits,
8699 : bool IsBigEndian) const {
8700 32900 : EVT VT = getValueType(0);
8701 : assert(VT.isVector() && "Expected a vector type");
8702 32900 : unsigned VecWidth = VT.getSizeInBits();
8703 32900 : if (MinSplatBits > VecWidth)
8704 : return false;
8705 :
8706 : // FIXME: The widths are based on this node's type, but build vectors can
8707 : // truncate their operands.
8708 32900 : SplatValue = APInt(VecWidth, 0);
8709 32900 : SplatUndef = APInt(VecWidth, 0);
8710 :
8711 : // Get the bits. Bits with undefined values (when the corresponding element
8712 : // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
8713 : // in SplatValue. If any of the values are not constant, give up and return
8714 : // false.
8715 32900 : unsigned int NumOps = getNumOperands();
8716 : assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
8717 : unsigned EltWidth = VT.getScalarSizeInBits();
8718 :
8719 318980 : for (unsigned j = 0; j < NumOps; ++j) {
8720 293061 : unsigned i = IsBigEndian ? NumOps - 1 - j : j;
8721 293061 : SDValue OpVal = getOperand(i);
8722 293061 : unsigned BitPos = j * EltWidth;
8723 :
8724 293061 : if (OpVal.isUndef())
8725 14089 : SplatUndef.setBits(BitPos, BitPos + EltWidth);
8726 : else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal))
8727 800268 : SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
8728 : else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
8729 15705 : SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
8730 : else
8731 : return false;
8732 : }
8733 :
8734 : // The build_vector is all constants or undefs. Find the smallest element
8735 : // size that splats the vector.
8736 25919 : HasAnyUndefs = (SplatUndef != 0);
8737 :
8738 : // FIXME: This does not work for vectors with elements less than 8 bits.
8739 66810 : while (VecWidth > 8) {
8740 63236 : unsigned HalfSize = VecWidth / 2;
8741 63236 : APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize);
8742 63236 : APInt LowValue = SplatValue.trunc(HalfSize);
8743 63236 : APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize);
8744 63236 : APInt LowUndef = SplatUndef.trunc(HalfSize);
8745 :
8746 : // If the two halves do not match (ignoring undef bits), stop here.
8747 136070 : if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
8748 : MinSplatBits > HalfSize)
8749 : break;
8750 :
8751 40891 : SplatValue = HighValue | LowValue;
8752 40891 : SplatUndef = HighUndef & LowUndef;
8753 :
8754 : VecWidth = HalfSize;
8755 : }
8756 :
8757 25919 : SplatBitSize = VecWidth;
8758 25919 : return true;
8759 : }
8760 :
8761 689930 : SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
8762 689930 : if (UndefElements) {
8763 : UndefElements->clear();
8764 696198 : UndefElements->resize(getNumOperands());
8765 : }
8766 : SDValue Splatted;
8767 4451388 : for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
8768 7747192 : SDValue Op = getOperand(i);
8769 3873596 : if (Op.isUndef()) {
8770 29941 : if (UndefElements)
8771 : (*UndefElements)[i] = true;
8772 3843655 : } else if (!Splatted) {
8773 : Splatted = Op;
8774 : } else if (Splatted != Op) {
8775 112138 : return SDValue();
8776 : }
8777 : }
8778 :
8779 577792 : if (!Splatted) {
8780 : assert(getOperand(0).isUndef() &&
8781 : "Can only have a splat without a constant for all undefs.");
8782 83 : return getOperand(0);
8783 : }
8784 :
8785 577709 : return Splatted;
8786 : }
8787 :
8788 : ConstantSDNode *
8789 321065 : BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
8790 321065 : return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
8791 : }
8792 :
8793 : ConstantFPSDNode *
8794 6882 : BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
8795 6882 : return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
8796 : }
8797 :
8798 : int32_t
8799 55 : BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
8800 : uint32_t BitWidth) const {
8801 : if (ConstantFPSDNode *CN =
8802 56 : dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
8803 : bool IsExact;
8804 54 : APSInt IntVal(BitWidth);
8805 54 : const APFloat &APF = CN->getValueAPF();
8806 54 : if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
8807 54 : APFloat::opOK ||
8808 48 : !IsExact)
8809 : return -1;
8810 :
8811 48 : return IntVal.exactLogBase2();
8812 : }
8813 : return -1;
8814 : }
8815 :
8816 421407 : bool BuildVectorSDNode::isConstant() const {
8817 2144254 : for (const SDValue &Op : op_values()) {
8818 1780984 : unsigned Opc = Op.getOpcode();
8819 1780984 : if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
8820 : return false;
8821 : }
8822 : return true;
8823 : }
8824 :
8825 78283 : bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
8826 : // Find the first non-undef value in the shuffle mask.
8827 : unsigned i, e;
8828 84853 : for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
8829 : /* search */;
8830 :
8831 : assert(i != e && "VECTOR_SHUFFLE node with all undef indices!");
8832 :
8833 : // Make sure all remaining elements are either undef or the same as the first
8834 : // non-undef value.
8835 301145 : for (int Idx = Mask[i]; i != e; ++i)
8836 285993 : if (Mask[i] >= 0 && Mask[i] != Idx)
8837 : return false;
8838 : return true;
8839 : }
8840 :
8841 : // Returns the SDNode if it is a constant integer BuildVector
8842 : // or constant integer.
8843 6393801 : SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) {
8844 : if (isa<ConstantSDNode>(N))
8845 : return N.getNode();
8846 5881037 : if (ISD::isBuildVectorOfConstantSDNodes(N.getNode()))
8847 : return N.getNode();
8848 : // Treat a GlobalAddress supporting constant offset folding as a
8849 : // constant integer.
8850 : if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N))
8851 1074614 : if (GA->getOpcode() == ISD::GlobalAddress &&
8852 537299 : TLI->isOffsetFoldingLegal(GA))
8853 1410 : return GA;
8854 : return nullptr;
8855 : }
8856 :
8857 6162 : SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) {
8858 : if (isa<ConstantFPSDNode>(N))
8859 : return N.getNode();
8860 :
8861 5824 : if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
8862 4 : return N.getNode();
8863 :
8864 : return nullptr;
8865 : }
8866 :
8867 42235002 : void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
8868 : assert(!Node->OperandList && "Node already has operands");
8869 42235002 : SDUse *Ops = OperandRecycler.allocate(
8870 42235002 : ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
8871 :
8872 : bool IsDivergent = false;
8873 186634097 : for (unsigned I = 0; I != Vals.size(); ++I) {
8874 144399095 : Ops[I].setUser(Node);
8875 144399095 : Ops[I].setInitial(Vals[I]);
8876 144399095 : if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence.
8877 108359808 : IsDivergent = IsDivergent || Ops[I].getNode()->isDivergent();
8878 : }
8879 42235002 : Node->NumOperands = Vals.size();
8880 42235002 : Node->OperandList = Ops;
8881 42235002 : IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, DA);
8882 42235002 : if (!TLI->isSDNodeAlwaysUniform(Node))
8883 42120827 : Node->SDNodeBits.IsDivergent = IsDivergent;
8884 42235002 : checkForCycles(Node);
8885 42235002 : }
8886 :
8887 : #ifndef NDEBUG
8888 : static void checkForCyclesHelper(const SDNode *N,
8889 : SmallPtrSetImpl<const SDNode*> &Visited,
8890 : SmallPtrSetImpl<const SDNode*> &Checked,
8891 : const llvm::SelectionDAG *DAG) {
8892 : // If this node has already been checked, don't check it again.
8893 : if (Checked.count(N))
8894 : return;
8895 :
8896 : // If a node has already been visited on this depth-first walk, reject it as
8897 : // a cycle.
8898 : if (!Visited.insert(N).second) {
8899 : errs() << "Detected cycle in SelectionDAG\n";
8900 : dbgs() << "Offending node:\n";
8901 : N->dumprFull(DAG); dbgs() << "\n";
8902 : abort();
8903 : }
8904 :
8905 : for (const SDValue &Op : N->op_values())
8906 : checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
8907 :
8908 : Checked.insert(N);
8909 : Visited.erase(N);
8910 : }
8911 : #endif
8912 :
8913 250893441 : void llvm::checkForCycles(const llvm::SDNode *N,
8914 : const llvm::SelectionDAG *DAG,
8915 : bool force) {
8916 : #ifndef NDEBUG
8917 : bool check = force;
8918 : #ifdef EXPENSIVE_CHECKS
8919 : check = true;
8920 : #endif // EXPENSIVE_CHECKS
8921 : if (check) {
8922 : assert(N && "Checking nonexistent SDNode");
8923 : SmallPtrSet<const SDNode*, 32> visited;
8924 : SmallPtrSet<const SDNode*, 32> checked;
8925 : checkForCyclesHelper(N, visited, checked, DAG);
8926 : }
8927 : #endif // !NDEBUG
8928 250893441 : }
8929 :
8930 20489843 : void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
8931 20489843 : checkForCycles(DAG->getRoot().getNode(), DAG, force);
8932 20489843 : }
|
