LLVM 17.0.0git
SelectionDAG.h
Go to the documentation of this file.
1//===- llvm/CodeGen/SelectionDAG.h - InstSelection DAG ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SelectionDAG class, and transitively defines the
10// SDNode class and subclasses.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CODEGEN_SELECTIONDAG_H
15#define LLVM_CODEGEN_SELECTIONDAG_H
16
17#include "llvm/ADT/APFloat.h"
18#include "llvm/ADT/APInt.h"
19#include "llvm/ADT/ArrayRef.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/DenseSet.h"
22#include "llvm/ADT/FoldingSet.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/ilist.h"
26#include "llvm/ADT/iterator.h"
34#include "llvm/IR/DebugLoc.h"
35#include "llvm/IR/Metadata.h"
42#include <cassert>
43#include <cstdint>
44#include <functional>
45#include <map>
46#include <string>
47#include <tuple>
48#include <utility>
49#include <vector>
50
51namespace llvm {
52
53class DIExpression;
54class DILabel;
55class DIVariable;
56class Function;
57class Pass;
58class Type;
59template <class GraphType> struct GraphTraits;
60template <typename T, unsigned int N> class SmallSetVector;
61template <typename T, typename Enable> struct FoldingSetTrait;
62class AAResults;
63class BlockAddress;
64class BlockFrequencyInfo;
65class Constant;
66class ConstantFP;
67class ConstantInt;
68class DataLayout;
69struct fltSemantics;
70class FunctionLoweringInfo;
71class FunctionVarLocs;
72class GlobalValue;
73struct KnownBits;
74class LegacyDivergenceAnalysis;
75class LLVMContext;
76class MachineBasicBlock;
77class MachineConstantPoolValue;
78class MCSymbol;
79class OptimizationRemarkEmitter;
80class ProfileSummaryInfo;
81class SDDbgValue;
82class SDDbgOperand;
83class SDDbgLabel;
84class SelectionDAG;
85class SelectionDAGTargetInfo;
86class TargetLibraryInfo;
87class TargetLowering;
88class TargetMachine;
89class TargetSubtargetInfo;
90class Value;
91
93 friend struct FoldingSetTrait<SDVTListNode>;
94
95 /// A reference to an Interned FoldingSetNodeID for this node.
96 /// The Allocator in SelectionDAG holds the data.
97 /// SDVTList contains all types which are frequently accessed in SelectionDAG.
98 /// The size of this list is not expected to be big so it won't introduce
99 /// a memory penalty.
100 FoldingSetNodeIDRef FastID;
101 const EVT *VTs;
102 unsigned int NumVTs;
103 /// The hash value for SDVTList is fixed, so cache it to avoid
104 /// hash calculation.
105 unsigned HashValue;
106
107public:
108 SDVTListNode(const FoldingSetNodeIDRef ID, const EVT *VT, unsigned int Num) :
109 FastID(ID), VTs(VT), NumVTs(Num) {
110 HashValue = ID.ComputeHash();
111 }
112
114 SDVTList result = {VTs, NumVTs};
115 return result;
116 }
117};
118
119/// Specialize FoldingSetTrait for SDVTListNode
120/// to avoid computing temp FoldingSetNodeID and hash value.
121template<> struct FoldingSetTrait<SDVTListNode> : DefaultFoldingSetTrait<SDVTListNode> {
122 static void Profile(const SDVTListNode &X, FoldingSetNodeID& ID) {
123 ID = X.FastID;
124 }
125
126 static bool Equals(const SDVTListNode &X, const FoldingSetNodeID &ID,
127 unsigned IDHash, FoldingSetNodeID &TempID) {
128 if (X.HashValue != IDHash)
129 return false;
130 return ID == X.FastID;
131 }
132
133 static unsigned ComputeHash(const SDVTListNode &X, FoldingSetNodeID &TempID) {
134 return X.HashValue;
135 }
136};
137
138template <> struct ilist_alloc_traits<SDNode> {
139 static void deleteNode(SDNode *) {
140 llvm_unreachable("ilist_traits<SDNode> shouldn't see a deleteNode call!");
141 }
142};
143
144/// Keeps track of dbg_value information through SDISel. We do
145/// not build SDNodes for these so as not to perturb the generated code;
146/// instead the info is kept off to the side in this structure. Each SDNode may
147/// have one or more associated dbg_value entries. This information is kept in
148/// DbgValMap.
149/// Byval parameters are handled separately because they don't use alloca's,
150/// which busts the normal mechanism. There is good reason for handling all
151/// parameters separately: they may not have code generated for them, they
152/// should always go at the beginning of the function regardless of other code
153/// motion, and debug info for them is potentially useful even if the parameter
154/// is unused. Right now only byval parameters are handled separately.
156 BumpPtrAllocator Alloc;
158 SmallVector<SDDbgValue*, 32> ByvalParmDbgValues;
161 DbgValMapType DbgValMap;
162
163public:
164 SDDbgInfo() = default;
165 SDDbgInfo(const SDDbgInfo &) = delete;
166 SDDbgInfo &operator=(const SDDbgInfo &) = delete;
167
168 void add(SDDbgValue *V, bool isParameter);
169
170 void add(SDDbgLabel *L) { DbgLabels.push_back(L); }
171
172 /// Invalidate all DbgValues attached to the node and remove
173 /// it from the Node-to-DbgValues map.
174 void erase(const SDNode *Node);
175
176 void clear() {
177 DbgValMap.clear();
178 DbgValues.clear();
179 ByvalParmDbgValues.clear();
180 DbgLabels.clear();
181 Alloc.Reset();
182 }
183
184 BumpPtrAllocator &getAlloc() { return Alloc; }
185
186 bool empty() const {
187 return DbgValues.empty() && ByvalParmDbgValues.empty() && DbgLabels.empty();
188 }
189
191 auto I = DbgValMap.find(Node);
192 if (I != DbgValMap.end())
193 return I->second;
194 return ArrayRef<SDDbgValue*>();
195 }
196
199
200 DbgIterator DbgBegin() { return DbgValues.begin(); }
201 DbgIterator DbgEnd() { return DbgValues.end(); }
202 DbgIterator ByvalParmDbgBegin() { return ByvalParmDbgValues.begin(); }
203 DbgIterator ByvalParmDbgEnd() { return ByvalParmDbgValues.end(); }
204 DbgLabelIterator DbgLabelBegin() { return DbgLabels.begin(); }
205 DbgLabelIterator DbgLabelEnd() { return DbgLabels.end(); }
206};
207
208void checkForCycles(const SelectionDAG *DAG, bool force = false);
209
210/// This is used to represent a portion of an LLVM function in a low-level
211/// Data Dependence DAG representation suitable for instruction selection.
212/// This DAG is constructed as the first step of instruction selection in order
213/// to allow implementation of machine specific optimizations
214/// and code simplifications.
215///
216/// The representation used by the SelectionDAG is a target-independent
217/// representation, which has some similarities to the GCC RTL representation,
218/// but is significantly more simple, powerful, and is a graph form instead of a
219/// linear form.
220///
222 const TargetMachine &TM;
223 const SelectionDAGTargetInfo *TSI = nullptr;
224 const TargetLowering *TLI = nullptr;
225 const TargetLibraryInfo *LibInfo = nullptr;
226 const FunctionVarLocs *FnVarLocs = nullptr;
227 MachineFunction *MF;
228 Pass *SDAGISelPass = nullptr;
229 LLVMContext *Context;
230 CodeGenOpt::Level OptLevel;
231
232 LegacyDivergenceAnalysis * DA = nullptr;
233 FunctionLoweringInfo * FLI = nullptr;
234
235 /// The function-level optimization remark emitter. Used to emit remarks
236 /// whenever manipulating the DAG.
238
239 ProfileSummaryInfo *PSI = nullptr;
240 BlockFrequencyInfo *BFI = nullptr;
241
242 /// List of non-single value types.
243 FoldingSet<SDVTListNode> VTListMap;
244
245 /// Pool allocation for misc. objects that are created once per SelectionDAG.
246 BumpPtrAllocator Allocator;
247
248 /// The starting token.
249 SDNode EntryNode;
250
251 /// The root of the entire DAG.
252 SDValue Root;
253
254 /// A linked list of nodes in the current DAG.
255 ilist<SDNode> AllNodes;
256
257 /// The AllocatorType for allocating SDNodes. We use
258 /// pool allocation with recycling.
260 sizeof(LargestSDNode),
261 alignof(MostAlignedSDNode)>;
262
263 /// Pool allocation for nodes.
264 NodeAllocatorType NodeAllocator;
265
266 /// This structure is used to memoize nodes, automatically performing
267 /// CSE with existing nodes when a duplicate is requested.
268 FoldingSet<SDNode> CSEMap;
269
270 /// Pool allocation for machine-opcode SDNode operands.
271 BumpPtrAllocator OperandAllocator;
272 ArrayRecycler<SDUse> OperandRecycler;
273
274 /// Tracks dbg_value and dbg_label information through SDISel.
275 SDDbgInfo *DbgInfo;
276
279
280 struct NodeExtraInfo {
281 CallSiteInfo CSInfo;
282 MDNode *HeapAllocSite = nullptr;
283 MDNode *PCSections = nullptr;
284 bool NoMerge = false;
285 };
286 /// Out-of-line extra information for SDNodes.
288
289 /// PersistentId counter to be used when inserting the next
290 /// SDNode to this SelectionDAG. We do not place that under
291 /// `#if LLVM_ENABLE_ABI_BREAKING_CHECKS` intentionally because
292 /// it adds unneeded complexity without noticeable
293 /// benefits (see discussion with @thakis in D120714).
294 uint16_t NextPersistentId = 0;
295
296public:
297 /// Clients of various APIs that cause global effects on
298 /// the DAG can optionally implement this interface. This allows the clients
299 /// to handle the various sorts of updates that happen.
300 ///
301 /// A DAGUpdateListener automatically registers itself with DAG when it is
302 /// constructed, and removes itself when destroyed in RAII fashion.
306
308 : Next(D.UpdateListeners), DAG(D) {
309 DAG.UpdateListeners = this;
310 }
311
313 assert(DAG.UpdateListeners == this &&
314 "DAGUpdateListeners must be destroyed in LIFO order");
315 DAG.UpdateListeners = Next;
316 }
317
318 /// The node N that was deleted and, if E is not null, an
319 /// equivalent node E that replaced it.
320 virtual void NodeDeleted(SDNode *N, SDNode *E);
321
322 /// The node N that was updated.
323 virtual void NodeUpdated(SDNode *N);
324
325 /// The node N that was inserted.
326 virtual void NodeInserted(SDNode *N);
327 };
328
330 std::function<void(SDNode *, SDNode *)> Callback;
331
333 std::function<void(SDNode *, SDNode *)> Callback)
335
336 void NodeDeleted(SDNode *N, SDNode *E) override { Callback(N, E); }
337
338 private:
339 virtual void anchor();
340 };
341
343 std::function<void(SDNode *)> Callback;
344
346 std::function<void(SDNode *)> Callback)
348
349 void NodeInserted(SDNode *N) override { Callback(N); }
350
351 private:
352 virtual void anchor();
353 };
354
355 /// Help to insert SDNodeFlags automatically in transforming. Use
356 /// RAII to save and resume flags in current scope.
358 SelectionDAG &DAG;
359 SDNodeFlags Flags;
360 FlagInserter *LastInserter;
361
362 public:
364 : DAG(SDAG), Flags(Flags),
365 LastInserter(SDAG.getFlagInserter()) {
366 SDAG.setFlagInserter(this);
367 }
369 : FlagInserter(SDAG, N->getFlags()) {}
370
371 FlagInserter(const FlagInserter &) = delete;
373 ~FlagInserter() { DAG.setFlagInserter(LastInserter); }
374
375 SDNodeFlags getFlags() const { return Flags; }
376 };
377
378 /// When true, additional steps are taken to
379 /// ensure that getConstant() and similar functions return DAG nodes that
380 /// have legal types. This is important after type legalization since
381 /// any illegally typed nodes generated after this point will not experience
382 /// type legalization.
384
385private:
386 /// DAGUpdateListener is a friend so it can manipulate the listener stack.
387 friend struct DAGUpdateListener;
388
389 /// Linked list of registered DAGUpdateListener instances.
390 /// This stack is maintained by DAGUpdateListener RAII.
391 DAGUpdateListener *UpdateListeners = nullptr;
392
393 /// Implementation of setSubgraphColor.
394 /// Return whether we had to truncate the search.
395 bool setSubgraphColorHelper(SDNode *N, const char *Color,
396 DenseSet<SDNode *> &visited,
397 int level, bool &printed);
398
399 template <typename SDNodeT, typename... ArgTypes>
400 SDNodeT *newSDNode(ArgTypes &&... Args) {
401 return new (NodeAllocator.template Allocate<SDNodeT>())
402 SDNodeT(std::forward<ArgTypes>(Args)...);
403 }
404
405 /// Build a synthetic SDNodeT with the given args and extract its subclass
406 /// data as an integer (e.g. for use in a folding set).
407 ///
408 /// The args to this function are the same as the args to SDNodeT's
409 /// constructor, except the second arg (assumed to be a const DebugLoc&) is
410 /// omitted.
411 template <typename SDNodeT, typename... ArgTypes>
412 static uint16_t getSyntheticNodeSubclassData(unsigned IROrder,
413 ArgTypes &&... Args) {
414 // The compiler can reduce this expression to a constant iff we pass an
415 // empty DebugLoc. Thankfully, the debug location doesn't have any bearing
416 // on the subclass data.
417 return SDNodeT(IROrder, DebugLoc(), std::forward<ArgTypes>(Args)...)
418 .getRawSubclassData();
419 }
420
421 template <typename SDNodeTy>
422 static uint16_t getSyntheticNodeSubclassData(unsigned Opc, unsigned Order,
423 SDVTList VTs, EVT MemoryVT,
424 MachineMemOperand *MMO) {
425 return SDNodeTy(Opc, Order, DebugLoc(), VTs, MemoryVT, MMO)
426 .getRawSubclassData();
427 }
428
429 void createOperands(SDNode *Node, ArrayRef<SDValue> Vals);
430
431 void removeOperands(SDNode *Node) {
432 if (!Node->OperandList)
433 return;
434 OperandRecycler.deallocate(
436 Node->OperandList);
437 Node->NumOperands = 0;
438 Node->OperandList = nullptr;
439 }
440 void CreateTopologicalOrder(std::vector<SDNode*>& Order);
441
442public:
443 // Maximum depth for recursive analysis such as computeKnownBits, etc.
444 static constexpr unsigned MaxRecursionDepth = 6;
445
447 SelectionDAG(const SelectionDAG &) = delete;
450
451 /// Prepare this SelectionDAG to process code in the given MachineFunction.
453 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
455 BlockFrequencyInfo *BFIin, FunctionVarLocs const *FnVarLocs);
456
458 FLI = FuncInfo;
459 }
460
461 /// Clear state and free memory necessary to make this
462 /// SelectionDAG ready to process a new block.
463 void clear();
464
465 MachineFunction &getMachineFunction() const { return *MF; }
466 const Pass *getPass() const { return SDAGISelPass; }
467
468 const DataLayout &getDataLayout() const { return MF->getDataLayout(); }
469 const TargetMachine &getTarget() const { return TM; }
470 const TargetSubtargetInfo &getSubtarget() const { return MF->getSubtarget(); }
471 template <typename STC> const STC &getSubtarget() const {
472 return MF->getSubtarget<STC>();
473 }
474 const TargetLowering &getTargetLoweringInfo() const { return *TLI; }
475 const TargetLibraryInfo &getLibInfo() const { return *LibInfo; }
476 const SelectionDAGTargetInfo &getSelectionDAGInfo() const { return *TSI; }
478 /// Returns the result of the AssignmentTrackingAnalysis pass if it's
479 /// available, otherwise return nullptr.
480 const FunctionVarLocs *getFunctionVarLocs() const { return FnVarLocs; }
481 LLVMContext *getContext() const { return Context; }
482 OptimizationRemarkEmitter &getORE() const { return *ORE; }
483 ProfileSummaryInfo *getPSI() const { return PSI; }
484 BlockFrequencyInfo *getBFI() const { return BFI; }
485
486 FlagInserter *getFlagInserter() { return Inserter; }
487 void setFlagInserter(FlagInserter *FI) { Inserter = FI; }
488
489 /// Just dump dot graph to a user-provided path and title.
490 /// This doesn't open the dot viewer program and
491 /// helps visualization when outside debugging session.
492 /// FileName expects absolute path. If provided
493 /// without any path separators then the file
494 /// will be created in the current directory.
495 /// Error will be emitted if the path is insane.
496#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
497 LLVM_DUMP_METHOD void dumpDotGraph(const Twine &FileName, const Twine &Title);
498#endif
499
500 /// Pop up a GraphViz/gv window with the DAG rendered using 'dot'.
501 void viewGraph(const std::string &Title);
502 void viewGraph();
503
504#if LLVM_ENABLE_ABI_BREAKING_CHECKS
505 std::map<const SDNode *, std::string> NodeGraphAttrs;
506#endif
507
508 /// Clear all previously defined node graph attributes.
509 /// Intended to be used from a debugging tool (eg. gdb).
510 void clearGraphAttrs();
511
512 /// Set graph attributes for a node. (eg. "color=red".)
513 void setGraphAttrs(const SDNode *N, const char *Attrs);
514
515 /// Get graph attributes for a node. (eg. "color=red".)
516 /// Used from getNodeAttributes.
517 std::string getGraphAttrs(const SDNode *N) const;
518
519 /// Convenience for setting node color attribute.
520 void setGraphColor(const SDNode *N, const char *Color);
521
522 /// Convenience for setting subgraph color attribute.
523 void setSubgraphColor(SDNode *N, const char *Color);
524
526
527 allnodes_const_iterator allnodes_begin() const { return AllNodes.begin(); }
528 allnodes_const_iterator allnodes_end() const { return AllNodes.end(); }
529
531
532 allnodes_iterator allnodes_begin() { return AllNodes.begin(); }
533 allnodes_iterator allnodes_end() { return AllNodes.end(); }
534
536 return AllNodes.size();
537 }
538
541 }
544 }
545
546 /// Return the root tag of the SelectionDAG.
547 const SDValue &getRoot() const { return Root; }
548
549 /// Return the token chain corresponding to the entry of the function.
551 return SDValue(const_cast<SDNode *>(&EntryNode), 0);
552 }
553
554 /// Set the current root tag of the SelectionDAG.
555 ///
557 assert((!N.getNode() || N.getValueType() == MVT::Other) &&
558 "DAG root value is not a chain!");
559 if (N.getNode())
560 checkForCycles(N.getNode(), this);
561 Root = N;
562 if (N.getNode())
563 checkForCycles(this);
564 return Root;
565 }
566
567#ifndef NDEBUG
568 void VerifyDAGDivergence();
569#endif
570
571 /// This iterates over the nodes in the SelectionDAG, folding
572 /// certain types of nodes together, or eliminating superfluous nodes. The
573 /// Level argument controls whether Combine is allowed to produce nodes and
574 /// types that are illegal on the target.
575 void Combine(CombineLevel Level, AAResults *AA,
576 CodeGenOpt::Level OptLevel);
577
578 /// This transforms the SelectionDAG into a SelectionDAG that
579 /// only uses types natively supported by the target.
580 /// Returns "true" if it made any changes.
581 ///
582 /// Note that this is an involved process that may invalidate pointers into
583 /// the graph.
584 bool LegalizeTypes();
585
586 /// This transforms the SelectionDAG into a SelectionDAG that is
587 /// compatible with the target instruction selector, as indicated by the
588 /// TargetLowering object.
589 ///
590 /// Note that this is an involved process that may invalidate pointers into
591 /// the graph.
592 void Legalize();
593
594 /// Transforms a SelectionDAG node and any operands to it into a node
595 /// that is compatible with the target instruction selector, as indicated by
596 /// the TargetLowering object.
597 ///
598 /// \returns true if \c N is a valid, legal node after calling this.
599 ///
600 /// This essentially runs a single recursive walk of the \c Legalize process
601 /// over the given node (and its operands). This can be used to incrementally
602 /// legalize the DAG. All of the nodes which are directly replaced,
603 /// potentially including N, are added to the output parameter \c
604 /// UpdatedNodes so that the delta to the DAG can be understood by the
605 /// caller.
606 ///
607 /// When this returns false, N has been legalized in a way that make the
608 /// pointer passed in no longer valid. It may have even been deleted from the
609 /// DAG, and so it shouldn't be used further. When this returns true, the
610 /// N passed in is a legal node, and can be immediately processed as such.
611 /// This may still have done some work on the DAG, and will still populate
612 /// UpdatedNodes with any new nodes replacing those originally in the DAG.
613 bool LegalizeOp(SDNode *N, SmallSetVector<SDNode *, 16> &UpdatedNodes);
614
615 /// This transforms the SelectionDAG into a SelectionDAG
616 /// that only uses vector math operations supported by the target. This is
617 /// necessary as a separate step from Legalize because unrolling a vector
618 /// operation can introduce illegal types, which requires running
619 /// LegalizeTypes again.
620 ///
621 /// This returns true if it made any changes; in that case, LegalizeTypes
622 /// is called again before Legalize.
623 ///
624 /// Note that this is an involved process that may invalidate pointers into
625 /// the graph.
626 bool LegalizeVectors();
627
628 /// This method deletes all unreachable nodes in the SelectionDAG.
629 void RemoveDeadNodes();
630
631 /// Remove the specified node from the system. This node must
632 /// have no referrers.
633 void DeleteNode(SDNode *N);
634
635 /// Return an SDVTList that represents the list of values specified.
637 SDVTList getVTList(EVT VT1, EVT VT2);
638 SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3);
639 SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4);
641
642 //===--------------------------------------------------------------------===//
643 // Node creation methods.
644
645 /// Create a ConstantSDNode wrapping a constant value.
646 /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR.
647 ///
648 /// If only legal types can be produced, this does the necessary
649 /// transformations (e.g., if the vector element type is illegal).
650 /// @{
651 SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
652 bool isTarget = false, bool isOpaque = false);
653 SDValue getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
654 bool isTarget = false, bool isOpaque = false);
655
656 SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget = false,
657 bool IsOpaque = false) {
659 IsTarget, IsOpaque);
660 }
661
662 SDValue getConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT,
663 bool isTarget = false, bool isOpaque = false);
665 bool isTarget = false);
667 bool LegalTypes = true);
669 bool isTarget = false);
670
672 bool isOpaque = false) {
673 return getConstant(Val, DL, VT, true, isOpaque);
674 }
675 SDValue getTargetConstant(const APInt &Val, const SDLoc &DL, EVT VT,
676 bool isOpaque = false) {
677 return getConstant(Val, DL, VT, true, isOpaque);
678 }
680 bool isOpaque = false) {
681 return getConstant(Val, DL, VT, true, isOpaque);
682 }
683
684 /// Create a true or false constant of type \p VT using the target's
685 /// BooleanContent for type \p OpVT.
686 SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT);
687 /// @}
688
689 /// Create a ConstantFPSDNode wrapping a constant value.
690 /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR.
691 ///
692 /// If only legal types can be produced, this does the necessary
693 /// transformations (e.g., if the vector element type is illegal).
694 /// The forms that take a double should only be used for simple constants
695 /// that can be exactly represented in VT. No checks are made.
696 /// @{
697 SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT,
698 bool isTarget = false);
699 SDValue getConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT,
700 bool isTarget = false);
701 SDValue getConstantFP(const ConstantFP &V, const SDLoc &DL, EVT VT,
702 bool isTarget = false);
703 SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT) {
704 return getConstantFP(Val, DL, VT, true);
705 }
706 SDValue getTargetConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT) {
707 return getConstantFP(Val, DL, VT, true);
708 }
710 return getConstantFP(Val, DL, VT, true);
711 }
712 /// @}
713
714 SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT,
715 int64_t offset = 0, bool isTargetGA = false,
716 unsigned TargetFlags = 0);
718 int64_t offset = 0, unsigned TargetFlags = 0) {
719 return getGlobalAddress(GV, DL, VT, offset, true, TargetFlags);
720 }
721 SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false);
723 return getFrameIndex(FI, VT, true);
724 }
725 SDValue getJumpTable(int JTI, EVT VT, bool isTarget = false,
726 unsigned TargetFlags = 0);
727 SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags = 0) {
728 return getJumpTable(JTI, VT, true, TargetFlags);
729 }
731 MaybeAlign Align = std::nullopt, int Offs = 0,
732 bool isT = false, unsigned TargetFlags = 0);
734 MaybeAlign Align = std::nullopt, int Offset = 0,
735 unsigned TargetFlags = 0) {
736 return getConstantPool(C, VT, Align, Offset, true, TargetFlags);
737 }
739 MaybeAlign Align = std::nullopt, int Offs = 0,
740 bool isT = false, unsigned TargetFlags = 0);
742 MaybeAlign Align = std::nullopt, int Offset = 0,
743 unsigned TargetFlags = 0) {
744 return getConstantPool(C, VT, Align, Offset, true, TargetFlags);
745 }
746 SDValue getTargetIndex(int Index, EVT VT, int64_t Offset = 0,
747 unsigned TargetFlags = 0);
748 // When generating a branch to a BB, we don't in general know enough
749 // to provide debug info for the BB at that time, so keep this one around.
751 SDValue getExternalSymbol(const char *Sym, EVT VT);
752 SDValue getTargetExternalSymbol(const char *Sym, EVT VT,
753 unsigned TargetFlags = 0);
755
757 SDValue getRegister(unsigned Reg, EVT VT);
758 SDValue getRegisterMask(const uint32_t *RegMask);
759 SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label);
760 SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root,
761 MCSymbol *Label);
762 SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset = 0,
763 bool isTarget = false, unsigned TargetFlags = 0);
765 int64_t Offset = 0, unsigned TargetFlags = 0) {
766 return getBlockAddress(BA, VT, Offset, true, TargetFlags);
767 }
768
770 SDValue N) {
772 getRegister(Reg, N.getValueType()), N);
773 }
774
775 // This version of the getCopyToReg method takes an extra operand, which
776 // indicates that there is potentially an incoming glue value (if Glue is not
777 // null) and that there should be a glue result.
779 SDValue Glue) {
781 SDValue Ops[] = { Chain, getRegister(Reg, N.getValueType()), N, Glue };
782 return getNode(ISD::CopyToReg, dl, VTs,
783 ArrayRef(Ops, Glue.getNode() ? 4 : 3));
784 }
785
786 // Similar to last getCopyToReg() except parameter Reg is a SDValue
788 SDValue Glue) {
790 SDValue Ops[] = { Chain, Reg, N, Glue };
791 return getNode(ISD::CopyToReg, dl, VTs,
792 ArrayRef(Ops, Glue.getNode() ? 4 : 3));
793 }
794
796 SDVTList VTs = getVTList(VT, MVT::Other);
797 SDValue Ops[] = { Chain, getRegister(Reg, VT) };
798 return getNode(ISD::CopyFromReg, dl, VTs, Ops);
799 }
800
801 // This version of the getCopyFromReg method takes an extra operand, which
802 // indicates that there is potentially an incoming glue value (if Glue is not
803 // null) and that there should be a glue result.
805 SDValue Glue) {
807 SDValue Ops[] = { Chain, getRegister(Reg, VT), Glue };
808 return getNode(ISD::CopyFromReg, dl, VTs,
809 ArrayRef(Ops, Glue.getNode() ? 3 : 2));
810 }
811
813
814 /// Return an ISD::VECTOR_SHUFFLE node. The number of elements in VT,
815 /// which must be a vector type, must match the number of mask elements
816 /// NumElts. An integer mask element equal to -1 is treated as undefined.
818 ArrayRef<int> Mask);
819
820 /// Return an ISD::BUILD_VECTOR node. The number of elements in VT,
821 /// which must be a vector type, must match the number of operands in Ops.
822 /// The operands must have the same type as (or, for integers, a type wider
823 /// than) VT's element type.
825 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
826 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
827 }
828
829 /// Return an ISD::BUILD_VECTOR node. The number of elements in VT,
830 /// which must be a vector type, must match the number of operands in Ops.
831 /// The operands must have the same type as (or, for integers, a type wider
832 /// than) VT's element type.
834 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
835 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
836 }
837
838 /// Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all
839 /// elements. VT must be a vector type. Op's type must be the same as (or,
840 /// for integers, a type wider than) VT's element type.
842 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
843 if (Op.getOpcode() == ISD::UNDEF) {
844 assert((VT.getVectorElementType() == Op.getValueType() ||
845 (VT.isInteger() &&
846 VT.getVectorElementType().bitsLE(Op.getValueType()))) &&
847 "A splatted value must have a width equal or (for integers) "
848 "greater than the vector element type!");
849 return getNode(ISD::UNDEF, SDLoc(), VT);
850 }
851
853 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
854 }
855
856 // Return a splat ISD::SPLAT_VECTOR node, consisting of Op splatted to all
857 // elements.
859 if (Op.getOpcode() == ISD::UNDEF) {
860 assert((VT.getVectorElementType() == Op.getValueType() ||
861 (VT.isInteger() &&
862 VT.getVectorElementType().bitsLE(Op.getValueType()))) &&
863 "A splatted value must have a width equal or (for integers) "
864 "greater than the vector element type!");
865 return getNode(ISD::UNDEF, SDLoc(), VT);
866 }
867 return getNode(ISD::SPLAT_VECTOR, DL, VT, Op);
868 }
869
870 /// Returns a node representing a splat of one value into all lanes
871 /// of the provided vector type. This is a utility which returns
872 /// either a BUILD_VECTOR or SPLAT_VECTOR depending on the
873 /// scalability of the desired vector type.
874 SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op) {
875 assert(VT.isVector() && "Can't splat to non-vector type");
876 return VT.isScalableVector() ?
877 getSplatVector(VT, DL, Op) : getSplatBuildVector(VT, DL, Op);
878 }
879
880 /// Returns a vector of type ResVT whose elements contain the linear sequence
881 /// <0, Step, Step * 2, Step * 3, ...>
882 SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal);
883
884 /// Returns a vector of type ResVT whose elements contain the linear sequence
885 /// <0, 1, 2, 3, ...>
886 SDValue getStepVector(const SDLoc &DL, EVT ResVT);
887
888 /// Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to
889 /// the shuffle node in input but with swapped operands.
890 ///
891 /// Example: shuffle A, B, <0,5,2,7> -> shuffle B, A, <4,1,6,3>
893
894 /// Convert Op, which must be of float type, to the
895 /// float type VT, by either extending or rounding (by truncation).
897
898 /// Convert Op, which must be a STRICT operation of float type, to the
899 /// float type VT, by either extending or rounding (by truncation).
900 std::pair<SDValue, SDValue>
902
903 /// Convert *_EXTEND_VECTOR_INREG to *_EXTEND opcode.
904 static unsigned getOpcode_EXTEND(unsigned Opcode) {
905 switch (Opcode) {
906 case ISD::ANY_EXTEND:
908 return ISD::ANY_EXTEND;
909 case ISD::ZERO_EXTEND:
911 return ISD::ZERO_EXTEND;
912 case ISD::SIGN_EXTEND:
914 return ISD::SIGN_EXTEND;
915 }
916 llvm_unreachable("Unknown opcode");
917 }
918
919 /// Convert *_EXTEND to *_EXTEND_VECTOR_INREG opcode.
920 static unsigned getOpcode_EXTEND_VECTOR_INREG(unsigned Opcode) {
921 switch (Opcode) {
922 case ISD::ANY_EXTEND:
925 case ISD::ZERO_EXTEND:
928 case ISD::SIGN_EXTEND:
931 }
932 llvm_unreachable("Unknown opcode");
933 }
934
935 /// Convert Op, which must be of integer type, to the
936 /// integer type VT, by either any-extending or truncating it.
937 SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
938
939 /// Convert Op, which must be of integer type, to the
940 /// integer type VT, by either sign-extending or truncating it.
941 SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
942
943 /// Convert Op, which must be of integer type, to the
944 /// integer type VT, by either zero-extending or truncating it.
945 SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
946
947 /// Return the expression required to zero extend the Op
948 /// value assuming it was the smaller SrcTy value.
950
951 /// Convert Op, which must be of integer type, to the integer type VT, by
952 /// either truncating it or performing either zero or sign extension as
953 /// appropriate extension for the pointer's semantics.
954 SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
955
956 /// Return the expression required to extend the Op as a pointer value
957 /// assuming it was the smaller SrcTy value. This may be either a zero extend
958 /// or a sign extend.
960
961 /// Convert Op, which must be of integer type, to the integer type VT,
962 /// by using an extension appropriate for the target's
963 /// BooleanContent for type OpVT or truncating it.
964 SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT);
965
966 /// Create negative operation as (SUB 0, Val).
967 SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT);
968
969 /// Create a bitwise NOT operation as (XOR Val, -1).
970 SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT);
971
972 /// Create a logical NOT operation as (XOR Val, BooleanOne).
973 SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT);
974
975 /// Create a vector-predicated logical NOT operation as (VP_XOR Val,
976 /// BooleanOne, Mask, EVL).
977 SDValue getVPLogicalNOT(const SDLoc &DL, SDValue Val, SDValue Mask,
978 SDValue EVL, EVT VT);
979
980 /// Convert a vector-predicated Op, which must be an integer vector, to the
981 /// vector-type VT, by performing either vector-predicated zext or truncating
982 /// it. The Op will be returned as-is if Op and VT are vectors containing
983 /// integer with same width.
984 SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask,
985 SDValue EVL);
986
987 /// Convert a vector-predicated Op, which must be of integer type, to the
988 /// vector-type integer type VT, by either truncating it or performing either
989 /// vector-predicated zero or sign extension as appropriate extension for the
990 /// pointer's semantics. This function just redirects to getVPZExtOrTrunc
991 /// right now.
992 SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask,
993 SDValue EVL);
994
995 /// Returns sum of the base pointer and offset.
996 /// Unlike getObjectPtrOffset this does not set NoUnsignedWrap by default.
998 const SDNodeFlags Flags = SDNodeFlags());
1000 const SDNodeFlags Flags = SDNodeFlags());
1001
1002 /// Create an add instruction with appropriate flags when used for
1003 /// addressing some offset of an object. i.e. if a load is split into multiple
1004 /// components, create an add nuw from the base pointer to the offset.
1006 SDNodeFlags Flags;
1007 Flags.setNoUnsignedWrap(true);
1008 return getMemBasePlusOffset(Ptr, Offset, SL, Flags);
1009 }
1010
1012 // The object itself can't wrap around the address space, so it shouldn't be
1013 // possible for the adds of the offsets to the split parts to overflow.
1014 SDNodeFlags Flags;
1015 Flags.setNoUnsignedWrap(true);
1016 return getMemBasePlusOffset(Ptr, Offset, SL, Flags);
1017 }
1018
1019 /// Return a new CALLSEQ_START node, that starts new call frame, in which
1020 /// InSize bytes are set up inside CALLSEQ_START..CALLSEQ_END sequence and
1021 /// OutSize specifies part of the frame set up prior to the sequence.
1023 const SDLoc &DL) {
1025 SDValue Ops[] = { Chain,
1026 getIntPtrConstant(InSize, DL, true),
1027 getIntPtrConstant(OutSize, DL, true) };
1028 return getNode(ISD::CALLSEQ_START, DL, VTs, Ops);
1029 }
1030
1031 /// Return a new CALLSEQ_END node, which always must have a
1032 /// glue result (to ensure it's not CSE'd).
1033 /// CALLSEQ_END does not have a useful SDLoc.
1035 SDValue InGlue, const SDLoc &DL) {
1038 Ops.push_back(Chain);
1039 Ops.push_back(Op1);
1040 Ops.push_back(Op2);
1041 if (InGlue.getNode())
1042 Ops.push_back(InGlue);
1043 return getNode(ISD::CALLSEQ_END, DL, NodeTys, Ops);
1044 }
1045
1047 SDValue Glue, const SDLoc &DL) {
1048 return getCALLSEQ_END(
1049 Chain, getIntPtrConstant(Size1, DL, /*isTarget=*/true),
1050 getIntPtrConstant(Size2, DL, /*isTarget=*/true), Glue, DL);
1051 }
1052
1053 /// Return true if the result of this operation is always undefined.
1054 bool isUndef(unsigned Opcode, ArrayRef<SDValue> Ops);
1055
1056 /// Return an UNDEF node. UNDEF does not have a useful SDLoc.
1058 return getNode(ISD::UNDEF, SDLoc(), VT);
1059 }
1060
1061 /// Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
1062 SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
1063 assert(MulImm.getMinSignedBits() <= VT.getSizeInBits() &&
1064 "Immediate does not fit VT");
1065 return getNode(ISD::VSCALE, DL, VT,
1066 getConstant(MulImm.sextOrTrunc(VT.getSizeInBits()), DL, VT));
1067 }
1068
1069 /// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
1071 return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT);
1072 }
1073
1074 /// Gets or creates the specified node.
1075 ///
1076 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
1077 ArrayRef<SDUse> Ops);
1078 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
1079 ArrayRef<SDValue> Ops, const SDNodeFlags Flags);
1080 SDValue getNode(unsigned Opcode, const SDLoc &DL, ArrayRef<EVT> ResultTys,
1081 ArrayRef<SDValue> Ops);
1082 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
1083 ArrayRef<SDValue> Ops, const SDNodeFlags Flags);
1084
1085 // Use flags from current flag inserter.
1086 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
1087 ArrayRef<SDValue> Ops);
1088 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
1089 ArrayRef<SDValue> Ops);
1090 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand);
1091 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
1092 SDValue N2);
1093 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
1094 SDValue N2, SDValue N3);
1095
1096 // Specialize based on number of operands.
1097 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT);
1098 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand,
1099 const SDNodeFlags Flags);
1100 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
1101 SDValue N2, const SDNodeFlags Flags);
1102 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
1103 SDValue N2, SDValue N3, const SDNodeFlags Flags);
1104 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
1105 SDValue N2, SDValue N3, SDValue N4);
1106 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
1107 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
1108
1109 // Specialize again based on number of operands for nodes with a VTList
1110 // rather than a single VT.
1111 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList);
1112 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N);
1113 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
1114 SDValue N2);
1115 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
1116 SDValue N2, SDValue N3);
1117 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
1118 SDValue N2, SDValue N3, SDValue N4);
1119 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
1120 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
1121
1122 /// Compute a TokenFactor to force all the incoming stack arguments to be
1123 /// loaded from the stack. This is used in tail call lowering to protect
1124 /// stack arguments from being clobbered.
1126
1128 SDValue Size, Align Alignment, bool isVol,
1129 bool AlwaysInline, bool isTailCall,
1130 MachinePointerInfo DstPtrInfo,
1131 MachinePointerInfo SrcPtrInfo,
1132 const AAMDNodes &AAInfo = AAMDNodes(),
1133 AAResults *AA = nullptr);
1134
1136 SDValue Size, Align Alignment, bool isVol, bool isTailCall,
1137 MachinePointerInfo DstPtrInfo,
1138 MachinePointerInfo SrcPtrInfo,
1139 const AAMDNodes &AAInfo = AAMDNodes(),
1140 AAResults *AA = nullptr);
1141
1143 SDValue Size, Align Alignment, bool isVol,
1144 bool AlwaysInline, bool isTailCall,
1145 MachinePointerInfo DstPtrInfo,
1146 const AAMDNodes &AAInfo = AAMDNodes());
1147
1149 SDValue Src, SDValue Size, Type *SizeTy,
1150 unsigned ElemSz, bool isTailCall,
1151 MachinePointerInfo DstPtrInfo,
1152 MachinePointerInfo SrcPtrInfo);
1153
1155 SDValue Src, SDValue Size, Type *SizeTy,
1156 unsigned ElemSz, bool isTailCall,
1157 MachinePointerInfo DstPtrInfo,
1158 MachinePointerInfo SrcPtrInfo);
1159
1161 SDValue Value, SDValue Size, Type *SizeTy,
1162 unsigned ElemSz, bool isTailCall,
1163 MachinePointerInfo DstPtrInfo);
1164
1165 /// Helper function to make it easier to build SetCC's if you just have an
1166 /// ISD::CondCode instead of an SDValue.
1169 bool IsSignaling = false) {
1170 assert(LHS.getValueType().isVector() == RHS.getValueType().isVector() &&
1171 "Vector/scalar operand type mismatch for setcc");
1172 assert(LHS.getValueType().isVector() == VT.isVector() &&
1173 "Vector/scalar result type mismatch for setcc");
1175 "Cannot create a setCC of an invalid node.");
1176 if (Chain)
1177 return getNode(IsSignaling ? ISD::STRICT_FSETCCS : ISD::STRICT_FSETCC, DL,
1178 {VT, MVT::Other}, {Chain, LHS, RHS, getCondCode(Cond)});
1179 return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond));
1180 }
1181
1182 /// Helper function to make it easier to build VP_SETCCs if you just have an
1183 /// ISD::CondCode instead of an SDValue.
1185 ISD::CondCode Cond, SDValue Mask, SDValue EVL) {
1186 assert(LHS.getValueType().isVector() && RHS.getValueType().isVector() &&
1187 "Cannot compare scalars");
1189 "Cannot create a setCC of an invalid node.");
1190 return getNode(ISD::VP_SETCC, DL, VT, LHS, RHS, getCondCode(Cond), Mask,
1191 EVL);
1192 }
1193
1194 /// Helper function to make it easier to build Select's if you just have
1195 /// operands and don't want to check for vector.
1197 SDValue RHS) {
1198 assert(LHS.getValueType() == VT && RHS.getValueType() == VT &&
1199 "Cannot use select on differing types");
1200 auto Opcode = Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
1201 return getNode(Opcode, DL, VT, Cond, LHS, RHS);
1202 }
1203
1204 /// Helper function to make it easier to build SelectCC's if you just have an
1205 /// ISD::CondCode instead of an SDValue.
1207 SDValue False, ISD::CondCode Cond) {
1208 return getNode(ISD::SELECT_CC, DL, True.getValueType(), LHS, RHS, True,
1209 False, getCondCode(Cond));
1210 }
1211
1212 /// Try to simplify a select/vselect into 1 of its operands or a constant.
1214
1215 /// Try to simplify a shift into 1 of its operands or a constant.
1217
1218 /// Try to simplify a floating-point binary operation into 1 of its operands
1219 /// or a constant.
1220 SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
1221 SDNodeFlags Flags);
1222
1223 /// VAArg produces a result and token chain, and takes a pointer
1224 /// and a source value as input.
1226 SDValue SV, unsigned Align);
1227
1228 /// Gets a node for an atomic cmpxchg op. There are two
1229 /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces the value loaded and a
1230 /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded,
1231 /// a success flag (initially i1), and a chain.
1232 SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT,
1234 SDValue Cmp, SDValue Swp, MachineMemOperand *MMO);
1235
1236 /// Gets a node for an atomic op, produces result (if relevant)
1237 /// and chain and takes 2 operands.
1238 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain,
1240
1241 /// Gets a node for an atomic op, produces result and chain and
1242 /// takes 1 operand.
1243 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, EVT VT,
1245
1246 /// Gets a node for an atomic op, produces result and chain and takes N
1247 /// operands.
1248 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
1249 SDVTList VTList, ArrayRef<SDValue> Ops,
1250 MachineMemOperand *MMO);
1251
1252 /// Creates a MemIntrinsicNode that may produce a
1253 /// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
1254 /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not
1255 /// less than FIRST_TARGET_MEMORY_OPCODE.
1257 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
1261 uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes());
1262
1264 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
1266 MaybeAlign Alignment = std::nullopt,
1269 uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()) {
1270 // Ensure that codegen never sees alignment 0
1271 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, PtrInfo,
1272 Alignment.value_or(getEVTAlign(MemVT)), Flags,
1273 Size, AAInfo);
1274 }
1275
1276 SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList,
1277 ArrayRef<SDValue> Ops, EVT MemVT,
1278 MachineMemOperand *MMO);
1279
1280 /// Creates a LifetimeSDNode that starts (`IsStart==true`) or ends
1281 /// (`IsStart==false`) the lifetime of the portion of `FrameIndex` between
1282 /// offsets `Offset` and `Offset + Size`.
1283 SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain,
1284 int FrameIndex, int64_t Size, int64_t Offset = -1);
1285
1286 /// Creates a PseudoProbeSDNode with function GUID `Guid` and
1287 /// the index of the block `Index` it is probing, as well as the attributes
1288 /// `attr` of the probe.
1290 uint64_t Index, uint32_t Attr);
1291
1292 /// Create a MERGE_VALUES node from the given operands.
1294
1295 /// Loads are not normal binary operators: their result type is not
1296 /// determined by their operands, and they produce a value AND a token chain.
1297 ///
1298 /// This function will set the MOLoad flag on MMOFlags, but you can set it if
1299 /// you want. The MOStore flag must not be set.
1304 const AAMDNodes &AAInfo = AAMDNodes(),
1305 const MDNode *Ranges = nullptr);
1306 /// FIXME: Remove once transition to Align is over.
1307 LLVM_DEPRECATED("Use the getLoad function that takes a MaybeAlign instead",
1308 "")
1314 const MDNode *Ranges = nullptr) {
1316 AAInfo, Ranges);
1317 }
1319 MachineMemOperand *MMO);
1320 SDValue
1321 getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain,
1325 const AAMDNodes &AAInfo = AAMDNodes());
1326 SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT,
1327 SDValue Chain, SDValue Ptr, EVT MemVT,
1328 MachineMemOperand *MMO);
1329 SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base,
1335 const AAMDNodes &AAInfo = AAMDNodes(),
1336 const MDNode *Ranges = nullptr);
1338 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
1342 const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr) {
1343 // Ensures that codegen never sees a None Alignment.
1344 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT,
1345 Alignment.value_or(getEVTAlign(MemVT)), MMOFlags, AAInfo,
1346 Ranges);
1347 }
1348 /// FIXME: Remove once transition to Align is over.
1349 LLVM_DEPRECATED("Use the getLoad function that takes a MaybeAlign instead",
1350 "")
1352 getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
1354 MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment,
1357 const MDNode *Ranges = nullptr) {
1358 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT,
1360 }
1363 EVT MemVT, MachineMemOperand *MMO);
1364
1365 /// Helper function to build ISD::STORE nodes.
1366 ///
1367 /// This function will set the MOStore flag on MMOFlags, but you can set it if
1368 /// you want. The MOLoad and MOInvariant flags must not be set.
1369
1370 SDValue
1374 const AAMDNodes &AAInfo = AAMDNodes());
1375 inline SDValue
1379 const AAMDNodes &AAInfo = AAMDNodes()) {
1380 return getStore(Chain, dl, Val, Ptr, PtrInfo,
1381 Alignment.value_or(getEVTAlign(Val.getValueType())),
1382 MMOFlags, AAInfo);
1383 }
1384 /// FIXME: Remove once transition to Align is over.
1385 LLVM_DEPRECATED("Use the version that takes a MaybeAlign instead", "")
1391 return getStore(Chain, dl, Val, Ptr, PtrInfo, MaybeAlign(Alignment),
1392 MMOFlags, AAInfo);
1393 }
1395 MachineMemOperand *MMO);
1396 SDValue
1400 const AAMDNodes &AAInfo = AAMDNodes());
1401 inline SDValue
1406 const AAMDNodes &AAInfo = AAMDNodes()) {
1407 return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT,
1408 Alignment.value_or(getEVTAlign(SVT)), MMOFlags,
1409 AAInfo);
1410 }
1411 /// FIXME: Remove once transition to Align is over.
1412 LLVM_DEPRECATED("Use the version that takes a MaybeAlign instead", "")
1415 MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment,
1418 return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT,
1420 }
1422 SDValue Ptr, EVT SVT, MachineMemOperand *MMO);
1423 SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base,
1425
1429 EVT MemVT, Align Alignment,
1431 const MDNode *Ranges = nullptr, bool IsExpanding = false);
1432 inline SDValue
1435 SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT,
1438 const AAMDNodes &AAInfo = AAMDNodes(),
1439 const MDNode *Ranges = nullptr, bool IsExpanding = false) {
1440 // Ensures that codegen never sees a None Alignment.
1441 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL,
1442 PtrInfo, MemVT, Alignment.value_or(getEVTAlign(MemVT)),
1443 MMOFlags, AAInfo, Ranges, IsExpanding);
1444 }
1447 SDValue Mask, SDValue EVL, EVT MemVT,
1448 MachineMemOperand *MMO, bool IsExpanding = false);
1452 const AAMDNodes &AAInfo, const MDNode *Ranges = nullptr,
1453 bool IsExpanding = false);
1455 SDValue Mask, SDValue EVL, MachineMemOperand *MMO,
1456 bool IsExpanding = false);
1457 SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT,
1458 SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL,
1461 const AAMDNodes &AAInfo, bool IsExpanding = false);
1462 SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT,
1463 SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL,
1464 EVT MemVT, MachineMemOperand *MMO,
1465 bool IsExpanding = false);
1469 SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT,
1471 bool IsTruncating = false, bool IsCompressing = false);
1473 SDValue Ptr, SDValue Mask, SDValue EVL,
1476 const AAMDNodes &AAInfo, bool IsCompressing = false);
1478 SDValue Ptr, SDValue Mask, SDValue EVL, EVT SVT,
1479 MachineMemOperand *MMO, bool IsCompressing = false);
1482
1484 EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr,
1485 SDValue Offset, SDValue Stride, SDValue Mask,
1488 const AAMDNodes &AAInfo,
1489 const MDNode *Ranges = nullptr,
1490 bool IsExpanding = false);
1492 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
1497 const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr,
1498 bool IsExpanding = false) {
1499 // Ensures that codegen never sees a None Alignment.
1500 return getStridedLoadVP(AM, ExtType, VT, DL, Chain, Ptr, Offset, Stride,
1501 Mask, EVL, PtrInfo, MemVT,
1502 Alignment.value_or(getEVTAlign(MemVT)), MMOFlags,
1503 AAInfo, Ranges, IsExpanding);
1504 }
1506 EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr,
1507 SDValue Offset, SDValue Stride, SDValue Mask,
1508 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
1509 bool IsExpanding = false);
1511 SDValue Stride, SDValue Mask, SDValue EVL,
1514 const AAMDNodes &AAInfo,
1515 const MDNode *Ranges = nullptr,
1516 bool IsExpanding = false);
1518 SDValue Stride, SDValue Mask, SDValue EVL,
1519 MachineMemOperand *MMO, bool IsExpanding = false);
1520 SDValue
1521 getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT,
1522 SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask,
1525 const AAMDNodes &AAInfo, bool IsExpanding = false);
1527 SDValue Chain, SDValue Ptr, SDValue Stride,
1528 SDValue Mask, SDValue EVL, EVT MemVT,
1529 MachineMemOperand *MMO, bool IsExpanding = false);
1534 SDValue Ptr, SDValue Offset, SDValue Stride,
1535 SDValue Mask, SDValue EVL, EVT MemVT,
1537 bool IsTruncating = false,
1538 bool IsCompressing = false);
1540 SDValue Ptr, SDValue Stride, SDValue Mask,
1542 EVT SVT, Align Alignment,
1544 const AAMDNodes &AAInfo,
1545 bool IsCompressing = false);
1547 SDValue Ptr, SDValue Stride, SDValue Mask,
1548 SDValue EVL, EVT SVT, MachineMemOperand *MMO,
1549 bool IsCompressing = false);
1553
1554 SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
1556 ISD::MemIndexType IndexType);
1557 SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
1559 ISD::MemIndexType IndexType);
1560
1562 SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT,
1564 ISD::LoadExtType, bool IsExpanding = false);
1568 SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT,
1570 bool IsTruncating = false, bool IsCompressing = false);
1571 SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
1574 SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
1576 ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy);
1577 SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
1579 ISD::MemIndexType IndexType,
1580 bool IsTruncating = false);
1581
1582 /// Construct a node to track a Value* through the backend.
1583 SDValue getSrcValue(const Value *v);
1584
1585 /// Return an MDNodeSDNode which holds an MDNode.
1586 SDValue getMDNode(const MDNode *MD);
1587
1588 /// Return a bitcast using the SDLoc of the value operand, and casting to the
1589 /// provided type. Use getNode to set a custom SDLoc.
1591
1592 /// Return an AddrSpaceCastSDNode.
1593 SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS,
1594 unsigned DestAS);
1595
1596 /// Return a freeze using the SDLoc of the value operand.
1598
1599 /// Return an AssertAlignSDNode.
1601
1602 /// Swap N1 and N2 if Opcode is a commutative binary opcode
1603 /// and the canonical form expects the opposite order.
1604 void canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
1605 SDValue &N2) const;
1606
1607 /// Return the specified value casted to
1608 /// the target's desired shift amount type.
1610
1611 /// Expand the specified \c ISD::VAARG node as the Legalize pass would.
1613
1614 /// Expand the specified \c ISD::VACOPY node as the Legalize pass would.
1616
1617 /// Returs an GlobalAddress of the function from the current module with
1618 /// name matching the given ExternalSymbol. Additionally can provide the
1619 /// matched function.
1620 /// Panics the function doesn't exists.
1622 Function **TargetFunction = nullptr);
1623
1624 /// *Mutate* the specified node in-place to have the
1625 /// specified operands. If the resultant node already exists in the DAG,
1626 /// this does not modify the specified node, instead it returns the node that
1627 /// already exists. If the resultant node does not exist in the DAG, the
1628 /// input node is returned. As a degenerate case, if you specify the same
1629 /// input operands as the node already has, the input node is returned.
1633 SDValue Op3);
1635 SDValue Op3, SDValue Op4);
1637 SDValue Op3, SDValue Op4, SDValue Op5);
1639
1640 /// Creates a new TokenFactor containing \p Vals. If \p Vals contains 64k
1641 /// values or more, move values into new TokenFactors in 64k-1 blocks, until
1642 /// the final TokenFactor has less than 64k operands.
1644
1645 /// *Mutate* the specified machine node's memory references to the provided
1646 /// list.
1649
1650 // Calculate divergence of node \p N based on its operands.
1652
1653 // Propagates the change in divergence to users
1654 void updateDivergence(SDNode * N);
1655
1656 /// These are used for target selectors to *mutate* the
1657 /// specified node to have the specified return type, Target opcode, and
1658 /// operands. Note that target opcodes are stored as
1659 /// ~TargetOpcode in the node opcode field. The resultant node is returned.
1660 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT);
1661 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, SDValue Op1);
1662 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
1663 SDValue Op1, SDValue Op2);
1664 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
1665 SDValue Op1, SDValue Op2, SDValue Op3);
1666 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
1667 ArrayRef<SDValue> Ops);
1668 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, EVT VT2);
1669 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
1670 EVT VT2, ArrayRef<SDValue> Ops);
1671 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
1672 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops);
1673 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
1674 EVT VT2, SDValue Op1, SDValue Op2);
1675 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, SDVTList VTs,
1676 ArrayRef<SDValue> Ops);
1677
1678 /// This *mutates* the specified node to have the specified
1679 /// return type, opcode, and operands.
1680 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs,
1681 ArrayRef<SDValue> Ops);
1682
1683 /// Mutate the specified strict FP node to its non-strict equivalent,
1684 /// unlinking the node from its chain and dropping the metadata arguments.
1685 /// The node must be a strict FP node.
1687
1688 /// These are used for target selectors to create a new node
1689 /// with specified return type(s), MachineInstr opcode, and operands.
1690 ///
1691 /// Note that getMachineNode returns the resultant node. If there is already
1692 /// a node of the specified opcode and operands, it returns that node instead
1693 /// of the current one.
1694 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT);
1695 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1696 SDValue Op1);
1697 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1698 SDValue Op1, SDValue Op2);
1699 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1700 SDValue Op1, SDValue Op2, SDValue Op3);
1701 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1702 ArrayRef<SDValue> Ops);
1703 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1704 EVT VT2, SDValue Op1, SDValue Op2);
1705 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1706 EVT VT2, SDValue Op1, SDValue Op2, SDValue Op3);
1707 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1708 EVT VT2, ArrayRef<SDValue> Ops);
1709 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1710 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2);
1711 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1712 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2,
1713 SDValue Op3);
1714 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1715 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops);
1716 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl,
1717 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops);
1718 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, SDVTList VTs,
1719 ArrayRef<SDValue> Ops);
1720
1721 /// A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
1722 SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
1723 SDValue Operand);
1724
1725 /// A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
1726 SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
1727 SDValue Operand, SDValue Subreg);
1728
1729 /// Get the specified node if it's already available, or else return NULL.
1730 SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList,
1731 ArrayRef<SDValue> Ops, const SDNodeFlags Flags);
1732 SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList,
1733 ArrayRef<SDValue> Ops);
1734
1735 /// Check if a node exists without modifying its flags.
1736 bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef<SDValue> Ops);
1737
1738 /// Creates a SDDbgValue node.
1740 unsigned R, bool IsIndirect, const DebugLoc &DL,
1741 unsigned O);
1742
1743 /// Creates a constant SDDbgValue node.
1745 const Value *C, const DebugLoc &DL,
1746 unsigned O);
1747
1748 /// Creates a FrameIndex SDDbgValue node.
1750 unsigned FI, bool IsIndirect,
1751 const DebugLoc &DL, unsigned O);
1752
1753 /// Creates a FrameIndex SDDbgValue node.
1755 unsigned FI,
1756 ArrayRef<SDNode *> Dependencies,
1757 bool IsIndirect, const DebugLoc &DL,
1758 unsigned O);
1759
1760 /// Creates a VReg SDDbgValue node.
1762 unsigned VReg, bool IsIndirect,
1763 const DebugLoc &DL, unsigned O);
1764
1765 /// Creates a SDDbgValue node from a list of locations.
1768 ArrayRef<SDNode *> Dependencies, bool IsIndirect,
1769 const DebugLoc &DL, unsigned O, bool IsVariadic);
1770
1771 /// Creates a SDDbgLabel node.
1772 SDDbgLabel *getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O);
1773
1774 /// Transfer debug values from one node to another, while optionally
1775 /// generating fragment expressions for split-up values. If \p InvalidateDbg
1776 /// is set, debug values are invalidated after they are transferred.
1777 void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits = 0,
1778 unsigned SizeInBits = 0, bool InvalidateDbg = true);
1779
1780 /// Remove the specified node from the system. If any of its
1781 /// operands then becomes dead, remove them as well. Inform UpdateListener
1782 /// for each node deleted.
1783 void RemoveDeadNode(SDNode *N);
1784
1785 /// This method deletes the unreachable nodes in the
1786 /// given list, and any nodes that become unreachable as a result.
1788
1789 /// Modify anything using 'From' to use 'To' instead.
1790 /// This can cause recursive merging of nodes in the DAG. Use the first
1791 /// version if 'From' is known to have a single result, use the second
1792 /// if you have two nodes with identical results (or if 'To' has a superset
1793 /// of the results of 'From'), use the third otherwise.
1794 ///
1795 /// These methods all take an optional UpdateListener, which (if not null) is
1796 /// informed about nodes that are deleted and modified due to recursive
1797 /// changes in the dag.
1798 ///
1799 /// These functions only replace all existing uses. It's possible that as
1800 /// these replacements are being performed, CSE may cause the From node
1801 /// to be given new uses. These new uses of From are left in place, and
1802 /// not automatically transferred to To.
1803 ///
1806 void ReplaceAllUsesWith(SDNode *From, const SDValue *To);
1807
1808 /// Replace any uses of From with To, leaving
1809 /// uses of other values produced by From.getNode() alone.
1811
1812 /// Like ReplaceAllUsesOfValueWith, but for multiple values at once.
1813 /// This correctly handles the case where
1814 /// there is an overlap between the From values and the To values.
1815 void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To,
1816 unsigned Num);
1817
1818 /// If an existing load has uses of its chain, create a token factor node with
1819 /// that chain and the new memory node's chain and update users of the old
1820 /// chain to the token factor. This ensures that the new memory node will have
1821 /// the same relative memory dependency position as the old load. Returns the
1822 /// new merged load chain.
1823 SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain);
1824
1825 /// If an existing load has uses of its chain, create a token factor node with
1826 /// that chain and the new memory node's chain and update users of the old
1827 /// chain to the token factor. This ensures that the new memory node will have
1828 /// the same relative memory dependency position as the old load. Returns the
1829 /// new merged load chain.
1831
1832 /// Topological-sort the AllNodes list and a
1833 /// assign a unique node id for each node in the DAG based on their
1834 /// topological order. Returns the number of nodes.
1835 unsigned AssignTopologicalOrder();
1836
1837 /// Move node N in the AllNodes list to be immediately
1838 /// before the given iterator Position. This may be used to update the
1839 /// topological ordering when the list of nodes is modified.
1841 AllNodes.insert(Position, AllNodes.remove(N));
1842 }
1843
1844 /// Returns an APFloat semantics tag appropriate for the given type. If VT is
1845 /// a vector type, the element semantics are returned.
1847 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
1848 default: llvm_unreachable("Unknown FP format");
1849 case MVT::f16: return APFloat::IEEEhalf();
1850 case MVT::bf16: return APFloat::BFloat();
1851 case MVT::f32: return APFloat::IEEEsingle();
1852 case MVT::f64: return APFloat::IEEEdouble();
1853 case MVT::f80: return APFloat::x87DoubleExtended();
1854 case MVT::f128: return APFloat::IEEEquad();
1856 }
1857 }
1858
1859 /// Add a dbg_value SDNode. If SD is non-null that means the
1860 /// value is produced by SD.
1861 void AddDbgValue(SDDbgValue *DB, bool isParameter);
1862
1863 /// Add a dbg_label SDNode.
1864 void AddDbgLabel(SDDbgLabel *DB);
1865
1866 /// Get the debug values which reference the given SDNode.
1868 return DbgInfo->getSDDbgValues(SD);
1869 }
1870
1871public:
1872 /// Return true if there are any SDDbgValue nodes associated
1873 /// with this SelectionDAG.
1874 bool hasDebugValues() const { return !DbgInfo->empty(); }
1875
1876 SDDbgInfo::DbgIterator DbgBegin() const { return DbgInfo->DbgBegin(); }
1877 SDDbgInfo::DbgIterator DbgEnd() const { return DbgInfo->DbgEnd(); }
1878
1880 return DbgInfo->ByvalParmDbgBegin();
1881 }
1883 return DbgInfo->ByvalParmDbgEnd();
1884 }
1885
1887 return DbgInfo->DbgLabelBegin();
1888 }
1890 return DbgInfo->DbgLabelEnd();
1891 }
1892
1893 /// To be invoked on an SDNode that is slated to be erased. This
1894 /// function mirrors \c llvm::salvageDebugInfo.
1895 void salvageDebugInfo(SDNode &N);
1896
1897 void dump() const;
1898
1899 /// In most cases this function returns the ABI alignment for a given type,
1900 /// except for illegal vector types where the alignment exceeds that of the
1901 /// stack. In such cases we attempt to break the vector down to a legal type
1902 /// and return the ABI alignment for that instead.
1903 Align getReducedAlign(EVT VT, bool UseABI);
1904
1905 /// Create a stack temporary based on the size in bytes and the alignment
1907
1908 /// Create a stack temporary, suitable for holding the specified value type.
1909 /// If minAlign is specified, the slot size will have at least that alignment.
1910 SDValue CreateStackTemporary(EVT VT, unsigned minAlign = 1);
1911
1912 /// Create a stack temporary suitable for holding either of the specified
1913 /// value types.
1915
1916 SDValue FoldSymbolOffset(unsigned Opcode, EVT VT,
1917 const GlobalAddressSDNode *GA,
1918 const SDNode *N2);
1919
1920 SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
1921 ArrayRef<SDValue> Ops);
1922
1923 /// Fold floating-point operations with 2 operands when both operands are
1924 /// constants and/or undefined.
1925 SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT,
1926 SDValue N1, SDValue N2);
1927
1928 /// Constant fold a setcc to true or false.
1930 const SDLoc &dl);
1931
1932 /// Return true if the sign bit of Op is known to be zero.
1933 /// We use this predicate to simplify operations downstream.
1934 bool SignBitIsZero(SDValue Op, unsigned Depth = 0) const;
1935
1936 /// Return true if 'Op & Mask' is known to be zero. We
1937 /// use this predicate to simplify operations downstream. Op and Mask are
1938 /// known to be the same type.
1939 bool MaskedValueIsZero(SDValue Op, const APInt &Mask,
1940 unsigned Depth = 0) const;
1941
1942 /// Return true if 'Op & Mask' is known to be zero in DemandedElts. We
1943 /// use this predicate to simplify operations downstream. Op and Mask are
1944 /// known to be the same type.
1945 bool MaskedValueIsZero(SDValue Op, const APInt &Mask,
1946 const APInt &DemandedElts, unsigned Depth = 0) const;
1947
1948 /// Return true if 'Op' is known to be zero in DemandedElts. We
1949 /// use this predicate to simplify operations downstream.
1950 bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts,
1951 unsigned Depth = 0) const;
1952
1953 /// Return true if '(Op & Mask) == Mask'.
1954 /// Op and Mask are known to be the same type.
1955 bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask,
1956 unsigned Depth = 0) const;
1957
1958 /// For each demanded element of a vector, see if it is known to be zero.
1959 APInt computeVectorKnownZeroElements(SDValue Op, const APInt &DemandedElts,
1960 unsigned Depth = 0) const;
1961
1962 /// Determine which bits of Op are known to be either zero or one and return
1963 /// them in Known. For vectors, the known bits are those that are shared by
1964 /// every vector element.
1965 /// Targets can implement the computeKnownBitsForTargetNode method in the
1966 /// TargetLowering class to allow target nodes to be understood.
1967 KnownBits computeKnownBits(SDValue Op, unsigned Depth = 0) const;
1968
1969 /// Determine which bits of Op are known to be either zero or one and return
1970 /// them in Known. The DemandedElts argument allows us to only collect the
1971 /// known bits that are shared by the requested vector elements.
1972 /// Targets can implement the computeKnownBitsForTargetNode method in the
1973 /// TargetLowering class to allow target nodes to be understood.
1974 KnownBits computeKnownBits(SDValue Op, const APInt &DemandedElts,
1975 unsigned Depth = 0) const;
1976
1977 /// Used to represent the possible overflow behavior of an operation.
1978 /// Never: the operation cannot overflow.
1979 /// Always: the operation will always overflow.
1980 /// Sometime: the operation may or may not overflow.
1985 };
1986
1987 /// Determine if the result of the addition of 2 node can overflow.
1989
1990 /// Test if the given value is known to have exactly one bit set. This differs
1991 /// from computeKnownBits in that it doesn't necessarily determine which bit
1992 /// is set.
1993 bool isKnownToBeAPowerOfTwo(SDValue Val) const;
1994
1995 /// Return the number of times the sign bit of the register is replicated into
1996 /// the other bits. We know that at least 1 bit is always equal to the sign
1997 /// bit (itself), but other cases can give us information. For example,
1998 /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal
1999 /// to each other, so we return 3. Targets can implement the
2000 /// ComputeNumSignBitsForTarget method in the TargetLowering class to allow
2001 /// target nodes to be understood.
2002 unsigned ComputeNumSignBits(SDValue Op, unsigned Depth = 0) const;
2003
2004 /// Return the number of times the sign bit of the register is replicated into
2005 /// the other bits. We know that at least 1 bit is always equal to the sign
2006 /// bit (itself), but other cases can give us information. For example,
2007 /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal
2008 /// to each other, so we return 3. The DemandedElts argument allows
2009 /// us to only collect the minimum sign bits of the requested vector elements.
2010 /// Targets can implement the ComputeNumSignBitsForTarget method in the
2011 /// TargetLowering class to allow target nodes to be understood.
2012 unsigned ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
2013 unsigned Depth = 0) const;
2014
2015 /// Get the upper bound on bit size for this Value \p Op as a signed integer.
2016 /// i.e. x == sext(trunc(x to MaxSignedBits) to bitwidth(x)).
2017 /// Similar to the APInt::getSignificantBits function.
2018 /// Helper wrapper to ComputeNumSignBits.
2019 unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth = 0) const;
2020
2021 /// Get the upper bound on bit size for this Value \p Op as a signed integer.
2022 /// i.e. x == sext(trunc(x to MaxSignedBits) to bitwidth(x)).
2023 /// Similar to the APInt::getSignificantBits function.
2024 /// Helper wrapper to ComputeNumSignBits.
2025 unsigned ComputeMaxSignificantBits(SDValue Op, const APInt &DemandedElts,
2026 unsigned Depth = 0) const;
2027
2028 /// Return true if this function can prove that \p Op is never poison
2029 /// and, if \p PoisonOnly is false, does not have undef bits.
2030 bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly = false,
2031 unsigned Depth = 0) const;
2032
2033 /// Return true if this function can prove that \p Op is never poison
2034 /// and, if \p PoisonOnly is false, does not have undef bits. The DemandedElts
2035 /// argument limits the check to the requested vector elements.
2036 bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, const APInt &DemandedElts,
2037 bool PoisonOnly = false,
2038 unsigned Depth = 0) const;
2039
2040 /// Return true if this function can prove that \p Op is never poison.
2041 bool isGuaranteedNotToBePoison(SDValue Op, unsigned Depth = 0) const {
2042 return isGuaranteedNotToBeUndefOrPoison(Op, /*PoisonOnly*/ true, Depth);
2043 }
2044
2045 /// Return true if this function can prove that \p Op is never poison. The
2046 /// DemandedElts argument limits the check to the requested vector elements.
2047 bool isGuaranteedNotToBePoison(SDValue Op, const APInt &DemandedElts,
2048 unsigned Depth = 0) const {
2049 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts,
2050 /*PoisonOnly*/ true, Depth);
2051 }
2052
2053 /// Return true if Op can create undef or poison from non-undef & non-poison
2054 /// operands. The DemandedElts argument limits the check to the requested
2055 /// vector elements.
2056 ///
2057 /// \p ConsiderFlags controls whether poison producing flags on the
2058 /// instruction are considered. This can be used to see if the instruction
2059 /// could still introduce undef or poison even without poison generating flags
2060 /// which might be on the instruction. (i.e. could the result of
2061 /// Op->dropPoisonGeneratingFlags() still create poison or undef)
2062 bool canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
2063 bool PoisonOnly = false,
2064 bool ConsiderFlags = true,
2065 unsigned Depth = 0) const;
2066
2067 /// Return true if Op can create undef or poison from non-undef & non-poison
2068 /// operands.
2069 ///
2070 /// \p ConsiderFlags controls whether poison producing flags on the
2071 /// instruction are considered. This can be used to see if the instruction
2072 /// could still introduce undef or poison even without poison generating flags
2073 /// which might be on the instruction. (i.e. could the result of
2074 /// Op->dropPoisonGeneratingFlags() still create poison or undef)
2075 bool canCreateUndefOrPoison(SDValue Op, bool PoisonOnly = false,
2076 bool ConsiderFlags = true,
2077 unsigned Depth = 0) const;
2078
2079 /// Return true if the specified operand is an ISD::ADD with a ConstantSDNode
2080 /// on the right-hand side, or if it is an ISD::OR with a ConstantSDNode that
2081 /// is guaranteed to have the same semantics as an ADD. This handles the
2082 /// equivalence:
2083 /// X|Cst == X+Cst iff X&Cst = 0.
2084 bool isBaseWithConstantOffset(SDValue Op) const;
2085
2086 /// Test whether the given SDValue (or all elements of it, if it is a
2087 /// vector) is known to never be NaN. If \p SNaN is true, returns if \p Op is
2088 /// known to never be a signaling NaN (it may still be a qNaN).
2089 bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const;
2090
2091 /// \returns true if \p Op is known to never be a signaling NaN.
2092 bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const {
2093 return isKnownNeverNaN(Op, true, Depth);
2094 }
2095
2096 /// Test whether the given floating point SDValue is known to never be
2097 /// positive or negative zero.
2098 bool isKnownNeverZeroFloat(SDValue Op) const;
2099
2100 /// Test whether the given SDValue is known to contain non-zero value(s).
2101 bool isKnownNeverZero(SDValue Op) const;
2102
2103 /// Test whether two SDValues are known to compare equal. This
2104 /// is true if they are the same value, or if one is negative zero and the
2105 /// other positive zero.
2106 bool isEqualTo(SDValue A, SDValue B) const;
2107
2108 /// Return true if A and B have no common bits set. As an example, this can
2109 /// allow an 'add' to be transformed into an 'or'.
2110 bool haveNoCommonBitsSet(SDValue A, SDValue B) const;
2111
2112 /// Test whether \p V has a splatted value for all the demanded elements.
2113 ///
2114 /// On success \p UndefElts will indicate the elements that have UNDEF
2115 /// values instead of the splat value, this is only guaranteed to be correct
2116 /// for \p DemandedElts.
2117 ///
2118 /// NOTE: The function will return true for a demanded splat of UNDEF values.
2119 bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts,
2120 unsigned Depth = 0) const;
2121
2122 /// Test whether \p V has a splatted value.
2123 bool isSplatValue(SDValue V, bool AllowUndefs = false) const;
2124
2125 /// If V is a splatted value, return the source vector and its splat index.
2126 SDValue getSplatSourceVector(SDValue V, int &SplatIndex);
2127
2128 /// If V is a splat vector, return its scalar source operand by extracting
2129 /// that element from the source vector. If LegalTypes is true, this method
2130 /// may only return a legally-typed splat value. If it cannot legalize the
2131 /// splatted value it will return SDValue().
2132 SDValue getSplatValue(SDValue V, bool LegalTypes = false);
2133
2134 /// If a SHL/SRA/SRL node \p V has a constant or splat constant shift amount
2135 /// that is less than the element bit-width of the shift node, return it.
2137 const APInt &DemandedElts) const;
2138
2139 /// If a SHL/SRA/SRL node \p V has constant shift amounts that are all less
2140 /// than the element bit-width of the shift node, return the minimum value.
2141 const APInt *
2143 const APInt &DemandedElts) const;
2144
2145 /// If a SHL/SRA/SRL node \p V has constant shift amounts that are all less
2146 /// than the element bit-width of the shift node, return the maximum value.
2147 const APInt *
2149 const APInt &DemandedElts) const;
2150
2151 /// Match a binop + shuffle pyramid that represents a horizontal reduction
2152 /// over the elements of a vector starting from the EXTRACT_VECTOR_ELT node /p
2153 /// Extract. The reduction must use one of the opcodes listed in /p
2154 /// CandidateBinOps and on success /p BinOp will contain the matching opcode.
2155 /// Returns the vector that is being reduced on, or SDValue() if a reduction
2156 /// was not matched. If \p AllowPartials is set then in the case of a
2157 /// reduction pattern that only matches the first few stages, the extracted
2158 /// subvector of the start of the reduction is returned.
2160 ArrayRef<ISD::NodeType> CandidateBinOps,
2161 bool AllowPartials = false);
2162
2163 /// Utility function used by legalize and lowering to
2164 /// "unroll" a vector operation by splitting out the scalars and operating
2165 /// on each element individually. If the ResNE is 0, fully unroll the vector
2166 /// op. If ResNE is less than the width of the vector op, unroll up to ResNE.
2167 /// If the ResNE is greater than the width of the vector op, unroll the
2168 /// vector op and fill the end of the resulting vector with UNDEFS.
2169 SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0);
2170
2171 /// Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes.
2172 /// This is a separate function because those opcodes have two results.
2173 std::pair<SDValue, SDValue> UnrollVectorOverflowOp(SDNode *N,
2174 unsigned ResNE = 0);
2175
2176 /// Return true if loads are next to each other and can be
2177 /// merged. Check that both are nonvolatile and if LD is loading
2178 /// 'Bytes' bytes from a location that is 'Dist' units away from the
2179 /// location that the 'Base' load is loading from.
2181 unsigned Bytes, int Dist) const;
2182
2183 /// Infer alignment of a load / store address. Return std::nullopt if it
2184 /// cannot be inferred.
2186
2187 /// Compute the VTs needed for the low/hi parts of a type
2188 /// which is split (or expanded) into two not necessarily identical pieces.
2189 std::pair<EVT, EVT> GetSplitDestVTs(const EVT &VT) const;
2190
2191 /// Compute the VTs needed for the low/hi parts of a type, dependent on an
2192 /// enveloping VT that has been split into two identical pieces. Sets the
2193 /// HisIsEmpty flag when hi type has zero storage size.
2194 std::pair<EVT, EVT> GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
2195 bool *HiIsEmpty) const;
2196
2197 /// Split the vector with EXTRACT_SUBVECTOR using the provides
2198 /// VTs and return the low/high part.
2199 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL,
2200 const EVT &LoVT, const EVT &HiVT);
2201
2202 /// Split the vector with EXTRACT_SUBVECTOR and return the low/high part.
2203 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) {
2204 EVT LoVT, HiVT;
2205 std::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType());
2206 return SplitVector(N, DL, LoVT, HiVT);
2207 }
2208
2209 /// Split the explicit vector length parameter of a VP operation.
2210 std::pair<SDValue, SDValue> SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL);
2211
2212 /// Split the node's operand with EXTRACT_SUBVECTOR and
2213 /// return the low/high part.
2214 std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo)
2215 {
2216 return SplitVector(N->getOperand(OpNo), SDLoc(N));
2217 }
2218
2219 /// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
2220 SDValue WidenVector(const SDValue &N, const SDLoc &DL);
2221
2222 /// Append the extracted elements from Start to Count out of the vector Op in
2223 /// Args. If Count is 0, all of the elements will be extracted. The extracted
2224 /// elements will have type EVT if it is provided, and otherwise their type
2225 /// will be Op's element type.
2227 unsigned Start = 0, unsigned Count = 0,
2228 EVT EltVT = EVT());
2229
2230 /// Compute the default alignment value for the given type.
2231 Align getEVTAlign(EVT MemoryVT) const;
2232
2233 /// Test whether the given value is a constant int or similar node.
2235
2236 /// Test whether the given value is a constant FP or similar node.
2238
2239 /// \returns true if \p N is any kind of constant or build_vector of
2240 /// constants, int or float. If a vector, it may not necessarily be a splat.
2244 }
2245
2246 /// Set CallSiteInfo to be associated with Node.
2248 SDEI[Node].CSInfo = std::move(CallInfo);
2249 }
2250 /// Return CallSiteInfo associated with Node, or a default if none exists.
2252 auto I = SDEI.find(Node);
2253 return I != SDEI.end() ? std::move(I->second).CSInfo : CallSiteInfo();
2254 }
2255 /// Set HeapAllocSite to be associated with Node.
2257 SDEI[Node].HeapAllocSite = MD;
2258 }
2259 /// Return HeapAllocSite associated with Node, or nullptr if none exists.
2261 auto I = SDEI.find(Node);
2262 return I != SDEI.end() ? I->second.HeapAllocSite : nullptr;
2263 }
2264 /// Set PCSections to be associated with Node.
2265 void addPCSections(const SDNode *Node, MDNode *MD) {
2266 SDEI[Node].PCSections = MD;
2267 }
2268 /// Return PCSections associated with Node, or nullptr if none exists.
2270 auto It = SDEI.find(Node);
2271 return It != SDEI.end() ? It->second.PCSections : nullptr;
2272 }
2273 /// Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
2274 void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge) {
2275 if (NoMerge)
2276 SDEI[Node].NoMerge = NoMerge;
2277 }
2278 /// Return NoMerge info associated with Node.
2279 bool getNoMergeSiteInfo(const SDNode *Node) const {
2280 auto I = SDEI.find(Node);
2281 return I != SDEI.end() ? I->second.NoMerge : false;
2282 }
2283
2284 /// Copy extra info associated with one node to another.
2285 void copyExtraInfo(SDNode *From, SDNode *To);
2286
2287 /// Return the current function's default denormal handling kind for the given
2288 /// floating point type.
2290 return MF->getDenormalMode(EVTToAPFloatSemantics(VT));
2291 }
2292
2293 bool shouldOptForSize() const;
2294
2295 /// Get the (commutative) neutral element for the given opcode, if it exists.
2296 SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT,
2297 SDNodeFlags Flags);
2298
2299 /// Some opcodes may create immediate undefined behavior when used with some
2300 /// values (integer division-by-zero for example). Therefore, these operations
2301 /// are not generally safe to move around or change.
2302 bool isSafeToSpeculativelyExecute(unsigned Opcode) const {
2303 switch (Opcode) {
2304 case ISD::SDIV:
2305 case ISD::SREM:
2306 case ISD::SDIVREM:
2307 case ISD::UDIV:
2308 case ISD::UREM:
2309 case ISD::UDIVREM:
2310 return false;
2311 default:
2312 return true;
2313 }
2314 }
2315
2316private:
2317 void InsertNode(SDNode *N);
2318 bool RemoveNodeFromCSEMaps(SDNode *N);
2319 void AddModifiedNodeToCSEMaps(SDNode *N);
2320 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op, void *&InsertPos);
2321 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
2322 void *&InsertPos);
2323 SDNode *FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
2324 void *&InsertPos);
2325 SDNode *UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &loc);
2326
2327 void DeleteNodeNotInCSEMaps(SDNode *N);
2328 void DeallocateNode(SDNode *N);
2329
2330 void allnodes_clear();
2331
2332 /// Look up the node specified by ID in CSEMap. If it exists, return it. If
2333 /// not, return the insertion token that will make insertion faster. This
2334 /// overload is for nodes other than Constant or ConstantFP, use the other one
2335 /// for those.
2336 SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, void *&InsertPos);
2337
2338 /// Look up the node specified by ID in CSEMap. If it exists, return it. If
2339 /// not, return the insertion token that will make insertion faster. Performs
2340 /// additional processing for constant nodes.
2341 SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, const SDLoc &DL,
2342 void *&InsertPos);
2343
2344 /// Maps to auto-CSE operations.
2345 std::vector<CondCodeSDNode*> CondCodeNodes;
2346
2347 std::vector<SDNode*> ValueTypeNodes;
2348 std::map<EVT, SDNode*, EVT::compareRawBits> ExtendedValueTypeNodes;
2349 StringMap<SDNode*> ExternalSymbols;
2350
2351 std::map<std::pair<std::string, unsigned>, SDNode *> TargetExternalSymbols;
2353
2354 FlagInserter *Inserter = nullptr;
2355};
2356
2357template <> struct GraphTraits<SelectionDAG*> : public GraphTraits<SDNode*> {
2359
2361 return nodes_iterator(G->allnodes_begin());
2362 }
2363
2365 return nodes_iterator(G->allnodes_end());
2366 }
2367};
2368
2369} // end namespace llvm
2370
2371#endif // LLVM_CODEGEN_SELECTIONDAG_H
aarch64 promote const
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file defines the StringMap class.
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file defines the BumpPtrAllocator interface.
always inline
SmallVector< MachineOperand, 4 > Cond
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
RelocType Type
Definition: COFFYAML.cpp:390
#define LLVM_DEPRECATED(MSG, FIX)
Definition: Compiler.h:145
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition: Compiler.h:492
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
uint64_t Size
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This file defines a hash set that can be used to remove duplication of nodes in a graph.
Hexagon Vector Combine
print lazy value Lazy Value Info Printer Pass
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
unsigned Reg
This file contains the declarations for metadata subclasses.
LLVMContext & Context
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static void removeOperands(MachineInstr &MI, unsigned i)
Value * RHS
Value * LHS
Class for arbitrary precision integers.
Definition: APInt.h:75
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:214
unsigned getMinSignedBits() const
NOTE: This is soft-deprecated. Please use getSignificantBits() instead.
Definition: APInt.h:1487
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition: APInt.cpp:1002
static Capacity get(size_t N)
Get the capacity of an array that can hold at least N elements.
Definition: ArrayRecycler.h:79
Recycle small arrays allocated from a BumpPtrAllocator.
Definition: ArrayRecycler.h:28
void deallocate(Capacity Cap, T *Ptr)
Deallocate an array with the specified Capacity.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
The address of a basic block.
Definition: Constants.h:875
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Allocate memory in an ever growing pool, as if by bump-pointer.
Definition: Allocator.h:66
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:256
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
This is an important base class in LLVM.
Definition: Constant.h:41
DWARF expression.
Base class for variables.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:114
A debug info location.
Definition: DebugLoc.h:33
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:150
iterator end()
Definition: DenseMap.h:84
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Node - This class is used to maintain the singly linked bucket list in a folding set.
Definition: FoldingSet.h:136
FoldingSetNodeIDRef - This class describes a reference to an interned FoldingSetNodeID,...
Definition: FoldingSet.h:288
FoldingSetNodeID - This class is used to gather all the unique data bits of a node.
Definition: FoldingSet.h:318
FoldingSet - This template class is used to instantiate a specialized implementation of the folding s...
Definition: FoldingSet.h:520
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Data structure describing the variable locations in a function.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
This class is used to represent ISD::LOAD nodes.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
Metadata node.
Definition: Metadata.h:943
SimpleValueType SimpleTy
Abstract base class for all machine specific constantpool value subclasses.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
SmallVectorImpl< ArgRegPair > CallSiteInfoImpl
SmallVector< ArgRegPair, 1 > CallSiteInfo
Vector of call argument and its forwarding register.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
An SDNode that represents everything that will be needed to construct a MachineInstr.
The optimization diagnostic interface.
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:91
Analysis providing profile information.
Keeps track of dbg_value information through SDISel.
Definition: SelectionDAG.h:155
BumpPtrAllocator & getAlloc()
Definition: SelectionDAG.h:184
DbgIterator ByvalParmDbgBegin()
Definition: SelectionDAG.h:202
DbgIterator DbgEnd()
Definition: SelectionDAG.h:201
SDDbgInfo & operator=(const SDDbgInfo &)=delete
SmallVectorImpl< SDDbgLabel * >::iterator DbgLabelIterator
Definition: SelectionDAG.h:198
SDDbgInfo()=default
void add(SDDbgValue *V, bool isParameter)
bool empty() const
Definition: SelectionDAG.h:186
DbgLabelIterator DbgLabelEnd()
Definition: SelectionDAG.h:205
DbgIterator ByvalParmDbgEnd()
Definition: SelectionDAG.h:203
SmallVectorImpl< SDDbgValue * >::iterator DbgIterator
Definition: SelectionDAG.h:197
SDDbgInfo(const SDDbgInfo &)=delete
DbgLabelIterator DbgLabelBegin()
Definition: SelectionDAG.h:204
void add(SDDbgLabel *L)
Definition: SelectionDAG.h:170
DbgIterator DbgBegin()
Definition: SelectionDAG.h:200
void erase(const SDNode *Node)
Invalidate all DbgValues attached to the node and remove it from the Node-to-DbgValues map.
ArrayRef< SDDbgValue * > getSDDbgValues(const SDNode *Node) const
Definition: SelectionDAG.h:190
Holds the information from a dbg_label node through SDISel.
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
SDVTListNode(const FoldingSetNodeIDRef ID, const EVT *VT, unsigned int Num)
Definition: SelectionDAG.h:108
SDVTList getSDVTList()
Definition: SelectionDAG.h:113
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
EVT getValueType() const
Return the ValueType of the referenced return value.
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
Help to insert SDNodeFlags automatically in transforming.
Definition: SelectionDAG.h:357
SDNodeFlags getFlags() const
Definition: SelectionDAG.h:375
FlagInserter(SelectionDAG &SDAG, SDNodeFlags Flags)
Definition: SelectionDAG.h:363
FlagInserter(const FlagInserter &)=delete
FlagInserter(SelectionDAG &SDAG, SDNode *N)
Definition: SelectionDAG.h:368
FlagInserter & operator=(const FlagInserter &)=delete
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:221
static unsigned getOpcode_EXTEND_VECTOR_INREG(unsigned Opcode)
Convert *_EXTEND to *_EXTEND_VECTOR_INREG opcode.
Definition: SelectionDAG.h:920
Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op)
Return the specified value casted to the target's desired shift amount type.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsExpanding=false)
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:717
SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:547
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
bool isKnownNeverSNaN(SDValue Op, unsigned Depth=0) const
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
bool isKnownNeverZero(SDValue Op) const
Test whether the given SDValue is known to contain non-zero value(s).
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:470
const APInt * getValidMaximumShiftAmountConstant(SDValue V, const APInt &DemandedElts) const
If a SHL/SRA/SRL node V has constant shift amounts that are all less than the element bit-width of th...
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, MaybeAlign Alignment=std::nullopt, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
const Pass * getPass() const
Definition: SelectionDAG.h:466
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
OptimizationRemarkEmitter & getORE() const
Definition: SelectionDAG.h:482
BlockFrequencyInfo * getBFI() const
Definition: SelectionDAG.h:484
void updateDivergence(SDNode *N)
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
SDValue getTargetConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT)
Definition: SelectionDAG.h:706
SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N) const
Test whether the given value is a constant int or similar node.
SDDbgInfo::DbgIterator ByvalParmDbgEnd() const
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, unsigned VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getSymbolFunctionGlobalAddress(SDValue Op, Function **TargetFunction=nullptr)
Returs an GlobalAddress of the function from the current module with name matching the given External...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
SDDbgInfo::DbgIterator ByvalParmDbgBegin() const
void setFunctionLoweringInfo(FunctionLoweringInfo *FuncInfo)
Definition: SelectionDAG.h:457
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
OverflowKind
Used to represent the possible overflow behavior of an operation.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N, SDValue Glue)
Definition: SelectionDAG.h:778
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
const SDLoc SDValue SDValue MachinePointerInfo PtrInfo
bool calculateDivergence(SDNode *N)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDNode * mutateStrictFPToFP(SDNode *Node)
Mutate the specified strict FP node to its non-strict equivalent, unlinking the node from its chain a...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getGLOBAL_OFFSET_TABLE(EVT VT)
Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
SDValue getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsExpanding=false)
SelectionDAG(const SelectionDAG &)=delete
const SDLoc SDValue Chain
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
OverflowKind computeOverflowKind(SDValue N0, SDValue N1) const
Determine if the result of the addition of 2 node can overflow.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const STC & getSubtarget() const
Definition: SelectionDAG.h:471
SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:474
LLVM_DEPRECATED("Use the getLoad function that takes a MaybeAlign instead", "") inline SDValue getLoad(ISD
FIXME: Remove once transition to Align is over.
bool isEqualTo(SDValue A, SDValue B) const
Test whether two SDValues are known to compare equal.
static constexpr unsigned MaxRecursionDepth
Definition: SelectionDAG.h:444
SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsCompressing=false)
SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
bool isGuaranteedNotToBePoison(SDValue Op, unsigned Depth=0) const
Return true if this function can prove that Op is never poison.
SDValue expandVACopy(SDNode *Node)
Expand the specified ISD::VACOPY node as the Legalize pass would.
SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SelectionDAG & operator=(const SelectionDAG &)=delete
const SDLoc SDValue SDValue Ptr
SDValue getTargetConstant(const APInt &Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:675
APInt computeVectorKnownZeroElements(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
For each demanded element of a vector, see if it is known to be zero.
void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
Definition: SelectionDAG.h:383
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
MDNode * getHeapAllocSite(const SDNode *Node) const
Return HeapAllocSite associated with Node, or nullptr if none exists.
void salvageDebugInfo(SDNode &N)
To be invoked on an SDNode that is slated to be erased.
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
LLVM_DEPRECATED("Use the getLoad function that takes a MaybeAlign instead", "") inline SDValue getLoad(EVT VT
FIXME: Remove once transition to Align is over.
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
Definition: SelectionDAG.h:727
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, bool IsExpanding=false)
std::pair< SDValue, SDValue > UnrollVectorOverflowOp(SDNode *N, unsigned ResNE=0)
Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes.
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:527
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:824
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
allnodes_const_iterator allnodes_end() const
Definition: SelectionDAG.h:528
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
const LegacyDivergenceAnalysis * getDivergenceAnalysis() const
Definition: SelectionDAG.h:477
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDDbgInfo::DbgIterator DbgEnd() const
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
CallSiteInfo getCallSiteInfo(const SDNode *Node)
Return CallSiteInfo associated with Node, or a default if none exists.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:468
SDNode * isConstantFPBuildVectorOrConstantFP(SDValue N) const
Test whether the given value is a constant FP or similar node.
allnodes_iterator allnodes_begin()
Definition: SelectionDAG.h:532
iterator_range< allnodes_const_iterator > allnodes() const
Definition: SelectionDAG.h:542
MDNode * getPCSections(const SDNode *Node) const
Return PCSections associated with Node, or nullptr if none exists.
ProfileSummaryInfo * getPSI() const
Definition: SelectionDAG.h:483
SDValue expandVAArg(SDNode *Node)
Expand the specified ISD::VAARG node as the Legalize pass would.
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:722
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
void setGraphAttrs(const SDNode *N, const char *Attrs)
Set graph attributes for a node. (eg. "color=red".)
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, SDValue Reg, SDValue N, SDValue Glue)
Definition: SelectionDAG.h:787
void addCallSiteInfo(const SDNode *Node, CallSiteInfoImpl &&CallInfo)
Set CallSiteInfo to be associated with Node.
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
void addHeapAllocSite(const SDNode *Node, MDNode *MD)
Set HeapAllocSite to be associated with Node.
const SelectionDAGTargetInfo & getSelectionDAGInfo() const
Definition: SelectionDAG.h:476
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
SDValue getTargetConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:679
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL)
Split the vector with EXTRACT_SUBVECTOR and return the low/high part.
void copyExtraInfo(SDNode *From, SDNode *To)
Copy extra info associated with one node to another.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
void setGraphColor(const SDNode *N, const char *Color)
Convenience for setting node color attribute.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
Definition: SelectionDAG.h:656
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, bool IsExpanding=false)
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block.
SDValue getCALLSEQ_END(SDValue Chain, uint64_t Size1, uint64_t Size2, SDValue Glue, const SDLoc &DL)
SDValue getIndexedStridedStoreVP(SDValue OrigStore, const SDLoc &DL, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part.
const APInt * getValidShiftAmountConstant(SDValue V, const APInt &DemandedElts) const
If a SHL/SRA/SRL node V has a constant or splat constant shift amount that is less than the element b...
bool isKnownToBeAPowerOfTwo(SDValue Val) const
Test if the given value is known to have exactly one bit set.
bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly=false, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, SDValue N2)
Fold floating-point operations with 2 operands when both operands are constants and/or undefined.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
Definition: SelectionDAG.h:858
const SDLoc SDValue SDValue MachinePointerInfo unsigned MachineMemOperand::Flags const AAMDNodes const MDNode * Ranges
SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
FlagInserter * getFlagInserter()
Definition: SelectionDAG.h:486
bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if '(Op & Mask) == Mask'.
const SDLoc SDValue SDValue MachinePointerInfo unsigned MachineMemOperand::Flags const AAMDNodes & AAInfo
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
bool hasDebugValues() const
Return true if there are any SDDbgValue nodes associated with this SelectionDAG.
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
SDValue getRegister(unsigned Reg, EVT VT)
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDUse > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:833
void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
bool isConstantValueOfAnyType(SDValue N) const
SDDbgInfo::DbgLabelIterator DbgLabelEnd() const
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
allnodes_iterator allnodes_end()
Definition: SelectionDAG.h:533
SDDbgInfo::DbgLabelIterator DbgLabelBegin() const
SDValue getTargetIndex(int Index, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, LegacyDivergenceAnalysis *Divergence, ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin, FunctionVarLocs const *FnVarLocs)
Prepare this SelectionDAG to process code in the given MachineFunction.
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getBasicBlock(MachineBasicBlock *MBB)
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
std::string getGraphAttrs(const SDNode *N) const
Get graph attributes for a node.
SDValue getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:469
bool getNoMergeSiteInfo(const SDNode *Node) const
Return NoMerge info associated with Node.
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, SDValue Offset)
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getVPLogicalNOT(const SDLoc &DL, SDValue Val, SDValue Mask, SDValue EVL, EVT VT)
Create a vector-predicated logical NOT operation as (VP_XOR Val, BooleanOne, Mask,...
SDDbgInfo::DbgIterator DbgBegin() const
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:769
iterator_range< allnodes_iterator > allnodes()
Definition: SelectionDAG.h:539
const SDLoc & dl
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
bool isKnownNeverZeroFloat(SDValue Op) const
Test whether the given floating point SDValue is known to never be positive or negative zero.
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT)
Definition: SelectionDAG.h:703
ArrayRef< SDDbgValue * > GetDbgValues(const SDNode *SD) const
Get the debug values which reference the given SDNode.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
ilist< SDNode >::size_type allnodes_size() const
Definition: SelectionDAG.h:535
SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsCompressing=false)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:671
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:475
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
const SDLoc SDValue SDValue MachinePointerInfo unsigned Alignment
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Return true if 'Op' is known to be zero in DemandedElts.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:764
SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be of integer type, to the vector-type integer type VT,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:465
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:795
SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
bool canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, bool PoisonOnly=false, bool ConsiderFlags=true, unsigned Depth=0) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
void setFlagInserter(FlagInserter *FI)
Definition: SelectionDAG.h:487
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
Definition: SelectionDAG.h:841
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
const FunctionVarLocs * getFunctionVarLocs() const
Returns the result of the AssignmentTrackingAnalysis pass if it's available, otherwise return nullptr...
Definition: SelectionDAG.h:480
void canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1, SDValue &N2) const
Swap N1 and N2 if Opcode is a commutative binary opcode and the canonical form expects the opposite o...
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, int64_t Size, int64_t Offset=-1)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the por...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getIndexedStridedLoadVP(SDValue OrigLoad, const SDLoc &DL, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
Definition: SelectionDAG.h:481
SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, SDNodeFlags Flags)
Try to simplify a floating-point binary operation into 1 of its operands or a constant.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:556
const APInt * getValidMinimumShiftAmountConstant(SDValue V, const APInt &DemandedElts) const
If a SHL/SRA/SRL node V has constant shift amounts that are all less than the element bit-width of th...
void addPCSections(const SDNode *Node, MDNode *MD)
Set PCSections to be associated with Node.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
bool isGuaranteedNotToBePoison(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Return true if this function can prove that Op is never poison.
SDValue getTargetConstantPool(MachineConstantPoolValue *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:741
void clearGraphAttrs()
Clear all previously defined node graph attributes.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT, SDValue Glue)
Definition: SelectionDAG.h:804
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
bool isUndef(unsigned Opcode, ArrayRef< SDValue > Ops)
Return true if the result of this operation is always undefined.
SDValue getSetCCVP(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Mask, SDValue EVL)
Helper function to make it easier to build VP_SETCCs if you just have an ISD::CondCode instead of an ...
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
std::pair< EVT, EVT > GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, bool *HiIsEmpty) const
Compute the VTs needed for the low/hi parts of a type, dependent on an enveloping VT that has been sp...
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:733
SDValue FoldSymbolOffset(unsigned Opcode, EVT VT, const GlobalAddressSDNode *GA, const SDNode *N2)
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:550
SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
void setSubgraphColor(SDNode *N, const char *Color)
Convenience for setting subgraph color attribute.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getTargetConstantFP(const ConstantFP &Val, const SDLoc &DL, EVT VT)
Definition: SelectionDAG.h:709
DenormalMode getDenormalMode(EVT VT) const
Return the current function's default denormal handling kind for the given floating point type.
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
Definition: SelectionDAG.h:874
static unsigned getOpcode_EXTEND(unsigned Opcode)
Convert *_EXTEND_VECTOR_INREG to *_EXTEND opcode.
Definition: SelectionDAG.h:904
SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, ArrayRef< ISD::NodeType > CandidateBinOps, bool AllowPartials=false)
Match a binop + shuffle pyramid that represents a horizontal reduction over the elements of a vector ...
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
LLVM_DUMP_METHOD void dumpDotGraph(const Twine &FileName, const Twine &Title)
Just dump dot graph to a user-provided path and title.
SDValue simplifyShift(SDValue X, SDValue Y)
Try to simplify a shift into 1 of its operands or a constant.
const SDLoc SDValue SDValue MachinePointerInfo unsigned MachineMemOperand::Flags MMOFlags
void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits=0, unsigned SizeInBits=0, bool InvalidateDbg=true)
Transfer debug values from one node to another, while optionally generating fragment expressions for ...
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
bool LegalizeTypes()
This transforms the SelectionDAG into a SelectionDAG that only uses types natively supported by the t...
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:301
bool empty() const
Definition: SmallVector.h:94
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
typename SuperClass::iterator iterator
Definition: SmallVector.h:581
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringMap - This is an unconventional map that is specialized for handling keys that are "strings",...
Definition: StringMap.h:111
Provides information about what library functions are available for the current target.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:78
TargetSubtargetInfo - Generic base class for all target subtargets.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
LLVM Value Representation.
Definition: Value.h:74
Iterator for intrusive lists based on ilist_node.
pointer remove(iterator &IT)
Definition: ilist.h:252
iterator insert(iterator where, pointer New)
Definition: ilist.h:229
An intrusive list with ownership and callbacks specified/controlled by ilist_traits,...
Definition: ilist.h:392
A range adaptor for a pair of iterators.
This file defines classes to implement an intrusive doubly linked list class (i.e.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
Level
Code generation optimization level.
Definition: CodeGen.h:57
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:736
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:475
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:766
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:803
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:476
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:760
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:713
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:613
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:208
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:763
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:728
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
Definition: ISDOpcodes.h:1253
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:792
@ GLOBAL_OFFSET_TABLE
The address of the GOT.
Definition: ISDOpcodes.h:87
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:722
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:814
@ BlockAddress
Definition: ISDOpcodes.h:84
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
Definition: ISDOpcodes.h:1070
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:514
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
Definition: ISDOpcodes.h:1390
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Definition: ISDOpcodes.h:1377
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1428
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1408
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:406
AlignedCharArrayUnion< AtomicSDNode, TargetIndexSDNode, BlockAddressSDNode, GlobalAddressSDNode, PseudoProbeSDNode > LargestSDNode
A representation of the largest SDNode, for use in sizeof().
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void checkForCycles(const SelectionDAG *DAG, bool force=false)
BumpPtrAllocatorImpl BumpPtrAllocator
The standard BumpPtrAllocator which just uses the default template parameters.
Definition: Allocator.h:375
CombineLevel
Definition: DAGCombine.h:15
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1862
Definition: BitVector.h:851
#define N
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:651
static const fltSemantics & IEEEsingle() LLVM_READNONE
Definition: APFloat.cpp:201
static const fltSemantics & PPCDoubleDouble() LLVM_READNONE
Definition: APFloat.cpp:208
static const fltSemantics & x87DoubleExtended() LLVM_READNONE
Definition: APFloat.cpp:213
static const fltSemantics & IEEEquad() LLVM_READNONE
Definition: APFloat.cpp:207
static const fltSemantics & IEEEdouble() LLVM_READNONE
Definition: APFloat.cpp:204
static const fltSemantics & IEEEhalf() LLVM_READNONE
Definition: APFloat.cpp:195
static const fltSemantics & BFloat() LLVM_READNONE
Definition: APFloat.cpp:198
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
DefaultFoldingSetTrait - This class provides default implementations for FoldingSetTrait implementati...
Definition: FoldingSet.h:231
Represent subnormal handling kind for floating point instruction inputs and outputs.
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:340
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:352
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:288
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:154
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:295
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:160
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:300
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:308
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:280
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
static bool Equals(const SDVTListNode &X, const FoldingSetNodeID &ID, unsigned IDHash, FoldingSetNodeID &TempID)
Definition: SelectionDAG.h:126
static unsigned ComputeHash(const SDVTListNode &X, FoldingSetNodeID &TempID)
Definition: SelectionDAG.h:133
static void Profile(const SDVTListNode &X, FoldingSetNodeID &ID)
Definition: SelectionDAG.h:122
FoldingSetTrait - This trait class is used to define behavior of how to "profile" (in the FoldingSet ...
Definition: FoldingSet.h:261
static nodes_iterator nodes_begin(SelectionDAG *G)
static nodes_iterator nodes_end(SelectionDAG *G)
This class contains a discriminated union of information about pointers in memory operands,...
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
DAGNodeDeletedListener(SelectionDAG &DAG, std::function< void(SDNode *, SDNode *)> Callback)
Definition: SelectionDAG.h:332
void NodeDeleted(SDNode *N, SDNode *E) override
The node N that was deleted and, if E is not null, an equivalent node E that replaced it.
Definition: SelectionDAG.h:336
std::function< void(SDNode *, SDNode *)> Callback
Definition: SelectionDAG.h:330
std::function< void(SDNode *)> Callback
Definition: SelectionDAG.h:343
void NodeInserted(SDNode *N) override
The node N that was inserted.
Definition: SelectionDAG.h:349
DAGNodeInsertedListener(SelectionDAG &DAG, std::function< void(SDNode *)> Callback)
Definition: SelectionDAG.h:345
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
Definition: SelectionDAG.h:303
DAGUpdateListener *const Next
Definition: SelectionDAG.h:304
virtual void NodeDeleted(SDNode *N, SDNode *E)
The node N that was deleted and, if E is not null, an equivalent node E that replaced it.
virtual void NodeInserted(SDNode *N)
The node N that was inserted.
virtual void NodeUpdated(SDNode *N)
The node N that was updated.
static void deleteNode(SDNode *)
Definition: SelectionDAG.h:139
Use delete by default for iplist and ilist.
Definition: ilist.h:41