LLVM 24.0.0git
TargetLowering.h
Go to the documentation of this file.
1//===- llvm/CodeGen/TargetLowering.h - Target Lowering Info -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file describes how to lower LLVM code to machine code. This has two
11/// main components:
12///
13/// 1. Which ValueTypes are natively supported by the target.
14/// 2. Which operations are supported for supported ValueTypes.
15/// 3. Cost thresholds for alternative implementations of certain operations.
16///
17/// In addition it has a few other components, like information about FP
18/// immediates.
19///
20//===----------------------------------------------------------------------===//
21
22#ifndef LLVM_CODEGEN_TARGETLOWERING_H
23#define LLVM_CODEGEN_TARGETLOWERING_H
24
25#include "llvm/ADT/APInt.h"
26#include "llvm/ADT/ArrayRef.h"
27#include "llvm/ADT/DenseMap.h"
29#include "llvm/ADT/StringRef.h"
42#include "llvm/IR/Attributes.h"
43#include "llvm/IR/CallingConv.h"
44#include "llvm/IR/DataLayout.h"
46#include "llvm/IR/Function.h"
47#include "llvm/IR/InlineAsm.h"
48#include "llvm/IR/Instruction.h"
51#include "llvm/IR/Type.h"
58#include <algorithm>
59#include <cassert>
60#include <climits>
61#include <cstdint>
62#include <map>
63#include <string>
64#include <utility>
65#include <vector>
66
67namespace llvm {
68
69class AssumptionCache;
70class CCState;
71class CCValAssign;
74class Constant;
75enum class ExceptionHandling : int;
76class FastISel;
78class GlobalValue;
79class Loop;
81class IntrinsicInst;
82class IRBuilderBase;
83struct KnownBits;
84class LLVMContext;
86class MachineFunction;
87class MachineInstr;
89class MachineLoop;
91class MCContext;
92class MCExpr;
93class Module;
96class TargetMachine;
97class MCRegisterClass;
101class Value;
102class VPIntrinsic;
103
104namespace Sched {
105
107 None, // No preference
108 Source, // Follow source order.
109 RegPressure, // Scheduling for lowest register pressure.
110 Hybrid, // Scheduling for both latency and register pressure.
111 ILP, // Scheduling for ILP in low register pressure mode.
112 VLIW, // Scheduling for VLIW targets.
113 Fast, // Fast suboptimal list scheduling
114 Linearize, // Linearize DAG, no scheduling
115 Last = Linearize // Marker for the last Sched::Preference
116};
117
118} // end namespace Sched
119
120// MemOp models a memory operation, either memset or memcpy/memmove.
121struct MemOp {
122private:
123 enum class MemOpKind {
124 Memset,
125 MemsetWithZero, // memset the memory with zeros
126 Memcpy, // copy memory from source to destination, source and destination do
127 // not overlap
128 MemcpyStrSrc, // memcpy source is an in-register constant, so it does not
129 // need to be loaded
130 Memmove, // memmove: like memcpy, but source and destination regions may
131 // overlap
132 };
133
134 // Shared
135 uint64_t Size;
136 bool DstAlignCanChange; // true if destination alignment can satisfy any
137 // constraint.
138 Align DstAlign; // Specified alignment of the memory operation.
139
140 bool IsVolatile;
141 MemOpKind Kind;
142 Align SrcAlign; // Inferred alignment of the source or default value if the
143 // memory operation does not need to load the value.
144public:
145 static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
146 Align SrcAlign, bool IsVolatile,
147 bool MemcpyStrSrc = false) {
148 MemOp Op;
149 Op.Size = Size;
150 Op.DstAlignCanChange = DstAlignCanChange;
151 Op.DstAlign = DstAlign;
152 Op.IsVolatile = IsVolatile;
153 Op.Kind = MemcpyStrSrc ? MemOpKind::MemcpyStrSrc : MemOpKind::Memcpy;
154 Op.SrcAlign = SrcAlign;
155 return Op;
156 }
157
158 static MemOp Move(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
159 Align SrcAlign, bool IsVolatile) {
160 MemOp Op;
161 Op.Size = Size;
162 Op.DstAlignCanChange = DstAlignCanChange;
163 Op.DstAlign = DstAlign;
164 Op.IsVolatile = IsVolatile;
165 Op.Kind = MemOpKind::Memmove;
166 Op.SrcAlign = SrcAlign;
167 return Op;
168 }
169
170 static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
171 bool IsZeroMemset, bool IsVolatile) {
172 MemOp Op;
173 Op.Size = Size;
174 Op.DstAlignCanChange = DstAlignCanChange;
175 Op.DstAlign = DstAlign;
176 Op.IsVolatile = IsVolatile;
177 Op.Kind = IsZeroMemset ? MemOpKind::MemsetWithZero : MemOpKind::Memset;
178 return Op;
179 }
180
181 uint64_t size() const { return Size; }
183 assert(!DstAlignCanChange);
184 return DstAlign;
185 }
186 bool isFixedDstAlign() const { return !DstAlignCanChange; }
187 bool isVolatile() const { return IsVolatile; }
188 bool isMemset() const {
189 return Kind == MemOpKind::Memset || Kind == MemOpKind::MemsetWithZero;
190 }
191 bool isMemcpy() const {
192 return Kind == MemOpKind::Memcpy || Kind == MemOpKind::MemcpyStrSrc;
193 }
194 bool isMemmove() const { return Kind == MemOpKind::Memmove; }
195 bool isMemcpyOrMemmove() const { return isMemcpy() || isMemmove(); }
197 return isMemcpyOrMemmove() && !DstAlignCanChange;
198 }
199 bool isZeroMemset() const { return Kind == MemOpKind::MemsetWithZero; }
200 bool isMemcpyStrSrc() const { return Kind == MemOpKind::MemcpyStrSrc; }
202 assert(isMemcpyOrMemmove() && "Must be a memcpy or memmove");
203 return SrcAlign;
204 }
205 bool isSrcAligned(Align AlignCheck) const {
206 return isMemset() || llvm::isAligned(AlignCheck, SrcAlign.value());
207 }
208 bool isDstAligned(Align AlignCheck) const {
209 return DstAlignCanChange || llvm::isAligned(AlignCheck, DstAlign.value());
210 }
211 bool isAligned(Align AlignCheck) const {
212 return isSrcAligned(AlignCheck) && isDstAligned(AlignCheck);
213 }
214};
215
216/// This base class for TargetLowering contains the SelectionDAG-independent
217/// parts that can be used from the rest of CodeGen.
218///
219/// Virtual methods here should express a fundamental property of the
220/// target. These should be reusable in different contexts, and not be used to
221/// control a specific transformation.
223public:
224 /// This enum indicates whether operations are valid for a target, and if not,
225 /// what action should be used to make them valid.
227 Legal, // The target natively supports this operation.
228 Promote, // This operation should be executed in a larger type.
229 Expand, // Try to expand this to other ops, otherwise use a libcall.
230 LibCall, // Don't try to expand this to other ops, always use a libcall.
231 Custom // Use the LowerOperation hook to implement custom lowering.
232 };
233
234 /// This enum indicates whether a types are legal for a target, and if not,
235 /// what action should be used to make them valid.
237 TypeLegal, // The target natively supports this type.
238 TypePromoteInteger, // Replace this integer with a larger one.
239 TypeExpandInteger, // Split this integer into two of half the size.
240 TypeSoftenFloat, // Convert this float to a same size integer type.
241 TypeExpandFloat, // Split this float into two of half the size.
242 TypeScalarizeVector, // Replace this one-element vector with its element.
243 TypeSplitVector, // Split this vector into two of half the size.
244 TypeWidenVector, // This vector should be widened into a larger vector.
245 TypeSoftPromoteHalf, // Soften half to i16 and use float to do arithmetic.
246 TypeScalarizeScalableVector, // This action is explicitly left
247 // unimplemented. While it is theoretically
248 // possible to legalize operations on scalable
249 // types with a loop that handles the vscale *
250 // #lanes of the vector, this is non-trivial at
251 // SelectionDAG level and these types are
252 // better to be widened or promoted.
253 };
254
255 /// LegalizeKind holds the legalization kind that needs to happen to EVT
256 /// in order to type-legalize it.
257 using LegalizeKind = std::pair<LegalizeTypeAction, EVT>;
258
259 /// Enum that describes how the target represents true/false values.
261 UndefinedBooleanContent, // Only bit 0 counts, the rest can hold garbage.
262 ZeroOrOneBooleanContent, // All bits zero except for bit 0.
263 ZeroOrNegativeOneBooleanContent // All bits equal to bit 0.
264 };
265
266 /// Enum that describes what type of support for selects the target has.
268 ScalarValSelect, // The target supports scalar selects (ex: cmov).
269 ScalarCondVectorVal, // The target supports selects with a scalar condition
270 // and vector values (ex: cmov).
271 VectorMaskSelect // The target supports vector selects with a vector
272 // mask (ex: x86 blends).
273 };
274
275 /// Enum that specifies what an atomic load/AtomicRMWInst is expanded
276 /// to, if at all. Exists because different targets have different levels of
277 /// support for these atomic instructions, and also have different options
278 /// w.r.t. what they should expand to.
280 None, // Don't expand the instruction.
281 CastToInteger, // Cast the atomic instruction to another type, e.g. from
282 // floating-point to integer type.
283 LLSC, // Expand the instruction into loadlinked/storeconditional; used
284 // by ARM/AArch64/PowerPC.
285 LLOnly, // Expand the (load) instruction into just a load-linked, which has
286 // greater atomic guarantees than a normal load.
287 CmpXChg, // Expand the instruction into cmpxchg; used by at least X86.
288 MaskedIntrinsic, // Use a target-specific intrinsic for the LL/SC loop.
289 BitTestIntrinsic, // Use a target-specific intrinsic for special bit
290 // operations; used by X86.
291 CmpArithIntrinsic, // Use a target-specific intrinsic for special compare
292 // operations; used by X86.
293 Expand, // Generic expansion in terms of other atomic operations.
294 CustomExpand, // Custom target-specific expansion using TLI hooks.
295
296 // Rewrite to a non-atomic form for use in a known non-preemptible
297 // environment.
299 };
300
301 /// Enum that specifies when a multiplication should be expanded.
302 enum class MulExpansionKind {
303 Always, // Always expand the instruction.
304 OnlyLegalOrCustom, // Only expand when the resulting instructions are legal
305 // or custom.
306 };
307
308 /// Enum that specifies when a float negation is beneficial.
309 enum class NegatibleCost {
310 Cheaper = 0, // Negated expression is cheaper.
311 Neutral = 1, // Negated expression has the same cost.
312 Expensive = 2 // Negated expression is more expensive.
313 };
314
315 /// Enum that specifies how expensive lowering an EXTRACT_SUBVECTOR is.
317 Free = 0, // Lowers to no instruction at all, e.g. a subregister copy.
318 Cheap = 1, // Lowers to at most one instruction, and may still be free if
319 // the target can fold the extract into the instruction
320 // consuming it (e.g. a widening op that reads the high half of
321 // a register).
322 Expensive = 2 // Needs a shuffle sequence that cannot be folded away.
323 };
324
325 /// Enum of different potentially desirable ways to fold (and/or (setcc ...),
326 /// (setcc ...)).
328 None = 0, // No fold is preferable.
329 AddAnd = 1, // Fold with `Add` op and `And` op is preferable.
330 NotAnd = 2, // Fold with `Not` op and `And` op is preferable.
331 ABS = 4, // Fold with `llvm.abs` op is preferable.
332 };
333
335 public:
338 /// Original unlegalized argument type.
340 /// Same as OrigTy, or partially legalized for soft float libcalls.
342 bool IsSExt : 1;
343 bool IsZExt : 1;
344 bool IsNoExt : 1;
345 bool IsInReg : 1;
346 bool IsSRet : 1;
347 bool IsNest : 1;
348 bool IsByVal : 1;
349 bool IsByRef : 1;
350 bool IsInAlloca : 1;
352 bool IsReturned : 1;
353 bool IsSwiftSelf : 1;
354 bool IsSwiftAsync : 1;
355 bool IsSwiftError : 1;
357 MaybeAlign Alignment = std::nullopt;
358 Type *IndirectType = nullptr;
359
366
370 setExtAttribute(ExtAttr);
371 }
372
375 : ArgListEntry(nullptr, Node, Ty) {
376 setExtAttribute(ExtAttr);
377 }
378
379 LLVM_ABI void setAttributes(const CallBase *Call, unsigned ArgIdx);
380 LLVM_ABI void setAttributes(const AttributeList &Attrs, unsigned ArgIdx);
381 LLVM_ABI void setExtAttribute(Attribute::AttrKind Attr);
382 };
383 using ArgListTy = std::vector<ArgListEntry>;
384
386 switch (Content) {
388 // Extend by adding rubbish bits.
389 return ISD::ANY_EXTEND;
391 // Extend by adding zero bits.
392 return ISD::ZERO_EXTEND;
394 // Extend by copying the sign bit.
395 return ISD::SIGN_EXTEND;
396 }
397 llvm_unreachable("Invalid content kind");
398 }
399
400 explicit TargetLoweringBase(const TargetMachine &TM,
401 const TargetSubtargetInfo &STI);
405
406 /// Return true if the target support strict float operation
407 bool isStrictFPEnabled() const {
408 return IsStrictFPEnabled;
409 }
410
411protected:
412 /// Initialize all of the actions to default values.
413 void initActions();
414
415public:
416 const TargetMachine &getTargetMachine() const { return TM; }
417
418 virtual bool useSoftFloat() const { return false; }
419
420 /// Return the pointer type for the given address space, defaults to
421 /// the pointer type from the data layout.
422 /// FIXME: The default needs to be removed once all the code is updated.
423 virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const {
424 return MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
425 }
426
427 /// Return the in-memory pointer type for the given address space, defaults to
428 /// the pointer type from the data layout.
429 /// FIXME: The default needs to be removed once all the code is updated.
430 virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS = 0) const {
431 return MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
432 }
433
434 /// Return the type for frame index, which is determined by
435 /// the alloca address space specified through the data layout.
437 return getPointerTy(DL, DL.getAllocaAddrSpace());
438 }
439
440 /// Return the type for code pointers, which is determined by the program
441 /// address space specified through the data layout.
443 return getPointerTy(DL, DL.getProgramAddressSpace());
444 }
445
446 /// Return the type for operands of fence.
447 /// TODO: Let fence operands be of i32 type and remove this.
448 virtual MVT getFenceOperandTy(const DataLayout &DL) const {
449 return getPointerTy(DL);
450 }
451
452 /// Return the type to use for a scalar shift opcode, given the shifted amount
453 /// type. Targets should return a legal type if the input type is legal.
454 /// Targets can return a type that is too small if the input type is illegal.
455 virtual MVT getScalarShiftAmountTy(const DataLayout &, EVT) const;
456
457 /// Returns the type for the shift amount of a shift opcode. For vectors,
458 /// returns the input type. For scalars, calls getScalarShiftAmountTy.
459 /// If getScalarShiftAmountTy type cannot represent all possible shift
460 /// amounts, returns MVT::i32.
461 EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const;
462
463 /// Return the preferred type to use for a shift opcode, given the shifted
464 /// amount type is \p ShiftValueTy.
466 virtual LLT getPreferredShiftAmountTy(LLT ShiftValueTy) const {
467 return ShiftValueTy;
468 }
469
470 /// Returns the type to be used for the index operand vector operations. By
471 /// default we assume it will have the same size as an address space 0
472 /// pointer.
473 virtual unsigned getVectorIdxWidth(const DataLayout &DL) const {
474 return DL.getPointerSizeInBits(0);
475 }
476
477 /// Returns the type to be used for the index operand of:
478 /// ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
479 /// ISD::INSERT_SUBVECTOR, and ISD::EXTRACT_SUBVECTOR
483
484 /// Returns the type to be used for the index operand of:
485 /// G_INSERT_VECTOR_ELT, G_EXTRACT_VECTOR_ELT,
486 /// G_INSERT_SUBVECTOR, and G_EXTRACT_SUBVECTOR
489 }
490
491 /// Returns the type to be used for the EVL/AVL operand of VP nodes:
492 /// ISD::VP_UDIV, ISD::VP_SDIV, etc. It must be a legal scalar integer type,
493 /// and must be at least as large as i32. The EVL is implicitly zero-extended
494 /// to any larger type.
495 virtual MVT getVPExplicitVectorLengthTy() const { return MVT::i32; }
496
497 /// This callback is used to inspect load/store instructions and add
498 /// target-specific MachineMemOperand flags to them. The default
499 /// implementation does nothing.
503
504 /// This callback is used to inspect load/store SDNode.
505 /// The default implementation does nothing.
510
511 MachineMemOperand::Flags getLoadMemOperandFlags(
512 const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC = nullptr,
513 const TargetLibraryInfo *LibInfo = nullptr,
515 MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI,
516 const DataLayout &DL) const;
517 MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI,
518 const DataLayout &DL) const;
520 getVPIntrinsicMemOperandFlags(const VPIntrinsic &VPIntrin) const;
521
522 virtual bool isSelectSupported(SelectSupportKind /*kind*/) const {
523 return true;
524 }
525
526 /// Return true if the @llvm.get.active.lane.mask intrinsic should be expanded
527 /// using generic code in SelectionDAGBuilder.
528 virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const {
529 return true;
530 }
531
532 virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF,
533 bool IsScalable) const {
534 return true;
535 }
536
537 /// Return the minimum number of bits required to hold the maximum possible
538 /// number of trailing zero vector elements.
539 unsigned getBitWidthForCttzElements(EVT RetVT, ElementCount EC,
540 bool ZeroIsPoison,
541 const ConstantRange *VScaleRange) const;
542
543 // Return true if op(vecreduce(x), vecreduce(y)) should be reassociated to
544 // vecreduce(op(x, y)) for the reduction opcode RedOpc.
545 virtual bool shouldReassociateReduction(unsigned RedOpc, EVT VT) const {
546 return true;
547 }
548
549 /// Return true if it is profitable to convert a select of FP constants into
550 /// a constant pool load whose address depends on the select condition. The
551 /// parameter may be used to differentiate a select with FP compare from
552 /// integer compare.
553 virtual bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const {
554 return true;
555 }
556
557 /// Does the target have multiple (allocatable) condition registers that
558 /// can be used to store the results of comparisons for use by selects
559 /// and conditional branches. With multiple condition registers, the code
560 /// generator will not aggressively sink comparisons into the blocks of their
561 /// users. \p VT is the type of the condition value, e.g. the type of the
562 /// result of a comparison.
563 virtual bool hasMultipleConditionRegisters(EVT VT) const { return false; }
564
565 /// Return true if the target has BitExtract instructions.
566 bool hasExtractBitsInsn() const { return HasExtractBitsInsn; }
567
568 /// Return the preferred vector type legalization action.
571 // The default action for one element vectors is to scalarize
573 return TypeScalarizeVector;
574 // The default action for an odd-width vector is to widen.
575 if (!VT.isPow2VectorType())
576 return TypeWidenVector;
577 // The default action for other vectors is to promote
578 return TypePromoteInteger;
579 }
580
581 // Return true if, for soft-promoted half, the half type should be passed to
582 // and returned from functions as f32. The default behavior is to pass as
583 // i16. If soft-promoted half is not used, this function is ignored and
584 // values are always passed and returned as f32.
585 virtual bool useFPRegsForHalfType() const { return false; }
586
587 // There are two general methods for expanding a BUILD_VECTOR node:
588 // 1. Use SCALAR_TO_VECTOR on the defined scalar values and then shuffle
589 // them together.
590 // 2. Build the vector on the stack and then load it.
591 // If this function returns true, then method (1) will be used, subject to
592 // the constraint that all of the necessary shuffles are legal (as determined
593 // by isShuffleMaskLegal). If this function returns false, then method (2) is
594 // always used. The vector type, and the number of defined values, are
595 // provided.
596 virtual bool
598 unsigned DefinedValues) const {
599 return DefinedValues < 3;
600 }
601
602 /// Return true if integer divide is usually cheaper than a sequence of
603 /// several shifts, adds, and multiplies for this target.
604 /// The definition of "cheaper" may depend on whether we're optimizing
605 /// for speed or for size.
606 virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const { return false; }
607
608 /// Return true if the target can handle a standalone remainder operation.
609 virtual bool hasStandaloneRem(EVT VT) const {
610 return true;
611 }
612
613 /// Return true if SQRT(X) shouldn't be replaced with X*RSQRT(X).
614 virtual bool isFsqrtCheap(SDValue X, SelectionDAG &DAG) const {
615 // Default behavior is to replace SQRT(X) with X*RSQRT(X).
616 return false;
617 }
618
619 /// Reciprocal estimate status values used by the functions below.
624 };
625
626 /// Return a ReciprocalEstimate enum value for a square root of the given type
627 /// based on the function's attributes. If the operation is not overridden by
628 /// the function's attributes, "Unspecified" is returned and target defaults
629 /// are expected to be used for instruction selection.
630 int getRecipEstimateSqrtEnabled(EVT VT, const Function &F) const;
631
632 /// Return a ReciprocalEstimate enum value for a division of the given type
633 /// based on the function's attributes. If the operation is not overridden by
634 /// the function's attributes, "Unspecified" is returned and target defaults
635 /// are expected to be used for instruction selection.
636 int getRecipEstimateDivEnabled(EVT VT, const Function &MF) const;
637
638 /// Return the refinement step count for a square root of the given type based
639 /// on the function's attributes. If the operation is not overridden by
640 /// the function's attributes, "Unspecified" is returned and target defaults
641 /// are expected to be used for instruction selection.
642 int getSqrtRefinementSteps(EVT VT, const Function &MF) const;
643
644 /// Return the refinement step count for a division of the given type based
645 /// on the function's attributes. If the operation is not overridden by
646 /// the function's attributes, "Unspecified" is returned and target defaults
647 /// are expected to be used for instruction selection.
648 int getDivRefinementSteps(EVT VT, const Function &MF) const;
649
650 /// Returns true if target has indicated at least one type should be bypassed.
651 bool isSlowDivBypassed() const { return !BypassSlowDivWidths.empty(); }
652
653 /// Returns map of slow types for division or remainder with corresponding
654 /// fast types
656 return BypassSlowDivWidths;
657 }
658
659 /// Return true if Flow Control is an expensive operation that should be
660 /// avoided.
661 bool isJumpExpensive() const { return JumpIsExpensive; }
662
663 // Costs parameters used by
664 // SelectionDAGBuilder::shouldKeepJumpConditionsTogether.
665 // shouldKeepJumpConditionsTogether will use these parameter value to
666 // determine if two conditions in the form `br (and/or cond1, cond2)` should
667 // be split into two branches or left as one.
668 //
669 // BaseCost is the cost threshold (in latency). If the estimated latency of
670 // computing both `cond1` and `cond2` is below the cost of just computing
671 // `cond1` + BaseCost, the two conditions will be kept together. Otherwise
672 // they will be split.
673 //
674 // LikelyBias increases BaseCost if branch probability info indicates that it
675 // is likely that both `cond1` and `cond2` will be computed.
676 //
677 // UnlikelyBias decreases BaseCost if branch probability info indicates that
678 // it is likely that both `cond1` and `cond2` will be computed.
679 //
680 // Set any field to -1 to make it ignored (setting BaseCost to -1 results in
681 // `shouldKeepJumpConditionsTogether` always returning false).
687 // Return params for deciding if we should keep two branch conditions merged
688 // or split them into two separate branches.
689 // Arg0: The binary op joining the two conditions (and/or).
690 // Arg1: The first condition (cond1)
691 // Arg2: The second condition (cond2)
692 // Arg3: The containing function.
693 virtual CondMergingParams
695 const Value *, const Function *) const {
696 // -1 will always result in splitting.
697 return {-1, -1, -1};
698 }
699
700 /// Return true if selects are only cheaper than branches if the branch is
701 /// unlikely to be predicted right.
705
706 virtual bool fallBackToDAGISel(const Instruction &Inst) const {
707 return false;
708 }
709
710 /// Return true if the following transform is beneficial:
711 /// fold (conv (load x)) -> (load (conv*)x)
712 /// On architectures that don't natively support some vector loads
713 /// efficiently, casting the load to a smaller vector of larger types and
714 /// loading is more efficient, however, this can be undone by optimizations in
715 /// dag combiner.
716 virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
717 const SelectionDAG &DAG,
718 const MachineMemOperand &MMO) const;
719
720 /// Return true if the following transform is beneficial:
721 /// (store (y (conv x)), y*)) -> (store x, (x*))
722 virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT,
723 const SelectionDAG &DAG,
724 const MachineMemOperand &MMO) const {
725 // Default to the same logic as loads.
726 return isLoadBitCastBeneficial(StoreVT, BitcastVT, DAG, MMO);
727 }
728
729 /// Return true if it is expected to be cheaper to do a store of vector
730 /// constant with the given size and type for the address space than to
731 /// store the individual scalar element constants.
732 virtual bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT,
733 unsigned NumElem,
734 unsigned AddrSpace) const {
735 return IsZero;
736 }
737
738 /// Allow store merging for the specified type after legalization in addition
739 /// to before legalization. This may transform stores that do not exist
740 /// earlier (for example, stores created from intrinsics).
741 virtual bool mergeStoresAfterLegalization(EVT MemVT) const {
742 return true;
743 }
744
745 /// Returns if it's reasonable to merge stores to MemVT size.
746 virtual bool canMergeStoresTo(unsigned AS, EVT MemVT,
747 const MachineFunction &MF) const {
748 return true;
749 }
750
751 /// Return true if it is cheap to speculate a call to intrinsic cttz.
752 virtual bool isCheapToSpeculateCttz(Type *Ty) const {
753 return false;
754 }
755
756 /// Return true if it is cheap to speculate a call to intrinsic ctlz.
757 virtual bool isCheapToSpeculateCtlz(Type *Ty) const {
758 return false;
759 }
760
761 /// Return true if ctlz instruction is fast.
762 virtual bool isCtlzFast() const {
763 return false;
764 }
765
766 /// Return true if ctpop instruction is fast.
767 virtual bool isCtpopFast(EVT VT) const {
768 return isOperationLegal(ISD::CTPOP, VT);
769 }
770
771 /// Return the maximum number of "x & (x - 1)" operations that can be done
772 /// instead of deferring to a custom CTPOP.
773 virtual unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const {
774 return 1;
775 }
776
777 /// Return true if instruction generated for equality comparison is folded
778 /// with instruction generated for signed comparison.
779 virtual bool isEqualityCmpFoldedWithSignedCmp() const { return true; }
780
781 /// Return true if the heuristic to prefer icmp eq zero should be used in code
782 /// gen prepare.
783 virtual bool preferZeroCompareBranch() const { return false; }
784
785 /// Return true if it is cheaper to split the store of a merged int val
786 /// from a pair of smaller values into multiple stores.
787 virtual bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const {
788 return false;
789 }
790
791 /// Return if the target supports combining a
792 /// chain like:
793 /// \code
794 /// %andResult = and %val1, #mask
795 /// %icmpResult = icmp %andResult, 0
796 /// \endcode
797 /// into a single machine instruction of a form like:
798 /// \code
799 /// cc = test %register, #mask
800 /// \endcode
801 virtual bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const {
802 return false;
803 }
804
805 /// Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
806 virtual bool
808 const MemSDNode &NodeY) const {
809 return true;
810 }
811
812 /// Use bitwise logic to make pairs of compares more efficient. For example:
813 /// and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
814 /// This should be true when it takes more than one instruction to lower
815 /// setcc (cmp+set on x86 scalar), when bitwise ops are faster than logic on
816 /// condition bits (crand on PowerPC), and/or when reducing cmp+br is a win.
817 virtual bool convertSetCCLogicToBitwiseLogic(EVT VT) const {
818 return false;
819 }
820
821 /// Return the preferred operand type if the target has a quick way to compare
822 /// integer values of the given size. Assume that any legal integer type can
823 /// be compared efficiently. Targets may override this to allow illegal wide
824 /// types to return a vector type if there is support to compare that type.
825 virtual MVT hasFastEqualityCompare(unsigned NumBits) const {
826 MVT VT = MVT::getIntegerVT(NumBits);
828 }
829
830 /// Return true if the target should transform:
831 /// (X & Y) == Y ---> (~X & Y) == 0
832 /// (X & Y) != Y ---> (~X & Y) != 0
833 ///
834 /// This may be profitable if the target has a bitwise and-not operation that
835 /// sets comparison flags. A target may want to limit the transformation based
836 /// on the type of Y or if Y is a constant.
837 ///
838 /// Note that the transform will not occur if Y is known to be a power-of-2
839 /// because a mask and compare of a single bit can be handled by inverting the
840 /// predicate, for example:
841 /// (X & 8) == 8 ---> (X & 8) != 0
842 virtual bool hasAndNotCompare(SDValue Y) const {
843 return false;
844 }
845
846 /// Return true if the target has a bitwise and-not operation:
847 /// X = ~A & B
848 /// This can be used to simplify select or other instructions.
849 virtual bool hasAndNot(SDValue X) const {
850 // If the target has the more complex version of this operation, assume that
851 // it has this operation too.
852 return hasAndNotCompare(X);
853 }
854
855 /// Return true if the target has a bit-test instruction:
856 /// (X & (1 << Y)) ==/!= 0
857 /// This knowledge can be used to prevent breaking the pattern,
858 /// or creating it if it could be recognized.
859 virtual bool hasBitTest(SDValue X, SDValue Y) const { return false; }
860
861 /// There are two ways to clear extreme bits (either low or high):
862 /// Mask: x & (-1 << y) (the instcombine canonical form)
863 /// Shifts: x >> y << y
864 /// Return true if the variant with 2 variable shifts is preferred.
865 /// Return false if there is no preference.
867 // By default, let's assume that no one prefers shifts.
868 return false;
869 }
870
871 /// Return true if it is profitable to fold a pair of shifts into a mask.
872 /// This is usually true on most targets. But some targets, like Thumb1,
873 /// have immediate shift instructions, but no immediate "and" instruction;
874 /// this makes the fold unprofitable.
875 virtual bool shouldFoldConstantShiftPairToMask(const SDNode *N) const {
876 return true;
877 }
878
879 /// Should we tranform the IR-optimal check for whether given truncation
880 /// down into KeptBits would be truncating or not:
881 /// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
882 /// Into it's more traditional form:
883 /// ((%x << C) a>> C) dstcond %x
884 /// Return true if we should transform.
885 /// Return false if there is no preference.
887 unsigned KeptBits) const {
888 // By default, let's assume that no one prefers shifts.
889 return false;
890 }
891
892 /// Given the pattern
893 /// (X & (C l>>/<< Y)) ==/!= 0
894 /// return true if it should be transformed into:
895 /// ((X <</l>> Y) & C) ==/!= 0
896 /// WARNING: if 'X' is a constant, the fold may deadlock!
897 /// FIXME: we could avoid passing XC, but we can't use isConstOrConstSplat()
898 /// here because it can end up being not linked in.
901 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
902 SelectionDAG &DAG) const {
903 if (hasBitTest(X, Y)) {
904 // One interesting pattern that we'd want to form is 'bit test':
905 // ((1 << Y) & C) ==/!= 0
906 // But we also need to be careful not to try to reverse that fold.
907
908 // Is this '1 << Y' ?
909 if (OldShiftOpcode == ISD::SHL && CC->isOne())
910 return false; // Keep the 'bit test' pattern.
911
912 // Will it be '1 << Y' after the transform ?
913 if (XC && NewShiftOpcode == ISD::SHL && XC->isOne())
914 return true; // Do form the 'bit test' pattern.
915 }
916
917 // If 'X' is a constant, and we transform, then we will immediately
918 // try to undo the fold, thus causing endless combine loop.
919 // So by default, let's assume everyone prefers the fold
920 // iff 'X' is not a constant.
921 return !XC;
922 }
923
924 // Return true if its desirable to perform the following transform:
925 // (fmul C, (uitofp Pow2))
926 // -> (bitcast_to_FP (add (bitcast_to_INT C), Log2(Pow2) << mantissa))
927 // (fdiv C, (uitofp Pow2))
928 // -> (bitcast_to_FP (sub (bitcast_to_INT C), Log2(Pow2) << mantissa))
929 //
930 // This is only queried after we have verified the transform will be bitwise
931 // equals.
932 //
933 // SDNode *N : The FDiv/FMul node we want to transform.
934 // SDValue FPConst: The Float constant operand in `N`.
935 // SDValue IntPow2: The Integer power of 2 operand in `N`.
937 SDValue IntPow2) const {
938 // Default to avoiding fdiv which is often very expensive.
939 return N->getOpcode() == ISD::FDIV;
940 }
941
942 // Given:
943 // (icmp eq/ne (and X, C0), (shift X, C1))
944 // or
945 // (icmp eq/ne X, (rotate X, C1))
946
947 // If C0 is a mask or shifted mask and the shift amt (C1) isolates the
948 // remaining bits (i.e something like `(x64 & UINT32_MAX) == (x64 >> 32)`)
949 // Do we prefer the shift to be shift-right, shift-left, or rotate.
950 // Note: It's only valid to convert between the rotate and shift versions iff
951 // the shift-amt (`C1`) divides the bit width.
952 // If ShiftOpc (current Opcode) is returned, do nothing.
954 EVT VT, unsigned ShiftOpc, bool MayTransformRotate,
955 const APInt &ShiftOrRotateAmt,
956 const std::optional<APInt> &AndMask) const {
957 return ShiftOpc;
958 }
959
960 /// These two forms are equivalent:
961 /// sub %y, (xor %x, -1)
962 /// add (add %x, 1), %y
963 /// The variant with two add's is IR-canonical.
964 /// Some targets may prefer one to the other.
965 virtual bool preferIncOfAddToSubOfNot(EVT VT) const {
966 // By default, let's assume that everyone prefers the form with two add's.
967 return true;
968 }
969
970 // By default prefer folding (abs (sub nsw x, y)) -> abds(x, y). Some targets
971 // may want to avoid this to prevent loss of sub_nsw pattern.
972 virtual bool preferABDSToABSWithNSW(EVT VT) const {
973 return true;
974 }
975
976 // Return true if the target wants to transform Op(Splat(X)) -> Splat(Op(X))
977 virtual bool preferScalarizeSplat(SDNode *N) const { return true; }
978
979 // Return true if the target wants to transform:
980 // (TruncVT truncate(sext_in_reg(VT X, ExtVT))
981 // -> (TruncVT sext_in_reg(truncate(VT X), ExtVT))
982 // Some targets might prefer pre-sextinreg to improve truncation/saturation.
983 virtual bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT, EVT ExtVT) const {
984 return true;
985 }
986
987 /// Return true if the target wants to use the optimization that
988 /// turns ext(promotableInst1(...(promotableInstN(load)))) into
989 /// promotedInst1(...(promotedInstN(ext(load)))).
991
992 /// Return true if the target can combine store(extractelement VectorTy,
993 /// Idx).
994 /// \p Cost[out] gives the cost of that transformation when this is true.
995 virtual bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx,
996 unsigned &Cost) const {
997 return false;
998 }
999
1000 /// Return true if the target shall perform extract vector element and store
1001 /// given that the vector is known to be splat of constant.
1002 /// \p Index[out] gives the index of the vector element to be extracted when
1003 /// this is true.
1005 Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const {
1006 return false;
1007 }
1008
1009 /// Return true if inserting a scalar into a variable element of an undef
1010 /// vector is more efficiently handled by splatting the scalar instead.
1011 virtual bool shouldSplatInsEltVarIndex(EVT) const {
1012 return false;
1013 }
1014
1015 /// Return true if target always benefits from combining into FMA for a
1016 /// given value type. This must typically return false on targets where FMA
1017 /// takes more cycles to execute than FADD.
1018 virtual bool enableAggressiveFMAFusion(EVT VT) const { return false; }
1019
1020 /// Return true if target always benefits from combining into FMA for a
1021 /// given value type. This must typically return false on targets where FMA
1022 /// takes more cycles to execute than FADD.
1023 virtual bool enableAggressiveFMAFusion(LLT Ty) const { return false; }
1024
1025 /// Return the ValueType of the result of SETCC operations.
1026 virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
1027 EVT VT) const;
1028
1029 /// Return the integer type returned by comparison libcalls. Comparison
1030 /// libcalls include floating point comparison calls, and Ordered/Unordered
1031 /// check calls on floating point numbers.
1033 return Type::getInt32Ty(Ctx);
1034 }
1035
1036 /// For targets without i1 registers, this gives the nature of the high-bits
1037 /// of boolean values held in types wider than i1.
1038 ///
1039 /// "Boolean values" are special true/false values produced by nodes like
1040 /// SETCC and consumed (as the condition) by nodes like SELECT and BRCOND.
1041 /// Not to be confused with general values promoted from i1. Some cpus
1042 /// distinguish between vectors of boolean and scalars; the isVec parameter
1043 /// selects between the two kinds. For example on X86 a scalar boolean should
1044 /// be zero extended from i1, while the elements of a vector of booleans
1045 /// should be sign extended from i1.
1046 ///
1047 /// Some cpus also treat floating point types the same way as they treat
1048 /// vectors instead of the way they treat scalars.
1050 if (isVec)
1051 return BooleanVectorContents;
1052 return isFloat ? BooleanFloatContents : BooleanContents;
1053 }
1054
1056 return getBooleanContents(Type.isVector(), Type.isFloatingPoint());
1057 }
1058
1059 /// Promote the given target boolean to a target boolean of the given type.
1060 /// A target boolean is an integer value, not necessarily of type i1, the bits
1061 /// of which conform to getBooleanContents.
1062 ///
1063 /// ValVT is the type of values that produced the boolean.
1065 EVT ValVT) const {
1066 SDLoc dl(Bool);
1067 EVT BoolVT =
1068 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ValVT);
1070 return DAG.getNode(ExtendCode, dl, BoolVT, Bool);
1071 }
1072
1073 /// Return target scheduling preference.
1075 return SchedPreferenceInfo;
1076 }
1077
1078 /// Some scheduler, e.g. hybrid, can switch to different scheduling heuristics
1079 /// for different nodes. This function returns the preference (or none) for
1080 /// the given node.
1082 return Sched::None;
1083 }
1084
1085 /// Return the register class that should be used for the specified value
1086 /// type.
1087 virtual const TargetRegisterClass *getRegClassFor(MVT VT, bool isDivergent = false) const {
1088 (void)isDivergent;
1089 const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy];
1090 assert(RC && "This value type is not natively supported!");
1091 return RC;
1092 }
1093
1094 /// Allows target to decide about the register class of the
1095 /// specific value that is live outside the defining block.
1096 /// Returns true if the value needs uniform register class.
1098 const Value *) const {
1099 return false;
1100 }
1101
1102 /// Return the 'representative' register class for the specified value
1103 /// type.
1104 ///
1105 /// The 'representative' register class is the largest legal super-reg
1106 /// register class for the register class of the value type. For example, on
1107 /// i386 the rep register class for i8, i16, and i32 are GR32; while the rep
1108 /// register class is GR64 on x86_64.
1109 virtual const TargetRegisterClass *getRepRegClassFor(MVT VT) const {
1110 const TargetRegisterClass *RC = RepRegClassForVT[VT.SimpleTy];
1111 return RC;
1112 }
1113
1114 /// Return the cost of the 'representative' register class for the specified
1115 /// value type.
1117 return RepRegClassCostForVT[VT.SimpleTy];
1118 }
1119
1120 /// Return the preferred strategy to legalize tihs SHIFT instruction, with
1121 /// \p ExpansionFactor being the recursion depth - how many expansion needed.
1127 virtual ShiftLegalizationStrategy
1129 unsigned ExpansionFactor) const {
1130 if (ExpansionFactor == 1)
1133 }
1134
1135 /// Return true if the target has native support for the specified value type.
1136 /// This means that it has a register that directly holds it without
1137 /// promotions or expansions.
1138 bool isTypeLegal(EVT VT) const {
1139 assert(!VT.isSimple() ||
1140 (unsigned)VT.getSimpleVT().SimpleTy < std::size(RegClassForVT));
1141 return VT.isSimple() && RegClassForVT[VT.getSimpleVT().SimpleTy] != nullptr;
1142 }
1143
1145 /// ValueTypeActions - For each value type, keep a LegalizeTypeAction enum
1146 /// that indicates how instruction selection should deal with the type.
1147 LegalizeTypeAction ValueTypeActions[MVT::VALUETYPE_SIZE];
1148
1149 public:
1150 ValueTypeActionImpl() { llvm::fill(ValueTypeActions, TypeLegal); }
1151
1153 return ValueTypeActions[VT.SimpleTy];
1154 }
1155
1157 ValueTypeActions[VT.SimpleTy] = Action;
1158 }
1159 };
1160
1162 return ValueTypeActions;
1163 }
1164
1165 /// Return pair that represents the legalization kind (first) that needs to
1166 /// happen to EVT (second) in order to type-legalize it.
1167 ///
1168 /// First: how we should legalize values of this type, either it is already
1169 /// legal (return 'Legal') or we need to promote it to a larger type (return
1170 /// 'Promote'), or we need to expand it into multiple registers of smaller
1171 /// integer type (return 'Expand'). 'Custom' is not an option.
1172 ///
1173 /// Second: for types supported by the target, this is an identity function.
1174 /// For types that must be promoted to larger types, this returns the larger
1175 /// type to promote to. For integer types that are larger than the largest
1176 /// integer register, this contains one step in the expansion to get to the
1177 /// smaller register. For illegal floating point types, this returns the
1178 /// integer type to transform to.
1179 LegalizeKind getTypeConversion(LLVMContext &Context, EVT VT) const;
1180
1181 /// Return how we should legalize values of this type, either it is already
1182 /// legal (return 'Legal') or we need to promote it to a larger type (return
1183 /// 'Promote'), or we need to expand it into multiple registers of smaller
1184 /// integer type (return 'Expand'). 'Custom' is not an option.
1186 return getTypeConversion(Context, VT).first;
1187 }
1189 return ValueTypeActions.getTypeAction(VT);
1190 }
1191
1192 /// For types supported by the target, this is an identity function. For
1193 /// types that must be promoted to larger types, this returns the larger type
1194 /// to promote to. For integer types that are larger than the largest integer
1195 /// register, this contains one step in the expansion to get to the smaller
1196 /// register. For illegal floating point types, this returns the integer type
1197 /// to transform to.
1198 virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const {
1199 return getTypeConversion(Context, VT).second;
1200 }
1201
1202 /// Perform getTypeToTransformTo repeatedly until a legal type is obtained.
1203 /// Useful for vector operations that might take multiple steps to legalize.
1205 EVT LegalVT = getTypeToTransformTo(Context, VT);
1206 while (LegalVT != VT) {
1207 VT = LegalVT;
1208 LegalVT = getTypeToTransformTo(Context, VT);
1209 }
1210 return LegalVT;
1211 }
1212
1213 /// For types supported by the target, this is an identity function. For
1214 /// types that must be expanded (i.e. integer types that are larger than the
1215 /// largest integer register or illegal floating point types), this returns
1216 /// the largest legal type it will be expanded to.
1217 EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const {
1218 assert(!VT.isVector());
1219 while (true) {
1220 switch (getTypeAction(Context, VT)) {
1221 case TypeLegal:
1222 return VT;
1223 case TypeExpandInteger:
1224 VT = getTypeToTransformTo(Context, VT);
1225 break;
1226 default:
1227 llvm_unreachable("Type is not legal nor is it to be expanded!");
1228 }
1229 }
1230 }
1231
1232 /// For vector types that are widened, this returns the type reached by
1233 /// widening repeatedly, i.e. the first type whose action is not
1234 /// TypeWidenVector. Unlike getLegalTypeToTransformTo, this does not follow
1235 /// other actions, so the result may still be illegal (e.g. if it is split).
1236 /// For vector types that are not widened, this is an identity function.
1237 EVT getTypeToWidenTo(LLVMContext &Context, EVT VT) const {
1238 assert(VT.isVector());
1239 while (getTypeAction(Context, VT) == TypeWidenVector)
1240 VT = getTypeToTransformTo(Context, VT);
1241 return VT;
1242 }
1243
1244 /// Vector types are broken down into some number of legal first class types.
1245 /// For example, EVT::v8f32 maps to 2 EVT::v4f32 with Altivec or SSE1, or 8
1246 /// promoted EVT::f64 values with the X86 FP stack. Similarly, EVT::v2i64
1247 /// turns into 4 EVT::i32 values with both PPC and X86.
1248 ///
1249 /// This method returns the number of registers needed, and the VT for each
1250 /// register. It also returns the VT and quantity of the intermediate values
1251 /// before they are promoted/expanded.
1253 EVT &IntermediateVT,
1254 unsigned &NumIntermediates,
1255 MVT &RegisterVT) const {
1256 return getVectorTypeBreakdownImpl(Context, VT, IntermediateVT,
1257 NumIntermediates, RegisterVT,
1258 /*ForCallingConv=*/false);
1259 }
1260
1261 /// Return true if fixed-length, non-power-of-two vectors should be broken
1262 /// down into legal vector parts instead of scalars for internal values.
1264 return false;
1265 }
1266
1267 bool shouldUseDynamicVectorTypeBreakdown(EVT VT, bool ForCallingConv) const {
1268 return preferVectorizedNonPowerOfTwoTypeBreakdown() && !ForCallingConv &&
1269 VT.isFixedLengthVector() &&
1271 }
1272
1273 /// Certain targets such as MIPS require that some types such as vectors are
1274 /// always broken down into scalars in some contexts. This occurs even if the
1275 /// vector type is legal.
1277 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
1278 unsigned &NumIntermediates, MVT &RegisterVT) const {
1279 return getVectorTypeBreakdownImpl(Context, VT, IntermediateVT,
1280 NumIntermediates, RegisterVT,
1281 /*ForCallingConv=*/true);
1282 }
1283
1285 unsigned opc = 0; // target opcode
1286 EVT memVT; // memory VT
1287
1288 // value representing memory location
1290
1291 // Fallback address space for use if ptrVal is nullptr. std::nullopt means
1292 // unknown address space.
1293 std::optional<unsigned> fallbackAddressSpace;
1294
1295 int offset = 0; // offset off of ptrVal
1296 uint64_t size = 0; // the size of the memory location
1297 // (taken from memVT if zero)
1298 MaybeAlign align = Align(1); // alignment
1299
1304 IntrinsicInfo() = default;
1305 };
1306
1307 /// Given an intrinsic, checks if on the target the intrinsic will need to map
1308 /// to a MemIntrinsicNode (touches memory). If this is the case, it stores
1309 /// the intrinsic information into the IntrinsicInfo vector passed to the
1310 /// function. The vector may contain multiple entries for intrinsics that
1311 /// access multiple memory locations.
1313 const CallBase &I, MachineFunction &MF,
1314 unsigned Intrinsic) const {}
1315
1316 /// How the legalizer should materialize a floating-point immediate.
1317 enum class FPImmAction {
1318 /// The target can instruction select the immediate natively.
1320 /// Build the immediate as a bitcast of an integer constant with the same
1321 /// bits.
1323 /// Load the immediate from the constant pool.
1325 };
1326
1327 /// Returns how the legalizer should materialize the specified FP immediate.
1328 virtual FPImmAction getFPImmAction(const APFloat & /*Imm*/, EVT /*VT*/,
1329 bool ForCodeSize = false) const {
1331 }
1332
1333 /// \copydoc getFPImmAction(const APFloat&,EVT,bool) const
1334 virtual FPImmAction getFPImmAction(const APFloat & /*Imm*/, LLT /*Ty*/,
1335 bool ForCodeSize = false) const {
1337 }
1338
1339 /// Returns true if the specified FP immediate can be materialized without a
1340 /// load from the constant pool.
1341 bool isFPImmLegal(const APFloat &Imm, EVT VT,
1342 bool ForCodeSize = false) const {
1343 return getFPImmAction(Imm, VT, ForCodeSize) != FPImmAction::ConstantPool;
1344 }
1345
1346 /// Targets can use this to indicate that they only support *some*
1347 /// VECTOR_SHUFFLE operations, those with specific masks. By default, if a
1348 /// target supports the VECTOR_SHUFFLE node, all mask values are assumed to be
1349 /// legal.
1350 virtual bool isShuffleMaskLegal(ArrayRef<int> /*Mask*/, EVT /*VT*/) const {
1351 return true;
1352 }
1353
1354 /// Similar to isShuffleMaskLegal. Targets can use this to indicate if there
1355 /// is a suitable VECTOR_SHUFFLE that can be used to replace a VAND with a
1356 /// constant pool entry.
1358 EVT /*VT*/) const {
1359 return false;
1360 }
1361
1362 /// How to legalize this custom operation?
1364 return Legal;
1365 }
1366
1367 /// Return how this operation should be treated: either it is legal, needs to
1368 /// be promoted to a larger size, needs to be expanded to some other code
1369 /// sequence, or the target has a custom expander for it.
1371 // If a target-specific SDNode requires legalization, require the target
1372 // to provide custom legalization for it.
1373 if (Op >= std::size(OpActions[0]))
1374 return Custom;
1375 if (VT.isExtended())
1376 return Expand;
1377 return OpActions[(unsigned)VT.getSimpleVT().SimpleTy][Op];
1378 }
1379
1380 /// Custom method defined by each target to indicate if an operation which
1381 /// may require a scale is supported natively by the target.
1382 /// If not, the operation is illegal.
1383 virtual bool isSupportedFixedPointOperation(unsigned Op, EVT VT,
1384 unsigned Scale) const {
1385 return false;
1386 }
1387
1388 /// Some fixed point operations may be natively supported by the target but
1389 /// only for specific scales. This method allows for checking
1390 /// if the width is supported by the target for a given operation that may
1391 /// depend on scale.
1393 unsigned Scale) const {
1394 auto Action = getOperationAction(Op, VT);
1395 if (Action != Legal)
1396 return Action;
1397
1398 // This operation is supported in this type but may only work on specific
1399 // scales.
1400 bool Supported;
1401 switch (Op) {
1402 default:
1403 llvm_unreachable("Unexpected fixed point operation.");
1404 case ISD::SMULFIX:
1405 case ISD::SMULFIXSAT:
1406 case ISD::UMULFIX:
1407 case ISD::UMULFIXSAT:
1408 case ISD::SDIVFIX:
1409 case ISD::SDIVFIXSAT:
1410 case ISD::UDIVFIX:
1411 case ISD::UDIVFIXSAT:
1412 Supported = isSupportedFixedPointOperation(Op, VT, Scale);
1413 break;
1414 }
1415
1416 return Supported ? Action : Expand;
1417 }
1418
1419 // If Op is a strict floating-point operation, return the result
1420 // of getOperationAction for the equivalent non-strict operation.
1422 unsigned EqOpc;
1423 switch (Op) {
1424 default: llvm_unreachable("Unexpected FP pseudo-opcode");
1425#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1426 case ISD::STRICT_##DAGN: EqOpc = ISD::DAGN; break;
1427#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1428 case ISD::STRICT_##DAGN: EqOpc = ISD::SETCC; break;
1429#include "llvm/IR/ConstrainedOps.def"
1430 }
1431
1432 return getOperationAction(EqOpc, VT);
1433 }
1434
1435 /// Return true if the specified operation is legal on this target or can be
1436 /// made legal with custom lowering. This is used to help guide high-level
1437 /// lowering decisions. LegalOnly is an optional convenience for code paths
1438 /// traversed pre and post legalisation.
1440 bool LegalOnly = false) const {
1441 if (LegalOnly)
1442 return isOperationLegal(Op, VT);
1443
1444 return (VT == MVT::Other || isTypeLegal(VT)) &&
1445 (getOperationAction(Op, VT) == Legal ||
1446 getOperationAction(Op, VT) == Custom);
1447 }
1448
1449 /// Return true if the specified operation is legal on this target or can be
1450 /// made legal using promotion. This is used to help guide high-level lowering
1451 /// decisions. LegalOnly is an optional convenience for code paths traversed
1452 /// pre and post legalisation.
1454 bool LegalOnly = false) const {
1455 if (LegalOnly)
1456 return isOperationLegal(Op, VT);
1457
1458 return (VT == MVT::Other || isTypeLegal(VT)) &&
1459 (getOperationAction(Op, VT) == Legal ||
1460 getOperationAction(Op, VT) == Promote);
1461 }
1462
1463 /// Return true if the specified operation is legal on this target or can be
1464 /// made legal with custom lowering or using promotion. This is used to help
1465 /// guide high-level lowering decisions. LegalOnly is an optional convenience
1466 /// for code paths traversed pre and post legalisation.
1468 bool LegalOnly = false) const {
1469 if (LegalOnly)
1470 return isOperationLegal(Op, VT);
1471
1472 return (VT == MVT::Other || isTypeLegal(VT)) &&
1473 (getOperationAction(Op, VT) == Legal ||
1474 getOperationAction(Op, VT) == Custom ||
1475 getOperationAction(Op, VT) == Promote);
1476 }
1477
1478 /// Return true if the operation uses custom lowering, regardless of whether
1479 /// the type is legal or not.
1480 bool isOperationCustom(unsigned Op, EVT VT) const {
1481 return getOperationAction(Op, VT) == Custom;
1482 }
1483
1484 /// Return true if lowering to a jump table is allowed.
1485 virtual bool areJTsAllowed(const Function *Fn) const {
1486 if (Fn->getFnAttribute("no-jump-tables").getValueAsBool())
1487 return false;
1488
1489 return isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
1491 }
1492
1493 /// Check whether the range [Low,High] fits in a machine word.
1494 bool rangeFitsInWord(const APInt &Low, const APInt &High,
1495 const DataLayout &DL) const {
1496 // FIXME: Using the pointer type doesn't seem ideal.
1497 uint64_t BW = DL.getIndexSizeInBits(0u);
1498 uint64_t Range = (High - Low).getLimitedValue(UINT64_MAX - 1) + 1;
1499 return Range <= BW;
1500 }
1501
1502 /// Return true if lowering to a jump table is suitable for a set of case
1503 /// clusters which may contain \p NumCases cases, \p Range range of values.
1504 virtual bool isSuitableForJumpTable(const SwitchInst *SI, uint64_t NumCases,
1506 BlockFrequencyInfo *BFI) const;
1507
1508 /// Returns preferred type for switch condition.
1509 virtual MVT getPreferredSwitchConditionType(LLVMContext &Context,
1510 EVT ConditionVT) const;
1511
1512 /// Return true if lowering to a bit test is suitable for a set of case
1513 /// clusters which contains \p NumDests unique destinations, \p Low and
1514 /// \p High as its lowest and highest case values, and expects \p NumCmps
1515 /// case value comparisons. Check if the number of destinations, comparison
1516 /// metric, and range are all suitable.
1519 const APInt &Low, const APInt &High, const DataLayout &DL) const {
1520 // FIXME: I don't think NumCmps is the correct metric: a single case and a
1521 // range of cases both require only one branch to lower. Just looking at the
1522 // number of clusters and destinations should be enough to decide whether to
1523 // build bit tests.
1524
1525 // To lower a range with bit tests, the range must fit the bitwidth of a
1526 // machine word.
1527 if (!rangeFitsInWord(Low, High, DL))
1528 return false;
1529
1530 unsigned NumDests = DestCmps.size();
1531 unsigned NumCmps = 0;
1532 unsigned int MaxBitTestEntry = 0;
1533 for (auto &DestCmp : DestCmps) {
1534 NumCmps += DestCmp.second;
1535 if (DestCmp.second > MaxBitTestEntry)
1536 MaxBitTestEntry = DestCmp.second;
1537 }
1538
1539 // Comparisons might be cheaper for small number of comparisons, which can
1540 // be Arch Target specific.
1541 if (MaxBitTestEntry < getMinimumBitTestCmps())
1542 return false;
1543
1544 // Decide whether it's profitable to lower this range with bit tests. Each
1545 // destination requires a bit test and branch, and there is an overall range
1546 // check branch. For a small number of clusters, separate comparisons might
1547 // be cheaper, and for many destinations, splitting the range might be
1548 // better.
1549 return (NumDests == 1 && NumCmps >= 3) || (NumDests == 2 && NumCmps >= 5) ||
1550 (NumDests == 3 && NumCmps >= 6);
1551 }
1552
1553 /// Return true if the specified operation is illegal on this target or
1554 /// unlikely to be made legal with custom lowering. This is used to help guide
1555 /// high-level lowering decisions.
1556 bool isOperationExpand(unsigned Op, EVT VT) const {
1557 return (!isTypeLegal(VT) || getOperationAction(Op, VT) == Expand);
1558 }
1559
1560 /// Return true if the specified operation is legal on this target.
1561 bool isOperationLegal(unsigned Op, EVT VT) const {
1562 return (VT == MVT::Other || isTypeLegal(VT)) &&
1563 getOperationAction(Op, VT) == Legal;
1564 }
1565
1566 bool isOperationExpandOrLibCall(unsigned Op, EVT VT) const {
1567 return isOperationExpand(Op, VT) || getOperationAction(Op, VT) == LibCall;
1568 }
1569
1570 /// Returns an alternative action to use when the coarser lookups (configured
1571 /// through `setLoadExtAction` and `setAtomicLoadExtAction`) yield
1572 /// `LegalizeAction::Custom`. Allows targets to use builtin behaviors (e.g.
1573 /// Legal, Promote) specialized by Alignment and AddrSpace, rather than just
1574 /// types.
1575 virtual LegalizeAction
1576 getCustomLoadAction(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace,
1577 unsigned ExtType, bool Atomic) const {
1579 }
1580
1581 /// Return how this load with extension should be treated: either it is legal,
1582 /// needs to be promoted to a larger size, needs to be expanded to some other
1583 /// code sequence, or the target has a custom expander for it.
1584 LegalizeAction getLoadAction(EVT ValVT, EVT MemVT, Align Alignment,
1585 unsigned AddrSpace, unsigned ExtType,
1586 bool Atomic) const {
1587 if (ValVT.isExtended() || MemVT.isExtended())
1588 return Expand;
1589 unsigned ValI = (unsigned)ValVT.getSimpleVT().SimpleTy;
1590 unsigned MemI = (unsigned)MemVT.getSimpleVT().SimpleTy;
1592 MemI < MVT::VALUETYPE_SIZE && "Table isn't big enough!");
1593 unsigned Shift = 4 * ExtType;
1594
1595 LegalizeAction Action;
1596 if (Atomic) {
1597 Action =
1598 (LegalizeAction)((AtomicLoadExtActions[ValI][MemI] >> Shift) & 0xf);
1599 assert((Action == Legal || Action == Expand) &&
1600 "Unsupported atomic load extension action.");
1601 } else {
1602 Action = (LegalizeAction)((LoadExtActions[ValI][MemI] >> Shift) & 0xf);
1603 }
1604
1605 if (Action == LegalizeAction::Custom) {
1606 return getCustomLoadAction(ValVT, MemVT, Alignment, AddrSpace, ExtType,
1607 Atomic);
1608 }
1609
1610 return Action;
1611 }
1612
1613 /// Return true if the specified load with extension is legal on this target.
1614 bool isLoadLegal(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace,
1615 unsigned ExtType, bool Atomic) const {
1616 return getLoadAction(ValVT, MemVT, Alignment, AddrSpace, ExtType, Atomic) ==
1617 Legal;
1618 }
1619
1620 /// Return true if the specified load with extension is legal or custom
1621 /// on this target.
1622 bool isLoadLegalOrCustom(EVT ValVT, EVT MemVT, Align Alignment,
1623 unsigned AddrSpace, unsigned ExtType,
1624 bool Atomic) const {
1625 LegalizeAction Action =
1626 getLoadAction(ValVT, MemVT, Alignment, AddrSpace, ExtType, Atomic);
1627 return Action == Legal || Action == Custom;
1628 }
1629
1630 /// Returns an alternative action to use when the coarser lookups (configured
1631 /// through `setTruncStoreAction` yield
1632 /// `LegalizeAction::Custom`. Allows targets to use builtin behaviors (e.g.
1633 /// Legal, Promote) specialized by Alignment and AddrSpace, rather than just
1634 /// types.
1636 Align Alignment,
1637 unsigned AddrSpace) const {
1639 }
1640
1641 /// Return how this store with truncation should be treated: either it is
1642 /// legal, needs to be promoted to a larger size, needs to be expanded to some
1643 /// other code sequence, or the target has a custom expander for it.
1645 unsigned AddrSpace) const {
1646 if (ValVT.isExtended() || MemVT.isExtended())
1647 return Expand;
1648 unsigned ValI = (unsigned)ValVT.getSimpleVT().SimpleTy;
1649 unsigned MemI = (unsigned)MemVT.getSimpleVT().SimpleTy;
1651 "Table isn't big enough!");
1652
1653 LegalizeAction Action = TruncStoreActions[ValI][MemI];
1654
1655 if (Action == LegalizeAction::Custom) {
1656 return getCustomTruncStoreAction(ValVT, MemVT, Alignment, AddrSpace);
1657 }
1658
1659 return Action;
1660 }
1661
1662 /// Return true if the specified store with truncation is legal on this
1663 /// target.
1664 bool isTruncStoreLegal(EVT ValVT, EVT MemVT, Align Alignment,
1665 unsigned AddrSpace) const {
1666 return isTypeLegal(ValVT) &&
1667 getTruncStoreAction(ValVT, MemVT, Alignment, AddrSpace) == Legal;
1668 }
1669
1670 /// Return true if the specified store with truncation has solution on this
1671 /// target.
1672 bool isTruncStoreLegalOrCustom(EVT ValVT, EVT MemVT, Align Alignment,
1673 unsigned AddrSpace) const {
1674 if (!isTypeLegal(ValVT))
1675 return false;
1676
1677 LegalizeAction Action =
1678 getTruncStoreAction(ValVT, MemVT, Alignment, AddrSpace);
1679 return (Action == Legal || Action == Custom);
1680 }
1681
1682 virtual bool canCombineTruncStore(EVT ValVT, EVT MemVT, Align Alignment,
1683 unsigned AddrSpace, bool LegalOnly) const {
1684 if (LegalOnly)
1685 return isTruncStoreLegal(ValVT, MemVT, Alignment, AddrSpace);
1686
1687 return isTruncStoreLegalOrCustom(ValVT, MemVT, Alignment, AddrSpace);
1688 }
1689
1690 /// Return how the indexed load should be treated: either it is legal, needs
1691 /// to be promoted to a larger size, needs to be expanded to some other code
1692 /// sequence, or the target has a custom expander for it.
1693 LegalizeAction getIndexedLoadAction(unsigned IdxMode, MVT VT) const {
1694 return getIndexedModeAction(IdxMode, VT, IMAB_Load);
1695 }
1696
1697 /// Return true if the specified indexed load is legal on this target.
1698 bool isIndexedLoadLegal(unsigned IdxMode, EVT VT) const {
1699 return VT.isSimple() &&
1700 (getIndexedLoadAction(IdxMode, VT.getSimpleVT()) == Legal ||
1701 getIndexedLoadAction(IdxMode, VT.getSimpleVT()) == Custom);
1702 }
1703
1704 /// Return how the indexed store should be treated: either it is legal, needs
1705 /// to be promoted to a larger size, needs to be expanded to some other code
1706 /// sequence, or the target has a custom expander for it.
1707 LegalizeAction getIndexedStoreAction(unsigned IdxMode, MVT VT) const {
1708 return getIndexedModeAction(IdxMode, VT, IMAB_Store);
1709 }
1710
1711 /// Return true if the specified indexed load is legal on this target.
1712 bool isIndexedStoreLegal(unsigned IdxMode, EVT VT) const {
1713 return VT.isSimple() &&
1714 (getIndexedStoreAction(IdxMode, VT.getSimpleVT()) == Legal ||
1715 getIndexedStoreAction(IdxMode, VT.getSimpleVT()) == Custom);
1716 }
1717
1718 /// Return how the indexed load should be treated: either it is legal, needs
1719 /// to be promoted to a larger size, needs to be expanded to some other code
1720 /// sequence, or the target has a custom expander for it.
1721 LegalizeAction getIndexedMaskedLoadAction(unsigned IdxMode, MVT VT) const {
1722 return getIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad);
1723 }
1724
1725 /// Return true if the specified indexed load is legal on this target.
1726 bool isIndexedMaskedLoadLegal(unsigned IdxMode, EVT VT) const {
1727 return VT.isSimple() &&
1728 (getIndexedMaskedLoadAction(IdxMode, VT.getSimpleVT()) == Legal ||
1730 }
1731
1732 /// Return how the indexed store should be treated: either it is legal, needs
1733 /// to be promoted to a larger size, needs to be expanded to some other code
1734 /// sequence, or the target has a custom expander for it.
1735 LegalizeAction getIndexedMaskedStoreAction(unsigned IdxMode, MVT VT) const {
1736 return getIndexedModeAction(IdxMode, VT, IMAB_MaskedStore);
1737 }
1738
1739 /// Return true if the specified indexed load is legal on this target.
1740 bool isIndexedMaskedStoreLegal(unsigned IdxMode, EVT VT) const {
1741 return VT.isSimple() &&
1742 (getIndexedMaskedStoreAction(IdxMode, VT.getSimpleVT()) == Legal ||
1744 }
1745
1746 /// Returns true if the index type for a masked gather/scatter requires
1747 /// extending
1748 virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const { return false; }
1749
1750 // Returns true if Extend can be folded into the index of a masked gathers/scatters
1751 // on this target.
1752 virtual bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const {
1753 return false;
1754 }
1755
1756 // Return true if the target supports a scatter/gather instruction with
1757 // indices which are scaled by the particular value. Note that all targets
1758 // must by definition support scale of 1.
1760 uint64_t ElemSize) const {
1761 // MGATHER/MSCATTER are only required to support scaling by one or by the
1762 // element size.
1763 if (Scale != ElemSize && Scale != 1)
1764 return false;
1765 return true;
1766 }
1767
1768 /// Return how the condition code should be treated: either it is legal, needs
1769 /// to be expanded to some other code sequence, or the target has a custom
1770 /// expander for it.
1773 assert((unsigned)CC < std::size(CondCodeActions) &&
1774 ((unsigned)VT.SimpleTy >> 3) < std::size(CondCodeActions[0]) &&
1775 "Table isn't big enough!");
1776 // See setCondCodeAction for how this is encoded.
1777 uint32_t Shift = 4 * (VT.SimpleTy & 0x7);
1778 uint32_t Value = CondCodeActions[CC][VT.SimpleTy >> 3];
1779 LegalizeAction Action = (LegalizeAction) ((Value >> Shift) & 0xF);
1780 assert(Action != Promote && "Can't promote condition code!");
1781 return Action;
1782 }
1783
1784 /// Return true if the specified condition code is legal for a comparison of
1785 /// the specified types on this target.
1787 return getCondCodeAction(CC, VT) == Legal;
1788 }
1789
1790 /// Return true if the specified condition code is legal or custom for a
1791 /// comparison of the specified types on this target.
1793 return getCondCodeAction(CC, VT) == Legal ||
1794 getCondCodeAction(CC, VT) == Custom;
1795 }
1796
1797 /// Return how a PARTIAL_REDUCE_U/SMLA node with Acc type AccVT and Input type
1798 /// InputVT should be treated. Either it's legal, needs to be promoted to a
1799 /// larger size, needs to be expanded to some other code sequence, or the
1800 /// target has a custom expander for it.
1802 EVT InputVT) const {
1805 // If the inputs aren't simple (e.g. non-power-of-two element type), just
1806 // expand.
1807 if (!AccVT.isSimple() || !InputVT.isSimple())
1808 return Expand;
1809 PartialReduceActionTypes Key = {Opc, AccVT.getSimpleVT().SimpleTy,
1810 InputVT.getSimpleVT().SimpleTy};
1811 auto It = PartialReduceMLAActions.find(Key);
1812 return It != PartialReduceMLAActions.end() ? It->second : Expand;
1813 }
1814
1815 /// Return true if a PARTIAL_REDUCE_U/SMLA node with the specified types is
1816 /// legal or custom for this target.
1818 EVT InputVT) const {
1819 LegalizeAction Action = getPartialReduceMLAAction(Opc, AccVT, InputVT);
1820 return Action == Legal || Action == Custom;
1821 }
1822
1823 /// Return how a VECTOR_INTERLEAVE or VECTOR_DEINTERLEAVE node with the
1824 /// given interleave factor and VT should be handled.
1826 EVT VT) const {
1828 VectorInterleaveActionKey Key = {Opc, Factor, VT.getSimpleVT().SimpleTy};
1829 auto It = VectorInterleaveActions.find(Key);
1830 return It != VectorInterleaveActions.end() ? It->second : Expand;
1831 }
1832
1833 /// Return true if a VECTOR_INTERLEAVE or VECTOR_DEINTERLEAVE node with the
1834 /// given interleave factor and fragment type is legal or custom.
1835 bool isVectorInterleaveLegalOrCustom(unsigned Opc, unsigned Factor,
1836 EVT VT) const {
1837 LegalizeAction Action = getVectorInterleaveAction(Opc, Factor, VT);
1838 return Action == Legal || Action == Custom;
1839 }
1840
1841 /// If the action for this operation is to promote, this method returns the
1842 /// ValueType to promote to.
1843 MVT getTypeToPromoteTo(unsigned Op, MVT VT) const {
1845 "This operation isn't promoted!");
1846
1847 // See if this has an explicit type specified.
1848 std::map<std::pair<unsigned, MVT::SimpleValueType>,
1850 PromoteToType.find(std::make_pair(Op, VT.SimpleTy));
1851 if (PTTI != PromoteToType.end()) return PTTI->second;
1852
1853 assert((VT.isInteger() || VT.isFloatingPoint()) &&
1854 "Cannot autopromote this type, add it with AddPromotedToType.");
1855
1856 uint64_t VTBits = VT.getScalarSizeInBits();
1857 MVT NVT = VT;
1858 do {
1859 NVT = (MVT::SimpleValueType)(NVT.SimpleTy+1);
1860 assert(NVT.isInteger() == VT.isInteger() &&
1861 NVT.isFloatingPoint() == VT.isFloatingPoint() &&
1862 "Didn't find type to promote to!");
1863 } while (VTBits >= NVT.getScalarSizeInBits() || !isTypeLegal(NVT) ||
1864 getOperationAction(Op, NVT) == Promote);
1865 return NVT;
1866 }
1867
1869 bool AllowUnknown = false) const {
1870 return getValueType(DL, Ty, AllowUnknown);
1871 }
1872
1873 /// Return the EVT corresponding to this LLVM type. This is fixed by the LLVM
1874 /// operations except for the pointer size. If AllowUnknown is true, this
1875 /// will return MVT::Other for types with no EVT counterpart (e.g. structs),
1876 /// otherwise it will assert.
1878 bool AllowUnknown = false) const {
1879 // Lower scalar pointers to native pointer types.
1880 if (auto *PTy = dyn_cast<PointerType>(Ty))
1881 return getPointerTy(DL, PTy->getAddressSpace());
1882
1883 if (auto *VTy = dyn_cast<VectorType>(Ty)) {
1884 Type *EltTy = VTy->getElementType();
1885 // Lower vectors of pointers to native pointer types.
1886 EVT EltVT;
1887 if (auto *PTy = dyn_cast<PointerType>(EltTy))
1888 EltVT = getPointerTy(DL, PTy->getAddressSpace());
1889 else
1890 EltVT = EVT::getEVT(EltTy, false);
1891 return EVT::getVectorVT(Ty->getContext(), EltVT, VTy->getElementCount());
1892 }
1893
1894 return EVT::getEVT(Ty, AllowUnknown);
1895 }
1896
1898 bool AllowUnknown = false) const {
1899 // Lower scalar pointers to native pointer types.
1900 if (auto *PTy = dyn_cast<PointerType>(Ty))
1901 return getPointerMemTy(DL, PTy->getAddressSpace());
1902
1903 if (auto *VTy = dyn_cast<VectorType>(Ty)) {
1904 Type *EltTy = VTy->getElementType();
1905 EVT EltVT;
1906 if (auto *PTy = dyn_cast<PointerType>(EltTy))
1907 EltVT = getPointerMemTy(DL, PTy->getAddressSpace());
1908 else
1909 EltVT = EVT::getEVT(EltTy, false);
1910 return EVT::getVectorVT(Ty->getContext(), EltVT, VTy->getElementCount());
1911 }
1912
1913 return getValueType(DL, Ty, AllowUnknown);
1914 }
1915
1916
1917 /// Return the MVT corresponding to this LLVM type. See getValueType.
1919 bool AllowUnknown = false) const {
1920 return getValueType(DL, Ty, AllowUnknown).getSimpleVT();
1921 }
1922
1923 /// Returns the desired alignment for ByVal or InAlloca aggregate function
1924 /// arguments in the caller parameter area.
1925 virtual Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const;
1926
1927 /// Return the type of registers that this ValueType will eventually require.
1928 MVT getRegisterType(LLVMContext &Context, EVT VT) const {
1929 return getRegisterTypeImpl(Context, VT, /*ForCallingConv=*/false);
1930 }
1931
1932 /// Return the number of registers that this ValueType will eventually
1933 /// require.
1934 ///
1935 /// This is one for any types promoted to live in larger registers, but may be
1936 /// more than one for types (like i64) that are split into pieces. For types
1937 /// like i140, which are first promoted then expanded, it is the number of
1938 /// registers needed to hold all the bits of the original type. For an i140
1939 /// on a 32 bit machine this means 5 registers.
1940 ///
1941 /// RegisterVT may be passed as a way to override the default settings, for
1942 /// instance with i128 inline assembly operands on SystemZ.
1943 virtual unsigned
1945 std::optional<MVT> RegisterVT = std::nullopt) const {
1946 return getNumRegistersImpl(Context, VT, /*ForCallingConv=*/false);
1947 }
1948
1949 /// Certain combinations of ABIs, Targets and features require that types
1950 /// are legal for some operations and not for other operations.
1951 /// For MIPS all vector types must be passed through the integer register set.
1953 CallingConv::ID CC, EVT VT) const {
1954 return getRegisterTypeImpl(Context, VT, /*ForCallingConv=*/true);
1955 }
1956
1957 /// Certain targets require unusual breakdowns of certain types. For MIPS,
1958 /// this occurs when a vector type is used, as vector are passed through the
1959 /// integer register set.
1962 EVT VT) const {
1963 return getNumRegistersImpl(Context, VT, /*ForCallingConv=*/true);
1964 }
1965
1966 /// Certain targets have context sensitive alignment requirements, where one
1967 /// type has the alignment requirement of another type.
1969 const DataLayout &DL) const {
1970 return DL.getABITypeAlign(ArgTy);
1971 }
1972
1973 /// If true, then instruction selection should seek to shrink the FP constant
1974 /// of the specified type to a smaller type in order to save space and / or
1975 /// reduce runtime.
1976 virtual bool ShouldShrinkFPConstant(EVT) const { return true; }
1977
1978 /// Return true if it is profitable to reduce a load to a smaller type.
1979 /// \p ByteOffset is only set if we know the pointer offset at compile time
1980 /// otherwise we should assume that additional pointer math is required.
1981 /// Example: (i16 (trunc (i32 (load x))) -> i16 load x
1982 /// Example: (i16 (trunc (srl (i32 (load x)), 16)) -> i16 load x+2
1984 SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,
1985 std::optional<unsigned> ByteOffset = std::nullopt) const {
1986 // By default, assume that it is cheaper to extract a subvector from a wide
1987 // vector load rather than creating multiple narrow vector loads.
1988 if (NewVT.isVector() && !SDValue(Load, 0).hasOneUse())
1989 return false;
1990
1991 return true;
1992 }
1993
1994 /// Return true (the default) if it is profitable to remove a sext_inreg(x)
1995 /// where the sext is redundant, and use x directly.
1996 virtual bool shouldRemoveRedundantExtend(SDValue Op) const { return true; }
1997
1998 /// Indicates if any padding is guaranteed to go at the most significant bits
1999 /// when storing the type to memory and the type size isn't equal to the store
2000 /// size.
2002 return VT.isScalarInteger() && !VT.isByteSized();
2003 }
2004
2005 /// When splitting a value of the specified type into parts, does the Lo
2006 /// or Hi part come first? This usually follows the endianness, except
2007 /// for ppcf128, where the Hi part always comes first.
2009 return DL.isBigEndian() || VT == MVT::ppcf128;
2010 }
2011
2012 /// If true, the target has custom DAG combine transformations that it can
2013 /// perform for the specified node.
2015 assert(unsigned(NT >> 3) < std::size(TargetDAGCombineArray));
2016 return TargetDAGCombineArray[NT >> 3] & (1 << (NT&7));
2017 }
2018
2021 }
2022
2023 /// Returns the size of the platform's va_list object.
2024 virtual unsigned getVaListSizeInBits(const DataLayout &DL) const {
2025 return getPointerTy(DL).getSizeInBits();
2026 }
2027
2028 /// Get maximum # of store operations permitted for llvm.memset
2029 ///
2030 /// This function returns the maximum number of store operations permitted
2031 /// to replace a call to llvm.memset. The value is set by the target at the
2032 /// performance threshold for such a replacement. If OptSize is true,
2033 /// return the limit for functions that have OptSize attribute.
2034 unsigned getMaxStoresPerMemset(bool OptSize) const;
2035
2036 /// Get maximum # of store operations permitted for llvm.memcpy
2037 ///
2038 /// This function returns the maximum number of store operations permitted
2039 /// to replace a call to llvm.memcpy. The value is set by the target at the
2040 /// performance threshold for such a replacement. If OptSize is true,
2041 /// return the limit for functions that have OptSize attribute.
2042 unsigned getMaxStoresPerMemcpy(bool OptSize) const;
2043
2044 /// \brief Get maximum # of store operations to be glued together
2045 ///
2046 /// This function returns the maximum number of store operations permitted
2047 /// to glue together during lowering of llvm.memcpy. The value is set by
2048 // the target at the performance threshold for such a replacement.
2049 virtual unsigned getMaxGluedStoresPerMemcpy() const {
2051 }
2052
2053 /// Get maximum # of load operations permitted for memcmp
2054 ///
2055 /// This function returns the maximum number of load operations permitted
2056 /// to replace a call to memcmp. The value is set by the target at the
2057 /// performance threshold for such a replacement. If OptSize is true,
2058 /// return the limit for functions that have OptSize attribute.
2059 unsigned getMaxExpandSizeMemcmp(bool OptSize) const {
2061 }
2062
2063 /// Get maximum # of store operations permitted for llvm.memmove
2064 ///
2065 /// This function returns the maximum number of store operations permitted
2066 /// to replace a call to llvm.memmove. The value is set by the target at the
2067 /// performance threshold for such a replacement. If OptSize is true,
2068 /// return the limit for functions that have OptSize attribute.
2069 unsigned getMaxStoresPerMemmove(bool OptSize) const;
2070
2071 /// Determine if the target supports unaligned memory accesses.
2072 ///
2073 /// This function returns true if the target allows unaligned memory accesses
2074 /// of the specified type in the given address space. If true, it also returns
2075 /// a relative speed of the unaligned memory access in the last argument by
2076 /// reference. The higher the speed number the faster the operation comparing
2077 /// to a number returned by another such call. This is used, for example, in
2078 /// situations where an array copy/move/set is converted to a sequence of
2079 /// store operations. Its use helps to ensure that such replacements don't
2080 /// generate code that causes an alignment error (trap) on the target machine.
2082 EVT, unsigned AddrSpace = 0, Align Alignment = Align(1),
2084 unsigned * /*Fast*/ = nullptr) const {
2085 return false;
2086 }
2087
2088 /// LLT handling variant.
2090 LLT, unsigned AddrSpace = 0, Align Alignment = Align(1),
2092 unsigned * /*Fast*/ = nullptr) const {
2093 return false;
2094 }
2095
2096 /// This function returns true if the memory access is aligned or if the
2097 /// target allows this specific unaligned memory access. If the access is
2098 /// allowed, the optional final parameter returns a relative speed of the
2099 /// access (as defined by the target).
2100 bool allowsMemoryAccessForAlignment(
2101 LLVMContext &Context, const DataLayout &DL, EVT VT,
2102 unsigned AddrSpace = 0, Align Alignment = Align(1),
2104 unsigned *Fast = nullptr) const;
2105
2106 /// Return true if the memory access of this type is aligned or if the target
2107 /// allows this specific unaligned access for the given MachineMemOperand.
2108 /// If the access is allowed, the optional final parameter returns a relative
2109 /// speed of the access (as defined by the target).
2110 bool allowsMemoryAccessForAlignment(LLVMContext &Context,
2111 const DataLayout &DL, EVT VT,
2112 const MachineMemOperand &MMO,
2113 unsigned *Fast = nullptr) const;
2114
2115 /// Return true if the target supports a memory access of this type for the
2116 /// given address space and alignment. If the access is allowed, the optional
2117 /// final parameter returns the relative speed of the access (as defined by
2118 /// the target).
2119 virtual bool
2120 allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
2121 unsigned AddrSpace = 0, Align Alignment = Align(1),
2123 unsigned *Fast = nullptr) const;
2124
2125 /// Return true if the target supports a memory access of this type for the
2126 /// given MachineMemOperand. If the access is allowed, the optional
2127 /// final parameter returns the relative access speed (as defined by the
2128 /// target).
2129 bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
2130 const MachineMemOperand &MMO,
2131 unsigned *Fast = nullptr) const;
2132
2133 /// LLT handling variant.
2134 bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, LLT Ty,
2135 const MachineMemOperand &MMO,
2136 unsigned *Fast = nullptr) const;
2137
2138 /// Returns the target specific optimal type for load and store operations as
2139 /// a result of memset, memcpy, and memmove lowering.
2140 /// It returns EVT::Other if the type should be determined using generic
2141 /// target-independent logic.
2142 virtual EVT
2144 const AttributeList & /*FuncAttributes*/) const {
2145 return MVT::Other;
2146 }
2147
2148 /// LLT returning variant.
2149 virtual LLT
2151 const AttributeList & /*FuncAttributes*/) const {
2152 return LLT();
2153 }
2154
2155 /// Returns true if it's safe to use load / store of the specified type to
2156 /// expand memcpy / memset inline.
2157 ///
2158 /// This is mostly true for all types except for some special cases. For
2159 /// example, on X86 targets without SSE2 f64 load / store are done with fldl /
2160 /// fstpl which also does type conversion. Note the specified type doesn't
2161 /// have to be legal as the hook is used before type legalization.
2162 virtual bool isSafeMemOpType(MVT /*VT*/) const { return true; }
2163
2164 /// Return lower limit for number of blocks in a jump table.
2165 virtual unsigned getMinimumJumpTableEntries() const;
2166
2167 /// Return lower limit of the density in a jump table.
2168 unsigned getMinimumJumpTableDensity(bool OptForSize) const;
2169
2170 /// Return upper limit for number of entries in a jump table.
2171 /// Zero if no limit.
2172 unsigned getMaximumJumpTableSize() const;
2173
2174 virtual bool isJumpTableRelative() const;
2175
2176 /// Retuen the minimum of largest number of comparisons in BitTest.
2177 unsigned getMinimumBitTestCmps() const;
2178
2179 /// Return maximum known-legal store size, which can be guaranteed for
2180 /// scalable vectors.
2182 return MaximumLegalStoreInBits;
2183 }
2184
2185 /// If a physical register, this specifies the register that
2186 /// llvm.savestack/llvm.restorestack should save and restore.
2188 return StackPointerRegisterToSaveRestore;
2189 }
2190
2191 /// If a physical register, this returns the register that receives the
2192 /// exception address on entry to an EH pad.
2193 virtual Register
2195 const Constant *PersonalityFn) const {
2196 return Register();
2197 }
2198
2199 /// If a physical register, this returns the register that receives the
2200 /// exception typeid on entry to a landing pad.
2201 virtual Register
2203 const Constant *PersonalityFn) const {
2204 return Register();
2205 }
2206
2207 virtual bool needsFixedCatchObjects() const {
2208 reportFatalUsageError("Funclet EH is not implemented for this target");
2209 }
2210
2211 /// Return the minimum stack alignment of an argument.
2213 return MinStackArgumentAlignment;
2214 }
2215
2216 /// Return the minimum function alignment.
2217 Align getMinFunctionAlignment() const { return MinFunctionAlignment; }
2218
2219 /// Return the preferred function alignment.
2220 Align getPrefFunctionAlignment() const { return PrefFunctionAlignment; }
2221
2222 /// Return the preferred loop alignment. \p BlockToAlign, when non-null, is
2223 /// the block that will actually be aligned; after loop rotation this need not
2224 /// be the LoopInfo header. Targets whose alignment depends on the block
2225 /// contents should use it. Callers that are not aligning a particular block,
2226 /// such as llvm-exegesis and ARM constant islands, leave it null.
2227 virtual Align
2228 getPrefLoopAlignment(MachineLoop *ML = nullptr,
2229 const MachineBasicBlock *BlockToAlign = nullptr) const;
2230
2231 /// Return the maximum amount of bytes allowed to be emitted when padding for
2232 /// alignment
2233 virtual unsigned
2234 getMaxPermittedBytesForAlignment(MachineBasicBlock *MBB) const;
2235
2236 /// Should loops be aligned even when the function is marked OptSize (but not
2237 /// MinSize).
2238 virtual bool alignLoopsWithOptSize() const { return false; }
2239
2240 /// If the target has a standard location for the stack protector guard,
2241 /// returns the address of that location. Otherwise, returns nullptr.
2242 /// DEPRECATED: please override useLoadStackGuardNode and customize
2243 /// LOAD_STACK_GUARD, or customize \@llvm.stackguard().
2244 virtual Value *getIRStackGuard(IRBuilderBase &IRB,
2245 const LibcallLoweringInfo &Libcalls) const;
2246
2247 /// Inserts necessary declarations for SSP (stack protection) purpose.
2248 /// Should be used only when getIRStackGuard returns nullptr.
2249 virtual void insertSSPDeclarations(Module &M,
2250 const LibcallLoweringInfo &Libcalls) const;
2251
2252 /// Return the variable that's previously inserted by insertSSPDeclarations,
2253 /// if any, otherwise return nullptr. Should be used only when
2254 /// getIRStackGuard returns nullptr.
2255 virtual Value *getSDagStackGuard(const Module &M,
2256 const LibcallLoweringInfo &Libcalls) const;
2257
2258 /// If this function returns true, stack protection checks should mix the
2259 /// frame pointer (or whichever pointer is used to address locals) into the
2260 /// stack guard value before checking it. getIRStackGuard must return nullptr
2261 /// if this returns true.
2262 virtual bool useStackGuardMixFP() const { return false; }
2263
2264 /// If the target has a standard stack protection check function that
2265 /// performs validation and error handling, returns the function. Otherwise,
2266 /// returns nullptr. Must be previously inserted by insertSSPDeclarations.
2267 /// Should be used only when getIRStackGuard returns nullptr.
2268 Function *getSSPStackGuardCheck(const Module &M,
2269 const LibcallLoweringInfo &Libcalls) const;
2270
2271protected:
2272 Value *
2273 getDefaultSafeStackPointerLocation(IRBuilderBase &IRB,
2274 const LibcallLoweringInfo &Libcalls) const;
2275
2276public:
2277 /// Emits a call to the libcall returning the address of the unsafe stack
2278 /// pointer. Returns nullptr if the libcall is unavailable.
2279 Value *
2280 getSafeStackPointerAddressCall(IRBuilderBase &IRB,
2281 const LibcallLoweringInfo &Libcalls) const;
2282
2283 /// Returns the target-specific address of the unsafe stack pointer.
2284 virtual Value *
2285 getSafeStackPointerLocation(IRBuilderBase &IRB,
2286 const LibcallLoweringInfo &Libcalls) const;
2287
2288 /// Returns the name of the symbol used to emit stack probes or the empty
2289 /// string if not applicable.
2290 virtual bool hasStackProbeSymbol(const MachineFunction &MF) const { return false; }
2291
2292 virtual bool hasInlineStackProbe(const MachineFunction &MF) const { return false; }
2293
2295 return "";
2296 }
2297
2298 /// Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g. we
2299 /// are happy to sink it into basic blocks. A cast may be free, but not
2300 /// necessarily a no-op. e.g. a free truncate from a 64-bit to 32-bit pointer.
2301 virtual bool isFreeAddrSpaceCast(const DataLayout &DL, unsigned SrcAS,
2302 unsigned DestAS) const;
2303
2304 /// Return true if the pointer arguments to CI should be aligned by aligning
2305 /// the object whose address is being passed. If so then MinSize is set to the
2306 /// minimum size the object must be to be aligned and PrefAlign is set to the
2307 /// preferred alignment.
2308 virtual bool shouldAlignPointerArgs(CallInst * /*CI*/, unsigned & /*MinSize*/,
2309 Align & /*PrefAlign*/) const {
2310 return false;
2311 }
2312
2313 //===--------------------------------------------------------------------===//
2314 /// \name Helpers for TargetTransformInfo implementations
2315 /// @{
2316
2317 /// Get the ISD node that corresponds to the Instruction class opcode.
2318 int InstructionOpcodeToISD(unsigned Opcode) const;
2319
2320 /// Get the ISD node that corresponds to the Intrinsic ID. Returns
2321 /// ISD::DELETED_NODE by default for an unsupported Intrinsic ID.
2322 int IntrinsicIDToISD(Intrinsic::ID ID) const;
2323
2324 /// @}
2325
2326 //===--------------------------------------------------------------------===//
2327 /// \name Helpers for atomic expansion.
2328 /// @{
2329
2330 /// Returns the maximum atomic operation size (in bits) supported by
2331 /// the backend. Atomic operations greater than this size (as well
2332 /// as ones that are not naturally aligned), will be expanded by
2333 /// AtomicExpandPass into an __atomic_* library call.
2335 return MaxAtomicSizeInBitsSupported;
2336 }
2337
2338 /// Returns the size in bits of the maximum div/rem the backend supports.
2339 /// Larger operations will be expanded by ExpandIRInsts.
2341 return MaxDivRemBitWidthSupported;
2342 }
2343
2344 /// Returns the size in bits of the maximum fp to/from int conversion the
2345 /// backend supports. Larger operations will be expanded by ExpandIRInsts.
2347 return MaxLargeFPConvertBitWidthSupported;
2348 }
2349
2350 /// Returns the size of the smallest cmpxchg or ll/sc instruction
2351 /// the backend supports. Any smaller operations are widened in
2352 /// AtomicExpandPass.
2353 ///
2354 /// Note that *unlike* operations above the maximum size, atomic ops
2355 /// are still natively supported below the minimum; they just
2356 /// require a more complex expansion.
2357 unsigned getMinCmpXchgSizeInBits() const { return MinCmpXchgSizeInBits; }
2358
2359 /// Return true if the target supports an atomic access of \p SizeInBytes
2360 /// bytes at the given \p Alignment. The default implementation only allows
2361 /// naturally aligned atomics, unless setSupportsUnalignedAtomics(true) was
2362 /// called.
2363 virtual bool isAtomicAlignmentSupported(Align Alignment,
2364 uint64_t SizeInBytes) const {
2365 return SupportsUnalignedAtomics || Alignment.value() >= SizeInBytes;
2366 }
2367
2368 /// Whether AtomicExpandPass should automatically insert fences and reduce
2369 /// ordering for this atomic. This should be true for most architectures with
2370 /// weak memory ordering. Defaults to false.
2371 virtual bool shouldInsertFencesForAtomic(const Instruction *I) const {
2372 return false;
2373 }
2374
2375 /// Whether a fence between the load-linked and the store-conditional can
2376 /// clear the reservation. If true, AtomicExpandPass places the release fence
2377 /// of a weak cmpxchg before the load-linked. Defaults to false.
2378 virtual bool fenceClearsLoadLinkedReservation() const { return false; }
2379
2380 /// Whether AtomicExpandPass should automatically insert a seq_cst trailing
2381 /// fence without reducing the ordering for this atomic store. Defaults to
2382 /// false.
2383 virtual bool
2385 return false;
2386 }
2387
2388 // The memory ordering that AtomicExpandPass should assign to a atomic
2389 // instruction that it has lowered by adding fences. This can be used
2390 // to "fold" one of the fences into the atomic instruction.
2391 virtual AtomicOrdering
2395
2396 // Whether to issue an atomic load for the initial word value before the
2397 // atomicrmw/cmpxchg emulation loop.
2398 // TODO: For correctness, an atomic load should be issued for all targets.
2399 // Remove this API once this is achieved
2401 return true;
2402 }
2403
2404 /// Perform a load-linked operation on Addr, returning a "Value *" with the
2405 /// corresponding pointee type. This may entail some non-trivial operations to
2406 /// truncate or reconstruct types that will be illegal in the backend. See
2407 /// ARMISelLowering for an example implementation.
2408 virtual Value *emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
2409 Value *Addr, AtomicOrdering Ord) const {
2410 llvm_unreachable("Load linked unimplemented on this target");
2411 }
2412
2413 /// Perform a store-conditional operation to Addr. Return the status of the
2414 /// store. This should be 0 if the store succeeded, non-zero otherwise.
2416 Value *Addr, AtomicOrdering Ord) const {
2417 llvm_unreachable("Store conditional unimplemented on this target");
2418 }
2419
2420 /// Perform a masked atomicrmw using a target-specific intrinsic. This
2421 /// represents the core LL/SC loop which will be lowered at a late stage by
2422 /// the backend. The target-specific intrinsic returns the loaded value and
2423 /// is not responsible for masking and shifting the result.
2425 AtomicRMWInst *AI,
2426 Value *AlignedAddr, Value *Incr,
2427 Value *Mask, Value *ShiftAmt,
2428 AtomicOrdering Ord) const {
2429 llvm_unreachable("Masked atomicrmw expansion unimplemented on this target");
2430 }
2431
2432 /// Perform a atomicrmw expansion using a target-specific way. This is
2433 /// expected to be called when masked atomicrmw and bit test atomicrmw don't
2434 /// work, and the target supports another way to lower atomicrmw.
2435 virtual void emitExpandAtomicRMW(AtomicRMWInst *AI) const {
2437 "Generic atomicrmw expansion unimplemented on this target");
2438 }
2439
2440 /// Perform a atomic store using a target-specific way.
2441 virtual void emitExpandAtomicStore(StoreInst *SI) const {
2443 "Generic atomic store expansion unimplemented on this target");
2444 }
2445
2446 /// Perform a atomic load using a target-specific way.
2447 virtual void emitExpandAtomicLoad(LoadInst *LI) const {
2449 "Generic atomic load expansion unimplemented on this target");
2450 }
2451
2452 /// Perform a cmpxchg expansion using a target-specific method.
2454 llvm_unreachable("Generic cmpxchg expansion unimplemented on this target");
2455 }
2456
2457 /// Perform a bit test atomicrmw using a target-specific intrinsic. This
2458 /// represents the combined bit test intrinsic which will be lowered at a late
2459 /// stage by the backend.
2462 "Bit test atomicrmw expansion unimplemented on this target");
2463 }
2464
2465 /// Perform a atomicrmw which the result is only used by comparison, using a
2466 /// target-specific intrinsic. This represents the combined atomic and compare
2467 /// intrinsic which will be lowered at a late stage by the backend.
2470 "Compare arith atomicrmw expansion unimplemented on this target");
2471 }
2472
2473 /// Perform a masked cmpxchg using a target-specific intrinsic. This
2474 /// represents the core LL/SC loop which will be lowered at a late stage by
2475 /// the backend. The target-specific intrinsic returns the loaded value and
2476 /// is not responsible for masking and shifting the result.
2478 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
2479 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
2480 llvm_unreachable("Masked cmpxchg expansion unimplemented on this target");
2481 }
2482
2483 //===--------------------------------------------------------------------===//
2484 /// \name KCFI check lowering.
2485 /// @{
2486
2489 const TargetInstrInfo *TII) const {
2490 llvm_unreachable("KCFI is not supported on this target");
2491 }
2492
2493 /// @}
2494
2495 //===--------------------------------------------------------------------===//
2496 /// \name Speculative load lowering.
2497 /// @{
2498
2499 /// Emit code to check if a speculative load of the given size from Ptr is
2500 /// safe. Returns a Value* representing the check result (i1), or nullptr
2501 /// to use the default lowering (which returns false). Targets can override
2502 /// to provide their own safety check (e.g., alignment-based page boundary
2503 /// check).
2504 /// \param Builder IRBuilder positioned at the intrinsic call site
2505 /// \param Ptr the pointer operand
2506 /// \param Size the size in bytes (constant or runtime value for scalable)
2508 Value *Size) const {
2509 return nullptr;
2510 }
2511
2512 /// @}
2513
2514 /// Inserts in the IR a target-specific intrinsic specifying a fence.
2515 /// It is called by AtomicExpandPass before expanding an
2516 /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad
2517 /// if shouldInsertFencesForAtomic returns true.
2518 ///
2519 /// Inst is the original atomic instruction, prior to other expansions that
2520 /// may be performed.
2521 ///
2522 /// This function should either return a nullptr, or a pointer to an IR-level
2523 /// Instruction*. Even complex fence sequences can be represented by a
2524 /// single Instruction* through an intrinsic to be lowered later.
2525 ///
2526 /// The default implementation emits an IR fence before any release (or
2527 /// stronger) operation that stores, and after any acquire (or stronger)
2528 /// operation. This is generally a correct implementation, but backends may
2529 /// override if they wish to use alternative schemes (e.g. the PowerPC
2530 /// standard ABI uses a fence before a seq_cst load instead of after a
2531 /// seq_cst store).
2532 /// @{
2533 virtual Instruction *emitLeadingFence(IRBuilderBase &Builder,
2534 Instruction *Inst,
2535 AtomicOrdering Ord) const;
2536
2537 virtual Instruction *emitTrailingFence(IRBuilderBase &Builder,
2538 Instruction *Inst,
2539 AtomicOrdering Ord) const;
2540 /// @}
2541
2542 // Emits code that executes when the comparison result in the ll/sc
2543 // expansion of a cmpxchg instruction is such that the store-conditional will
2544 // not execute. This makes it possible to balance out the load-linked with
2545 // a dedicated instruction, if desired.
2546 // E.g., on ARM, if ldrex isn't followed by strex, the exclusive monitor would
2547 // be unnecessarily held, except if clrex, inserted by this hook, is executed.
2548 virtual void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const {}
2549
2550 /// Returns true if arguments should be sign-extended in lib calls.
2551 virtual bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const {
2552 return IsSigned;
2553 }
2554
2555 /// Returns true if arguments should be extended in lib calls.
2556 virtual bool shouldExtendTypeInLibCall(EVT Type) const {
2557 return true;
2558 }
2559
2560 /// Returns how the given (atomic) load should be expanded by the
2561 /// IR-level AtomicExpand pass.
2565
2566 /// Returns how the given (atomic) load should be cast by the IR-level
2567 /// AtomicExpand pass.
2573
2574 /// Returns how the given (atomic) store should be expanded by the IR-level
2575 /// AtomicExpand pass into. For instance AtomicExpansionKind::CustomExpand
2576 /// will try to use an atomicrmw xchg.
2580
2581 /// Returns how the given (atomic) store should be cast by the IR-level
2582 /// AtomicExpand pass into. For instance AtomicExpansionKind::CastToInteger
2583 /// will try to cast the operands to integer values.
2585 if (SI->getValueOperand()->getType()->isFloatingPointTy())
2588 }
2589
2590 /// Returns how the given atomic cmpxchg should be expanded by the IR-level
2591 /// AtomicExpand pass.
2592 virtual AtomicExpansionKind
2596
2597 /// Returns how the IR-level AtomicExpand pass should expand the given
2598 /// AtomicRMW, if at all. By default, elementwise operations are expanded and
2599 /// floating-point operations use are expanded to a compare-exchange loop.
2600 virtual AtomicExpansionKind
2602 if (RMW->isElementwise())
2604 if (RMW->isFloatingPointOperation())
2606 if (RMW->getType()->isVectorTy())
2609 }
2610
2611 /// Returns how the given atomic atomicrmw should be cast by the IR-level
2612 /// AtomicExpand pass.
2613 virtual AtomicExpansionKind
2615 Type *ValTy = RMWI->getValOperand()->getType();
2616 if (RMWI->getOperation() == AtomicRMWInst::Xchg &&
2617 (ValTy->isFloatingPointTy() || ValTy->isPointerTy() ||
2618 ValTy->isVectorTy()))
2620
2622 }
2623
2624 /// On some platforms, an AtomicRMW that never actually modifies the value
2625 /// (such as fetch_add of 0) can be turned into a fence followed by an
2626 /// atomic load. This may sound useless, but it makes it possible for the
2627 /// processor to keep the cacheline shared, dramatically improving
2628 /// performance. And such idempotent RMWs are useful for implementing some
2629 /// kinds of locks, see for example (justification + benchmarks):
2630 /// http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf
2631 /// This method tries doing that transformation, returning the atomic load if
2632 /// it succeeds, and nullptr otherwise.
2633 /// If shouldExpandAtomicLoadInIR returns true on that load, it will undergo
2634 /// another round of expansion.
2635 virtual LoadInst *
2637 return nullptr;
2638 }
2639
2640 /// Returns how the platform's atomic operations are extended (ZERO_EXTEND,
2641 /// SIGN_EXTEND, or ANY_EXTEND).
2643 return ISD::ZERO_EXTEND;
2644 }
2645
2646 /// Returns how the platform's atomic compare and swap expects its comparison
2647 /// value to be extended (ZERO_EXTEND, SIGN_EXTEND, or ANY_EXTEND). This is
2648 /// separate from getExtendForAtomicOps, which is concerned with the
2649 /// sign-extension of the instruction's output, whereas here we are concerned
2650 /// with the sign-extension of the input. For targets with compare-and-swap
2651 /// instructions (or sub-word comparisons in their LL/SC loop expansions),
2652 /// the input can be ANY_EXTEND, but the output will still have a specific
2653 /// extension.
2655 return ISD::ANY_EXTEND;
2656 }
2657
2658 /// Returns how the platform's atomic rmw operations expect their input
2659 /// argument to be extended (ZERO_EXTEND, SIGN_EXTEND, or ANY_EXTEND).
2661 return ISD::ANY_EXTEND;
2662 }
2663
2664 /// @}
2665
2666 /// Returns true if we should normalize
2667 /// select(N0&N1, X, Y) => select(N0, select(N1, X, Y), Y) and
2668 /// select(N0|N1, X, Y) => select(N0, select(N1, X, Y, Y)) if it is likely
2669 /// that it saves us from materializing N0 and N1 in an integer register.
2670 /// Targets that are able to perform and/or on flags should return false here.
2671 /// \p VT is the type of the select (and X and Y). \p CCVT is the type of its
2672 /// condition (N0 and N1).
2674 EVT CCVT) const {
2675 // If a target has multiple condition registers, then it likely has logical
2676 // operations on those registers.
2678 return false;
2679 // Only do the transform if the value won't be split into multiple
2680 // registers.
2681 LegalizeTypeAction Action = getTypeAction(Context, VT);
2682 return Action != TypeExpandInteger && Action != TypeExpandFloat &&
2683 Action != TypeSplitVector;
2684 }
2685
2686 virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const { return true; }
2687
2688 /// Return true if a select of constants (select Cond, C1, C2) should be
2689 /// transformed into simple math ops with the condition value. For example:
2690 /// select Cond, C1, C1-1 --> add (zext Cond), C1-1
2691 virtual bool convertSelectOfConstantsToMath(EVT VT) const {
2692 return false;
2693 }
2694
2695 /// Return true if it is profitable to transform an integer
2696 /// multiplication-by-constant into simpler operations like shifts and adds.
2697 /// This may be true if the target does not directly support the
2698 /// multiplication operation for the specified type or the sequence of simpler
2699 /// ops is faster than the multiply.
2701 EVT VT, SDValue C) const {
2702 return false;
2703 }
2704
2705 /// Return true if it may be profitable to transform
2706 /// (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
2707 /// This may not be true if c1 and c2 can be represented as immediates but
2708 /// c1*c2 cannot, for example.
2709 /// The target should check if c1, c2 and c1*c2 can be represented as
2710 /// immediates, or have to be materialized into registers. If it is not sure
2711 /// about some cases, a default true can be returned to let the DAGCombiner
2712 /// decide.
2713 /// AddNode is (add x, c1), and ConstNode is c2.
2715 SDValue ConstNode) const {
2716 return true;
2717 }
2718
2719 /// Return true if it is more correct/profitable to use strict FP_TO_INT
2720 /// conversion operations - canonicalizing the FP source value instead of
2721 /// converting all cases and then selecting based on value.
2722 /// This may be true if the target throws exceptions for out of bounds
2723 /// conversions or has fast FP CMOV.
2724 virtual bool shouldUseStrictFP_TO_INT(EVT FpVT, EVT IntVT,
2725 bool IsSigned) const {
2726 return false;
2727 }
2728
2729 /// Return true if it is beneficial to expand an @llvm.powi.* intrinsic.
2730 /// If not optimizing for size, expanding @llvm.powi.* intrinsics is always
2731 /// considered beneficial.
2732 /// If optimizing for size, expansion is only considered beneficial for upto
2733 /// 5 multiplies and a divide (if the exponent is negative).
2734 bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const {
2735 if (Exponent < 0)
2736 Exponent = -Exponent;
2737 uint64_t E = static_cast<uint64_t>(Exponent);
2738 return !OptForSize || (llvm::popcount(E) + Log2_64(E) < 7);
2739 }
2740
2741 //===--------------------------------------------------------------------===//
2742 // TargetLowering Configuration Methods - These methods should be invoked by
2743 // the derived class constructor to configure this object for the target.
2744 //
2745protected:
2746 /// Specify how the target extends the result of integer and floating point
2747 /// boolean values from i1 to a wider type. See getBooleanContents.
2749 BooleanContents = Ty;
2750 BooleanFloatContents = Ty;
2751 }
2752
2753 /// Specify how the target extends the result of integer and floating point
2754 /// boolean values from i1 to a wider type. See getBooleanContents.
2756 BooleanContents = IntTy;
2757 BooleanFloatContents = FloatTy;
2758 }
2759
2760 /// Specify how the target extends the result of a vector boolean value from a
2761 /// vector of i1 to a wider type. See getBooleanContents.
2763 BooleanVectorContents = Ty;
2764 }
2765
2766 /// Specify the target scheduling preference.
2768 SchedPreferenceInfo = Pref;
2769 }
2770
2771 /// Indicate the minimum number of blocks to generate jump tables.
2772 void setMinimumJumpTableEntries(unsigned Val);
2773
2774 /// Indicate the maximum number of entries in jump tables.
2775 /// Set to zero to generate unlimited jump tables.
2776 void setMaximumJumpTableSize(unsigned);
2777
2778 /// Set the minimum of largest of number of comparisons to generate BitTest.
2779 void setMinimumBitTestCmps(unsigned Val);
2780
2781 /// If set to a physical register, this specifies the register that
2782 /// llvm.savestack/llvm.restorestack should save and restore.
2784 StackPointerRegisterToSaveRestore = R;
2785 }
2786
2787 /// Tells the code generator that the target has BitExtract instructions.
2788 /// The code generator will aggressively sink "shift"s into the blocks of
2789 /// their users if the users will generate "and" instructions which can be
2790 /// combined with "shift" to BitExtract instructions.
2791 void setHasExtractBitsInsn(bool hasExtractInsn = true) {
2792 HasExtractBitsInsn = hasExtractInsn;
2793 }
2794
2795 /// Tells the code generator not to expand logic operations on comparison
2796 /// predicates into separate sequences that increase the amount of flow
2797 /// control.
2798 void setJumpIsExpensive(bool isExpensive = true);
2799
2800 /// Tells the code generator which bitwidths to bypass.
2801 void addBypassSlowDiv(unsigned int SlowBitWidth, unsigned int FastBitWidth) {
2802 BypassSlowDivWidths[SlowBitWidth] = FastBitWidth;
2803 }
2804
2805 /// Add the specified register class as an available regclass for the
2806 /// specified value type. This indicates the selector can handle values of
2807 /// that class natively.
2809 assert((unsigned)VT.SimpleTy < std::size(RegClassForVT));
2810 RegClassForVT[VT.SimpleTy] = RC;
2811 }
2812
2813 /// Return the largest legal super-reg register class of the register class
2814 /// for the specified type and its associated "cost".
2815 virtual std::pair<const TargetRegisterClass *, uint8_t>
2816 findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const;
2817
2818 /// Once all of the register classes are added, this allows us to compute
2819 /// derived properties we expose.
2820 void computeRegisterProperties(const TargetRegisterInfo *TRI);
2821
2822 /// Indicate that the specified operation does not work with the specified
2823 /// type and indicate what to do about it. Note that VT may refer to either
2824 /// the type of a result or that of an operand of Op.
2825 void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action) {
2826 assert(Op < std::size(OpActions[0]) && "Table isn't big enough!");
2827 OpActions[(unsigned)VT.SimpleTy][Op] = Action;
2828 }
2830 LegalizeAction Action) {
2831 for (auto Op : Ops)
2832 setOperationAction(Op, VT, Action);
2833 }
2835 LegalizeAction Action) {
2836 for (auto VT : VTs)
2837 setOperationAction(Ops, VT, Action);
2838 }
2839
2840 /// Indicate that the specified load with extension does not work with the
2841 /// specified type and indicate what to do about it.
2842 void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT,
2843 LegalizeAction Action) {
2844 assert(ExtType < ISD::LAST_LOADEXT_TYPE && ValVT.isValid() &&
2845 MemVT.isValid() && "Table isn't big enough!");
2846 assert((unsigned)Action < 0x10 && "too many bits for bitfield array");
2847 unsigned Shift = 4 * ExtType;
2848 LoadExtActions[ValVT.SimpleTy][MemVT.SimpleTy] &= ~((uint16_t)0xF << Shift);
2849 LoadExtActions[ValVT.SimpleTy][MemVT.SimpleTy] |= (uint16_t)Action << Shift;
2850 }
2851 void setLoadExtAction(ArrayRef<unsigned> ExtTypes, MVT ValVT, MVT MemVT,
2852 LegalizeAction Action) {
2853 for (auto ExtType : ExtTypes)
2854 setLoadExtAction(ExtType, ValVT, MemVT, Action);
2855 }
2857 ArrayRef<MVT> MemVTs, LegalizeAction Action) {
2858 for (auto MemVT : MemVTs)
2859 setLoadExtAction(ExtTypes, ValVT, MemVT, Action);
2860 }
2861
2862 /// Let target indicate that an extending atomic load of the specified type
2863 /// is legal.
2864 void setAtomicLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT,
2865 LegalizeAction Action) {
2866 assert(ExtType < ISD::LAST_LOADEXT_TYPE && ValVT.isValid() &&
2867 MemVT.isValid() && "Table isn't big enough!");
2868 assert((unsigned)Action < 0x10 && "too many bits for bitfield array");
2869 unsigned Shift = 4 * ExtType;
2870 AtomicLoadExtActions[ValVT.SimpleTy][MemVT.SimpleTy] &=
2871 ~((uint16_t)0xF << Shift);
2872 AtomicLoadExtActions[ValVT.SimpleTy][MemVT.SimpleTy] |=
2873 ((uint16_t)Action << Shift);
2874 }
2876 LegalizeAction Action) {
2877 for (auto ExtType : ExtTypes)
2878 setAtomicLoadExtAction(ExtType, ValVT, MemVT, Action);
2879 }
2881 ArrayRef<MVT> MemVTs, LegalizeAction Action) {
2882 for (auto MemVT : MemVTs)
2883 setAtomicLoadExtAction(ExtTypes, ValVT, MemVT, Action);
2884 }
2885
2886 /// Indicate that the specified truncating store does not work with the
2887 /// specified type and indicate what to do about it.
2888 void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action) {
2889 assert(ValVT.isValid() && MemVT.isValid() && "Table isn't big enough!");
2890 TruncStoreActions[(unsigned)ValVT.SimpleTy][MemVT.SimpleTy] = Action;
2891 }
2892
2893 /// Indicate that the specified indexed load does or does not work with the
2894 /// specified type and indicate what to do abort it.
2895 ///
2896 /// NOTE: All indexed mode loads are initialized to Expand in
2897 /// TargetLowering.cpp
2899 LegalizeAction Action) {
2900 for (auto IdxMode : IdxModes)
2901 setIndexedModeAction(IdxMode, VT, IMAB_Load, Action);
2902 }
2903
2905 LegalizeAction Action) {
2906 for (auto VT : VTs)
2907 setIndexedLoadAction(IdxModes, VT, Action);
2908 }
2909
2910 /// Indicate that the specified indexed store does or does not work with the
2911 /// specified type and indicate what to do about it.
2912 ///
2913 /// NOTE: All indexed mode stores are initialized to Expand in
2914 /// TargetLowering.cpp
2916 LegalizeAction Action) {
2917 for (auto IdxMode : IdxModes)
2918 setIndexedModeAction(IdxMode, VT, IMAB_Store, Action);
2919 }
2920
2922 LegalizeAction Action) {
2923 for (auto VT : VTs)
2924 setIndexedStoreAction(IdxModes, VT, Action);
2925 }
2926
2927 /// Indicate that the specified indexed masked load does or does not work with
2928 /// the specified type and indicate what to do about it.
2929 ///
2930 /// NOTE: All indexed mode masked loads are initialized to Expand in
2931 /// TargetLowering.cpp
2932 void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT,
2933 LegalizeAction Action) {
2934 setIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad, Action);
2935 }
2936
2937 /// Indicate that the specified indexed masked store does or does not work
2938 /// with the specified type and indicate what to do about it.
2939 ///
2940 /// NOTE: All indexed mode masked stores are initialized to Expand in
2941 /// TargetLowering.cpp
2942 void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT,
2943 LegalizeAction Action) {
2944 setIndexedModeAction(IdxMode, VT, IMAB_MaskedStore, Action);
2945 }
2946
2947 /// Indicate that the specified condition code is or isn't supported on the
2948 /// target and indicate what to do about it.
2950 LegalizeAction Action) {
2951 for (auto CC : CCs) {
2952 assert(VT.isValid() && (unsigned)CC < std::size(CondCodeActions) &&
2953 "Table isn't big enough!");
2954 assert((unsigned)Action < 0x10 && "too many bits for bitfield array");
2955 /// The lower 3 bits of the SimpleTy index into Nth 4bit set from the
2956 /// 32-bit value and the upper 29 bits index into the second dimension of
2957 /// the array to select what 32-bit value to use.
2958 uint32_t Shift = 4 * (VT.SimpleTy & 0x7);
2959 CondCodeActions[CC][VT.SimpleTy >> 3] &= ~((uint32_t)0xF << Shift);
2960 CondCodeActions[CC][VT.SimpleTy >> 3] |= (uint32_t)Action << Shift;
2961 }
2962 }
2964 LegalizeAction Action) {
2965 for (auto VT : VTs)
2966 setCondCodeAction(CCs, VT, Action);
2967 }
2968
2969 /// Indicate how a PARTIAL_REDUCE_U/SMLA node with Acc type AccVT and Input
2970 /// type InputVT should be treated by the target. Either it's legal, needs to
2971 /// be promoted to a larger size, needs to be expanded to some other code
2972 /// sequence, or the target has a custom expander for it.
2973 void setPartialReduceMLAAction(unsigned Opc, MVT AccVT, MVT InputVT,
2974 LegalizeAction Action) {
2977 assert(AccVT.isValid() && InputVT.isValid() &&
2978 "setPartialReduceMLAAction types aren't valid");
2979 PartialReduceActionTypes Key = {Opc, AccVT.SimpleTy, InputVT.SimpleTy};
2980 PartialReduceMLAActions[Key] = Action;
2981 }
2983 MVT InputVT, LegalizeAction Action) {
2984 for (unsigned Opc : Opcodes)
2985 setPartialReduceMLAAction(Opc, AccVT, InputVT, Action);
2986 }
2987
2988 /// Indicate how a VECTOR_INTERLEAVE or VECTOR_DEINTERLEAVE node with the
2989 /// given interleave factor Factor and type VT should be treated.
2990 void setVectorInterleaveAction(unsigned Opc, unsigned Factor, MVT VT,
2991 LegalizeAction Action) {
2993 VectorInterleaveActionKey Key = {Opc, Factor, VT.SimpleTy};
2994 VectorInterleaveActions[Key] = Action;
2995 }
2996
2998 ArrayRef<unsigned> Factors, MVT VT,
2999 LegalizeAction Action) {
3000 for (unsigned Opc : Opcodes)
3001 for (unsigned Factor : Factors)
3002 setVectorInterleaveAction(Opc, Factor, VT, Action);
3003 }
3004
3005 /// If Opc/OrigVT is specified as being promoted, the promotion code defaults
3006 /// to trying a larger integer/fp until it can find one that works. If that
3007 /// default is insufficient, this method can be used by the target to override
3008 /// the default.
3009 void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT) {
3010 PromoteToType[std::make_pair(Opc, OrigVT.SimpleTy)] = DestVT.SimpleTy;
3011 }
3012
3013 /// Convenience method to set an operation to Promote and specify the type
3014 /// in a single call.
3015 void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT) {
3016 setOperationAction(Opc, OrigVT, Promote);
3017 AddPromotedToType(Opc, OrigVT, DestVT);
3018 }
3020 MVT DestVT) {
3021 for (auto Op : Ops) {
3022 setOperationAction(Op, OrigVT, Promote);
3023 AddPromotedToType(Op, OrigVT, DestVT);
3024 }
3025 }
3026
3027 /// Targets should invoke this method for each target independent node that
3028 /// they want to provide a custom DAG combiner for by implementing the
3029 /// PerformDAGCombine virtual method.
3031 for (auto NT : NTs) {
3032 assert(unsigned(NT >> 3) < std::size(TargetDAGCombineArray));
3033 TargetDAGCombineArray[NT >> 3] |= 1 << (NT & 7);
3034 }
3035 }
3036
3037 /// Set the target's minimum function alignment.
3039 MinFunctionAlignment = Alignment;
3040 }
3041
3042 /// Set the target's preferred function alignment. This should be set if
3043 /// there is a performance benefit to higher-than-minimum alignment
3045 PrefFunctionAlignment = Alignment;
3046 }
3047
3048 /// Set the target's preferred loop alignment. Default alignment is one, it
3049 /// means the target does not care about loop alignment. The target may also
3050 /// override getPrefLoopAlignment to provide per-loop values.
3051 void setPrefLoopAlignment(Align Alignment) { PrefLoopAlignment = Alignment; }
3052 void setMaxBytesForAlignment(unsigned MaxBytes) {
3053 MaxBytesForAlignment = MaxBytes;
3054 }
3055
3056 /// Set the minimum stack alignment of an argument.
3058 MinStackArgumentAlignment = Alignment;
3059 }
3060
3061 /// Set the maximum atomic operation size supported by the
3062 /// backend. Atomic operations greater than this size (as well as
3063 /// ones that are not naturally aligned), will be expanded by
3064 /// AtomicExpandPass into an __atomic_* library call.
3065 void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits) {
3066 MaxAtomicSizeInBitsSupported = SizeInBits;
3067 }
3068
3069 /// Set the size in bits of the maximum div/rem the backend supports.
3070 /// Larger operations will be expanded by ExpandIRInsts.
3071 void setMaxDivRemBitWidthSupported(unsigned SizeInBits) {
3072 MaxDivRemBitWidthSupported = SizeInBits;
3073 }
3074
3075 /// Set the size in bits of the maximum fp to/from int conversion the backend
3076 /// supports. Larger operations will be expanded by ExpandIRInsts.
3077 void setMaxLargeFPConvertBitWidthSupported(unsigned SizeInBits) {
3078 MaxLargeFPConvertBitWidthSupported = SizeInBits;
3079 }
3080
3081 /// Sets the minimum cmpxchg or ll/sc size supported by the backend.
3082 void setMinCmpXchgSizeInBits(unsigned SizeInBits) {
3083 MinCmpXchgSizeInBits = SizeInBits;
3084 }
3085
3086 /// Sets whether unaligned atomic operations are supported.
3087 void setSupportsUnalignedAtomics(bool UnalignedSupported) {
3088 SupportsUnalignedAtomics = UnalignedSupported;
3089 }
3090
3091public:
3092 //===--------------------------------------------------------------------===//
3093 // Addressing mode description hooks (used by LSR etc).
3094 //
3095
3096 /// CodeGenPrepare sinks address calculations into the same BB as Load/Store
3097 /// instructions reading the address. This allows as much computation as
3098 /// possible to be done in the address mode for that operand. This hook lets
3099 /// targets also pass back when this should be done on intrinsics which
3100 /// load/store.
3101 virtual bool getAddrModeArguments(const IntrinsicInst * /*I*/,
3102 SmallVectorImpl<Value *> & /*Ops*/,
3103 Type *& /*AccessTy*/) const {
3104 return false;
3105 }
3106
3107 /// This represents an addressing mode of:
3108 /// BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*vscale
3109 /// If BaseGV is null, there is no BaseGV.
3110 /// If BaseOffs is zero, there is no base offset.
3111 /// If HasBaseReg is false, there is no base register.
3112 /// If Scale is zero, there is no ScaleReg. Scale of 1 indicates a reg with
3113 /// no scale.
3114 /// If ScalableOffset is zero, there is no scalable offset.
3115 struct AddrMode {
3117 int64_t BaseOffs = 0;
3118 bool HasBaseReg = false;
3119 int64_t Scale = 0;
3120 int64_t ScalableOffset = 0;
3121 AddrMode() = default;
3122 };
3123
3124 /// Return true if the addressing mode represented by AM is legal for this
3125 /// target, for a load/store of the specified type.
3126 ///
3127 /// The type may be VoidTy, in which case only return true if the addressing
3128 /// mode is legal for a load/store of any legal type. TODO: Handle
3129 /// pre/postinc as well.
3130 ///
3131 /// If the address space cannot be determined, it will be -1.
3132 ///
3133 /// TODO: Remove default argument
3134 virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
3135 Type *Ty, unsigned AddrSpace,
3136 Instruction *I = nullptr) const;
3137
3138 /// Returns true if the targets addressing mode can target thread local
3139 /// storage (TLS).
3140 virtual bool addressingModeSupportsTLS(const GlobalValue &) const {
3141 return false;
3142 }
3143
3144 /// Return the prefered common base offset.
3145 virtual int64_t getPreferredLargeGEPBaseOffset(int64_t MinOffset,
3146 int64_t MaxOffset) const {
3147 return 0;
3148 }
3149
3150 /// Return true if the specified immediate is legal icmp immediate, that is
3151 /// the target has icmp instructions which can compare a register against the
3152 /// immediate without having to materialize the immediate into a register.
3153 virtual bool isLegalICmpImmediate(int64_t) const {
3154 return true;
3155 }
3156
3157 /// Return true if the specified immediate is legal add immediate, that is the
3158 /// target has add instructions which can add a register with the immediate
3159 /// without having to materialize the immediate into a register.
3160 virtual bool isLegalAddImmediate(int64_t) const {
3161 return true;
3162 }
3163
3164 /// Return true if adding the specified scalable immediate is legal, that is
3165 /// the target has add instructions which can add a register with the
3166 /// immediate (multiplied by vscale) without having to materialize the
3167 /// immediate into a register.
3168 virtual bool isLegalAddScalableImmediate(int64_t) const { return false; }
3169
3170 /// Return true if the specified immediate is legal for the value input of a
3171 /// store instruction.
3172 virtual bool isLegalStoreImmediate(int64_t Value) const {
3173 // Default implementation assumes that at least 0 works since it is likely
3174 // that a zero register exists or a zero immediate is allowed.
3175 return Value == 0;
3176 }
3177
3178 /// Given a shuffle vector SVI representing a vector splat, return a new
3179 /// scalar type of size equal to SVI's scalar type if the new type is more
3180 /// profitable. Returns nullptr otherwise. For example under MVE float splats
3181 /// are converted to integer to prevent the need to move from SPR to GPR
3182 /// registers.
3184 return nullptr;
3185 }
3186
3187 /// Given a set in interconnected phis of type 'From' that are loaded/stored
3188 /// or bitcast to type 'To', return true if the set should be converted to
3189 /// 'To'.
3190 virtual bool shouldConvertPhiType(Type *From, Type *To) const {
3191 return (From->isIntegerTy() || From->isFloatingPointTy()) &&
3192 (To->isIntegerTy() || To->isFloatingPointTy());
3193 }
3194
3195 /// Returns true if the opcode is a commutative binary operation.
3196 virtual bool isCommutativeBinOp(unsigned Opcode) const {
3197 // FIXME: This should get its info from the td file.
3198 switch (Opcode) {
3199 case ISD::ADD:
3200 case ISD::SMIN:
3201 case ISD::SMAX:
3202 case ISD::UMIN:
3203 case ISD::UMAX:
3204 case ISD::MUL:
3205 case ISD::CLMUL:
3206 case ISD::CLMULH:
3207 case ISD::CLMULR:
3208 case ISD::MULHU:
3209 case ISD::MULHS:
3210 case ISD::SMUL_LOHI:
3211 case ISD::UMUL_LOHI:
3212 case ISD::FADD:
3213 case ISD::FMUL:
3214 case ISD::AND:
3215 case ISD::OR:
3216 case ISD::XOR:
3217 case ISD::SADDO:
3218 case ISD::UADDO:
3219 case ISD::ADDC:
3220 case ISD::SADDSAT:
3221 case ISD::UADDSAT:
3222 case ISD::FMINNUM:
3223 case ISD::FMAXNUM:
3224 case ISD::FMINNUM_IEEE:
3225 case ISD::FMAXNUM_IEEE:
3226 case ISD::FMINIMUM:
3227 case ISD::FMAXIMUM:
3228 case ISD::FMINIMUMNUM:
3229 case ISD::FMAXIMUMNUM:
3230 case ISD::AVGFLOORS:
3231 case ISD::AVGFLOORU:
3232 case ISD::AVGCEILS:
3233 case ISD::AVGCEILU:
3234 case ISD::ABDS:
3235 case ISD::ABDU:
3236 return true;
3237 default: return false;
3238 }
3239 }
3240
3241 /// Return true if the node is a math/logic binary operator.
3242 virtual bool isBinOp(unsigned Opcode) const {
3243 // A commutative binop must be a binop.
3244 if (isCommutativeBinOp(Opcode))
3245 return true;
3246 // These are non-commutative binops.
3247 switch (Opcode) {
3248 case ISD::SUB:
3249 case ISD::SHL:
3250 case ISD::SRL:
3251 case ISD::SRA:
3252 case ISD::ROTL:
3253 case ISD::ROTR:
3254 case ISD::SDIV:
3255 case ISD::UDIV:
3256 case ISD::SREM:
3257 case ISD::UREM:
3258 case ISD::SSUBSAT:
3259 case ISD::USUBSAT:
3260 case ISD::AVGSUBFLOORS:
3261 case ISD::AVGSUBFLOORU:
3262 case ISD::FSUB:
3263 case ISD::FDIV:
3264 case ISD::FREM:
3265 case ISD::PSEUDO_FMIN:
3266 case ISD::PSEUDO_FMAX:
3267 return true;
3268 default:
3269 return false;
3270 }
3271 }
3272
3273 /// Return true if it's free to truncate a value of type FromTy to type
3274 /// ToTy. e.g. On x86 it's free to truncate a i32 value in register EAX to i16
3275 /// by referencing its sub-register AX.
3276 /// Targets must return false when FromTy <= ToTy.
3277 virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const {
3278 return false;
3279 }
3280
3281 /// Return true if a truncation from FromTy to ToTy is permitted when deciding
3282 /// whether a call is in tail position. Typically this means that both results
3283 /// would be assigned to the same register or stack slot, but it could mean
3284 /// the target performs adequate checks of its own before proceeding with the
3285 /// tail call. Targets must return false when FromTy <= ToTy.
3286 virtual bool allowTruncateForTailCall(Type *FromTy, Type *ToTy) const {
3287 return false;
3288 }
3289
3290 virtual bool isTruncateFree(EVT FromVT, EVT ToVT) const { return false; }
3291 virtual bool isTruncateFree(LLT FromTy, LLT ToTy, LLVMContext &Ctx) const {
3292 return isTruncateFree(getApproximateEVTForLLT(FromTy, Ctx),
3293 getApproximateEVTForLLT(ToTy, Ctx));
3294 }
3295
3296 /// Return true if truncating the specific node Val to type VT2 is free.
3297 virtual bool isTruncateFree(SDValue Val, EVT VT2) const {
3298 // Fallback to type matching.
3299 return isTruncateFree(Val.getValueType(), VT2);
3300 }
3301
3302 virtual bool isProfitableToHoist(Instruction *I) const { return true; }
3303
3304 /// Return true if the extension represented by \p I is free.
3305 /// Unlikely the is[Z|FP]ExtFree family which is based on types,
3306 /// this method can use the context provided by \p I to decide
3307 /// whether or not \p I is free.
3308 /// This method extends the behavior of the is[Z|FP]ExtFree family.
3309 /// In other words, if is[Z|FP]Free returns true, then this method
3310 /// returns true as well. The converse is not true.
3311 /// The target can perform the adequate checks by overriding isExtFreeImpl.
3312 /// \pre \p I must be a sign, zero, or fp extension.
3313 bool isExtFree(const Instruction *I) const {
3314 switch (I->getOpcode()) {
3315 case Instruction::FPExt:
3316 if (isFPExtFree(EVT::getEVT(I->getType()),
3317 EVT::getEVT(I->getOperand(0)->getType())))
3318 return true;
3319 break;
3320 case Instruction::ZExt:
3321 if (isZExtFree(I->getOperand(0)->getType(), I->getType()))
3322 return true;
3323 break;
3324 case Instruction::SExt:
3325 break;
3326 default:
3327 llvm_unreachable("Instruction is not an extension");
3328 }
3329 return isExtFreeImpl(I);
3330 }
3331
3332 /// Return true if \p Load and \p Ext can form an ExtLoad.
3333 /// For example, in AArch64
3334 /// %L = load i8, i8* %ptr
3335 /// %E = zext i8 %L to i32
3336 /// can be lowered into one load instruction
3337 /// ldrb w0, [x0]
3338 bool isExtLoad(const LoadInst *Load, const Instruction *Ext,
3339 const DataLayout &DL) const {
3340 EVT VT = getValueType(DL, Ext->getType());
3341 EVT LoadVT = getValueType(DL, Load->getType());
3342
3343 // If the load has other users and the truncate is not free, the ext
3344 // probably isn't free.
3345 if (!Load->hasOneUse() && (isTypeLegal(LoadVT) || !isTypeLegal(VT)) &&
3346 !isTruncateFree(Ext->getType(), Load->getType()))
3347 return false;
3348
3349 // Check whether the target supports casts folded into loads.
3350 unsigned LType;
3351 if (isa<ZExtInst>(Ext))
3352 LType = ISD::ZEXTLOAD;
3353 else {
3354 assert(isa<SExtInst>(Ext) && "Unexpected ext type!");
3355 LType = ISD::SEXTLOAD;
3356 }
3357
3358 return isLoadLegal(VT, LoadVT, Load->getAlign(),
3359 Load->getPointerAddressSpace(), LType, false);
3360 }
3361
3362 /// Return true if any actual instruction that defines a value of type FromTy
3363 /// implicitly zero-extends the value to ToTy in the result register.
3364 ///
3365 /// The function should return true when it is likely that the truncate can
3366 /// be freely folded with an instruction defining a value of FromTy. If
3367 /// the defining instruction is unknown (because you're looking at a
3368 /// function argument, PHI, etc.) then the target may require an
3369 /// explicit truncate, which is not necessarily free, but this function
3370 /// does not deal with those cases.
3371 /// Targets must return false when FromTy >= ToTy.
3372 virtual bool isZExtFree(Type *FromTy, Type *ToTy) const {
3373 return false;
3374 }
3375
3376 virtual bool isZExtFree(EVT FromTy, EVT ToTy) const { return false; }
3377 virtual bool isZExtFree(LLT FromTy, LLT ToTy, LLVMContext &Ctx) const {
3378 return isZExtFree(getApproximateEVTForLLT(FromTy, Ctx),
3379 getApproximateEVTForLLT(ToTy, Ctx));
3380 }
3381
3382 /// Return true if zero-extending the specific node Val to type VT2 is free
3383 /// (either because it's implicitly zero-extended such as ARM ldrb / ldrh or
3384 /// because it's folded such as X86 zero-extending loads).
3385 virtual bool isZExtFree(SDValue Val, EVT VT2) const {
3386 return isZExtFree(Val.getValueType(), VT2);
3387 }
3388
3389 /// Return true is an anyext is free from FromTy to ToTy. Usually true for
3390 /// scalar types when not trying to pack elements into vector lanes.
3391 virtual bool isAnyExtFree(EVT FromTy, EVT ToTy) const {
3392 return !FromTy.isVector();
3393 }
3394
3395 /// Return true if sign-extension from FromTy to ToTy is cheaper than
3396 /// zero-extension.
3397 virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const {
3398 return false;
3399 }
3400
3401 /// Return true if this constant should be sign extended when promoting to
3402 /// a larger type.
3403 virtual bool signExtendConstant(const ConstantInt *C) const { return false; }
3404
3405 /// Try to optimize extending or truncating conversion instructions (like
3406 /// zext, trunc, fptoui, uitofp) for the target.
3407 virtual bool
3409 const TargetTransformInfo &TTI) const {
3410 return false;
3411 }
3412
3414 return false;
3415 }
3416
3417 /// Return true if the target supplies and combines to a paired load
3418 /// two loaded values of type LoadedType next to each other in memory.
3419 /// RequiredAlignment gives the minimal alignment constraints that must be met
3420 /// to be able to select this paired load.
3421 ///
3422 /// This information is *not* used to generate actual paired loads, but it is
3423 /// used to generate a sequence of loads that is easier to combine into a
3424 /// paired load.
3425 /// For instance, something like this:
3426 /// a = load i64* addr
3427 /// b = trunc i64 a to i32
3428 /// c = lshr i64 a, 32
3429 /// d = trunc i64 c to i32
3430 /// will be optimized into:
3431 /// b = load i32* addr1
3432 /// d = load i32* addr2
3433 /// Where addr1 = addr2 +/- sizeof(i32).
3434 ///
3435 /// In other words, unless the target performs a post-isel load combining,
3436 /// this information should not be provided because it will generate more
3437 /// loads.
3438 virtual bool hasPairedLoad(EVT /*LoadedType*/,
3439 Align & /*RequiredAlignment*/) const {
3440 return false;
3441 }
3442
3443 /// Return true if the target has a vector blend instruction.
3444 virtual bool hasVectorBlend() const { return false; }
3445
3446 /// Get the maximum supported factor for interleaved memory accesses.
3447 /// Default to be the minimum interleave factor: 2.
3448 virtual unsigned getMaxSupportedInterleaveFactor() const { return 2; }
3449
3450 /// Lower an interleaved load to target specific intrinsics. Return
3451 /// true on success.
3452 ///
3453 /// \p Load is the vector load instruction. Can be either a plain load
3454 /// instruction or a vp.load intrinsic.
3455 /// \p Mask is a per-segment (i.e. number of lanes equal to that of one
3456 /// component being interwoven) mask. Can be nullptr, in which case the
3457 /// result is uncondiitional.
3458 /// \p Shuffles is the shufflevector list to DE-interleave the loaded vector.
3459 /// \p Indices is the corresponding indices for each shufflevector.
3460 /// \p Factor is the interleave factor.
3461 /// \p GapMask is a mask with zeros for components / fields that may not be
3462 /// accessed.
3465 ArrayRef<unsigned> Indices, unsigned Factor,
3466 const APInt &GapMask) const {
3467 return false;
3468 }
3469
3470 /// Lower an interleaved store to target specific intrinsics. Return
3471 /// true on success.
3472 ///
3473 /// \p SI is the vector store instruction. Can be either a plain store
3474 /// or a vp.store.
3475 /// \p Mask is a per-segment (i.e. number of lanes equal to that of one
3476 /// component being interwoven) mask. Can be nullptr, in which case the
3477 /// result is unconditional.
3478 /// \p SVI is the shufflevector to RE-interleave the stored vector.
3479 /// \p Factor is the interleave factor.
3480 /// \p GapMask is a mask with zeros for components / fields that may not be
3481 /// accessed.
3483 ShuffleVectorInst *SVI, unsigned Factor,
3484 const APInt &GapMask) const {
3485 return false;
3486 }
3487
3488 /// Lower a deinterleave intrinsic to a target specific load intrinsic.
3489 /// Return true on success. Currently only supports
3490 /// llvm.vector.deinterleave{2,3,5,7}
3491 ///
3492 /// \p Load is the accompanying load instruction. Can be either a plain load
3493 /// instruction or a vp.load intrinsic.
3494 /// \p DI represents the deinterleaveN intrinsic.
3495 /// \p GapMask is a mask with zeros for components / fields that may not be
3496 /// accessed.
3498 IntrinsicInst *DI,
3499 const APInt &GapMask) const {
3500 return false;
3501 }
3502
3503 /// Lower an interleave intrinsic to a target specific store intrinsic.
3504 /// Return true on success. Currently only supports
3505 /// llvm.vector.interleave{2,3,5,7}
3506 ///
3507 /// \p Store is the accompanying store instruction. Can be either a plain
3508 /// store or a vp.store intrinsic.
3509 /// \p Mask is a per-segment (i.e. number of lanes equal to that of one
3510 /// component being interwoven) mask. Can be nullptr, in which case the
3511 /// result is uncondiitional.
3512 /// \p InterleaveValues contains the interleaved values.
3513 virtual bool
3515 ArrayRef<Value *> InterleaveValues) const {
3516 return false;
3517 }
3518
3519 /// Return true if an fpext operation is free (for instance, because
3520 /// single-precision floating-point numbers are implicitly extended to
3521 /// double-precision).
3522 virtual bool isFPExtFree(EVT DestVT, EVT SrcVT) const {
3523 assert(SrcVT.isFloatingPoint() && DestVT.isFloatingPoint() &&
3524 "invalid fpext types");
3525 return false;
3526 }
3527
3528 /// Return true if an fpext operation input to an \p Opcode operation is free
3529 /// (for instance, because half-precision floating-point numbers are
3530 /// implicitly extended to float-precision) for an FMA instruction.
3531 virtual bool isFPExtFoldable(const MachineInstr &MI, unsigned Opcode,
3532 LLT DestTy, LLT SrcTy) const {
3533 return false;
3534 }
3535
3536 /// Return true if an fpext operation input to an \p Opcode operation is free
3537 /// (for instance, because half-precision floating-point numbers are
3538 /// implicitly extended to float-precision) for an FMA instruction.
3539 virtual bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
3540 EVT DestVT, EVT SrcVT) const {
3541 assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() &&
3542 "invalid fpext types");
3543 return isFPExtFree(DestVT, SrcVT);
3544 }
3545
3546 /// Return true if folding a vector load into ExtVal (a sign, zero, or any
3547 /// extend node) is profitable.
3548 virtual bool isVectorLoadExtDesirable(SDValue ExtVal) const { return false; }
3549
3550 /// Return true if an fneg operation is free to the point where it is never
3551 /// worthwhile to replace it with a bitwise operation.
3552 virtual bool isFNegFree(EVT VT) const {
3553 assert(VT.isFloatingPoint());
3554 return false;
3555 }
3556
3557 /// Return true if an fabs operation is free to the point where it is never
3558 /// worthwhile to replace it with a bitwise operation.
3559 virtual bool isFAbsFree(EVT VT) const {
3560 assert(VT.isFloatingPoint());
3561 return false;
3562 }
3563
3564 /// Return true if an FMA operation is faster than a pair of fmul and fadd
3565 /// instructions. fmuladd intrinsics will be expanded to FMAs when this method
3566 /// returns true, otherwise fmuladd is expanded to fmul + fadd.
3567 ///
3568 /// NOTE: This may be called before legalization on types for which FMAs are
3569 /// not legal, but should return true if those types will eventually legalize
3570 /// to types that support FMAs. After legalization, it will only be called on
3571 /// types that support FMAs (via Legal or Custom actions)
3572 ///
3573 /// Targets that care about soft float support should return false when soft
3574 /// float code is being generated (i.e. use-soft-float).
3576 EVT) const {
3577 return false;
3578 }
3579
3580 /// Return true if an FMA operation is faster than a pair of fmul and fadd
3581 /// instructions. fmuladd intrinsics will be expanded to FMAs when this method
3582 /// returns true, otherwise fmuladd is expanded to fmul + fadd.
3583 ///
3584 /// NOTE: This may be called before legalization on types for which FMAs are
3585 /// not legal, but should return true if those types will eventually legalize
3586 /// to types that support FMAs. After legalization, it will only be called on
3587 /// types that support FMAs (via Legal or Custom actions)
3589 LLT) const {
3590 return false;
3591 }
3592
3593 /// IR version
3594 virtual bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *) const {
3595 return false;
3596 }
3597
3598 /// Returns true if \p MI can be combined with another instruction to
3599 /// form TargetOpcode::G_FMAD. \p N may be an TargetOpcode::G_FADD,
3600 /// TargetOpcode::G_FSUB, or an TargetOpcode::G_FMUL which will be
3601 /// distributed into an fadd/fsub.
3602 virtual bool isFMADLegal(const MachineInstr &MI, LLT Ty) const {
3603 assert((MI.getOpcode() == TargetOpcode::G_FADD ||
3604 MI.getOpcode() == TargetOpcode::G_FSUB ||
3605 MI.getOpcode() == TargetOpcode::G_FMUL) &&
3606 "unexpected node in FMAD forming combine");
3607 switch (Ty.getScalarSizeInBits()) {
3608 case 16:
3609 return isOperationLegal(TargetOpcode::G_FMAD, MVT::f16);
3610 case 32:
3611 return isOperationLegal(TargetOpcode::G_FMAD, MVT::f32);
3612 case 64:
3613 return isOperationLegal(TargetOpcode::G_FMAD, MVT::f64);
3614 default:
3615 break;
3616 }
3617
3618 return false;
3619 }
3620
3621 /// Returns true if be combined with to form an ISD::FMAD. \p N may be an
3622 /// ISD::FADD, ISD::FSUB, or an ISD::FMUL which will be distributed into an
3623 /// fadd/fsub.
3624 virtual bool isFMADLegal(const SelectionDAG &DAG, const SDNode *N) const {
3625 assert((N->getOpcode() == ISD::FADD || N->getOpcode() == ISD::FSUB ||
3626 N->getOpcode() == ISD::FMUL) &&
3627 "unexpected node in FMAD forming combine");
3628 return isOperationLegal(ISD::FMAD, N->getValueType(0));
3629 }
3630
3631 // Return true when the decision to generate FMA's (or FMS, FMLA etc) rather
3632 // than FMUL and ADD is delegated to the machine combiner.
3634 CodeGenOptLevel OptLevel) const {
3635 return false;
3636 }
3637
3638 /// Return true if it's profitable to narrow operations of type SrcVT to
3639 /// DestVT. e.g. on x86, it's profitable to narrow from i32 to i8 but not from
3640 /// i32 to i16.
3641 virtual bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const {
3642 return false;
3643 }
3644
3645 /// Return true if pulling a binary operation into a select with an identity
3646 /// constant is profitable. This is the inverse of an IR transform.
3647 /// Example: X + (Cond ? Y : 0) --> Cond ? (X + Y) : X
3648 virtual bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT,
3649 unsigned SelectOpcode,
3650 SDValue X,
3651 SDValue Y) const {
3652 return false;
3653 }
3654
3655 /// Return true if it is beneficial to convert a load of a constant to
3656 /// just the constant itself.
3657 /// On some targets it might be more efficient to use a combination of
3658 /// arithmetic instructions to materialize the constant instead of loading it
3659 /// from a constant pool.
3661 Type *Ty) const {
3662 return false;
3663 }
3664
3665 /// Return the cost of extracting a subvector of type \p ResVT from a vector
3666 /// of type \p SrcVT, starting at element \p Index.
3667 ///
3668 /// Most callers only create a new EXTRACT_SUBVECTOR when the cost is at most
3669 /// ExtractSubvectorCost::Cheap. This hook exists because EXTRACT_SUBVECTOR
3670 /// usually has custom lowering that depends on the index of the first
3671 /// element, so only the target knows which lowering is cheap.
3673 unsigned Index) const {
3675 }
3676
3677 /// Try to convert an extract element of a vector binary operation into an
3678 /// extract element followed by a scalar operation.
3679 virtual bool shouldScalarizeBinop(SDValue VecOp) const {
3680 return false;
3681 }
3682
3683 /// Return true if extraction of a scalar element from the given vector type
3684 /// at the given index is cheap. For example, if scalar operations occur on
3685 /// the same register file as vector operations, then an extract element may
3686 /// be a sub-register rename rather than an actual instruction.
3687 virtual bool isExtractVecEltCheap(EVT VT, unsigned Index) const {
3688 return false;
3689 }
3690
3691 /// Try to convert math with an overflow comparison into the corresponding DAG
3692 /// node operation. Targets may want to override this independently of whether
3693 /// the operation is legal/custom for the given type because it may obscure
3694 /// matching of other patterns.
3695 virtual bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
3696 bool MathUsed) const {
3697 // Form it if it is legal.
3698 if (isOperationLegal(Opcode, VT))
3699 return true;
3700
3701 // TODO: The default logic is inherited from code in CodeGenPrepare.
3702 // The opcode should not make a difference by default?
3703 if (Opcode != ISD::UADDO)
3704 return false;
3705
3706 // Allow the transform as long as we have an integer type that is not
3707 // obviously illegal and unsupported and if the math result is used
3708 // besides the overflow check. On some targets (e.g. SPARC), it is
3709 // not profitable to form on overflow op if the math result has no
3710 // concrete users.
3711 if (VT.isVector())
3712 return false;
3713 return MathUsed && (VT.isSimple() || !isOperationExpand(Opcode, VT));
3714 }
3715
3716 // Return true if the target wants to optimize the mul overflow intrinsic
3717 // for the given \p VT.
3719 EVT VT) const {
3720 return false;
3721 }
3722
3723 // Return true if it is profitable to use a scalar input to a BUILD_VECTOR
3724 // even if the vector itself has multiple uses.
3725 virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const {
3726 return false;
3727 }
3728
3729 // Return true if CodeGenPrepare should consider splitting large offset of a
3730 // GEP to make the GEP fit into the addressing mode and can be sunk into the
3731 // same blocks of its users.
3732 virtual bool shouldConsiderGEPOffsetSplit() const { return false; }
3733
3734 /// Return true if creating a shift of the type by the given
3735 /// amount is not profitable.
3736 virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const {
3737 return false;
3738 }
3739
3740 // Should we fold (select_cc seteq (and x, y), 0, 0, A) -> (and (sra (shl x))
3741 // A) where y has a single bit set?
3743 const APInt &AndMask) const {
3744 unsigned ShCt = AndMask.getBitWidth() - 1;
3745 return !shouldAvoidTransformToShift(VT, ShCt);
3746 }
3747
3748 /// Does this target require the clearing of high-order bits in a register
3749 /// passed to the fp16 to fp conversion library function.
3750 virtual bool shouldKeepZExtForFP16Conv() const { return false; }
3751
3752 /// Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type
3753 /// VT. Used when folding idioms into a saturating fp-to-int conversion, such
3754 /// as min(max(fptoi)) clamps or NaN-guarded selects.
3755 virtual bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const {
3756 return isOperationLegalOrCustom(Op, VT);
3757 }
3758
3759 /// Should we prefer selects to doing arithmetic on boolean types
3761 return false;
3762 }
3763
3764 /// True if target has some particular form of dealing with pointer arithmetic
3765 /// semantics for pointers with the given value type. False if pointer
3766 /// arithmetic should not be preserved for passes such as instruction
3767 /// selection, and can fallback to regular arithmetic.
3768 /// This should be removed when PTRADD nodes are widely supported by backends.
3769 virtual bool shouldPreservePtrArith(const Function &F, EVT PtrVT) const {
3770 return false;
3771 }
3772
3773 /// True if the target allows transformations of in-bounds pointer
3774 /// arithmetic that cause out-of-bounds intermediate results.
3776 EVT PtrVT) const {
3777 return false;
3778 }
3779
3780 /// Does this target support complex deinterleaving
3781 virtual bool isComplexDeinterleavingSupported() const { return false; }
3782
3783 /// Does this target support complex deinterleaving with the given operation
3784 /// and type
3787 return false;
3788 }
3789
3790 // Get the preferred opcode for FP_TO_XINT nodes.
3791 // By default, this checks if the provded operation is an illegal FP_TO_UINT
3792 // and if so, checks if FP_TO_SINT is legal or custom for use as a
3793 // replacement. If both UINT and SINT conversions are Custom, we choose SINT
3794 // by default because that's the right thing on PPC.
3795 virtual unsigned getPreferredFPToIntOpcode(unsigned Op, EVT FromVT,
3796 EVT ToVT) const {
3797 if (isOperationLegal(Op, ToVT))
3798 return Op;
3799 switch (Op) {
3800 case ISD::FP_TO_UINT:
3802 return ISD::FP_TO_SINT;
3803 break;
3807 break;
3808 default:
3809 break;
3810 }
3811 return Op;
3812 }
3813
3814 /// Create the IR node for the given complex deinterleaving operation.
3815 /// If one cannot be created using all the given inputs, nullptr should be
3816 /// returned.
3819 ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB,
3820 Value *Accumulator = nullptr) const {
3821 return nullptr;
3822 }
3823
3825 return RuntimeLibcallInfo;
3826 }
3827
3828 const LibcallLoweringInfo &getLibcallLoweringInfo() const { return Libcalls; }
3829
3830 void setLibcallImpl(RTLIB::Libcall Call, RTLIB::LibcallImpl Impl) {
3831 Libcalls.setLibcallImpl(Call, Impl);
3832 }
3833
3834 /// Get the libcall impl routine name for the specified libcall.
3835 RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const {
3836 return Libcalls.getLibcallImpl(Call);
3837 }
3838
3839 /// Get the libcall routine name for the specified libcall.
3840 // FIXME: This should be removed. Only LibcallImpl should have a name.
3841 const char *getLibcallName(RTLIB::Libcall Call) const {
3842 return Libcalls.getLibcallName(Call);
3843 }
3844
3845 /// Get the libcall routine name for the specified libcall implementation
3849
3850 /// Get the CallingConv that should be used for the specified libcall.
3851 // FIXME: Remove this wrapper and directly use the used LibcallImpl
3853 return Libcalls.getLibcallCallingConv(Call);
3854 }
3855
3856 /// Execute target specific actions to finalize target lowering.
3857 /// This is used to set extra flags in MachineFrameInformation and freezing
3858 /// the set of reserved registers.
3859 /// The default implementation just freezes the set of reserved registers.
3860 virtual void finalizeLowering(MachineFunction &MF) const;
3861
3862 /// Returns true if it's profitable to allow merging store of loads when there
3863 /// are functions calls between the load and the store.
3864 virtual bool shouldMergeStoreOfLoadsOverCall(EVT, EVT) const { return true; }
3865
3866 //===----------------------------------------------------------------------===//
3867 // GlobalISel Hooks
3868 //===----------------------------------------------------------------------===//
3869 /// Check whether or not \p MI needs to be moved close to its uses.
3870 virtual bool shouldLocalize(const MachineInstr &MI, const TargetTransformInfo *TTI) const;
3871
3872
3873private:
3874 const TargetMachine &TM;
3875
3876 /// Tells the code generator that the target has BitExtract instructions.
3877 /// The code generator will aggressively sink "shift"s into the blocks of
3878 /// their users if the users will generate "and" instructions which can be
3879 /// combined with "shift" to BitExtract instructions.
3880 bool HasExtractBitsInsn;
3881
3882 /// Tells the code generator to bypass slow divide or remainder
3883 /// instructions. For example, BypassSlowDivWidths[32,8] tells the code
3884 /// generator to bypass 32-bit integer div/rem with an 8-bit unsigned integer
3885 /// div/rem when the operands are positive and less than 256.
3886 DenseMap <unsigned int, unsigned int> BypassSlowDivWidths;
3887
3888 /// Tells the code generator that it shouldn't generate extra flow control
3889 /// instructions and should attempt to combine flow control instructions via
3890 /// predication.
3891 bool JumpIsExpensive;
3892
3893 /// Information about the contents of the high-bits in boolean values held in
3894 /// a type wider than i1. See getBooleanContents.
3895 BooleanContent BooleanContents;
3896
3897 /// Information about the contents of the high-bits in boolean values held in
3898 /// a type wider than i1. See getBooleanContents.
3899 BooleanContent BooleanFloatContents;
3900
3901 /// Information about the contents of the high-bits in boolean vector values
3902 /// when the element type is wider than i1. See getBooleanContents.
3903 BooleanContent BooleanVectorContents;
3904
3905 /// The target scheduling preference: shortest possible total cycles or lowest
3906 /// register usage.
3907 Sched::Preference SchedPreferenceInfo;
3908
3909 /// The minimum alignment that any argument on the stack needs to have.
3910 Align MinStackArgumentAlignment;
3911
3912 /// The minimum function alignment (used when optimizing for size, and to
3913 /// prevent explicitly provided alignment from leading to incorrect code).
3914 Align MinFunctionAlignment;
3915
3916 /// The preferred function alignment (used when alignment unspecified and
3917 /// optimizing for speed).
3918 Align PrefFunctionAlignment;
3919
3920 /// The preferred loop alignment (in log2 bot in bytes).
3921 Align PrefLoopAlignment;
3922 /// The maximum amount of bytes permitted to be emitted for alignment.
3923 unsigned MaxBytesForAlignment;
3924
3925 /// Size in bits of the maximum atomics size the backend supports.
3926 /// Accesses larger than this will be expanded by AtomicExpandPass.
3927 unsigned MaxAtomicSizeInBitsSupported;
3928
3929 /// Size in bits of the maximum div/rem size the backend supports.
3930 /// Larger operations will be expanded by ExpandIRInsts.
3931 unsigned MaxDivRemBitWidthSupported;
3932
3933 /// Size in bits of the maximum fp to/from int conversion size the
3934 /// backend supports. Larger operations will be expanded by
3935 /// ExpandIRInsts.
3936 unsigned MaxLargeFPConvertBitWidthSupported;
3937
3938 /// Size in bits of the minimum cmpxchg or ll/sc operation the
3939 /// backend supports.
3940 unsigned MinCmpXchgSizeInBits;
3941
3942 /// The minimum of largest number of comparisons to use bit test for switch.
3943 unsigned MinimumBitTestCmps;
3944
3945 /// Maximum known-legal store size, which can be guaranteed for scalable
3946 /// vectors.
3947 unsigned MaximumLegalStoreInBits;
3948
3949 /// This indicates if the target supports unaligned atomic operations.
3950 bool SupportsUnalignedAtomics;
3951
3952 /// If set to a physical register, this specifies the register that
3953 /// llvm.savestack/llvm.restorestack should save and restore.
3954 Register StackPointerRegisterToSaveRestore;
3955
3956 /// This indicates the default register class to use for each ValueType the
3957 /// target supports natively.
3958 const TargetRegisterClass *RegClassForVT[MVT::VALUETYPE_SIZE];
3959 uint16_t NumRegistersForVT[MVT::VALUETYPE_SIZE];
3960 MVT RegisterTypeForVT[MVT::VALUETYPE_SIZE];
3961
3962 /// This indicates the "representative" register class to use for each
3963 /// ValueType the target supports natively. This information is used by the
3964 /// scheduler to track register pressure. By default, the representative
3965 /// register class is the largest legal super-reg register class of the
3966 /// register class of the specified type. e.g. On x86, i8, i16, and i32's
3967 /// representative class would be GR32.
3968 const TargetRegisterClass *RepRegClassForVT[MVT::VALUETYPE_SIZE] = {nullptr};
3969
3970 /// This indicates the "cost" of the "representative" register class for each
3971 /// ValueType. The cost is used by the scheduler to approximate register
3972 /// pressure.
3973 uint8_t RepRegClassCostForVT[MVT::VALUETYPE_SIZE];
3974
3975 /// For any value types we are promoting or expanding, this contains the value
3976 /// type that we are changing to. For Expanded types, this contains one step
3977 /// of the expand (e.g. i64 -> i32), even if there are multiple steps required
3978 /// (e.g. i64 -> i16). For types natively supported by the system, this holds
3979 /// the same type (e.g. i32 -> i32).
3980 MVT TransformToType[MVT::VALUETYPE_SIZE];
3981
3982 /// For each operation and each value type, keep a LegalizeAction that
3983 /// indicates how instruction selection should deal with the operation. Most
3984 /// operations are Legal (aka, supported natively by the target), but
3985 /// operations that are not should be described. Note that operations on
3986 /// non-legal value types are not described here.
3987 LegalizeAction OpActions[MVT::VALUETYPE_SIZE][ISD::BUILTIN_OP_END];
3988
3989 /// For each load extension type and each value type, keep a LegalizeAction
3990 /// that indicates how instruction selection should deal with a load of a
3991 /// specific value type and extension type. Uses 4-bits to store the action
3992 /// for each of the 4 load ext types.
3993 uint16_t LoadExtActions[MVT::VALUETYPE_SIZE][MVT::VALUETYPE_SIZE];
3994
3995 /// Similar to LoadExtActions, but for atomic loads. Only Legal or Expand
3996 /// (default) values are supported.
3997 uint16_t AtomicLoadExtActions[MVT::VALUETYPE_SIZE][MVT::VALUETYPE_SIZE];
3998
3999 /// For each value type pair keep a LegalizeAction that indicates whether a
4000 /// truncating store of a specific value type and truncating type is legal.
4001 LegalizeAction TruncStoreActions[MVT::VALUETYPE_SIZE][MVT::VALUETYPE_SIZE];
4002
4003 /// For each indexed mode and each value type, keep a quad of LegalizeAction
4004 /// that indicates how instruction selection should deal with the load /
4005 /// store / maskedload / maskedstore.
4006 ///
4007 /// The first dimension is the value_type for the reference. The second
4008 /// dimension represents the various modes for load store.
4009 uint16_t IndexedModeActions[MVT::VALUETYPE_SIZE][ISD::LAST_INDEXED_MODE];
4010
4011 /// For each condition code (ISD::CondCode) keep a LegalizeAction that
4012 /// indicates how instruction selection should deal with the condition code.
4013 ///
4014 /// Because each CC action takes up 4 bits, we need to have the array size be
4015 /// large enough to fit all of the value types. This can be done by rounding
4016 /// up the MVT::VALUETYPE_SIZE value to the next multiple of 8.
4017 uint32_t CondCodeActions[ISD::SETCC_INVALID][(MVT::VALUETYPE_SIZE + 7) / 8];
4018
4020 std::tuple<unsigned, MVT::SimpleValueType, MVT::SimpleValueType>;
4021 /// For each partial reduce opcode, result type and input type combination,
4022 /// keep a LegalizeAction which indicates how instruction selection should
4023 /// deal with this operation.
4024 DenseMap<PartialReduceActionTypes, LegalizeAction> PartialReduceMLAActions;
4025
4027 std::tuple<unsigned, unsigned, MVT::SimpleValueType>;
4028 /// For each vector (de)interleave opcode, interleave factor and fragment
4029 /// type combination, keep the corresponding LegalizeAction.
4030 DenseMap<VectorInterleaveActionKey, LegalizeAction> VectorInterleaveActions;
4031
4032 ValueTypeActionImpl ValueTypeActions;
4033
4034private:
4035 /// Targets can specify ISD nodes that they would like PerformDAGCombine
4036 /// callbacks for by calling setTargetDAGCombine(), which sets a bit in this
4037 /// array.
4038 unsigned char
4039 TargetDAGCombineArray[(ISD::BUILTIN_OP_END+CHAR_BIT-1)/CHAR_BIT];
4040
4041 /// For operations that must be promoted to a specific type, this holds the
4042 /// destination type. This map should be sparse, so don't hold it as an
4043 /// array.
4044 ///
4045 /// Targets add entries to this map with AddPromotedToType(..), clients access
4046 /// this with getTypeToPromoteTo(..).
4047 std::map<std::pair<unsigned, MVT::SimpleValueType>, MVT::SimpleValueType>
4048 PromoteToType;
4049
4050 /// FIXME: This should not live here; it should come from an analysis.
4051 const RTLIB::RuntimeLibcallsInfo RuntimeLibcallInfo;
4052
4053 /// The list of libcalls that the target will use.
4054 /// FIXME: This should not live here; it should come from an analysis.
4055 LibcallLoweringInfo Libcalls;
4056
4057 /// The bits of IndexedModeActions used to store the legalisation actions
4058 /// We store the data as | ML | MS | L | S | each taking 4 bits.
4059 enum IndexedModeActionsBits {
4060 IMAB_Store = 0,
4061 IMAB_Load = 4,
4062 IMAB_MaskedStore = 8,
4063 IMAB_MaskedLoad = 12
4064 };
4065
4066 void setIndexedModeAction(unsigned IdxMode, MVT VT, unsigned Shift,
4067 LegalizeAction Action) {
4068 assert(VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE &&
4069 (unsigned)Action < 0xf && "Table isn't big enough!");
4070 unsigned Ty = (unsigned)VT.SimpleTy;
4071 IndexedModeActions[Ty][IdxMode] &= ~(0xf << Shift);
4072 IndexedModeActions[Ty][IdxMode] |= ((uint16_t)Action) << Shift;
4073 }
4074
4075 LegalizeAction getIndexedModeAction(unsigned IdxMode, MVT VT,
4076 unsigned Shift) const {
4077 assert(IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid() &&
4078 "Table isn't big enough!");
4079 unsigned Ty = (unsigned)VT.SimpleTy;
4080 return (LegalizeAction)((IndexedModeActions[Ty][IdxMode] >> Shift) & 0xf);
4081 }
4082
4083 unsigned getVectorTypeBreakdownImpl(LLVMContext &Context, EVT VT,
4084 EVT &IntermediateVT,
4085 unsigned &NumIntermediates,
4086 MVT &RegisterVT,
4087 bool ForCallingConv) const;
4088
4089 unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
4090 unsigned &NumIntermediates,
4091 MVT &RegisterVT);
4092
4093 /// Return the type of registers that this ValueType will eventually require.
4094 MVT getCachedRegisterType(MVT VT) const {
4095 assert((unsigned)VT.SimpleTy < std::size(RegisterTypeForVT));
4096 return RegisterTypeForVT[VT.SimpleTy];
4097 }
4098
4099 MVT getRegisterTypeImpl(LLVMContext &Context, EVT VT,
4100 bool ForCallingConv) const {
4101 if (VT.isSimple() &&
4102 !shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv))
4103 return getCachedRegisterType(VT.getSimpleVT());
4104 if (VT.isVector()) {
4105 EVT VT1;
4106 MVT RegisterVT;
4107 unsigned NumIntermediates;
4108 (void)getVectorTypeBreakdownImpl(Context, VT, VT1, NumIntermediates,
4109 RegisterVT, ForCallingConv);
4110 return RegisterVT;
4111 }
4112 if (VT.isInteger()) {
4113 return getRegisterTypeImpl(Context, getTypeToTransformTo(Context, VT),
4114 ForCallingConv);
4115 }
4116 llvm_unreachable("Unsupported extended type!");
4117 }
4118
4119 unsigned getNumRegistersImpl(LLVMContext &Context, EVT VT,
4120 bool ForCallingConv) const {
4121 if (VT.isSimple() &&
4122 !shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv)) {
4123 assert((unsigned)VT.getSimpleVT().SimpleTy <
4124 std::size(NumRegistersForVT));
4125 return NumRegistersForVT[VT.getSimpleVT().SimpleTy];
4126 }
4127 if (VT.isVector()) {
4128 EVT VT1;
4129 MVT VT2;
4130 unsigned NumIntermediates;
4131 return getVectorTypeBreakdownImpl(Context, VT, VT1, NumIntermediates, VT2,
4132 ForCallingConv);
4133 }
4134 if (VT.isInteger()) {
4135 unsigned BitWidth = VT.getSizeInBits();
4136 unsigned RegWidth =
4137 getRegisterTypeImpl(Context, VT, ForCallingConv).getSizeInBits();
4138 return (BitWidth + RegWidth - 1) / RegWidth;
4139 }
4140 llvm_unreachable("Unsupported extended type!");
4141 }
4142
4143protected:
4144 /// Return true if the extension represented by \p I is free.
4145 /// \pre \p I is a sign, zero, or fp extension and
4146 /// is[Z|FP]ExtFree of the related types is not true.
4147 virtual bool isExtFreeImpl(const Instruction *I) const { return false; }
4148
4149 /// Depth that GatherAllAliases should continue looking for chain
4150 /// dependencies when trying to find a more preferable chain. As an
4151 /// approximation, this should be more than the number of consecutive stores
4152 /// expected to be merged.
4154
4155 /// \brief Specify maximum number of store instructions per memset call.
4156 ///
4157 /// When lowering \@llvm.memset this field specifies the maximum number of
4158 /// store operations that may be substituted for the call to memset. Targets
4159 /// must set this value based on the cost threshold for that target. Targets
4160 /// should assume that the memset will be done using as many of the largest
4161 /// store operations first, followed by smaller ones, if necessary, per
4162 /// alignment restrictions. For example, storing 9 bytes on a 32-bit machine
4163 /// with 16-bit alignment would result in four 2-byte stores and one 1-byte
4164 /// store. This only applies to setting a constant array of a constant size.
4166 /// Likewise for functions with the OptSize attribute.
4168
4169 /// \brief Specify maximum number of store instructions per memcpy call.
4170 ///
4171 /// When lowering \@llvm.memcpy this field specifies the maximum number of
4172 /// store operations that may be substituted for a call to memcpy. Targets
4173 /// must set this value based on the cost threshold for that target. Targets
4174 /// should assume that the memcpy will be done using as many of the largest
4175 /// store operations first, followed by smaller ones, if necessary, per
4176 /// alignment restrictions. For example, storing 7 bytes on a 32-bit machine
4177 /// with 32-bit alignment would result in one 4-byte store, a one 2-byte store
4178 /// and one 1-byte store. This only applies to copying a constant array of
4179 /// constant size.
4181 /// Likewise for functions with the OptSize attribute.
4183 /// \brief Specify max number of store instructions to glue in inlined memcpy.
4184 ///
4185 /// When memcpy is inlined based on MaxStoresPerMemcpy, specify maximum number
4186 /// of store instructions to keep together. This helps in pairing and
4187 // vectorization later on.
4189
4190 /// \brief Specify maximum number of load instructions per memcmp call.
4191 ///
4192 /// When lowering \@llvm.memcmp this field specifies the maximum number of
4193 /// pairs of load operations that may be substituted for a call to memcmp.
4194 /// Targets must set this value based on the cost threshold for that target.
4195 /// Targets should assume that the memcmp will be done using as many of the
4196 /// largest load operations first, followed by smaller ones, if necessary, per
4197 /// alignment restrictions. For example, loading 7 bytes on a 32-bit machine
4198 /// with 32-bit alignment would result in one 4-byte load, a one 2-byte load
4199 /// and one 1-byte load. This only applies to copying a constant array of
4200 /// constant size.
4202 /// Likewise for functions with the OptSize attribute.
4204
4205 /// \brief Specify maximum number of store instructions per memmove call.
4206 ///
4207 /// When lowering \@llvm.memmove this field specifies the maximum number of
4208 /// store instructions that may be substituted for a call to memmove. Targets
4209 /// must set this value based on the cost threshold for that target. Targets
4210 /// should assume that the memmove will be done using as many of the largest
4211 /// store operations first, followed by smaller ones, if necessary, per
4212 /// alignment restrictions. For example, moving 9 bytes on a 32-bit machine
4213 /// with 8-bit alignment would result in nine 1-byte stores. This only
4214 /// applies to copying a constant array of constant size.
4216 /// Likewise for functions with the OptSize attribute.
4218
4219 /// Tells the code generator that select is more expensive than a branch if
4220 /// the branch is usually predicted right.
4222
4223 /// \see enableExtLdPromotion.
4225
4226 /// Return true if the value types that can be represented by the specified
4227 /// register class are all legal.
4228 bool isLegalRC(const TargetRegisterInfo &TRI,
4229 const TargetRegisterClass &RC) const;
4230
4231 /// Replace/modify any TargetFrameIndex operands with a targte-dependent
4232 /// sequence of memory operands that is recognized by PrologEpilogInserter.
4234 MachineBasicBlock *MBB) const;
4235
4237};
4238
4239/// This class defines information used to lower LLVM code to legal SelectionDAG
4240/// operators that the target instruction selector can accept natively.
4241///
4242/// This class also defines callbacks that targets must implement to lower
4243/// target-specific constructs to SelectionDAG operators.
4245public:
4246 struct DAGCombinerInfo;
4247 struct MakeLibCallOptions;
4248
4251
4252 explicit TargetLowering(const TargetMachine &TM,
4253 const TargetSubtargetInfo &STI);
4255
4256 bool isPositionIndependent() const;
4257
4258 // If set to true, SelectionDAG nodes will be consistently processed in
4259 // topological order. This is a temporary hook until sorting can be
4260 // enabled globally.
4261 virtual bool useTopologicalSorting() const { return false; }
4262
4265 UniformityInfo *UA) const {
4266 return false;
4267 }
4268
4269 // Lets target to control the following reassociation of operands: (op (op x,
4270 // c1), y) -> (op (op x, y), c1) where N0 is (op x, c1) and N1 is y. By
4271 // default consider profitable any case where N0 has single use. This
4272 // behavior reflects the condition replaced by this target hook call in the
4273 // DAGCombiner. Any particular target can implement its own heuristic to
4274 // restrict common combiner.
4276 SDValue N1) const {
4277 return N0.hasOneUse();
4278 }
4279
4280 // Lets target to control the following reassociation of operands: (op (op x,
4281 // c1), y) -> (op (op x, y), c1) where N0 is (op x, c1) and N1 is y. By
4282 // default consider profitable any case where N0 has single use. This
4283 // behavior reflects the condition replaced by this target hook call in the
4284 // combiner. Any particular target can implement its own heuristic to
4285 // restrict common combiner.
4287 Register N1) const {
4288 return MRI.hasOneNonDBGUse(N0);
4289 }
4290
4291 virtual bool isSDNodeAlwaysUniform(const SDNode * N) const {
4292 return false;
4293 }
4294
4295 /// Returns true by value, base pointer and offset pointer and addressing mode
4296 /// by reference if the node's address can be legally represented as
4297 /// pre-indexed load / store address.
4298 virtual bool getPreIndexedAddressParts(SDNode * /*N*/, SDValue &/*Base*/,
4299 SDValue &/*Offset*/,
4300 ISD::MemIndexedMode &/*AM*/,
4301 SelectionDAG &/*DAG*/) const {
4302 return false;
4303 }
4304
4305 /// Returns true by value, base pointer and offset pointer and addressing mode
4306 /// by reference if this node can be combined with a load / store to form a
4307 /// post-indexed load / store.
4308 virtual bool getPostIndexedAddressParts(SDNode * /*N*/, SDNode * /*Op*/,
4309 SDValue &/*Base*/,
4310 SDValue &/*Offset*/,
4311 ISD::MemIndexedMode &/*AM*/,
4312 SelectionDAG &/*DAG*/) const {
4313 return false;
4314 }
4315
4316 /// Returns true if the specified base+offset is a legal indexed addressing
4317 /// mode for this target. \p MI is the load or store instruction that is being
4318 /// considered for transformation.
4320 bool IsPre, MachineRegisterInfo &MRI) const {
4321 return false;
4322 }
4323
4324 /// Return the entry encoding for a jump table in the current function. The
4325 /// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
4326 virtual unsigned getJumpTableEncoding() const;
4327
4328 virtual MVT getJumpTableRegTy(const DataLayout &DL) const {
4329 return getPointerTy(DL);
4330 }
4331
4332 virtual const MCExpr *
4334 const MachineBasicBlock * /*MBB*/, unsigned /*uid*/,
4335 MCContext &/*Ctx*/) const {
4336 llvm_unreachable("Need to implement this hook if target has custom JTIs");
4337 }
4338
4339 /// Returns relocation base for the given PIC jumptable.
4340 virtual SDValue getPICJumpTableRelocBase(SDValue Table,
4341 SelectionDAG &DAG) const;
4342
4343 /// This returns the relocation base for the given PIC jumptable, the same as
4344 /// getPICJumpTableRelocBase, but as an MCExpr.
4345 virtual const MCExpr *
4346 getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
4347 unsigned JTI, MCContext &Ctx) const;
4348
4349 /// Return true if folding a constant offset with the given GlobalAddress is
4350 /// legal. It is frequently not legal in PIC relocation models.
4351 virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const;
4352
4353 /// On x86, return true if the operand with index OpNo is a CALL or JUMP
4354 /// instruction, which can use either a memory constraint or an address
4355 /// constraint. -fasm-blocks "__asm call foo" lowers to
4356 /// call void asm sideeffect inteldialect "call ${0:P}", "*m..."
4357 ///
4358 /// This function is used by a hack to choose the address constraint,
4359 /// lowering to a direct call.
4360 virtual bool
4362 unsigned OpNo) const {
4363 return false;
4364 }
4365
4366 /// Return true if a function used as an inline asm address operand (e.g.
4367 /// "p", or a call target that isInlineAsmTargetBranch() recognizes) can be
4368 /// the bare function symbol, in an operand of kind InlineAsm::Kind::Func.
4369 /// The target then has to select the symbol as an address and print it with
4370 /// AsmPrinter::PrintAsmOperand(). Otherwise the function's address is
4371 /// computed like that of any other global.
4372 virtual bool supportsInlineAsmFuncOperands() const { return false; }
4373
4375 SDValue &Chain) const;
4376
4377 void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS,
4378 SDValue &NewRHS, ISD::CondCode &CCCode,
4379 const SDLoc &DL, const SDValue OldLHS,
4380 const SDValue OldRHS) const;
4381
4382 void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS,
4383 SDValue &NewRHS, ISD::CondCode &CCCode,
4384 const SDLoc &DL, const SDValue OldLHS,
4385 const SDValue OldRHS, SDValue &Chain,
4386 bool IsSignaling = false) const;
4387
4389 SDValue Chain, MachineMemOperand *MMO,
4390 SDValue &NewLoad, SDValue Ptr,
4391 SDValue PassThru, SDValue Mask) const {
4392 llvm_unreachable("Not Implemented");
4393 }
4394
4396 SDValue Chain, MachineMemOperand *MMO,
4397 SDValue Ptr, SDValue Val,
4398 SDValue Mask) const {
4399 llvm_unreachable("Not Implemented");
4400 }
4401
4402 /// Returns a pair of (return value, chain).
4403 /// It is an error to pass RTLIB::Unsupported as \p LibcallImpl
4404 std::pair<SDValue, SDValue>
4405 makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl, EVT RetVT,
4406 ArrayRef<SDValue> Ops, MakeLibCallOptions CallOptions,
4407 const SDLoc &dl, SDValue Chain = SDValue()) const;
4408
4409 /// It is an error to pass RTLIB::UNKNOWN_LIBCALL as \p LC.
4410 std::pair<SDValue, SDValue> makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC,
4411 EVT RetVT, ArrayRef<SDValue> Ops,
4412 MakeLibCallOptions CallOptions,
4413 const SDLoc &dl,
4414 SDValue Chain = SDValue()) const;
4415
4416 /// Build a call argument list for \p FuncTy, taking the argument node values
4417 /// from \p Ops and the parameter types and ABI attributes from \p FuncTy and
4418 /// \p FuncAttrs. \p Ops must have one entry per parameter of \p FuncTy.
4419 static ArgListTy getArgListForFunctionType(FunctionType *FuncTy,
4420 const AttributeList &FuncAttrs,
4422
4423 /// Check whether parameters to a call that are passed in callee saved
4424 /// registers are the same as from the calling function. This needs to be
4425 /// checked for tail call eligibility.
4426 bool parametersInCSRMatch(const MachineRegisterInfo &MRI,
4427 const uint32_t *CallerPreservedMask,
4428 const SmallVectorImpl<CCValAssign> &ArgLocs,
4429 const SmallVectorImpl<SDValue> &OutVals) const;
4430
4431 //===--------------------------------------------------------------------===//
4432 // TargetLowering Optimization Methods
4433 //
4434
4435 /// A convenience struct that encapsulates a DAG, and two SDValues for
4436 /// returning information from TargetLowering to its clients that want to
4437 /// combine.
4444
4446 bool LT, bool LO) :
4447 DAG(InDAG), LegalTys(LT), LegalOps(LO) {}
4448
4449 bool LegalTypes() const { return LegalTys; }
4450 bool LegalOperations() const { return LegalOps; }
4451
4453 Old = O;
4454 New = N;
4455 return true;
4456 }
4457 };
4458
4459 /// Determines the optimal series of memory ops to replace the memset /
4460 /// memcpy. Return true if the number of memory ops is below the threshold
4461 /// (Limit). Note that this is always the case when Limit is ~0. It returns
4462 /// the types of the sequence of memory ops to perform memset / memcpy by
4463 /// reference. If LargestVT is non-null, the target may set it to the largest
4464 /// EVT that should be used for generating the memset value (e.g., for vector
4465 /// splats). If LargestVT is null or left unchanged, the caller will compute
4466 /// it from MemOps.
4467 virtual bool findOptimalMemOpLowering(LLVMContext &Context,
4468 std::vector<EVT> &MemOps,
4469 unsigned Limit, const MemOp &Op,
4470 unsigned DstAS, unsigned SrcAS,
4471 const AttributeList &FuncAttributes,
4472 EVT *LargestVT = nullptr) const;
4473
4474 /// Check to see if the specified operand of the specified instruction is a
4475 /// constant integer. If so, check to see if there are any bits set in the
4476 /// constant that are not demanded. If so, shrink the constant and return
4477 /// true.
4479 const APInt &DemandedElts,
4480 TargetLoweringOpt &TLO) const;
4481
4482 /// Helper wrapper around ShrinkDemandedConstant, demanding all elements.
4484 TargetLoweringOpt &TLO) const;
4485
4486 // Target hook to do target-specific const optimization, which is called by
4487 // ShrinkDemandedConstant. This function should return true if the target
4488 // doesn't want ShrinkDemandedConstant to further optimize the constant.
4490 const APInt &DemandedBits,
4491 const APInt &DemandedElts,
4492 TargetLoweringOpt &TLO) const {
4493 return false;
4494 }
4495
4496 /// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
4497 /// This uses isTruncateFree/isZExtFree and ANY_EXTEND for the widening cast,
4498 /// but it could be generalized for targets with other types of implicit
4499 /// widening casts.
4500 bool ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
4501 const APInt &DemandedBits,
4502 TargetLoweringOpt &TLO) const;
4503
4504 /// Look at Op. At this point, we know that only the DemandedBits bits of the
4505 /// result of Op are ever used downstream. If we can use this information to
4506 /// simplify Op, create a new simplified DAG node and return true, returning
4507 /// the original and new nodes in Old and New. Otherwise, analyze the
4508 /// expression and return a mask of KnownOne and KnownZero bits for the
4509 /// expression (used to simplify the caller). The KnownZero/One bits may only
4510 /// be accurate for those bits in the Demanded masks.
4511 /// \p AssumeSingleUse When this parameter is true, this function will
4512 /// attempt to simplify \p Op even if there are multiple uses.
4513 /// Callers are responsible for correctly updating the DAG based on the
4514 /// results of this function, because simply replacing TLO.Old
4515 /// with TLO.New will be incorrect when this parameter is true and TLO.Old
4516 /// has multiple uses.
4517 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
4518 const APInt &DemandedElts, KnownBits &Known,
4519 TargetLoweringOpt &TLO, unsigned Depth = 0,
4520 bool AssumeSingleUse = false) const;
4521
4522 /// Helper wrapper around SimplifyDemandedBits, demanding all elements.
4523 /// Adds Op back to the worklist upon success.
4524 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
4525 KnownBits &Known, TargetLoweringOpt &TLO,
4526 unsigned Depth = 0,
4527 bool AssumeSingleUse = false) const;
4528
4529 /// Helper wrapper around SimplifyDemandedBits.
4530 /// Adds Op back to the worklist upon success.
4531 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
4532 DAGCombinerInfo &DCI) const;
4533
4534 /// Helper wrapper around SimplifyDemandedBits.
4535 /// Adds Op back to the worklist upon success.
4536 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
4537 const APInt &DemandedElts,
4538 DAGCombinerInfo &DCI) const;
4539
4540 /// More limited version of SimplifyDemandedBits that can be used to "look
4541 /// through" ops that don't contribute to the DemandedBits/DemandedElts -
4542 /// bitwise ops etc.
4543 SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits,
4544 const APInt &DemandedElts,
4545 SelectionDAG &DAG,
4546 unsigned Depth = 0) const;
4547
4548 /// Helper wrapper around SimplifyMultipleUseDemandedBits, demanding all
4549 /// elements.
4550 SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits,
4551 SelectionDAG &DAG,
4552 unsigned Depth = 0) const;
4553
4554 /// Helper wrapper around SimplifyMultipleUseDemandedBits, demanding all
4555 /// bits from only some vector elements.
4556 SDValue SimplifyMultipleUseDemandedVectorElts(SDValue Op,
4557 const APInt &DemandedElts,
4558 SelectionDAG &DAG,
4559 unsigned Depth = 0) const;
4560
4561 /// Look at Vector Op. At this point, we know that only the DemandedElts
4562 /// elements of the result of Op are ever used downstream. If we can use
4563 /// this information to simplify Op, create a new simplified DAG node and
4564 /// return true, storing the original and new nodes in TLO.
4565 /// Otherwise, analyze the expression and return a mask of KnownUndef and
4566 /// KnownZero elements for the expression (used to simplify the caller).
4567 /// The KnownUndef/Zero elements may only be accurate for those bits
4568 /// in the DemandedMask.
4569 /// \p AssumeSingleUse When this parameter is true, this function will
4570 /// attempt to simplify \p Op even if there are multiple uses.
4571 /// Callers are responsible for correctly updating the DAG based on the
4572 /// results of this function, because simply replacing TLO.Old
4573 /// with TLO.New will be incorrect when this parameter is true and TLO.Old
4574 /// has multiple uses.
4575 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask,
4576 APInt &KnownUndef, APInt &KnownZero,
4577 TargetLoweringOpt &TLO, unsigned Depth = 0,
4578 bool AssumeSingleUse = false) const;
4579
4580 /// Helper wrapper around SimplifyDemandedVectorElts.
4581 /// Adds Op back to the worklist upon success.
4582 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts,
4583 DAGCombinerInfo &DCI) const;
4584
4585 /// Return true if the target supports simplifying demanded vector elements by
4586 /// converting them to undefs.
4587 virtual bool
4589 const TargetLoweringOpt &TLO) const {
4590 return true;
4591 }
4592
4593 /// If only low elements of a vector are demanded, shrink the operation to the
4594 /// returned size in bits by converting
4595 /// (op x) to insert_subvector (op (extract_subvector x)).
4596 ///
4597 /// The returned size must be a multiple of the element size, greater than or
4598 /// equal to the demanded part of the vector and less than the original
4599 /// vector size. Return 0 to disable shrinking.
4600 virtual unsigned
4602 const APInt &DemandedElts) const {
4603 return 0;
4604 }
4605
4606 /// Determine which of the bits specified in Mask are known to be either zero
4607 /// or one and return them in the KnownZero/KnownOne bitsets. The DemandedElts
4608 /// argument allows us to only collect the known bits that are shared by the
4609 /// requested vector elements.
4610 virtual void computeKnownBitsForTargetNode(const SDValue Op,
4612 const APInt &DemandedElts,
4613 const SelectionDAG &DAG,
4614 unsigned Depth = 0) const;
4615
4616 /// Determine which of the bits specified in Mask are known to be either zero
4617 /// or one and return them in the KnownZero/KnownOne bitsets. The DemandedElts
4618 /// argument allows us to only collect the known bits that are shared by the
4619 /// requested vector elements. This is for GISel.
4620 virtual void computeKnownBitsForTargetInstr(GISelValueTracking &Analysis,
4622 const APInt &DemandedElts,
4623 const MachineRegisterInfo &MRI,
4624 unsigned Depth = 0) const;
4625
4626 virtual void computeKnownFPClassForTargetInstr(GISelValueTracking &Analysis,
4627 Register R,
4629 const APInt &DemandedElts,
4630 const MachineRegisterInfo &MRI,
4631 unsigned Depth = 0) const;
4632
4633 /// Determine the known alignment for the pointer value \p R. This is can
4634 /// typically be inferred from the number of low known 0 bits. However, for a
4635 /// pointer with a non-integral address space, the alignment value may be
4636 /// independent from the known low bits.
4637 virtual Align computeKnownAlignForTargetInstr(GISelValueTracking &Analysis,
4638 Register R,
4639 const MachineRegisterInfo &MRI,
4640 unsigned Depth = 0) const;
4641
4642 /// Determine known bits of a pointer to a known valid stack object.
4643 /// The default implementation computes low bits based on alignment.
4644 virtual void computeKnownBitsForStackObjectPointer(KnownBits &Known,
4645 const MachineFunction &MF,
4646 Align Alignment) const;
4647
4648 /// This method can be implemented by targets that want to expose additional
4649 /// information about sign bits to the DAG Combiner. The DemandedElts
4650 /// argument allows us to only collect the minimum sign bits that are shared
4651 /// by the requested vector elements.
4652 virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
4653 const APInt &DemandedElts,
4654 const SelectionDAG &DAG,
4655 unsigned Depth = 0) const;
4656
4657 /// This method can be implemented by targets that want to expose additional
4658 /// information about sign bits to GlobalISel combiners. The DemandedElts
4659 /// argument allows us to only collect the minimum sign bits that are shared
4660 /// by the requested vector elements.
4661 virtual unsigned computeNumSignBitsForTargetInstr(
4662 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
4663 const MachineRegisterInfo &MRI, unsigned Depth = 0) const;
4664
4665 /// Attempt to simplify any target nodes based on the demanded vector
4666 /// elements, returning true on success. Otherwise, analyze the expression and
4667 /// return a mask of KnownUndef and KnownZero elements for the expression
4668 /// (used to simplify the caller). The KnownUndef/Zero elements may only be
4669 /// accurate for those bits in the DemandedMask.
4670 virtual bool SimplifyDemandedVectorEltsForTargetNode(
4671 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef,
4672 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth = 0) const;
4673
4674 /// Attempt to simplify any target nodes based on the demanded bits/elts,
4675 /// returning true on success. Otherwise, analyze the
4676 /// expression and return a mask of KnownOne and KnownZero bits for the
4677 /// expression (used to simplify the caller). The KnownZero/One bits may only
4678 /// be accurate for those bits in the Demanded masks.
4679 virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op,
4680 const APInt &DemandedBits,
4681 const APInt &DemandedElts,
4683 TargetLoweringOpt &TLO,
4684 unsigned Depth = 0) const;
4685
4686 /// More limited version of SimplifyDemandedBits that can be used to "look
4687 /// through" ops that don't contribute to the DemandedBits/DemandedElts -
4688 /// bitwise ops etc.
4689 virtual SDValue SimplifyMultipleUseDemandedBitsForTargetNode(
4690 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
4691 SelectionDAG &DAG, unsigned Depth) const;
4692
4693 /// Return true if this function can prove that \p Op is never poison
4694 /// and, \p Kind can be used to track poison and/or undef bits. The
4695 /// DemandedElts argument limits the check to the requested vector elements.
4696 virtual bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(
4697 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4698 UndefPoisonKind Kind, unsigned Depth) const;
4699
4700 /// Return true if Op can create undef or poison from non-undef & non-poison
4701 /// operands. The DemandedElts argument limits the check to the requested
4702 /// vector elements.
4703 virtual bool canCreateUndefOrPoisonForTargetNode(
4704 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4705 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const;
4706
4707 /// Tries to build a legal vector shuffle using the provided parameters
4708 /// or equivalent variations. The Mask argument maybe be modified as the
4709 /// function tries different variations.
4710 /// Returns an empty SDValue if the operation fails.
4711 SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
4713 SelectionDAG &DAG) const;
4714
4715 /// This method returns the constant pool value that will be loaded by LD.
4716 /// NOTE: You must check for implicit extensions of the constant by LD.
4717 virtual const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const;
4718
4719 /// Determine floating-point class information for a target node. The
4720 /// DemandedElts argument allows us to only collect the known FP classes
4721 /// that are shared by the requested vector elements.
4722 virtual void computeKnownFPClassForTargetNode(const SDValue Op,
4724 const APInt &DemandedElts,
4725 const SelectionDAG &DAG,
4726 unsigned Depth = 0) const;
4727
4728 /// If \p SNaN is false, \returns true if \p Op is known to never be any
4729 /// NaN. If \p sNaN is true, returns if \p Op is known to never be a signaling
4730 /// NaN.
4731 virtual bool isKnownNeverNaNForTargetNode(SDValue Op,
4732 const APInt &DemandedElts,
4733 const SelectionDAG &DAG,
4734 bool SNaN = false,
4735 unsigned Depth = 0) const;
4736
4737 /// Return true if vector \p Op has the same value across all \p DemandedElts,
4738 /// indicating any elements which may be undef in the output \p UndefElts.
4739 virtual bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts,
4740 APInt &UndefElts,
4741 const SelectionDAG &DAG,
4742 unsigned Depth = 0) const;
4743
4744 /// Returns true if the given Opc is considered a canonical constant for the
4745 /// target, which should not be transformed back into a BUILD_VECTOR.
4747 return Op.getOpcode() == ISD::SPLAT_VECTOR ||
4748 Op.getOpcode() == ISD::SPLAT_VECTOR_PARTS;
4749 }
4750
4751 /// Return true if the given select/vselect should be considered canonical and
4752 /// not be transformed. Currently only used for "vselect (not Cond), N1, N2 ->
4753 /// vselect Cond, N2, N1".
4754 virtual bool isTargetCanonicalSelect(SDNode *N) const { return false; }
4755
4757 void *DC; // The DAG Combiner object.
4760
4761 public:
4763
4764 DAGCombinerInfo(SelectionDAG &dag, CombineLevel level, bool cl, void *dc)
4765 : DC(dc), Level(level), CalledByLegalizer(cl), DAG(dag) {}
4766
4767 bool isBeforeLegalize() const { return Level == BeforeLegalizeTypes; }
4769 bool isAfterLegalizeDAG() const { return Level >= AfterLegalizeDAG; }
4772
4773 LLVM_ABI void AddToWorklist(SDNode *N);
4774 LLVM_ABI SDValue CombineTo(SDNode *N, ArrayRef<SDValue> To,
4775 bool AddTo = true);
4776 LLVM_ABI SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true);
4777 LLVM_ABI SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
4778 bool AddTo = true);
4779
4780 LLVM_ABI bool recursivelyDeleteUnusedNodes(SDNode *N);
4781
4782 LLVM_ABI void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO);
4783 };
4784
4785 /// Return if the N is a constant or constant vector equal to the true value
4786 /// from getBooleanContents().
4787 bool isConstTrueVal(SDValue N) const;
4788
4789 /// Return if the N is a constant or constant vector equal to the false value
4790 /// from getBooleanContents().
4791 bool isConstFalseVal(SDValue N) const;
4792
4793 /// Return if \p N is a True value when extended to \p VT.
4794 bool isExtendedTrueVal(const ConstantSDNode *N, EVT VT, bool SExt) const;
4795
4796 /// Try to simplify a setcc built with the specified operands and cc. If it is
4797 /// unable to simplify it, return a null SDValue.
4798 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
4799 bool foldBooleans, DAGCombinerInfo &DCI,
4800 const SDLoc &dl) const;
4801
4802 // For targets which wrap address, unwrap for analysis.
4803 virtual SDValue unwrapAddress(SDValue N) const { return N; }
4804
4805 /// Returns true (and the GlobalValue and the offset) if the node is a
4806 /// GlobalAddress + offset.
4807 virtual bool
4808 isGAPlusOffset(SDNode *N, const GlobalValue* &GA, int64_t &Offset) const;
4809
4810 /// This method will be invoked for all target nodes and for any
4811 /// target-independent nodes that the target has registered with invoke it
4812 /// for.
4813 ///
4814 /// The semantics are as follows:
4815 /// Return Value:
4816 /// SDValue.Val == 0 - No change was made
4817 /// SDValue.Val == N - N was replaced, is dead, and is already handled.
4818 /// otherwise - N should be replaced by the returned Operand.
4819 ///
4820 /// In addition, methods provided by DAGCombinerInfo may be used to perform
4821 /// more complex transformations.
4822 ///
4823 virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const;
4824
4825 /// Return true if it is profitable to move this shift by a constant amount
4826 /// through its operand, adjusting any immediate operands as necessary to
4827 /// preserve semantics. This transformation may not be desirable if it
4828 /// disrupts a particularly auspicious target-specific tree (e.g. bitfield
4829 /// extraction in AArch64). By default, it returns true.
4830 ///
4831 /// @param N the shift node
4832 /// @param Level the current DAGCombine legalization level.
4834 CombineLevel Level) const {
4835 SDValue ShiftLHS = N->getOperand(0);
4836 if (!ShiftLHS->hasOneUse())
4837 return false;
4838 if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
4839 !ShiftLHS.getOperand(0)->hasOneUse())
4840 return false;
4841 return true;
4842 }
4843
4844 /// fold (A + vscale(C1)) + vscale(C2) -> A + vscale(C1+C2)
4845 /// If (A + vscale(C1)) is used multiple times, the fold results in a
4846 /// redundant addition instruction on the RISC-V architecture, whereas it
4847 /// does not have this effect on other architectures (e.g. AArch64).
4848 /// By default, it returns true.
4849 virtual bool isProfitableToFoldVScaleAdd(SDValue N) const { return true; }
4850
4851 /// GlobalISel - return true if it is profitable to move this shift by a
4852 /// constant amount through its operand, adjusting any immediate operands as
4853 /// necessary to preserve semantics. This transformation may not be desirable
4854 /// if it disrupts a particularly auspicious target-specific tree (e.g.
4855 /// bitfield extraction in AArch64). By default, it returns true.
4856 ///
4857 /// @param MI the shift instruction
4858 /// @param IsAfterLegal true if running after legalization.
4860 bool IsAfterLegal) const {
4861 return true;
4862 }
4863
4864 /// GlobalISel - return true if it's profitable to perform the combine:
4865 /// shl ([sza]ext x), y => zext (shl x, y)
4866 virtual bool isDesirableToPullExtFromShl(const MachineInstr &MI) const {
4867 return true;
4868 }
4869
4870 // Return AndOrSETCCFoldKind::{AddAnd, ABS} if its desirable to try and
4871 // optimize LogicOp(SETCC0, SETCC1). An example (what is implemented as of
4872 // writing this) is:
4873 // With C as a power of 2 and C != 0 and C != INT_MIN:
4874 // AddAnd:
4875 // (icmp eq A, C) | (icmp eq A, -C)
4876 // -> (icmp eq and(add(A, C), ~(C + C)), 0)
4877 // (icmp ne A, C) & (icmp ne A, -C)w
4878 // -> (icmp ne and(add(A, C), ~(C + C)), 0)
4879 // ABS:
4880 // (icmp eq A, C) | (icmp eq A, -C)
4881 // -> (icmp eq Abs(A), C)
4882 // (icmp ne A, C) & (icmp ne A, -C)w
4883 // -> (icmp ne Abs(A), C)
4884 //
4885 // @param LogicOp the logic op
4886 // @param SETCC0 the first of the SETCC nodes
4887 // @param SETCC0 the second of the SETCC nodes
4889 const SDNode *LogicOp, const SDNode *SETCC0, const SDNode *SETCC1) const {
4891 }
4892
4893 /// Return true if it is profitable to combine an XOR of a logical shift
4894 /// to create a logical shift of NOT. This transformation may not be desirable
4895 /// if it disrupts a particularly auspicious target-specific tree (e.g.
4896 /// BIC on ARM/AArch64). By default, it returns true.
4897 virtual bool isDesirableToCommuteXorWithShift(const SDNode *N) const {
4898 return true;
4899 }
4900
4901 /// Return true if the target has native support for the specified value type
4902 /// and it is 'desirable' to use the type for the given node type. e.g. On x86
4903 /// i16 is legal, but undesirable since i16 instruction encodings are longer
4904 /// and some i16 instructions are slow.
4905 virtual bool isTypeDesirableForOp(unsigned /*Opc*/, EVT VT) const {
4906 // By default, assume all legal types are desirable.
4907 return isTypeLegal(VT);
4908 }
4909
4910 /// Overload that takes the specific node being optimized.
4911 virtual bool isTypeDesirableForOp(SDNode *N, EVT VT) const {
4912 return isTypeDesirableForOp(N->getOpcode(), VT);
4913 }
4914
4915 /// Return true if it is profitable for dag combiner to transform a floating
4916 /// point op of specified opcode to a equivalent op of an integer
4917 /// type. e.g. f32 load -> i32 load can be profitable on ARM.
4918 virtual bool isDesirableToTransformToIntegerOp(unsigned /*Opc*/,
4919 EVT /*VT*/) const {
4920 return false;
4921 }
4922
4923 /// This method query the target whether it is beneficial for dag combiner to
4924 /// promote the specified node. If true, it should return the desired
4925 /// promotion type by reference.
4926 virtual bool IsDesirableToPromoteOp(SDValue /*Op*/, EVT &/*PVT*/) const {
4927 return false;
4928 }
4929
4930 /// Return true if the target supports swifterror attribute. It optimizes
4931 /// loads and stores to reading and writing a specific register.
4932 virtual bool supportSwiftError() const {
4933 return false;
4934 }
4935
4936 /// Return true if the target supports that a subset of CSRs for the given
4937 /// machine function is handled explicitly via copies.
4938 virtual bool supportSplitCSR(MachineFunction *MF) const {
4939 return false;
4940 }
4941
4942 /// Return true if the target supports kcfi operand bundles.
4943 virtual bool supportKCFIBundles() const { return false; }
4944
4945 /// Return true if the target supports ptrauth operand bundles.
4946 virtual bool supportPtrAuthBundles() const { return false; }
4947
4948 /// Perform necessary initialization to handle a subset of CSRs explicitly
4949 /// via copies. This function is called at the beginning of instruction
4950 /// selection.
4951 virtual void initializeSplitCSR(MachineBasicBlock *Entry) const {
4952 llvm_unreachable("Not Implemented");
4953 }
4954
4955 /// Insert explicit copies in entry and exit blocks. We copy a subset of
4956 /// CSRs to virtual registers in the entry block, and copy them back to
4957 /// physical registers in the exit blocks. This function is called at the end
4958 /// of instruction selection.
4960 MachineBasicBlock *Entry,
4961 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
4962 llvm_unreachable("Not Implemented");
4963 }
4964
4965 /// Return the newly negated expression if the cost is not expensive and
4966 /// set the cost in \p Cost to indicate that if it is cheaper or neutral to
4967 /// do the negation.
4968 virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG,
4969 bool LegalOps, bool OptForSize,
4970 NegatibleCost &Cost,
4971 unsigned Depth = 0) const;
4972
4974 SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize,
4975 const NegatibleCost CostThreshold = NegatibleCost::Neutral,
4976 unsigned Depth = 0) const {
4978 SDValue Neg =
4979 getNegatedExpression(Op, DAG, LegalOps, OptForSize, Cost, Depth);
4980 if (!Neg)
4981 return SDValue();
4982
4983 if (Cost <= CostThreshold)
4984 return Neg;
4985
4986 // Remove the new created node to avoid the side effect to the DAG.
4987 if (Neg->use_empty())
4988 DAG.RemoveDeadNode(Neg.getNode());
4989 return SDValue();
4990 }
4991
4992 /// This is the helper function to return the newly negated expression only
4993 /// when the cost is cheaper.
4995 bool LegalOps, bool OptForSize,
4996 unsigned Depth = 0) const {
4997 return getCheaperOrNeutralNegatedExpression(Op, DAG, LegalOps, OptForSize,
4999 }
5000
5001 /// This is the helper function to return the newly negated expression if
5002 /// the cost is not expensive.
5004 bool OptForSize, unsigned Depth = 0) const {
5006 return getNegatedExpression(Op, DAG, LegalOps, OptForSize, Cost, Depth);
5007 }
5008
5009 //===--------------------------------------------------------------------===//
5010 // Lowering methods - These methods must be implemented by targets so that
5011 // the SelectionDAGBuilder code knows how to lower these.
5012 //
5013
5014 /// Target-specific splitting of values into parts that fit a register
5015 /// storing a legal type
5017 SelectionDAG & DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
5018 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
5019 return false;
5020 }
5021
5022 /// Target-specific combining of register parts into its original value
5023 virtual SDValue
5025 const SDValue *Parts, unsigned NumParts,
5026 MVT PartVT, EVT ValueVT,
5027 std::optional<CallingConv::ID> CC) const {
5028 return SDValue();
5029 }
5030
5031 /// This hook must be implemented to lower the incoming (formal) arguments,
5032 /// described by the Ins array, into the specified DAG. The implementation
5033 /// should fill in the InVals array with legal-type argument values, and
5034 /// return the resulting token chain value.
5036 SDValue /*Chain*/, CallingConv::ID /*CallConv*/, bool /*isVarArg*/,
5037 const SmallVectorImpl<ISD::InputArg> & /*Ins*/, const SDLoc & /*dl*/,
5038 SelectionDAG & /*DAG*/, SmallVectorImpl<SDValue> & /*InVals*/) const {
5039 llvm_unreachable("Not Implemented");
5040 }
5041
5042 /// Optional target hook to add target-specific actions when entering EH pad
5043 /// blocks. The implementation should return the resulting token chain value.
5044 virtual SDValue lowerEHPadEntry(SDValue Chain, const SDLoc &DL,
5045 SelectionDAG &DAG) const {
5046 return SDValue();
5047 }
5048
5049 virtual void markLibCallAttributes(MachineFunction *MF, unsigned CC,
5050 ArgListTy &Args) const {}
5051
5052 /// This structure contains the information necessary for lowering
5053 /// pointer-authenticating indirect calls. It is equivalent to the "ptrauth"
5054 /// operand bundle found on the call instruction, if any.
5059
5060 /// This structure contains all information that is necessary for lowering
5061 /// calls. It is passed to TLI::LowerCallTo when the SelectionDAG builder
5062 /// needs to lower a call, and targets will see this struct in their LowerCall
5063 /// implementation.
5066 /// Original unlegalized return type.
5067 Type *OrigRetTy = nullptr;
5068 /// Same as OrigRetTy, or partially legalized for soft float libcalls.
5069 Type *RetTy = nullptr;
5070 bool RetSExt : 1;
5071 bool RetZExt : 1;
5072 bool IsVarArg : 1;
5073 bool IsInReg : 1;
5079 bool NoMerge : 1;
5080
5081 // IsTailCall should be modified by implementations of
5082 // TargetLowering::LowerCall that perform tail call conversions.
5083 bool IsTailCall = false;
5084
5085 // Is Call lowering done post SelectionDAG type legalization.
5087
5088 unsigned NumFixedArgs = -1;
5094 const CallBase *CB = nullptr;
5099 const ConstantInt *CFIType = nullptr;
5102
5103 std::optional<PtrAuthInfo> PAI;
5104
5110
5112 DL = dl;
5113 return *this;
5114 }
5115
5117 Chain = InChain;
5118 return *this;
5119 }
5120
5121 // setCallee with target/module-specific attributes
5123 SDValue Target, ArgListTy &&ArgsList) {
5124 return setLibCallee(CC, ResultType, ResultType, Target,
5125 std::move(ArgsList));
5126 }
5127
5129 Type *OrigResultType, SDValue Target,
5130 ArgListTy &&ArgsList) {
5131 OrigRetTy = OrigResultType;
5132 RetTy = ResultType;
5133 Callee = Target;
5134 CallConv = CC;
5135 NumFixedArgs = ArgsList.size();
5136 Args = std::move(ArgsList);
5137
5138 DAG.getTargetLoweringInfo().markLibCallAttributes(
5139 &(DAG.getMachineFunction()), CC, Args);
5140 return *this;
5141 }
5142
5144 SDValue Target, ArgListTy &&ArgsList,
5145 AttributeSet ResultAttrs = {}) {
5146 RetTy = OrigRetTy = ResultType;
5147 IsInReg = ResultAttrs.hasAttribute(Attribute::InReg);
5148 RetSExt = ResultAttrs.hasAttribute(Attribute::SExt);
5149 RetZExt = ResultAttrs.hasAttribute(Attribute::ZExt);
5150 NoMerge = ResultAttrs.hasAttribute(Attribute::NoMerge);
5151
5152 Callee = Target;
5153 CallConv = CC;
5154 NumFixedArgs = ArgsList.size();
5155 Args = std::move(ArgsList);
5156 return *this;
5157 }
5158
5160 SDValue Target, ArgListTy &&ArgsList,
5161 const CallBase &Call) {
5162 RetTy = OrigRetTy = ResultType;
5163
5164 IsInReg = Call.hasRetAttr(Attribute::InReg);
5166 Call.doesNotReturn() ||
5167 (!isa<InvokeInst>(Call) && isa<UnreachableInst>(Call.getNextNode()));
5168 IsVarArg = FTy->isVarArg();
5169 IsReturnValueUsed = !Call.use_empty();
5170 RetSExt = Call.hasRetAttr(Attribute::SExt);
5171 RetZExt = Call.hasRetAttr(Attribute::ZExt);
5172 NoMerge = Call.hasFnAttr(Attribute::NoMerge);
5173
5174 Callee = Target;
5175
5176 CallConv = Call.getCallingConv();
5177 NumFixedArgs = FTy->getNumParams();
5178 Args = std::move(ArgsList);
5179
5180 CB = &Call;
5181
5182 return *this;
5183 }
5184
5186 IsInReg = Value;
5187 return *this;
5188 }
5189
5192 return *this;
5193 }
5194
5196 IsVarArg = Value;
5197 return *this;
5198 }
5199
5201 IsTailCall = Value;
5202 return *this;
5203 }
5204
5207 return *this;
5208 }
5209
5212 return *this;
5213 }
5214
5216 RetSExt = Value;
5217 return *this;
5218 }
5219
5221 RetZExt = Value;
5222 return *this;
5223 }
5224
5227 return *this;
5228 }
5229
5232 return *this;
5233 }
5234
5236 PAI = Value;
5237 return *this;
5238 }
5239
5242 return *this;
5243 }
5244
5246 CFIType = Type;
5247 return *this;
5248 }
5249
5252 return *this;
5253 }
5254
5256 DeactivationSymbol = Sym;
5257 return *this;
5258 }
5259
5261 return Args;
5262 }
5263 };
5264
5265 /// This structure is used to pass arguments to makeLibCall function.
5267 // By passing type list before soften to makeLibCall, the target hook
5268 // shouldExtendTypeInLibCall can get the original type before soften.
5272
5273 bool IsSigned : 1;
5277 bool IsSoften : 1;
5278
5282
5284 IsSigned = Value;
5285 return *this;
5286 }
5287
5290 return *this;
5291 }
5292
5295 return *this;
5296 }
5297
5300 return *this;
5301 }
5302
5304 OpsVTBeforeSoften = OpsVT;
5305 RetVTBeforeSoften = RetVT;
5306 IsSoften = true;
5307 return *this;
5308 }
5309
5310 /// Override the argument type for an operand. Leave the type as null to use
5311 /// the type from the operand's node.
5313 OpsTypeOverrides = OpsTypes;
5314 return *this;
5315 }
5316 };
5317
5318 /// This function lowers an abstract call to a function into an actual call.
5319 /// This returns a pair of operands. The first element is the return value
5320 /// for the function (if RetTy is not VoidTy). The second element is the
5321 /// outgoing token chain. It calls LowerCall to do the actual lowering.
5322 std::pair<SDValue, SDValue> LowerCallTo(CallLoweringInfo &CLI) const;
5323
5324 /// This hook must be implemented to lower calls into the specified
5325 /// DAG. The outgoing arguments to the call are described by the Outs array,
5326 /// and the values to be returned by the call are described by the Ins
5327 /// array. The implementation should fill in the InVals array with legal-type
5328 /// return values from the call, and return the resulting token chain value.
5329 virtual SDValue
5331 SmallVectorImpl<SDValue> &/*InVals*/) const {
5332 llvm_unreachable("Not Implemented");
5333 }
5334
5335 /// Target-specific cleanup for formal ByVal parameters.
5336 virtual void HandleByVal(CCState *, unsigned &, Align) const {}
5337
5338 /// This hook should be implemented to check whether the return values
5339 /// described by the Outs array can fit into the return registers. If false
5340 /// is returned, an sret-demotion is performed.
5341 virtual bool CanLowerReturn(CallingConv::ID /*CallConv*/,
5342 MachineFunction &/*MF*/, bool /*isVarArg*/,
5343 const SmallVectorImpl<ISD::OutputArg> &/*Outs*/,
5344 LLVMContext &/*Context*/, const Type *RetTy) const
5345 {
5346 // Return true by default to get preexisting behavior.
5347 return true;
5348 }
5349
5350 /// Annotate a stack object pointer with known-bits assertions.
5351 SDValue annotateStackObjectPointer(SDValue Ptr, SelectionDAG &DAG,
5352 const SDLoc &DL, Align Alignment) const;
5353
5354 /// This hook must be implemented to lower outgoing return values, described
5355 /// by the Outs array, into the specified DAG. The implementation should
5356 /// return the resulting token chain value.
5357 virtual SDValue LowerReturn(SDValue /*Chain*/, CallingConv::ID /*CallConv*/,
5358 bool /*isVarArg*/,
5359 const SmallVectorImpl<ISD::OutputArg> & /*Outs*/,
5360 const SmallVectorImpl<SDValue> & /*OutVals*/,
5361 const SDLoc & /*dl*/,
5362 SelectionDAG & /*DAG*/) const {
5363 llvm_unreachable("Not Implemented");
5364 }
5365
5366 /// Return true if result of the specified node is used by a return node
5367 /// only. It also compute and return the input chain for the tail call.
5368 ///
5369 /// This is used to determine whether it is possible to codegen a libcall as
5370 /// tail call at legalization time.
5371 virtual bool isUsedByReturnOnly(SDNode *, SDValue &/*Chain*/) const {
5372 return false;
5373 }
5374
5375 /// Return true if the target may be able emit the call instruction as a tail
5376 /// call. This is used by optimization passes to determine if it's profitable
5377 /// to duplicate return instructions to enable tailcall optimization.
5378 virtual bool mayBeEmittedAsTailCall(const CallInst *) const {
5379 return false;
5380 }
5381
5382 /// Return the register ID of the name passed in. Used by named register
5383 /// global variables extension. There is no target-independent behaviour
5384 /// so the default action is to bail.
5385 virtual Register getRegisterByName(const char* RegName, LLT Ty,
5386 const MachineFunction &MF) const {
5387 reportFatalUsageError("Named registers not implemented for this target");
5388 }
5389
5390 /// Return the type that should be used to zero or sign extend a
5391 /// zeroext/signext integer return value. FIXME: Some C calling conventions
5392 /// require the return type to be promoted, but this is not true all the time,
5393 /// e.g. i1/i8/i16 on x86/x86_64. It is also not necessary for non-C calling
5394 /// conventions. The frontend should handle this and include all of the
5395 /// necessary information.
5397 ISD::NodeType /*ExtendKind*/) const {
5398 EVT MinVT = getRegisterType(Context, MVT::i32);
5399 return VT.bitsLT(MinVT) ? MinVT : VT;
5400 }
5401
5402 /// For some targets, an LLVM struct type must be broken down into multiple
5403 /// simple types, but the calling convention specifies that the entire struct
5404 /// must be passed in a block of consecutive registers.
5405 virtual bool
5407 bool isVarArg,
5408 const DataLayout &DL) const {
5409 return false;
5410 }
5411
5412 /// For most targets, an LLVM type must be broken down into multiple
5413 /// smaller types. Usually the halves are ordered according to the endianness
5414 /// but for some platform that would break. So this method will default to
5415 /// matching the endianness but can be overridden.
5416 virtual bool
5418 return DL.isLittleEndian();
5419 }
5420
5421 /// Returns a 0 terminated array of registers that can be safely used as
5422 /// scratch registers.
5424 return nullptr;
5425 }
5426
5427 /// Returns a 0 terminated array of rounding control registers that can be
5428 /// attached into strict FP call.
5432
5433 /// This callback is used to prepare for a volatile or atomic load.
5434 /// It takes a chain node as input and returns the chain for the load itself.
5435 ///
5436 /// Having a callback like this is necessary for targets like SystemZ,
5437 /// which allows a CPU to reuse the result of a previous load indefinitely,
5438 /// even if a cache-coherent store is performed by another CPU. The default
5439 /// implementation does nothing.
5441 SelectionDAG &DAG) const {
5442 return Chain;
5443 }
5444
5445 /// This callback is invoked by the type legalizer to legalize nodes with an
5446 /// illegal operand type but legal result types. It replaces the
5447 /// LowerOperation callback in the type Legalizer. The reason we can not do
5448 /// away with LowerOperation entirely is that LegalizeDAG isn't yet ready to
5449 /// use this callback.
5450 ///
5451 /// TODO: Consider merging with ReplaceNodeResults.
5452 ///
5453 /// The target places new result values for the node in Results (their number
5454 /// and types must exactly match those of the original return values of
5455 /// the node), or leaves Results empty, which indicates that the node is not
5456 /// to be custom lowered after all.
5457 /// The default implementation calls LowerOperation.
5458 virtual void LowerOperationWrapper(SDNode *N,
5460 SelectionDAG &DAG) const;
5461
5462 /// This callback is invoked for operations that are unsupported by the
5463 /// target, which are registered to use 'custom' lowering, and whose defined
5464 /// values are all legal. If the target has no operations that require custom
5465 /// lowering, it need not implement this. The default implementation of this
5466 /// aborts.
5467 virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const;
5468
5469 /// This callback is invoked when a node result type is illegal for the
5470 /// target, and the operation was registered to use 'custom' lowering for that
5471 /// result type. The target places new result values for the node in Results
5472 /// (their number and types must exactly match those of the original return
5473 /// values of the node), or leaves Results empty, which indicates that the
5474 /// node is not to be custom lowered after all.
5475 ///
5476 /// If the target has no operations that require custom lowering, it need not
5477 /// implement this. The default implementation aborts.
5478 virtual void ReplaceNodeResults(SDNode * /*N*/,
5479 SmallVectorImpl<SDValue> &/*Results*/,
5480 SelectionDAG &/*DAG*/) const {
5481 llvm_unreachable("ReplaceNodeResults not implemented for this target!");
5482 }
5483
5484 /// This method returns the name of a target specific DAG node.
5485 virtual const char *getTargetNodeName(unsigned Opcode) const;
5486
5487 /// This method returns a target specific FastISel object, or null if the
5488 /// target does not support "fast" ISel.
5490 const TargetLibraryInfo *,
5491 const LibcallLoweringInfo *) const {
5492 return nullptr;
5493 }
5494
5495 //===--------------------------------------------------------------------===//
5496 // Inline Asm Support hooks
5497 //
5498
5500 C_Register, // Constraint represents specific register(s).
5501 C_RegisterClass, // Constraint represents any of register(s) in class.
5502 C_Memory, // Memory constraint.
5503 C_Address, // Address constraint.
5504 C_Immediate, // Requires an immediate.
5505 C_Other, // Something else.
5506 C_Unknown // Unsupported constraint.
5507 };
5508
5510 // Generic weights.
5511 CW_Invalid = -1, // No match.
5512 CW_Okay = 0, // Acceptable.
5513 CW_Good = 1, // Good weight.
5514 CW_Better = 2, // Better weight.
5515 CW_Best = 3, // Best weight.
5516
5517 // Well-known weights.
5518 CW_SpecificReg = CW_Okay, // Specific register operands.
5519 CW_Register = CW_Good, // Register operands.
5520 CW_Memory = CW_Better, // Memory operands.
5521 CW_Constant = CW_Best, // Constant operand.
5522 CW_Default = CW_Okay // Default or don't know type.
5523 };
5524
5525 /// This contains information for each constraint that we are lowering.
5527 /// This contains the actual string for the code, like "m". TargetLowering
5528 /// picks the 'best' code from ConstraintInfo::Codes that most closely
5529 /// matches the operand.
5530 std::string ConstraintCode;
5531
5532 /// Information about the constraint code, e.g. Register, RegisterClass,
5533 /// Memory, Other, Unknown.
5535
5536 /// If this is the result output operand or a clobber, this is null,
5537 /// otherwise it is the incoming operand to the CallInst. This gets
5538 /// modified as the asm is processed.
5540
5541 /// The ValueType for the operand value.
5542 MVT ConstraintVT = MVT::Other;
5543
5544 /// Copy constructor for copying from a ConstraintInfo.
5547
5548 /// Return true of this is an input operand that is a matching constraint
5549 /// like "4".
5550 LLVM_ABI bool isMatchingInputConstraint() const;
5551
5552 /// If this is an input matching constraint, this method returns the output
5553 /// operand it matches.
5554 LLVM_ABI unsigned getMatchedOperand() const;
5555 };
5556
5557 using AsmOperandInfoVector = std::vector<AsmOperandInfo>;
5558
5559 /// Split up the constraint string from the inline assembly value into the
5560 /// specific constraints and their prefixes, and also tie in the associated
5561 /// operand values. If this returns an empty vector, and if the constraint
5562 /// string itself isn't empty, there was an error parsing.
5564 const TargetRegisterInfo *TRI,
5565 const CallBase &Call) const;
5566
5567 /// Examine constraint type and operand type and determine a weight value.
5568 /// The operand object must already have been set up with the operand type.
5570 AsmOperandInfo &info, int maIndex) const;
5571
5572 /// Examine constraint string and operand type and determine a weight value.
5573 /// The operand object must already have been set up with the operand type.
5575 AsmOperandInfo &info, const char *constraint) const;
5576
5577 /// Determines the constraint code and constraint type to use for the specific
5578 /// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
5579 /// If the actual operand being passed in is available, it can be passed in as
5580 /// Op, otherwise an empty SDValue can be passed.
5581 virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo,
5582 SDValue Op,
5583 SelectionDAG *DAG = nullptr) const;
5584
5585 /// Given a constraint, return the type of constraint it is for this target.
5586 virtual ConstraintType getConstraintType(StringRef Constraint) const;
5587
5588 using ConstraintPair = std::pair<StringRef, TargetLowering::ConstraintType>;
5590 /// Given an OpInfo with list of constraints codes as strings, return a
5591 /// sorted Vector of pairs of constraint codes and their types in priority of
5592 /// what we'd prefer to lower them as. This may contain immediates that
5593 /// cannot be lowered, but it is meant to be a machine agnostic order of
5594 /// preferences.
5596
5597 /// Given a physical register constraint (e.g. {edx}), return the register
5598 /// number and the register class for the register.
5599 ///
5600 /// Given a register class constraint, like 'r', if this corresponds directly
5601 /// to an LLVM register class, return a register of 0 and the register class
5602 /// pointer.
5603 ///
5604 /// This should only be used for C_Register constraints. On error, this
5605 /// returns a register number of 0 and a null register class pointer.
5606 virtual std::pair<unsigned, const TargetRegisterClass *>
5608 StringRef Constraint, MVT VT) const;
5609
5611 getInlineAsmMemConstraint(StringRef ConstraintCode) const {
5612 if (ConstraintCode == "m")
5614 if (ConstraintCode == "o")
5616 if (ConstraintCode == "X")
5618 if (ConstraintCode == "p")
5621 }
5622
5623 /// Try to replace an X constraint, which matches anything, with another that
5624 /// has more specific requirements based on the type of the corresponding
5625 /// operand. This returns null if there is no replacement to make.
5626 virtual const char *LowerXConstraint(EVT ConstraintVT) const;
5627
5628 /// Lower the specified operand into the Ops vector. If it is invalid, don't
5629 /// add anything to Ops.
5630 virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint,
5631 std::vector<SDValue> &Ops,
5632 SelectionDAG &DAG) const;
5633
5634 // Lower custom output constraints. If invalid, return SDValue().
5635 virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue,
5636 const SDLoc &DL,
5637 const AsmOperandInfo &OpInfo,
5638 SelectionDAG &DAG) const;
5639
5640 // Targets may override this function to collect operands from the CallInst
5641 // and for example, lower them into the SelectionDAG operands.
5642 virtual void CollectTargetIntrinsicOperands(const CallInst &I,
5644 SelectionDAG &DAG) const;
5645
5646 //===--------------------------------------------------------------------===//
5647 // Div utility functions
5648 //
5649
5650 SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization,
5651 bool IsAfterLegalTypes,
5652 SmallVectorImpl<SDNode *> &Created) const;
5653 SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization,
5654 bool IsAfterLegalTypes,
5655 SmallVectorImpl<SDNode *> &Created) const;
5656 // Build sdiv by power-of-2 with conditional move instructions
5657 SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor,
5658 SelectionDAG &DAG,
5659 SmallVectorImpl<SDNode *> &Created) const;
5660
5661 /// Targets may override this function to provide custom SDIV lowering for
5662 /// power-of-2 denominators. If the target returns an empty SDValue, LLVM
5663 /// assumes SDIV is expensive and replaces it with a series of other integer
5664 /// operations.
5665 virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor,
5666 SelectionDAG &DAG,
5667 SmallVectorImpl<SDNode *> &Created) const;
5668
5669 /// Targets may override this function to provide custom SREM lowering for
5670 /// power-of-2 denominators. If the target returns an empty SDValue, LLVM
5671 /// assumes SREM is expensive and replaces it with a series of other integer
5672 /// operations.
5673 virtual SDValue BuildSREMPow2(SDNode *N, const APInt &Divisor,
5674 SelectionDAG &DAG,
5675 SmallVectorImpl<SDNode *> &Created) const;
5676
5677 /// Indicate whether this target prefers to combine FDIVs with the same
5678 /// divisor. If the transform should never be done, return zero. If the
5679 /// transform should be done, return the minimum number of divisor uses
5680 /// that must exist.
5681 virtual unsigned combineRepeatedFPDivisors() const {
5682 return 0;
5683 }
5684
5685 /// Hooks for building estimates in place of slower divisions and square
5686 /// roots.
5687
5688 /// Return either a square root or its reciprocal estimate value for the input
5689 /// operand.
5690 /// \p Enabled is a ReciprocalEstimate enum with value either 'Unspecified' or
5691 /// 'Enabled' as set by a potential default override attribute.
5692 /// If \p RefinementSteps is 'Unspecified', the number of Newton-Raphson
5693 /// refinement iterations required to generate a sufficient (though not
5694 /// necessarily IEEE-754 compliant) estimate is returned in that parameter.
5695 /// The boolean UseOneConstNR output is used to select a Newton-Raphson
5696 /// algorithm implementation that uses either one or two constants.
5697 /// The boolean Reciprocal is used to select whether the estimate is for the
5698 /// square root of the input operand or the reciprocal of its square root.
5699 /// A target may choose to implement its own refinement within this function.
5700 /// If that's true, then return '0' as the number of RefinementSteps to avoid
5701 /// any further refinement of the estimate.
5702 /// An empty SDValue return means no estimate sequence can be created.
5704 int Enabled, int &RefinementSteps,
5705 bool &UseOneConstNR, bool Reciprocal) const {
5706 return SDValue();
5707 }
5708
5709 /// Try to convert the fminnum/fmaxnum to a compare/select sequence. This is
5710 /// required for correctness since InstCombine might have canonicalized a
5711 /// fcmp+select sequence to a FMINNUM/FMAXNUM intrinsic. If we were to fall
5712 /// through to the default expansion/soften to libcall, we might introduce a
5713 /// link-time dependency on libm into a file that originally did not have one.
5714 SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const;
5715
5716 /// Return a reciprocal estimate value for the input operand.
5717 /// \p Enabled is a ReciprocalEstimate enum with value either 'Unspecified' or
5718 /// 'Enabled' as set by a potential default override attribute.
5719 /// If \p RefinementSteps is 'Unspecified', the number of Newton-Raphson
5720 /// refinement iterations required to generate a sufficient (though not
5721 /// necessarily IEEE-754 compliant) estimate is returned in that parameter.
5722 /// A target may choose to implement its own refinement within this function.
5723 /// If that's true, then return '0' as the number of RefinementSteps to avoid
5724 /// any further refinement of the estimate.
5725 /// An empty SDValue return means no estimate sequence can be created.
5727 int Enabled, int &RefinementSteps) const {
5728 return SDValue();
5729 }
5730
5731 /// Return a target-dependent comparison result if the input operand is
5732 /// suitable for use with a square root estimate calculation. For example, the
5733 /// comparison may check if the operand is NAN, INF, zero, normal, etc. The
5734 /// result should be used as the condition operand for a select or branch.
5735 virtual SDValue getSqrtInputTest(SDValue Operand, SelectionDAG &DAG,
5736 const DenormalMode &Mode,
5737 SDNodeFlags Flags = {}) const;
5738
5739 /// Return a target-dependent result if the input operand is not suitable for
5740 /// use with a square root estimate calculation.
5742 SelectionDAG &DAG) const {
5743 return DAG.getConstantFP(0.0, SDLoc(Operand), Operand.getValueType());
5744 }
5745
5746 //===--------------------------------------------------------------------===//
5747 // Legalization utility functions
5748 //
5749
5750 /// Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes,
5751 /// respectively, each computing an n/2-bit part of the result.
5752 /// \param Result A vector that will be filled with the parts of the result
5753 /// in little-endian order.
5754 /// \param LL Low bits of the LHS of the MUL. You can use this parameter
5755 /// if you want to control how low bits are extracted from the LHS.
5756 /// \param LH High bits of the LHS of the MUL. See LL for meaning.
5757 /// \param RL Low bits of the RHS of the MUL. See LL for meaning
5758 /// \param RH High bits of the RHS of the MUL. See LL for meaning.
5759 /// \returns true if the node has been expanded, false if it has not
5760 bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS,
5761 SDValue RHS, SmallVectorImpl<SDValue> &Result, EVT HiLoVT,
5762 SelectionDAG &DAG, MulExpansionKind Kind,
5763 SDValue LL = SDValue(), SDValue LH = SDValue(),
5764 SDValue RL = SDValue(), SDValue RH = SDValue()) const;
5765
5766 /// Expand a MUL into two nodes. One that computes the high bits of
5767 /// the result and one that computes the low bits.
5768 /// \param HiLoVT The value type to use for the Lo and Hi nodes.
5769 /// \param LL Low bits of the LHS of the MUL. You can use this parameter
5770 /// if you want to control how low bits are extracted from the LHS.
5771 /// \param LH High bits of the LHS of the MUL. See LL for meaning.
5772 /// \param RL Low bits of the RHS of the MUL. See LL for meaning
5773 /// \param RH High bits of the RHS of the MUL. See LL for meaning.
5774 /// \returns true if the node has been expanded. false if it has not
5775 bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
5776 SelectionDAG &DAG, MulExpansionKind Kind,
5777 SDValue LL = SDValue(), SDValue LH = SDValue(),
5778 SDValue RL = SDValue(), SDValue RH = SDValue()) const;
5779
5780 /// Attempt to expand an n-bit div/rem/divrem by constant using an n/2-bit
5781 /// algorithm. First, attempt to expand the division using a n/2-bit urem by
5782 /// constant and other arithmetic ops. The n/2-bit urem by constant will be
5783 /// expanded by DAGCombiner. As this is not possible for all constant
5784 /// divisors, this method falls back to an implementation of the magic
5785 /// algorithm using n/2-bit operations.
5786 /// \param N Node to expand
5787 /// \param Result A vector that will be filled with the lo and high parts of
5788 /// the results. For *DIVREM, this will be the quotient parts followed
5789 /// by the remainder parts.
5790 /// \param HiLoVT The value type to use for the Lo and Hi parts. Should be
5791 /// half of VT.
5792 /// \param LL Low bits of the LHS of the operation. You can use this
5793 /// parameter if you want to control how low bits are extracted from
5794 /// the LHS.
5795 /// \param LH High bits of the LHS of the operation. See LL for meaning.
5796 /// \returns true if the node has been expanded, false if it has not.
5797 bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl<SDValue> &Result,
5798 EVT HiLoVT, SelectionDAG &DAG,
5799 SDValue LL = SDValue(),
5800 SDValue LH = SDValue()) const;
5801
5802 /// Expand funnel shift.
5803 /// \param N Node to expand
5804 /// \returns The expansion if successful, SDValue() otherwise
5805 SDValue expandFunnelShift(SDNode *N, SelectionDAG &DAG) const;
5806
5807 /// Expand carryless multiply.
5808 /// \param N Node to expand
5809 /// \returns The expansion if successful, SDValue() otherwise
5810 SDValue expandCLMUL(SDNode *N, SelectionDAG &DAG) const;
5811
5812 /// Expand MODF node into fractional and integer parts.
5813 /// \param N Node to expand
5814 /// \param Results Vector to store the resulting fractional and integer parts
5815 /// \returns true if the node has been expanded, false if it has not
5816 bool expandMODF(SDNode *N, SelectionDAG &DAG,
5818
5819 /// Expand parallel bit extract (compress).
5820 /// \param N Node to expand
5821 /// \returns The expansion if successful, SDValue() otherwise
5822 SDValue expandPEXT(SDNode *N, SelectionDAG &DAG) const;
5823
5824 /// Expand parallel bit deposit (expand).
5825 /// \param N Node to expand
5826 /// \returns The expansion if successful, SDValue() otherwise
5827 SDValue expandPDEP(SDNode *N, SelectionDAG &DAG) const;
5828
5829 /// Expand rotations.
5830 /// \param N Node to expand
5831 /// \param AllowVectorOps expand vector rotate, this should only be performed
5832 /// if the legalization is happening outside of LegalizeVectorOps
5833 /// \returns The expansion if successful, SDValue() otherwise
5834 SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const;
5835
5836 /// Expand shift-by-parts.
5837 /// \param N Node to expand
5838 /// \param Lo lower-output-part after conversion
5839 /// \param Hi upper-output-part after conversion
5840 void expandShiftParts(SDNode *N, SDValue &Lo, SDValue &Hi,
5841 SelectionDAG &DAG) const;
5842
5843 /// Expand float(f32) to SINT(i64) conversion
5844 /// \param N Node to expand
5845 /// \param Result output after conversion
5846 /// \returns True, if the expansion was successful, false otherwise
5847 bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
5848
5849 /// Expand float to UINT conversion
5850 /// \param N Node to expand
5851 /// \param Result output after conversion
5852 /// \param Chain output chain after conversion
5853 /// \returns True, if the expansion was successful, false otherwise
5854 bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SDValue &Chain,
5855 SelectionDAG &DAG) const;
5856
5857 /// Expand UINT(i64) to double(f64) conversion
5858 /// \param N Node to expand
5859 /// \param Result output after conversion
5860 /// \param Chain output chain after conversion
5861 /// \returns True, if the expansion was successful, false otherwise
5862 bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SDValue &Chain,
5863 SelectionDAG &DAG) const;
5864
5865 /// Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs.
5866 SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const;
5867
5868 /// Expand fminimum/fmaximum into multiple comparison with selects.
5869 SDValue expandFMINIMUM_FMAXIMUM(SDNode *N, SelectionDAG &DAG) const;
5870
5871 /// Expand fminimumnum/fmaximumnum into multiple comparison with selects.
5872 SDValue expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *N, SelectionDAG &DAG) const;
5873
5874 /// Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
5875 /// \param N Node to expand
5876 /// \returns The expansion result
5877 SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const;
5878
5879 /// Truncate Op to ResultVT. If the result is exact, leave it alone. If it is
5880 /// not exact, force the result to be odd.
5881 /// \param ResultVT The type of result.
5882 /// \param Op The value to round.
5883 /// \returns The expansion result
5884 SDValue expandRoundInexactToOdd(EVT ResultVT, SDValue Op, const SDLoc &DL,
5885 SelectionDAG &DAG) const;
5886
5887 /// Expand round(fp) to fp conversion
5888 /// \param N Node to expand
5889 /// \returns The expansion result
5890 SDValue expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const;
5891
5892 /// Expand check for floating point class.
5893 /// \param ResultVT The type of intrinsic call result.
5894 /// \param Op The tested value.
5895 /// \param Test The test to perform.
5896 /// \param Flags The optimization flags.
5897 /// \returns The expansion result or SDValue() if it fails.
5898 SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test,
5899 SDNodeFlags Flags, const SDLoc &DL,
5900 SelectionDAG &DAG) const;
5901
5902 /// Expand FCANONICALIZE to FMUL with 1.
5903 /// \param NodeNode to expand
5904 /// \returns The expansion result
5905 SDValue expandFCANONICALIZE(SDNode *Node, SelectionDAG &DAG) const;
5906
5907 /// Expand CONVERT_TO_ARBITRARY_FP using bit manipulation.
5908 /// \param Node Node to expand.
5909 /// \returns The expansion result, or SDValue() if fails.
5910 SDValue expandCONVERT_TO_ARBITRARY_FP(SDNode *Node, SelectionDAG &DAG) const;
5911
5912 /// Expand CONVERT_FROM_ARBITRARY_FP using bit manipulation.
5913 /// \param Node Node to expand.
5914 /// \returns The expansion result, or SDValue() if fails.
5915 SDValue expandCONVERT_FROM_ARBITRARY_FP(SDNode *Node,
5916 SelectionDAG &DAG) const;
5917
5918 /// Expand CTPOP nodes. Expands vector/scalar CTPOP nodes,
5919 /// vector nodes can only succeed if all operations are legal/custom.
5920 /// \param N Node to expand
5921 /// \returns The expansion result or SDValue() if it fails.
5922 SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const;
5923
5924 /// Expand CTLZ/CTLZ_ZERO_POISON nodes. Expands vector/scalar CTLZ nodes,
5925 /// vector nodes can only succeed if all operations are legal/custom.
5926 /// \param N Node to expand
5927 /// \returns The expansion result or SDValue() if it fails.
5928 SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const;
5929
5930 /// Expand CTLS (count leading sign bits) nodes.
5931 /// CTLS(x) = CTLZ(OR(SHL(XOR(x, SRA(x, BW-1)), 1), 1))
5932 /// \param N Node to expand
5933 /// \returns The expansion result or SDValue() if it fails.
5934 SDValue expandCTLS(SDNode *N, SelectionDAG &DAG) const;
5935
5936 /// Expand CTTZ via Table Lookup.
5937 /// \param N Node to expand
5938 /// \returns The expansion result or SDValue() if it fails.
5939 SDValue CTTZTableLookup(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, EVT VT,
5940 SDValue Op, unsigned NumBitsPerElt) const;
5941
5942 /// Expand CTTZ/CTTZ_ZERO_POISON nodes. Expands vector/scalar CTTZ nodes,
5943 /// vector nodes can only succeed if all operations are legal/custom.
5944 /// \param N Node to expand
5945 /// \returns The expansion result or SDValue() if it fails.
5946 SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const;
5947
5948 /// Expand VP_CTTZ_ELTS/VP_CTTZ_ELTS_ZERO_POISON nodes.
5949 /// \param N Node to expand
5950 /// \returns The expansion result or SDValue() if it fails.
5951 SDValue expandVPCTTZElements(SDNode *N, SelectionDAG &DAG) const;
5952
5953 /// Expand VECTOR_MATCH nodes.
5954 /// \param N Node to expand
5955 /// \returns The expansion result or SDValue() if it fails.
5956 SDValue expandVectorMatch(SDNode *N, SelectionDAG &DAG) const;
5957
5958 /// Expand VECTOR_FIND_LAST_ACTIVE nodes
5959 /// \param N Node to expand
5960 /// \returns The expansion result or SDValue() if it fails.
5961 SDValue expandVectorFindLastActive(SDNode *N, SelectionDAG &DAG) const;
5962
5963 /// Expand GET_ACTIVE_LANE_MASK nodes
5964 /// \param N Node to expand
5965 /// \returns The expansion result or SDValue() if it fails.
5966 SDValue expandGetActiveLaneMask(SDNode *N, SelectionDAG &DAG) const;
5967
5968 /// Expand LOOP_DEPENDENCE_MASK nodes
5969 /// \param N Node to expand
5970 /// \returns The expansion result or SDValue() if it fails.
5971 SDValue expandLoopDependenceMask(SDNode *N, SelectionDAG &DAG) const;
5972
5973 /// Expand ABS nodes. Expands vector/scalar ABS nodes,
5974 /// vector nodes can only succeed if all operations are legal/custom.
5975 /// (ABS x) -> (XOR (ADD x, (SRA x, type_size)), (SRA x, type_size))
5976 /// \param N Node to expand
5977 /// \param IsNegative indicate negated abs
5978 /// \returns The expansion result or SDValue() if it fails.
5979 SDValue expandABS(SDNode *N, SelectionDAG &DAG,
5980 bool IsNegative = false) const;
5981
5982 /// Expand ABDS/ABDU nodes. Expands vector/scalar ABDS/ABDU nodes.
5983 /// \param N Node to expand
5984 /// \returns The expansion result or SDValue() if it fails.
5985 SDValue expandABD(SDNode *N, SelectionDAG &DAG) const;
5986
5987 /// Expand vector/scalar AVGCEILS/AVGCEILU/AVGFLOORS/AVGFLOORU nodes.
5988 /// \param N Node to expand
5989 /// \returns The expansion result or SDValue() if it fails.
5990 SDValue expandAVG(SDNode *N, SelectionDAG &DAG) const;
5991
5992 /// Expand BSWAP nodes. Expands scalar/vector BSWAP nodes with i16/i32/i64
5993 /// scalar types. Returns SDValue() if expand fails.
5994 /// \param N Node to expand
5995 /// \returns The expansion result or SDValue() if it fails.
5996 SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const;
5997
5998 /// Expand BITREVERSE nodes. Expands scalar/vector BITREVERSE nodes.
5999 /// Returns SDValue() if expand fails.
6000 /// \param N Node to expand
6001 /// \returns The expansion result or SDValue() if it fails.
6002 SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const;
6003
6004 /// Turn load of vector type into a load of the individual elements.
6005 /// \param LD load to expand
6006 /// \returns BUILD_VECTOR and TokenFactor nodes.
6007 std::pair<SDValue, SDValue> scalarizeVectorLoad(LoadSDNode *LD,
6008 SelectionDAG &DAG) const;
6009
6010 // Turn a store of a vector type into stores of the individual elements.
6011 /// \param ST Store with a vector value type
6012 /// \returns TokenFactor of the individual store chains.
6014
6015 /// Expands an unaligned load to 2 half-size loads for an integer, and
6016 /// possibly more for vectors.
6017 std::pair<SDValue, SDValue> expandUnalignedLoad(LoadSDNode *LD,
6018 SelectionDAG &DAG) const;
6019
6020 /// Expands an unaligned store to 2 half-size stores for integer values, and
6021 /// possibly more for vectors.
6022 SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const;
6023
6024 /// Increments memory address \p Addr according to the type of the value
6025 /// \p DataVT that should be stored. If the data is stored in compressed
6026 /// form, the memory address should be incremented according to the number of
6027 /// the stored elements. This number is equal to the number of '1's bits
6028 /// in the \p Mask.
6029 /// \p DataVT is a vector type. \p Mask is a vector value.
6030 /// \p DataVT and \p Mask have the same number of vector elements.
6031 SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL,
6032 EVT DataVT, SelectionDAG &DAG,
6033 bool IsCompressedMemory) const;
6034
6035 /// Get a pointer to vector element \p Idx located in memory for a vector of
6036 /// type \p VecVT starting at a base address of \p VecPtr. If \p Idx is out of
6037 /// bounds the returned pointer is unspecified, but will be within the vector
6038 /// bounds. \p PtrArithFlags can be used to mark that arithmetic within the
6039 /// vector in memory is known to not wrap or to be inbounds.
6040 SDValue getVectorElementPointer(
6041 SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index,
6042 const SDNodeFlags PtrArithFlags = SDNodeFlags()) const;
6043
6044 /// Get a pointer to vector element \p Idx located in memory for a vector of
6045 /// type \p VecVT starting at a base address of \p VecPtr. If \p Idx is out of
6046 /// bounds the returned pointer is unspecified, but will be within the vector
6047 /// bounds. \p VecPtr is guaranteed to point to the beginning of a memory
6048 /// location large enough for the vector.
6050 EVT VecVT, SDValue Index) const {
6051 return getVectorElementPointer(DAG, VecPtr, VecVT, Index,
6054 }
6055
6056 /// Get a pointer to a sub-vector of type \p SubVecVT at index \p Idx located
6057 /// in memory for a vector of type \p VecVT starting at a base address of
6058 /// \p VecPtr. If \p Idx plus the size of \p SubVecVT is out of bounds the
6059 /// returned pointer is unspecified, but the value returned will be such that
6060 /// the entire subvector would be within the vector bounds. \p PtrArithFlags
6061 /// can be used to mark that arithmetic within the vector in memory is known
6062 /// to not wrap or to be inbounds.
6063 SDValue
6064 getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT,
6065 EVT SubVecVT, SDValue Index,
6066 const SDNodeFlags PtrArithFlags = SDNodeFlags()) const;
6067
6068 /// Method for building the DAG expansion of ISD::[US][MIN|MAX]. This
6069 /// method accepts integers as its arguments.
6070 SDValue expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const;
6071
6072 /// Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT. This
6073 /// method accepts integers as its arguments.
6074 SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const;
6075
6076 /// Method for building the DAG expansion of ISD::[US]CMP. This
6077 /// method accepts integers as its arguments
6078 SDValue expandCMP(SDNode *Node, SelectionDAG &DAG) const;
6079
6080 /// Method for building the DAG expansion of ISD::[US]SHLSAT. This
6081 /// method accepts integers as its arguments.
6082 SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const;
6083
6084 /// Method for building the DAG expansion of ISD::[U|S]MULFIX[SAT]. This
6085 /// method accepts integers as its arguments.
6086 SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const;
6087
6088 /// Method for building the DAG expansion of ISD::[US]DIVFIX[SAT]. This
6089 /// method accepts integers as its arguments.
6090 /// Note: This method may fail if the division could not be performed
6091 /// within the type. Clients must retry with a wider type if this happens.
6092 SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,
6094 unsigned Scale, SelectionDAG &DAG) const;
6095
6096 /// Method for building the DAG expansion of ISD::U(ADD|SUB)O. Expansion
6097 /// always suceeds and populates the Result and Overflow arguments.
6098 void expandUADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow,
6099 SelectionDAG &DAG) const;
6100
6101 /// Method for building the DAG expansion of ISD::S(ADD|SUB)O. Expansion
6102 /// always suceeds and populates the Result and Overflow arguments.
6103 void expandSADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow,
6104 SelectionDAG &DAG) const;
6105
6106 /// Method for building the DAG expansion of ISD::[US]MULO. Returns whether
6107 /// expansion was successful and populates the Result and Overflow arguments.
6108 bool expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow,
6109 SelectionDAG &DAG) const;
6110
6111 // Expand ISD::MULH[SU]. Can expand to MUL_LOHI or wide MUL if available.
6112 SDValue expandMULH(SDNode *Node, SelectionDAG &DAG) const;
6113
6114 /// Calculate the product twice the width of LHS and RHS. If HiLHS/HiRHS are
6115 /// non-null they will be included in the multiplication. The expansion works
6116 /// by splitting the 2 inputs into 4 pieces that we can multiply and add
6117 /// together without neding MULH or MUL_LOHI.
6118 void forceExpandMultiply(SelectionDAG &DAG, const SDLoc &dl, bool Signed,
6120 SDValue HiLHS = SDValue(),
6121 SDValue HiRHS = SDValue()) const;
6122
6123 /// Calculate full product of LHS and RHS either via a libcall or through
6124 /// brute force expansion of the multiplication. The expansion works by
6125 /// splitting the 2 inputs into 4 pieces that we can multiply and add together
6126 /// without needing MULH or MUL_LOHI.
6127 void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed,
6128 const SDValue LHS, const SDValue RHS, SDValue &Lo,
6129 SDValue &Hi) const;
6130
6131 /// Expand a VECREDUCE_* into an explicit calculation. If Count is specified,
6132 /// only the first Count elements of the vector are used.
6133 SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const;
6134
6135 /// Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
6136 SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const;
6137
6138 /// Expand an SREM or UREM using SDIV/UDIV or SDIVREM/UDIVREM, if legal.
6139 /// Returns true if the expansion was successful.
6140 bool expandREM(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const;
6141
6142 /// Method for building the DAG expansion of ISD::VECTOR_SPLICE. This
6143 /// method accepts vectors as its arguments.
6144 SDValue expandVectorSplice(SDNode *Node, SelectionDAG &DAG) const;
6145
6146 /// Expand a VECTOR_INTERLEAVE without introducing wider vector types.
6147 void expandVectorInterleave(SDNode *Node, SmallVectorImpl<SDValue> &Results,
6148 SelectionDAG &DAG) const;
6149
6150 /// Expand a VECTOR_DEINTERLEAVE without introducing wider vector types.
6151 void expandVectorDeinterleave(SDNode *Node, SmallVectorImpl<SDValue> &Results,
6152 SelectionDAG &DAG) const;
6153
6154 /// Expand a vector VECTOR_COMPRESS into a sequence of extract element, store
6155 /// temporarily, advance store position, before re-loading the final vector.
6156 SDValue expandVECTOR_COMPRESS(SDNode *Node, SelectionDAG &DAG) const;
6157
6158 /// Expand a VECTOR_SHUFFLE_VAR into variable-index element extracts.
6159 SDValue expandVECTOR_SHUFFLE_VAR(SDNode *Node, SelectionDAG &DAG) const;
6160
6161 /// Expand a CTTZ_ELTS or CTTZ_ELTS_ZERO_POISON by calculating (VL - i) for
6162 /// each active lane (i), getting the maximum and subtracting it from VL.
6163 SDValue expandCttzElts(SDNode *Node, SelectionDAG &DAG) const;
6164
6165 /// Expands PARTIAL_REDUCE_S/UMLA nodes to a series of simpler operations,
6166 /// consisting of zext/sext, extract_subvector, mul and add operations.
6167 SDValue expandPartialReduceMLA(SDNode *Node, SelectionDAG &DAG) const;
6168
6169 /// Expands a node with multiple results to an FP or vector libcall. The
6170 /// libcall is expected to take all the operands of the \p Node followed by
6171 /// output pointers for each of the results. \p CallRetResNo can be optionally
6172 /// set to indicate that one of the results comes from the libcall's return
6173 /// value.
6174 bool expandMultipleResultFPLibCall(
6175 SelectionDAG &DAG, RTLIB::Libcall LC, SDNode *Node,
6177 std::optional<unsigned> CallRetResNo = {}) const;
6178
6179 /// Legalize a SETCC with given LHS and RHS and condition code CC on the
6180 /// current target.
6181 ///
6182 /// If the SETCC has been legalized using AND / OR, then the legalized node
6183 /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
6184 /// will be set to false.
6185 ///
6186 /// If the SETCC has been legalized by using getSetCCSwappedOperands(), then
6187 /// the values of LHS and RHS will be swapped, CC will be set to the new
6188 /// condition, and NeedInvert will be set to false.
6189 ///
6190 /// If the SETCC has been legalized using the inverse condcode, then LHS and
6191 /// RHS will be unchanged, CC will set to the inverted condcode, and
6192 /// NeedInvert will be set to true. The caller must invert the result of the
6193 /// SETCC with SelectionDAG::getLogicalNOT() or take equivalent action to swap
6194 /// the effect of a true/false result.
6195 ///
6196 /// \returns true if the SETCC has been legalized, false if it hasn't.
6197 bool LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, SDValue &LHS,
6198 SDValue &RHS, SDValue &CC, bool &NeedInvert,
6199 const SDLoc &dl, SDValue &Chain,
6200 bool IsSignaling = false) const;
6201
6202 //===--------------------------------------------------------------------===//
6203 // Instruction Emitting Hooks
6204 //
6205
6206 /// This method should be implemented by targets that mark instructions with
6207 /// the 'usesCustomInserter' flag. These instructions are special in various
6208 /// ways, which require special support to insert. The specified MachineInstr
6209 /// is created but not inserted into any basic blocks, and this method is
6210 /// called to expand it into a sequence of instructions, potentially also
6211 /// creating new basic blocks and control flow.
6212 /// As long as the returned basic block is different (i.e., we created a new
6213 /// one), the custom inserter is free to modify the rest of \p MBB.
6214 virtual MachineBasicBlock *
6215 EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const;
6216
6217 /// This method should be implemented by targets that mark instructions with
6218 /// the 'hasPostISelHook' flag. These instructions must be adjusted after
6219 /// instruction selection by target hooks. e.g. To fill in optional defs for
6220 /// ARM 's' setting instructions.
6221 virtual void AdjustInstrPostInstrSelection(MachineInstr &MI,
6222 SDNode *Node) const;
6223
6224 /// If this function returns true, SelectionDAGBuilder emits a
6225 /// LOAD_STACK_GUARD node when it is lowering Intrinsic::stackprotector.
6226 virtual bool useLoadStackGuardNode(const Module &M) const { return false; }
6227
6229 const SDLoc &DL) const {
6230 llvm_unreachable("not implemented for this target");
6231 }
6232
6233 /// Lower TLS global address SDNode for target independent emulated TLS model.
6234 virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
6235 SelectionDAG &DAG) const;
6236
6237 /// Expands target specific indirect branch for the case of JumpTable
6238 /// expansion.
6239 virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value,
6240 SDValue Addr, int JTI,
6241 SelectionDAG &DAG) const;
6242
6243 // seteq(x, 0) -> truncate(srl(ctlz(zext(x)), log2(#bits)))
6244 // If we're comparing for equality to zero and isCtlzFast is true, expose the
6245 // fact that this can be implemented as a ctlz/srl pair, so that the dag
6246 // combiner can fold the new nodes.
6247 SDValue lowerCmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) const;
6248
6249 // Return true if `X & Y eq/ne 0` is preferable to `X & Y ne/eq Y`
6251 return true;
6252 }
6253
6254 // Expand vector operation by dividing it into smaller length operations and
6255 // joining their results. SDValue() is returned when expansion did not happen.
6256 SDValue expandVectorNaryOpBySplitting(SDNode *Node, SelectionDAG &DAG) const;
6257
6258 /// Replace an extraction of a load with a narrowed load.
6259 ///
6260 /// \param ResultVT type of the result extraction.
6261 /// \param InVecVT type of the input vector to with bitcasts resolved.
6262 /// \param EltNo index of the vector element to load.
6263 /// \param OriginalLoad vector load that to be replaced.
6264 /// \returns \p ResultVT Load on success SDValue() on failure.
6265 SDValue scalarizeExtractedVectorLoad(EVT ResultVT, const SDLoc &DL,
6266 EVT InVecVT, SDValue EltNo,
6267 LoadSDNode *OriginalLoad,
6268 SelectionDAG &DAG) const;
6269
6270protected:
6271 void setTypeIdForCallsiteInfo(const CallBase *CB, MachineFunction &MF,
6272 MachineFunction::CallSiteInfo &CSInfo) const;
6273
6274private:
6275 SDValue foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
6276 const SDLoc &DL, DAGCombinerInfo &DCI) const;
6277 SDValue foldSetCCWithOr(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
6278 const SDLoc &DL, DAGCombinerInfo &DCI) const;
6279 SDValue foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
6280 const SDLoc &DL, DAGCombinerInfo &DCI) const;
6281
6282 SDValue optimizeSetCCOfSignedTruncationCheck(EVT SCCVT, SDValue N0,
6284 DAGCombinerInfo &DCI,
6285 const SDLoc &DL) const;
6286
6287 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
6288 SDValue optimizeSetCCByHoistingAndByConstFromLogicalShift(
6289 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
6290 DAGCombinerInfo &DCI, const SDLoc &DL) const;
6291
6292 SDValue prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
6293 SDValue CompTargetNode, ISD::CondCode Cond,
6294 DAGCombinerInfo &DCI, const SDLoc &DL,
6295 SmallVectorImpl<SDNode *> &Created) const;
6296 SDValue buildUREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode,
6297 ISD::CondCode Cond, DAGCombinerInfo &DCI,
6298 const SDLoc &DL) const;
6299
6300 SDValue prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
6301 SDValue CompTargetNode, ISD::CondCode Cond,
6302 DAGCombinerInfo &DCI, const SDLoc &DL,
6303 SmallVectorImpl<SDNode *> &Created) const;
6304 SDValue buildSREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode,
6305 ISD::CondCode Cond, DAGCombinerInfo &DCI,
6306 const SDLoc &DL) const;
6307
6308 bool expandUDIVREMByConstantViaUREMDecomposition(
6309 SDNode *N, APInt Divisor, SmallVectorImpl<SDValue> &Result, EVT HiLoVT,
6310 SelectionDAG &DAG, SDValue LL, SDValue LH) const;
6311
6312 bool expandUDIVREMByConstantViaUMulHiMagic(SDNode *N, const APInt &Divisor,
6314 EVT HiLoVT, SelectionDAG &DAG,
6315 SDValue LL, SDValue LH) const;
6316};
6317
6318/// Given an LLVM IR type and return type attributes, compute the return value
6319/// EVTs and flags, and optionally also the offsets, if the return value is
6320/// being lowered to memory.
6321LLVM_ABI void GetReturnInfo(CallingConv::ID CC, Type *ReturnType,
6322 AttributeList attr,
6323 SmallVectorImpl<ISD::OutputArg> &Outs,
6324 const TargetLowering &TLI, const DataLayout &DL);
6325
6326} // end namespace llvm
6327
6328#endif // LLVM_CODEGEN_TARGETLOWERING_H
unsigned NumParts
return SDValue()
bool IsScalable
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
uint64_t Imm
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
block Block Frequency Analysis
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_ABI
Definition Compiler.h:211
#define LLVM_READONLY
Definition Compiler.h:334
This file defines the DenseMap class.
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
static bool ShrinkDemandedConstant(Instruction *I, unsigned OpNo, const APInt &Demanded)
Check to see if the specified operand of the specified instruction is a constant integer.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define RegName(no)
lazy value info
static bool isFloat(MachineBasicBlock::iterator I)
Implement a low-level type suitable for MachineInstr level instruction selection.
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
const uint64_t BitWidth
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t High
PowerPC Reduce CR logical Operation
SmallVector< MachineOperand, 4 > Cond
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
This file defines the SmallVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT, SelectionDAG &DAG)
Scalarize a vector store, bitcasting to TargetVT to determine the scalar type.
Value * RHS
Value * LHS
Class for arbitrary precision integers.
Definition APInt.h:78
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
bool isElementwise() const
Return true if this RMW has elementwise vector semantics.
bool isFloatingPointOperation() const
BinOp getOperation() const
This class holds the attributes for a particular argument, parameter, function, or return value.
Definition Attributes.h:448
LLVM_ABI bool getValueAsBool() const
Return the attribute's value as a boolean.
AttrKind
This enumeration lists the attributes that can be associated with parameters, function results,...
Definition Attributes.h:126
@ None
No attributes have been set.
Definition Attributes.h:128
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
CCState - This class holds information needed while lowering arguments and return values.
CCValAssign - Represent assignment of one arg/retval to a location.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
This class represents a function call, abstracting a target machine's calling convention.
This is the shared class of boolean and integer constants.
Definition Constants.h:87
This class represents a range of values.
This is an important base class in LLVM.
Definition Constant.h:44
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
unsigned size() const
Definition DenseMap.h:717
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:317
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition FastISel.h:67
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Class to represent function types.
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
bool isVarArg() const
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition Function.cpp:766
Common base class shared among various IRBuilders.
Definition IRBuilder.h:111
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
static LLT integer(unsigned SizeInBits)
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
Tracks which library functions to use for a particular subtarget or function.
An instruction for reading from memory.
This class is used to represent ISD::LOAD nodes.
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
Context object for machine code objects.
Definition MCContext.h:83
Base class for the full range of assembler expressions which are needed for parsing.
Definition MCExpr.h:34
MCRegisterClass - Base class of TargetRegisterClass.
Machine Value Type.
@ INVALID_SIMPLE_VALUE_TYPE
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
ElementCount getVectorElementCount() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
Instructions::iterator instr_iterator
Representation of each machine instruction.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This is an abstract virtual class for memory operations.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:294
A discriminated union of two or more pointer types, with the discriminator in the low bits of the poi...
Analysis providing profile information.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool use_empty() const
Return true if there are no uses of this node.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
const DataLayout & getDataLayout() const
LLVM_ABI void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVMContext * getContext() const
This instruction constructs a fixed permutation of two input vectors.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
Multiway switch.
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
LLVM_ABI void setExtAttribute(Attribute::AttrKind Attr)
ArgListEntry(Value *Val, SDValue Node, Type *Ty)
Type * Ty
Same as OrigTy, or partially legalized for soft float libcalls.
ArgListEntry(Value *Val, SDValue Node=SDValue(), Attribute::AttrKind ExtAttr=Attribute::None)
ArgListEntry(SDValue Node, Type *Ty, Attribute::AttrKind ExtAttr=Attribute::None)
Type * OrigTy
Original unlegalized argument type.
LegalizeTypeAction getTypeAction(MVT VT) const
void setTypeAction(MVT VT, LegalizeTypeAction Action)
This base class for TargetLowering contains the SelectionDAG-independent parts that can be used from ...
virtual Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const
Perform a store-conditional operation to Addr.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
virtual IntegerType * getCmpLibcallReturnType(LLVMContext &Ctx) const
Return the integer type returned by comparison libcalls.
virtual bool enableAggressiveFMAFusion(LLT Ty) const
Return true if target always benefits from combining into FMA for a given value type.
virtual void emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const
Perform a bit test atomicrmw using a target-specific intrinsic.
void setOperationAction(ArrayRef< unsigned > Ops, ArrayRef< MVT > VTs, LegalizeAction Action)
virtual bool requiresUniformRegister(MachineFunction &MF, const Value *) const
Allows target to decide about the register class of the specific value that is live outside the defin...
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
virtual unsigned getVaListSizeInBits(const DataLayout &DL) const
Returns the size of the platform's va_list object.
virtual bool lowerDeinterleaveIntrinsicToLoad(Instruction *Load, Value *Mask, IntrinsicInst *DI, const APInt &GapMask) const
Lower a deinterleave intrinsic to a target specific load intrinsic.
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT, EVT ExtVT) const
virtual bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
virtual bool hasAndNot(SDValue X) const
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
ReciprocalEstimate
Reciprocal estimate status values used by the functions below.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
virtual bool isAtomicAlignmentSupported(Align Alignment, uint64_t SizeInBytes) const
Return true if the target supports an atomic access of SizeInBytes bytes at the given Alignment.
virtual bool enableAggressiveFMAFusion(EVT VT) const
Return true if target always benefits from combining into FMA for a given value type.
virtual bool isComplexDeinterleavingOperationSupported(ComplexDeinterleavingOperation Operation, Type *Ty) const
Does this target support complex deinterleaving with the given operation and type.
virtual bool shouldRemoveRedundantExtend(SDValue Op) const
Return true (the default) if it is profitable to remove a sext_inreg(x) where the sext is redundant,...
bool isIndexedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
SDValue promoteTargetBoolean(SelectionDAG &DAG, SDValue Bool, EVT ValVT) const
Promote the given target boolean to a target boolean of the given type.
virtual bool isFMADLegal(const SelectionDAG &DAG, const SDNode *N) const
Returns true if be combined with to form an ISD::FMAD.
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT, std::optional< unsigned > ByteOffset=std::nullopt) const
Return true if it is profitable to reduce a load to a smaller type.
virtual bool hasStandaloneRem(EVT VT) const
Return true if the target can handle a standalone remainder operation.
virtual bool isExtFreeImpl(const Instruction *I) const
Return true if the extension represented by I is free.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual bool shouldExpandBuildVectorWithShuffles(EVT, unsigned DefinedValues) const
LegalizeAction getIndexedMaskedStoreAction(unsigned IdxMode, MVT VT) const
Return how the indexed store should be treated: either it is legal, needs to be promoted to a larger ...
virtual bool isSelectSupported(SelectSupportKind) const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual bool isEqualityCmpFoldedWithSignedCmp() const
Return true if instruction generated for equality comparison is folded with instruction generated for...
virtual bool preferSelectsOverBooleanArithmetic(EVT VT) const
Should we prefer selects to doing arithmetic on boolean types.
virtual bool isLegalICmpImmediate(int64_t) const
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
virtual bool convertSetCCLogicToBitwiseLogic(EVT VT) const
Use bitwise logic to make pairs of compares more efficient.
void setAtomicLoadExtAction(ArrayRef< unsigned > ExtTypes, MVT ValVT, ArrayRef< MVT > MemVTs, LegalizeAction Action)
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool shouldFormOverflowOp(unsigned Opcode, EVT VT, bool MathUsed) const
Try to convert math with an overflow comparison into the corresponding DAG node operation.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
virtual bool isVectorLoadExtDesirable(SDValue ExtVal) const
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
virtual bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const
Return if the target supports combining a chain like:
virtual Value * createComplexDeinterleavingIR(IRBuilderBase &B, ComplexDeinterleavingOperation OperationType, ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB, Value *Accumulator=nullptr) const
Create the IR node for the given complex deinterleaving operation.
virtual bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const
Return true if it is beneficial to convert a load of a constant to just the constant itself.
virtual bool isSupportedFixedPointOperation(unsigned Op, EVT VT, unsigned Scale) const
Custom method defined by each target to indicate if an operation which may require a scale is support...
void setLoadExtAction(ArrayRef< unsigned > ExtTypes, MVT ValVT, MVT MemVT, LegalizeAction Action)
unsigned getMaximumLegalStoreInBits() const
Return maximum known-legal store size, which can be guaranteed for scalable vectors.
virtual bool shouldOptimizeMulOverflowWithZeroHighBits(LLVMContext &Context, EVT VT) const
virtual AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
virtual Sched::Preference getSchedulingPreference(SDNode *) const
Some scheduler, e.g.
virtual MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const
void setMinStackArgumentAlignment(Align Alignment)
Set the minimum stack alignment of an argument.
bool isExtLoad(const LoadInst *Load, const Instruction *Ext, const DataLayout &DL) const
Return true if Load and Ext can form an ExtLoad.
LegalizeTypeAction getTypeAction(MVT VT) const
virtual bool isLegalScaleForGatherScatter(uint64_t Scale, uint64_t ElemSize) const
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual bool shouldInsertFencesForAtomic(const Instruction *I) const
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
virtual AtomicOrdering atomicOperationOrderAfterFenceSplit(const Instruction *I) const
MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
bool isOperationExpandOrLibCall(unsigned Op, EVT VT) const
virtual bool allowsMisalignedMemoryAccesses(LLT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
LLT handling variant.
virtual bool isSafeMemOpType(MVT) const
Returns true if it's safe to use load / store of the specified type to expand memcpy / memset inline.
virtual void emitExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) const
Perform a cmpxchg expansion using a target-specific method.
virtual ISD::NodeType getExtendForAtomicRMWArg(unsigned Op) const
Returns how the platform's atomic rmw operations expect their input argument to be extended (ZERO_EXT...
const TargetMachine & getTargetMachine() const
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
bool rangeFitsInWord(const APInt &Low, const APInt &High, const DataLayout &DL) const
Check whether the range [Low,High] fits in a machine word.
virtual bool isCtpopFast(EVT VT) const
Return true if ctpop instruction is fast.
virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
virtual Register getExceptionSelectorRegister(ExceptionHandling EH, const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
unsigned MaxGluedStoresPerMemcpy
Specify max number of store instructions to glue in inlined memcpy.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
bool isPaddedAtMostSignificantBitsWhenStored(EVT VT) const
Indicates if any padding is guaranteed to go at the most significant bits when storing the type to me...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
virtual bool useStackGuardMixFP() const
If this function returns true, stack protection checks should mix the frame pointer (or whichever poi...
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
virtual Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const
Perform a masked atomicrmw using a target-specific intrinsic.
virtual bool areJTsAllowed(const Function *Fn) const
Return true if lowering to a jump table is allowed.
virtual LegalizeAction getCustomTruncStoreAction(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace) const
Returns an alternative action to use when the coarser lookups (configured through setTruncStoreAction...
bool enableExtLdPromotion() const
Return true if the target wants to use the optimization that turns ext(promotableInst1(....
virtual bool isFPExtFoldable(const MachineInstr &MI, unsigned Opcode, LLT DestTy, LLT SrcTy) const
Return true if an fpext operation input to an Opcode operation is free (for instance,...
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const
Returns true if the specified FP immediate can be materialized without a load from the constant pool.
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
void setMaxBytesForAlignment(unsigned MaxBytes)
bool isOperationLegalOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal using promotion.
void setHasExtractBitsInsn(bool hasExtractInsn=true)
Tells the code generator that the target has BitExtract instructions.
void addBypassSlowDiv(unsigned int SlowBitWidth, unsigned int FastBitWidth)
Tells the code generator which bitwidths to bypass.
virtual bool hasBitTest(SDValue X, SDValue Y) const
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
MVT getRegisterType(LLVMContext &Context, EVT VT) const
Return the type of registers that this ValueType will eventually require.
virtual AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(const AtomicCmpXchgInst *AI) const
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
virtual bool needsFixedCatchObjects() const
virtual bool isAnyExtFree(EVT FromTy, EVT ToTy) const
Return true is an anyext is free from FromTy to ToTy.
EVT getLegalTypeToTransformTo(LLVMContext &Context, EVT VT) const
Perform getTypeToTransformTo repeatedly until a legal type is obtained.
virtual Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
void setMaxLargeFPConvertBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum fp to/from int conversion the backend supports.
const LibcallLoweringInfo & getLibcallLoweringInfo() const
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
virtual bool isCheapToSpeculateCttz(Type *Ty) const
Return true if it is cheap to speculate a call to intrinsic cttz.
unsigned getMinimumBitTestCmps() const
Retuen the minimum of largest number of comparisons in BitTest.
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
virtual bool useFPRegsForHalfType() const
LegalizeAction getCondCodeAction(ISD::CondCode CC, MVT VT) const
Return how the condition code should be treated: either it is legal, needs to be expanded to some oth...
bool hasExtractBitsInsn() const
Return true if the target has BitExtract instructions.
virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
LegalizeAction getIndexedStoreAction(unsigned IdxMode, MVT VT) const
Return how the indexed store should be treated: either it is legal, needs to be promoted to a larger ...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
virtual bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
LegalizeAction getTruncStoreAction(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace) const
Return how this store with truncation should be treated: either it is legal, needs to be promoted to ...
virtual unsigned getPreferredFPToIntOpcode(unsigned Op, EVT FromVT, EVT ToVT) const
virtual bool isExtractVecEltCheap(EVT VT, unsigned Index) const
Return true if extraction of a scalar element from the given vector type at the given index is cheap.
virtual Value * emitCanLoadSpeculatively(IRBuilderBase &Builder, Value *Ptr, Value *Size) const
Emit code to check if a speculative load of the given size from Ptr is safe.
void setOperationAction(ArrayRef< unsigned > Ops, MVT VT, LegalizeAction Action)
virtual bool optimizeFMulOrFDivAsShiftAddBitcast(SDNode *N, SDValue FPConst, SDValue IntPow2) const
SelectSupportKind
Enum that describes what type of support for selects the target has.
LegalizeAction getIndexedLoadAction(unsigned IdxMode, MVT VT) const
Return how the indexed load should be treated: either it is legal, needs to be promoted to a larger s...
virtual bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
virtual bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode, EVT DestVT, EVT SrcVT) const
Return true if an fpext operation input to an Opcode operation is free (for instance,...
bool isLegalRC(const TargetRegisterInfo &TRI, const TargetRegisterClass &RC) const
Return true if the value types that can be represented by the specified register class are all legal.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
virtual bool allowTruncateForTailCall(Type *FromTy, Type *ToTy) const
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
void setAtomicLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Let target indicate that an extending atomic load of the specified type is legal.
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
virtual StringRef getStackProbeSymbolName(const MachineFunction &MF) const
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
virtual bool preferScalarizeSplat(SDNode *N) const
void setVectorInterleaveAction(unsigned Opc, unsigned Factor, MVT VT, LegalizeAction Action)
Indicate how a VECTOR_INTERLEAVE or VECTOR_DEINTERLEAVE node with the given interleave factor Factor ...
bool isIndexedMaskedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
virtual LLT getOptimalMemOpLLT(const MemOp &Op, const AttributeList &) const
LLT returning variant.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
virtual ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, unsigned Index) const
Return the cost of extracting a subvector of type ResVT from a vector of type SrcVT,...
virtual void emitExpandAtomicRMW(AtomicRMWInst *AI) const
Perform a atomicrmw expansion using a target-specific way.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
virtual bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const
Return true if it is profitable to convert a select of FP constants into a constant pool load whose a...
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?
virtual bool hasStackProbeSymbol(const MachineFunction &MF) const
Returns the name of the symbol used to emit stack probes or the empty string if not applicable.
bool isSlowDivBypassed() const
Returns true if target has indicated at least one type should be bypassed.
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
virtual bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
virtual bool shouldExtendTypeInLibCall(EVT Type) const
Returns true if arguments should be extended in lib calls.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void setVectorInterleaveAction(ArrayRef< unsigned > Opcodes, ArrayRef< unsigned > Factors, MVT VT, LegalizeAction Action)
bool isPartialReduceMLALegalOrCustom(unsigned Opc, EVT AccVT, EVT InputVT) const
Return true if a PARTIAL_REDUCE_U/SMLA node with the specified types is legal or custom for this targ...
virtual bool isFsqrtCheap(SDValue X, SelectionDAG &DAG) const
Return true if SQRT(X) shouldn't be replaced with X*RSQRT(X).
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
virtual bool shouldNormalizeToSelectSequence(LLVMContext &Context, EVT VT, EVT CCVT) const
Returns true if we should normalize select(N0&N1, X, Y) => select(N0, select(N1, X,...
bool isSuitableForBitTests(const DenseMap< const BasicBlock *, unsigned int > &DestCmps, const APInt &Low, const APInt &High, const DataLayout &DL) const
Return true if lowering to a bit test is suitable for a set of case clusters which contains NumDests ...
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual bool shallExtractConstSplatVectorElementToStore(Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const
Return true if the target shall perform extract vector element and store given that the vector is kno...
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool hasMultipleConditionRegisters(EVT VT) const
Does the target have multiple (allocatable) condition registers that can be used to store the results...
unsigned getMaxExpandSizeMemcmp(bool OptSize) const
Get maximum # of load operations permitted for memcmp.
bool isStrictFPEnabled() const
Return true if the target support strict float operation.
virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const
Return true if creating a shift of the type by the given amount is not profitable.
virtual bool shouldPreservePtrArith(const Function &F, EVT PtrVT) const
True if target has some particular form of dealing with pointer arithmetic semantics for pointers wit...
virtual bool isFPExtFree(EVT DestVT, EVT SrcVT) const
Return true if an fpext operation is free (for instance, because single-precision floating-point numb...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
virtual bool lowerInterleavedStore(Instruction *Store, Value *Mask, ShuffleVectorInst *SVI, unsigned Factor, const APInt &GapMask) const
Lower an interleaved store to target specific intrinsics.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
MVT getSimpleValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the MVT corresponding to this LLVM type. See getValueType.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isVectorInterleaveLegalOrCustom(unsigned Opc, unsigned Factor, EVT VT) const
Return true if a VECTOR_INTERLEAVE or VECTOR_DEINTERLEAVE node with the given interleave factor and f...
LegalizeAction getVectorInterleaveAction(unsigned Opc, unsigned Factor, EVT VT) const
Return how a VECTOR_INTERLEAVE or VECTOR_DEINTERLEAVE node with the given interleave factor and VT sh...
virtual bool shouldReassociateReduction(unsigned RedOpc, EVT VT) const
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
virtual CondMergingParams getJumpConditionMergingParams(Instruction::BinaryOps, const Value *, const Value *, const Function *) const
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal for a comparison of the specified types on this ...
virtual bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const
Return true if the target can combine store(extractelement VectorTy,Idx).
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual bool shouldFoldConstantShiftPairToMask(const SDNode *N) const
Return true if it is profitable to fold a pair of shifts into a mask.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
ExtractSubvectorCost
Enum that specifies how expensive lowering an EXTRACT_SUBVECTOR is.
virtual void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const
void setSupportsUnalignedAtomics(bool UnalignedSupported)
Sets whether unaligned atomic operations are supported.
void setLoadExtAction(ArrayRef< unsigned > ExtTypes, MVT ValVT, ArrayRef< MVT > MemVTs, LegalizeAction Action)
virtual void emitExpandAtomicStore(StoreInst *SI) const
Perform a atomic store using a target-specific way.
virtual bool preferIncOfAddToSubOfNot(EVT VT) const
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
virtual bool ShouldShrinkFPConstant(EVT) const
If true, then instruction selection should seek to shrink the FP constant of the specified type to a ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
unsigned getMaxDivRemBitWidthSupported() const
Returns the size in bits of the maximum div/rem the backend supports.
virtual bool isLegalAddImmediate(int64_t) const
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
virtual unsigned getMaxSupportedInterleaveFactor() const
Get the maximum supported factor for interleaved memory accesses.
virtual FPImmAction getFPImmAction(const APFloat &, LLT, bool ForCodeSize=false) const
Returns how the legalizer should materialize the specified FP immediate.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool shouldKeepZExtForFP16Conv() const
Does this target require the clearing of high-order bits in a register passed to the fp16 to fp conve...
virtual AtomicExpansionKind shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const
Returns how the given atomic atomicrmw should be cast by the IR-level AtomicExpand pass.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
virtual bool canTransformPtrArithOutOfBounds(const Function &F, EVT PtrVT) const
True if the target allows transformations of in-bounds pointer arithmetic that cause out-of-bounds in...
virtual bool shouldConsiderGEPOffsetSplit() const
const ValueTypeActionImpl & getValueTypeActions() const
virtual bool canCombineTruncStore(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace, bool LegalOnly) const
TargetLoweringBase(const TargetMachine &TM, const TargetSubtargetInfo &STI)
NOTE: The TargetMachine owns TLOF.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
virtual bool isTruncateFree(SDValue Val, EVT VT2) const
Return true if truncating the specific node Val to type VT2 is free.
virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const
virtual unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
virtual bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
virtual bool shouldInsertTrailingSeqCstFenceForAtomicStore(const Instruction *I) const
Whether AtomicExpandPass should automatically insert a seq_cst trailing fence without reducing the or...
virtual bool isFNegFree(EVT VT) const
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
void setPartialReduceMLAAction(unsigned Opc, MVT AccVT, MVT InputVT, LegalizeAction Action)
Indicate how a PARTIAL_REDUCE_U/SMLA node with Acc type AccVT and Input type InputVT should be treate...
virtual AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
FPImmAction
How the legalizer should materialize a floating-point immediate.
@ ConstantPool
Load the immediate from the constant pool.
@ Bitcast
Build the immediate as a bitcast of an integer constant with the same bits.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
bool isExtFree(const Instruction *I) const
Return true if the extension represented by I is free.
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const
Perform a masked cmpxchg using a target-specific intrinsic.
virtual bool isZExtFree(EVT FromTy, EVT ToTy) const
virtual ISD::NodeType getExtendForAtomicCmpSwapArg() const
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
virtual bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X, SDValue Y) const
Return true if pulling a binary operation into a select with an identity constant is profitable.
BooleanContent
Enum that describes how the target represents true/false values.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const
Return true if integer divide is usually cheaper than a sequence of several shifts,...
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
virtual uint8_t getRepRegClassCostFor(MVT VT) const
Return the cost of the 'representative' register class for the specified value type.
virtual bool isZExtFree(LLT FromTy, LLT ToTy, LLVMContext &Ctx) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
LegalizeAction getPartialReduceMLAAction(unsigned Opc, EVT AccVT, EVT InputVT) const
Return how a PARTIAL_REDUCE_U/SMLA node with Acc type AccVT and Input type InputVT should be treated.
bool isPredictableSelectExpensive() const
Return true if selects are only cheaper than branches if the branch is unlikely to be predicted right...
virtual bool mergeStoresAfterLegalization(EVT MemVT) const
Allow store merging for the specified type after legalization in addition to before legalization.
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
virtual bool shouldIssueAtomicLoadForAtomicEmulationLoop(void) const
virtual bool shouldMergeStoreOfLoadsOverCall(EVT, EVT) const
Returns true if it's profitable to allow merging store of loads when there are functions calls betwee...
virtual bool isProfitableToHoist(Instruction *I) const
unsigned getGatherAllAliasesMaxDepth() const
virtual LegalizeAction getCustomOperationAction(SDNode &Op) const
How to legalize this custom operation?
virtual bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *) const
IR version.
virtual bool hasAndNotCompare(SDValue Y) const
Return true if the target should transform: (X & Y) == Y ---> (~X & Y) == 0 (X & Y) !...
virtual bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem, unsigned AddrSpace) const
Return true if it is expected to be cheaper to do a store of vector constant with the given size and ...
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
virtual const TargetRegisterClass * getRepRegClassFor(MVT VT) const
Return the 'representative' register class for the specified value type.
virtual bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
LegalizeAction getLoadAction(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace, unsigned ExtType, bool Atomic) const
Return how this load with extension should be treated: either it is legal, needs to be promoted to a ...
virtual bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const
Return true if it is cheaper to split the store of a merged int val from a pair of smaller values int...
bool shouldUseDynamicVectorTypeBreakdown(EVT VT, bool ForCallingConv) const
TargetLoweringBase(const TargetLoweringBase &)=delete
EVT getTypeToWidenTo(LLVMContext &Context, EVT VT) const
For vector types that are widened, this returns the type reached by widening repeatedly,...
virtual unsigned getMaxGluedStoresPerMemcpy() const
Get maximum # of store operations to be glued together.
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
virtual bool shouldFoldMaskToVariableShiftPair(SDValue X) const
There are two ways to clear extreme bits (either low or high): Mask: x & (-1 << y) (the instcombine c...
virtual bool alignLoopsWithOptSize() const
Should loops be aligned even when the function is marked OptSize (but not MinSize).
unsigned getMaxAtomicSizeInBitsSupported() const
Returns the maximum atomic operation size (in bits) supported by the backend.
virtual FPImmAction getFPImmAction(const APFloat &, EVT, bool ForCodeSize=false) const
Returns how the legalizer should materialize the specified FP immediate.
bool isIndexedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
virtual bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const
Returns if it's reasonable to merge stores to MemVT size.
void setPartialReduceMLAAction(ArrayRef< unsigned > Opcodes, MVT AccVT, MVT InputVT, LegalizeAction Action)
LegalizeAction getStrictFPOperationAction(unsigned Op, EVT VT) const
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
virtual bool preferABDSToABSWithNSW(EVT VT) const
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
virtual bool getAddrModeArguments(const IntrinsicInst *, SmallVectorImpl< Value * > &, Type *&) const
CodeGenPrepare sinks address calculations into the same BB as Load/Store instructions reading the add...
virtual bool hasInlineStackProbe(const MachineFunction &MF) const
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setBooleanContents(BooleanContent IntTy, BooleanContent FloatTy)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
const DenseMap< unsigned int, unsigned int > & getBypassSlowDivWidths() const
Returns map of slow types for division or remainder with corresponding fast types.
void setOperationPromotedToType(ArrayRef< unsigned > Ops, MVT OrigVT, MVT DestVT)
unsigned getMaxLargeFPConvertBitWidthSupported() const
Returns the size in bits of the maximum fp to/from int conversion the backend supports.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, LLT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
virtual bool isTruncateFree(EVT FromVT, EVT ToVT) const
bool isTruncStoreLegal(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace) const
Return true if the specified store with truncation is legal on this target.
virtual bool isCheapToSpeculateCtlz(Type *Ty) const
Return true if it is cheap to speculate a call to intrinsic ctlz.
virtual void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &Infos, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
virtual bool lowerInterleaveIntrinsicToStore(Instruction *Store, Value *Mask, ArrayRef< Value * > InterleaveValues) const
Lower an interleave intrinsic to a target specific store intrinsic.
virtual bool isTruncateFree(LLT FromTy, LLT ToTy, LLVMContext &Ctx) const
AndOrSETCCFoldKind
Enum of different potentially desirable ways to fold (and/or (setcc ...), (setcc ....
virtual bool shouldScalarizeBinop(SDValue VecOp) const
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
Align getPrefFunctionAlignment() const
Return the preferred function alignment.
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Get the libcall impl routine name for the specified libcall.
virtual void emitExpandAtomicLoad(LoadInst *LI) const
Perform a atomic load using a target-specific way.
Align getMinFunctionAlignment() const
Return the minimum function alignment.
virtual AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
virtual bool preferVectorizedNonPowerOfTwoTypeBreakdown() const
Return true if fixed-length, non-power-of-two vectors should be broken down into legal vector parts i...
static StringRef getLibcallImplName(RTLIB::LibcallImpl Call)
Get the libcall routine name for the specified libcall implementation.
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
virtual bool isCtlzFast() const
Return true if ctlz instruction is fast.
virtual bool useSoftFloat() const
virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: (store (y (conv x)), y*)) -> (store x,...
BooleanContent getBooleanContents(EVT Type) const
virtual LegalizeAction getCustomLoadAction(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace, unsigned ExtType, bool Atomic) const
Returns an alternative action to use when the coarser lookups (configured through setLoadExtAction an...
bool isIndexedMaskedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
virtual int64_t getPreferredLargeGEPBaseOffset(int64_t MinOffset, int64_t MaxOffset) const
Return the prefered common base offset.
virtual bool isVectorClearMaskLegal(ArrayRef< int >, EVT) const
Similar to isShuffleMaskLegal.
LegalizeKind getTypeConversion(LLVMContext &Context, EVT VT) const
Return pair that represents the legalization kind (first) that needs to happen to EVT (second) in ord...
Align getMinStackArgumentAlignment() const
Return the minimum stack alignment of an argument.
virtual bool shouldUseStrictFP_TO_INT(EVT FpVT, EVT IntVT, bool IsSigned) const
Return true if it is more correct/profitable to use strict FP_TO_INT conversion operations - canonica...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
bool hasTargetDAGCombine(ISD::NodeType NT) const
If true, the target has custom DAG combine transformations that it can perform for the specified node...
void setLibcallImpl(RTLIB::Libcall Call, RTLIB::LibcallImpl Impl)
virtual bool fallBackToDAGISel(const Instruction &Inst) const
unsigned GatherAllAliasesMaxDepth
Depth that GatherAllAliases should continue looking for chain dependencies when trying to find a more...
virtual bool shouldSplatInsEltVarIndex(EVT) const
Return true if inserting a scalar into a variable element of an undef vector is more efficiently hand...
LegalizeAction getIndexedMaskedLoadAction(unsigned IdxMode, MVT VT) const
Return how the indexed load should be treated: either it is legal, needs to be promoted to a larger s...
NegatibleCost
Enum that specifies when a float negation is beneficial.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual unsigned preferedOpcodeForCmpEqPiecesOfOperand(EVT VT, unsigned ShiftOpc, bool MayTransformRotate, const APInt &ShiftOrRotateAmt, const std::optional< APInt > &AndMask) const
virtual bool fenceClearsLoadLinkedReservation() const
Whether a fence between the load-linked and the store-conditional can clear the reservation.
virtual void emitCmpArithAtomicRMWIntrinsic(AtomicRMWInst *AI) const
Perform a atomicrmw which the result is only used by comparison, using a target-specific intrinsic.
virtual bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const
Returns true if arguments should be sign-extended in lib calls.
virtual bool optimizeVectorCrossLaneOperation(Instruction *I, Loop *L) const
virtual bool isFMADLegal(const MachineInstr &MI, LLT Ty) const
Returns true if MI can be combined with another instruction to form TargetOpcode::G_FMAD.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, ArrayRef< MVT > VTs, LegalizeAction Action)
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
virtual bool isLegalAddScalableImmediate(int64_t) const
Return true if adding the specified scalable immediate is legal, that is the target has add instructi...
std::vector< ArgListEntry > ArgListTy
virtual bool shouldAlignPointerArgs(CallInst *, unsigned &, Align &) const
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
virtual bool hasVectorBlend() const
Return true if the target has a vector blend instruction.
virtual AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const
Returns how the given (atomic) store should be cast by the IR-level AtomicExpand pass into.
bool isTruncStoreLegalOrCustom(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace) const
Return true if the specified store with truncation has solution on this target.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, ArrayRef< MVT > VTs, LegalizeAction Action)
virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const
virtual Register getExceptionPointerRegister(ExceptionHandling EH, const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
virtual MachineMemOperand::Flags getTargetMMOFlags(const MemSDNode &Node) const
This callback is used to inspect load/store SDNode.
virtual EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op, const AttributeList &) const
Returns the target specific optimal type for load and store operations as a result of memset,...
virtual Type * shouldConvertSplatType(ShuffleVectorInst *SVI) const
Given a shuffle vector SVI representing a vector splat, return a new scalar type of size equal to SVI...
virtual bool isZExtFree(SDValue Val, EVT VT2) const
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
void setAtomicLoadExtAction(ArrayRef< unsigned > ExtTypes, MVT ValVT, MVT MemVT, LegalizeAction Action)
virtual bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const
virtual LLVM_READONLY LLT getPreferredShiftAmountTy(LLT ShiftValueTy) const
Return the preferred type to use for a shift opcode, given the shifted amount type is ShiftValueTy.
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
LLT getVectorIdxLLT(const DataLayout &DL) const
Returns the type to be used for the index operand of: G_INSERT_VECTOR_ELT, G_EXTRACT_VECTOR_ELT,...
virtual EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, ArrayRef< MVT > VTs, LegalizeAction Action)
virtual AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const
Returns how the given (atomic) load should be cast by the IR-level AtomicExpand pass.
bool isCondCodeLegalOrCustom(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal or custom for a comparison of the specified type...
virtual bool isComplexDeinterleavingSupported() const
Does this target support complex deinterleaving.
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
bool isLoadLegal(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace, unsigned ExtType, bool Atomic) const
Return true if the specified load with extension is legal on this target.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
virtual bool addressingModeSupportsTLS(const GlobalValue &) const
Returns true if the targets addressing mode can target thread local storage (TLS).
bool isLoadLegalOrCustom(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace, unsigned ExtType, bool Atomic) const
Return true if the specified load with extension is legal or custom on this target.
virtual bool shouldConvertPhiType(Type *From, Type *To) const
Given a set in interconnected phis of type 'From' that are loaded/stored or bitcast to type 'To',...
virtual bool isFAbsFree(EVT VT) const
Return true if an fabs operation is free to the point where it is never worthwhile to replace it with...
virtual bool isLegalStoreImmediate(int64_t Value) const
Return true if the specified immediate is legal for the value input of a store instruction.
virtual bool preferZeroCompareBranch() const
Return true if the heuristic to prefer icmp eq zero should be used in code gen prepare.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
virtual bool lowerInterleavedLoad(Instruction *Load, Value *Mask, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor, const APInt &GapMask) const
Lower an interleaved load to target specific intrinsics.
virtual unsigned getVectorIdxWidth(const DataLayout &DL) const
Returns the type to be used for the index operand vector operations.
MVT getTypeToPromoteTo(unsigned Op, MVT VT) const
If the action for this operation is to promote, this method returns the ValueType to promote to.
virtual bool generateFMAsInMachineCombiner(EVT VT, CodeGenOptLevel OptLevel) const
virtual LoadInst * lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *RMWI) const
On some platforms, an AtomicRMW that never actually modifies the value (such as fetch_add of 0) can b...
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
virtual bool hasPairedLoad(EVT, Align &) const
Return true if the target supplies and combines to a paired load two loaded values of type LoadedType...
virtual bool convertSelectOfConstantsToMath(EVT VT) const
Return true if a select of constants (select Cond, C1, C2) should be transformed into simple math ops...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool optimizeExtendOrTruncateConversion(Instruction *I, Loop *L, const TargetTransformInfo &TTI) const
Try to optimize extending or truncating conversion instructions (like zext, trunc,...
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_UDIV, ISD::VP_SDIV,...
std::pair< LegalizeTypeAction, EVT > LegalizeKind
LegalizeKind holds the legalization kind that needs to happen to EVT in order to type-legalize it.
TargetLoweringBase & operator=(const TargetLoweringBase &)=delete
MulExpansionKind
Enum that specifies when a multiplication should be expanded.
static ISD::NodeType getExtendForContent(BooleanContent Content)
const RTLIB::RuntimeLibcallsInfo & getRuntimeLibcallsInfo() const
virtual bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT.
virtual bool supportKCFIBundles() const
Return true if the target supports kcfi operand bundles.
virtual ConstraintWeight getMultipleConstraintMatchWeight(AsmOperandInfo &info, int maIndex) const
Examine constraint type and operand type and determine a weight value.
SmallVector< ConstraintPair > ConstraintGroup
virtual SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const
Hooks for building estimates in place of slower divisions and square roots.
virtual bool isDesirableToCommuteWithShift(const MachineInstr &MI, bool IsAfterLegal) const
GlobalISel - return true if it is profitable to move this shift by a constant amount through its oper...
virtual bool supportPtrAuthBundles() const
Return true if the target supports ptrauth operand bundles.
virtual void ReplaceNodeResults(SDNode *, SmallVectorImpl< SDValue > &, SelectionDAG &) const
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
virtual bool isUsedByReturnOnly(SDNode *, SDValue &) const
Return true if result of the specified node is used by a return node only.
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue &NewLoad, SDValue Ptr, SDValue PassThru, SDValue Mask) const
virtual unsigned getPreferredShrunkVectorSizeInBits(SDValue Op, const APInt &DemandedElts) const
If only low elements of a vector are demanded, shrink the operation to the returned size in bits by c...
SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, unsigned Depth=0) const
This is the helper function to return the newly negated expression if the cost is not expensive.
virtual bool isProfitableToFoldVScaleAdd(SDValue N) const
fold (A + vscale(C1)) + vscale(C2) -> A + vscale(C1+C2) If (A + vscale(C1)) is used multiple times,...
virtual bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, SDValue N1) const
virtual bool isTypeDesirableForOp(SDNode *N, EVT VT) const
Overload that takes the specific node being optimized.
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
SDValue getCheaperOrNeutralNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, const NegatibleCost CostThreshold=NegatibleCost::Neutral, unsigned Depth=0) const
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
virtual bool isTargetCanonicalConstantNode(SDValue Op) const
Returns true if the given Opc is considered a canonical constant for the target, which should not be ...
virtual bool isTargetCanonicalSelect(SDNode *N) const
Return true if the given select/vselect should be considered canonical and not be transformed.
SDValue getCheaperNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, unsigned Depth=0) const
This is the helper function to return the newly negated expression only when the cost is cheaper.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual SDValue emitStackGuardMixFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
virtual SDValue lowerEHPadEntry(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
Optional target hook to add target-specific actions when entering EH pad blocks.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue unwrapAddress(SDValue N) const
virtual bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const
Target-specific splitting of values into parts that fit a register storing a legal type.
virtual bool IsDesirableToPromoteOp(SDValue, EVT &) const
This method query the target whether it is beneficial for dag combiner to promote the specified node.
virtual SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const
Target-specific combining of register parts into its original value.
virtual void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const
Insert explicit copies in entry and exit blocks.
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
~TargetLowering() override
TargetLowering & operator=(const TargetLowering &)=delete
virtual bool isDesirableToPullExtFromShl(const MachineInstr &MI) const
GlobalISel - return true if it's profitable to perform the combine: shl ([sza]ext x),...
bool isPositionIndependent() const
std::pair< StringRef, TargetLowering::ConstraintType > ConstraintPair
virtual bool supportsInlineAsmFuncOperands() const
Return true if a function used as an inline asm address operand (e.g.
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
virtual bool isIndexingLegal(MachineInstr &MI, Register Base, Register Offset, bool IsPre, MachineRegisterInfo &MRI) const
Returns true if the specified base+offset is a legal indexed addressing mode for this target.
ConstraintGroup getConstraintPreferences(AsmOperandInfo &OpInfo) const
Given an OpInfo with list of constraints codes as strings, return a sorted Vector of pairs of constra...
virtual void initializeSplitCSR(MachineBasicBlock *Entry) const
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
virtual bool isSDNodeSourceOfDivergence(const SDNode *N, FunctionLoweringInfo *FLI, UniformityInfo *UA) const
virtual SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const
Return a reciprocal estimate value for the input operand.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
virtual bool isSDNodeAlwaysUniform(const SDNode *N) const
virtual bool isDesirableToCommuteXorWithShift(const SDNode *N) const
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
TargetLowering(const TargetLowering &)=delete
virtual bool shouldSimplifyDemandedVectorElts(SDValue Op, const TargetLoweringOpt &TLO) const
Return true if the target supports simplifying demanded vector elements by converting them to undefs.
virtual SDValue LowerFormalArguments(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::InputArg > &, const SDLoc &, SelectionDAG &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue getSqrtResultForDenormInput(SDValue Operand, SelectionDAG &DAG) const
Return a target-dependent result if the input operand is not suitable for use with a square root esti...
virtual bool getPostIndexedAddressParts(SDNode *, SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
virtual bool shouldSplitFunctionArgumentsAsLittleEndian(const DataLayout &DL) const
For most targets, an LLVM type must be broken down into multiple smaller types.
virtual ArrayRef< MCPhysReg > getRoundingControlRegisters() const
Returns a 0 terminated array of rounding control registers that can be attached into strict FP call.
virtual SDValue LowerReturn(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::OutputArg > &, const SmallVectorImpl< SDValue > &, const SDLoc &, SelectionDAG &) const
This hook must be implemented to lower outgoing return values, described by the Outs array,...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to move this shift by a constant amount through its operand,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue Ptr, SDValue Val, SDValue Mask) const
virtual const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *, const MachineBasicBlock *, unsigned, MCContext &) const
virtual bool useTopologicalSorting() const
virtual bool useLoadStackGuardNode(const Module &M) const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
virtual FastISel * createFastISel(FunctionLoweringInfo &, const TargetLibraryInfo *, const LibcallLoweringInfo *) const
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
virtual unsigned combineRepeatedFPDivisors() const
Indicate whether this target prefers to combine FDIVs with the same divisor.
virtual AndOrSETCCFoldKind isDesirableToCombineLogicOpOfSETCC(const SDNode *LogicOp, const SDNode *SETCC0, const SDNode *SETCC1) const
virtual void HandleByVal(CCState *, unsigned &, Align) const
Target-specific cleanup for formal ByVal parameters.
virtual const MCPhysReg * getScratchRegisters(CallingConv::ID CC) const
Returns a 0 terminated array of registers that can be safely used as scratch registers.
virtual bool getPreIndexedAddressParts(SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index, const SDNodeFlags PtrArithFlags=SDNodeFlags()) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
virtual bool supportSplitCSR(MachineFunction *MF) const
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
virtual bool isReassocProfitable(MachineRegisterInfo &MRI, Register N0, Register N1) const
virtual bool mayBeEmittedAsTailCall(const CallInst *) const
Return true if the target may be able emit the call instruction as a tail call.
virtual bool isInlineAsmTargetBranch(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) const
On x86, return true if the operand with index OpNo is a CALL or JUMP instruction, which can use eithe...
SDValue getInboundsVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
virtual MVT getJumpTableRegTy(const DataLayout &DL) const
virtual void markLibCallAttributes(MachineFunction *MF, unsigned CC, ArgListTy &Args) const
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &, const Type *RetTy) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
virtual bool isXAndYEqZeroPreferableToXAndYEqY(ISD::CondCode, EVT) const
virtual bool isDesirableToTransformToIntegerOp(unsigned, EVT) const
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
Primary interface to the complete machine description for the target machine.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Target - Wrapper for Target specific information.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
This is the common base class for vector predication intrinsics.
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
CallInst * Call
#define UINT64_MAX
Definition DataTypes.h:77
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition ISDOpcodes.h:43
@ PARTIAL_REDUCE_SMLA
PARTIAL_REDUCE_[U|S]MLA(Accumulator, Input1, Input2) The partial reduction nodes sign or zero extend ...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition ISDOpcodes.h:277
@ AVGSUBFLOORS
AVGSUBFLOORS/AVGSUBFLOORU - Averaging subtract - Subtract two integers using an integer of type i[N+1...
Definition ISDOpcodes.h:739
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition ISDOpcodes.h:397
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:296
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition ISDOpcodes.h:527
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:403
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ PSEUDO_FMIN
PSEUDO_FMIN is strictly equivalent to op0 olt op1 ?
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ CLMUL
Carry-less multiplication operations.
Definition ISDOpcodes.h:802
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition ISDOpcodes.h:410
@ PARTIAL_REDUCE_UMLA
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition ISDOpcodes.h:731
@ PARTIAL_REDUCE_FMLA
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
Definition ISDOpcodes.h:640
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:377
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition ISDOpcodes.h:688
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition ISDOpcodes.h:351
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Same as FMINNUM/FMAXNUM, except that a signaling NaN operand deterministi...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:751
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:416
@ STRICT_FP_TO_UINT
Definition ISDOpcodes.h:483
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:482
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition ISDOpcodes.h:726
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
Definition ISDOpcodes.h:697
@ PARTIAL_REDUCE_SUMLA
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:368
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
Definition ISDOpcodes.h:629
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
Definition ISDOpcodes.h:746
static const int LAST_LOADEXT_TYPE
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
@ System
Synchronized with respect to all concurrently executing threads.
Definition LLVMContext.h:66
This namespace contains all of the command line option processing machinery.
This is an optimization pass for GlobalISel generic memory operations.
GenericUniformityInfo< SSAContext > UniformityInfo
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
@ Offset
Definition DWP.cpp:577
void fill(R &&Range, T &&Value)
Provide wrappers to std::fill which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1775
LLVM_ABI void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
InstructionCost Cost
@ Known
Known to have no common set bits.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
Definition Alignment.h:139
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:332
LLVM_ABI bool isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector, bool IsFP)
Returns true if given the TargetLowering's boolean contents information, the value Val contains a tru...
Definition Utils.cpp:1765
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
CodeGenOptLevel
Code generation optimization level.
Definition CodeGen.h:280
@ Default
-O2, -Os, -Oz
Definition CodeGen.h:283
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
AtomicOrdering
Atomic ordering for LLVM's memory model.
LLVM_ABI EVT getApproximateEVTForLLT(LLT Ty, LLVMContext &Ctx)
TargetTransformInfo TTI
CombineLevel
Definition DAGCombine.h:15
@ AfterLegalizeDAG
Definition DAGCombine.h:19
@ AfterLegalizeVectorOps
Definition DAGCombine.h:18
@ BeforeLegalizeTypes
Definition DAGCombine.h:16
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
@ Fast
Assign the register banks as fast as possible (default).
LLVM_ABI bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition Analysis.cpp:709
DWARFExpression::Operation Op
LLVM_ABI bool isConstFalseVal(const TargetLowering &TLI, int64_t Val, bool IsVector, bool IsFP)
Definition Utils.cpp:1778
constexpr unsigned BitWidth
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1933
ExceptionHandling
Definition CodeGen.h:54
UndefPoisonKind
Enumeration to track whether we are interested in Undef, Poison, or both.
Definition UndefPoison.h:20
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
Implement std::hash so that hash_code can be used in STL containers.
Definition BitVector.h:879
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:82
Represent subnormal handling kind for floating point instruction inputs and outputs.
Extended Value Type.
Definition ValueTypes.h:35
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:70
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition ValueTypes.h:338
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:170
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition ValueTypes.h:281
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
bool isFixedLengthVector() const
Definition ValueTypes.h:214
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
Definition ValueTypes.h:165
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition ValueTypes.h:180
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
ConstraintInfo()=default
Default constructor.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:111
bool isDstAligned(Align AlignCheck) const
bool isFixedDstAlign() const
uint64_t size() const
static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign, bool IsZeroMemset, bool IsVolatile)
Align getDstAlign() const
bool isMemcpyStrSrc() const
bool isAligned(Align AlignCheck) const
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile, bool MemcpyStrSrc=false)
bool isSrcAligned(Align AlignCheck) const
bool isMemcpyOrMemmoveWithFixedDstAlign() const
bool isMemcpyOrMemmove() const
bool isMemmove() const
bool isMemset() const
bool isMemcpy() const
static MemOp Move(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile)
bool isZeroMemset() const
bool isVolatile() const
Align getSrcAlign() const
A simple container for information about the supported runtime calls.
static StringRef getLibcallImplName(RTLIB::LibcallImpl CallImpl)
Get the libcall routine name for the specified libcall implementation.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
std::optional< unsigned > fallbackAddressSpace
PointerUnion< const Value *, const PseudoSourceValue * > ptrVal
This contains information for each constraint that we are lowering.
AsmOperandInfo(InlineAsm::ConstraintInfo Info)
Copy constructor for copying from a ConstraintInfo.
MVT ConstraintVT
The ValueType for the operand value.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
std::string ConstraintCode
This contains the actual string for the code, like "m".
Value * CallOperandVal
If this is the result output operand or a clobber, this is null, otherwise it is the incoming operand...
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setIsPostTypeLegalization(bool Value=true)
CallLoweringInfo & setDeactivationSymbol(GlobalValue *Sym)
CallLoweringInfo & setCallee(Type *ResultType, FunctionType *FTy, SDValue Target, ArgListTy &&ArgsList, const CallBase &Call)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
CallLoweringInfo & setInRegister(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setVarArg(bool Value=true)
Type * OrigRetTy
Original unlegalized return type.
std::optional< PtrAuthInfo > PAI
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, Type *OrigResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setSExtResult(bool Value=true)
CallLoweringInfo & setNoReturn(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
Type * RetTy
Same as OrigRetTy, or partially legalized for soft float libcalls.
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setPtrAuth(PtrAuthInfo Value)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
DAGCombinerInfo(SelectionDAG &dag, CombineLevel level, bool cl, void *dc)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setIsPostTypeLegalization(bool Value=true)
MakeLibCallOptions & setDiscardResult(bool Value=true)
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT)
MakeLibCallOptions & setIsSigned(bool Value=true)
MakeLibCallOptions & setNoReturn(bool Value=true)
MakeLibCallOptions & setOpsTypeOverrides(ArrayRef< Type * > OpsTypes)
Override the argument type for an operand.
This structure contains the information necessary for lowering pointer-authenticating indirect calls.
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
TargetLoweringOpt(SelectionDAG &InDAG, bool LT, bool LO)