LLVM 24.0.0git
TargetTransformInfo.h
Go to the documentation of this file.
1//===- TargetTransformInfo.h ------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This pass exposes codegen information to IR-level passes. Every
10/// transformation that uses codegen information is broken into three parts:
11/// 1. The IR-level analysis pass.
12/// 2. The IR-level transformation interface which provides the needed
13/// information.
14/// 3. Codegen-level implementation which uses target-specific hooks.
15///
16/// This file defines #2, which is the interface that IR-level transformations
17/// use for querying the codegen.
18///
19//===----------------------------------------------------------------------===//
20
21#ifndef LLVM_ANALYSIS_TARGETTRANSFORMINFO_H
22#define LLVM_ANALYSIS_TARGETTRANSFORMINFO_H
23
24#include "llvm/ADT/APInt.h"
25#include "llvm/ADT/ArrayRef.h"
28#include "llvm/ADT/Uniformity.h"
31#include "llvm/IR/FMF.h"
32#include "llvm/IR/InstrTypes.h"
33#include "llvm/IR/PassManager.h"
34#include "llvm/Pass.h"
39#include <functional>
40#include <limits>
41#include <optional>
42#include <utility>
43
44namespace llvm {
45
46namespace Intrinsic {
47typedef unsigned ID;
48}
49
50class AllocaInst;
51class AssumptionCache;
53class DominatorTree;
54class CondBrInst;
55class Function;
56class GlobalValue;
57class InstCombiner;
60class ICmpInst;
61class IntrinsicInst;
62class LoadInst;
63class Loop;
64class LoopInfo;
68class SCEV;
69class ScalarEvolution;
70class SmallBitVector;
71class StoreInst;
72class SwitchInst;
74class Type;
75class VPIntrinsic;
76struct KnownBits;
77
78/// Information about a load/store intrinsic defined by the target.
80 /// This is the pointer that the intrinsic is loading from or storing to.
81 /// If this is non-null, then analysis/optimization passes can assume that
82 /// this intrinsic is functionally equivalent to a load/store from this
83 /// pointer.
84 Value *PtrVal = nullptr;
85
86 // Ordering for atomic operations.
88
89 // Same Id is set by the target for corresponding load/store intrinsics.
90 unsigned short MatchingId = 0;
91
92 bool ReadMem = false;
93 bool WriteMem = false;
94 bool IsVolatile = false;
95
97
98 bool isUnordered() const {
101 !IsVolatile;
102 }
103};
104
105/// Attributes of a target dependent hardware loop.
109 Loop *L = nullptr;
112 const SCEV *ExitCount = nullptr;
114 Value *LoopDecrement = nullptr; // Decrement the loop counter by this
115 // value in every iteration.
116 bool IsNestingLegal = false; // Can a hardware loop be a parent to
117 // another hardware loop?
118 bool CounterInReg = false; // Should loop counter be updated in
119 // the loop via a phi?
120 bool PerformEntryTest = false; // Generate the intrinsic which also performs
121 // icmp ne zero on the loop counter value and
122 // produces an i1 to guard the loop entry.
124 DominatorTree &DT,
125 bool ForceNestedLoop = false,
126 bool ForceHardwareLoopPHI = false);
127 LLVM_ABI bool canAnalyze(LoopInfo &LI);
128};
129
130/// Information for memory intrinsic cost model.
132 /// Optional context instruction, if one exists, e.g. the
133 /// load/store to transform to the intrinsic.
134 const Instruction *I = nullptr;
135
136 /// Address in memory.
137 const Value *Ptr = nullptr;
138
139 /// Vector type of the data to be loaded or stored.
140 Type *DataTy = nullptr;
141
142 /// ID of the memory intrinsic.
143 Intrinsic::ID IID;
144
145 /// True when the memory access is predicated with a mask
146 /// that is not a compile-time constant.
147 bool VariableMask = true;
148
149 /// Address space of the pointer.
150 unsigned AddressSpace = 0;
151
152 /// Alignment of single element.
153 Align Alignment;
154
155 const Value *StrideVal;
156
157public:
159 bool VariableMask, Align Alignment,
160 const Instruction *I = nullptr,
161 const Value *StrideVal = nullptr)
162
163 : I(I), Ptr(Ptr), DataTy(DataTy), IID(Id), VariableMask(VariableMask),
164 Alignment(Alignment), StrideVal(StrideVal) {}
165
167 unsigned AddressSpace = 0,
168 const Value *StrideVal = nullptr)
169 : DataTy(DataTy), IID(Id), AddressSpace(AddressSpace),
170 Alignment(Alignment), StrideVal(StrideVal) {}
171
172 MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy, bool VariableMask,
173 Align Alignment, const Instruction *I = nullptr,
174 const Value *StrideVal = nullptr)
175
176 : I(I), DataTy(DataTy), IID(Id), VariableMask(VariableMask),
177 Alignment(Alignment), StrideVal(StrideVal) {}
178
179 Intrinsic::ID getID() const { return IID; }
180 const Instruction *getInst() const { return I; }
181 const Value *getPointer() const { return Ptr; }
182 Type *getDataType() const { return DataTy; }
183 bool getVariableMask() const { return VariableMask; }
184 unsigned getAddressSpace() const { return AddressSpace; }
185 Align getAlignment() const { return Alignment; }
186 const Value *getStrideVal() const { return StrideVal; }
187};
188
189/// Represents a hint about the context in which a vector instruction or
190/// intrinsic is used.
191///
192/// On some targets, inserts/extracts can cheaply be folded into loads/stores.
193/// Similarly, vp.merge can also be folded into binary ops on some targets.
194///
195/// This enum allows the vectorizer to give getVectorInstrCost and
196/// getIntrinsicInstrCost an idea of how the values are used.
197///
198/// See \c getVectorInstrContextHint to compute a VectorInstrContext from an
199/// insert/extract Instruction*.
201 None, ///< The instruction is not folded.
202 Load, ///< The value being inserted comes from a load (InsertElement only).
203 Store, ///< The extracted value is stored (ExtractElement only).
204 BinaryOp, ///< One of the operands is a binary op.
205 SplatOpFolded, ///< All of the value's users support splatting the value.
206};
207
209 const IntrinsicInst *II = nullptr;
210 Type *RetTy = nullptr;
211 Intrinsic::ID IID;
212 SmallVector<Type *, 4> ParamTys;
214 FastMathFlags FMF;
215 // If ScalarizationCost is UINT_MAX, the cost of scalarizing the
216 // arguments and the return value will be computed based on types.
217 InstructionCost ScalarizationCost = InstructionCost::getInvalid();
219
220public:
222 Intrinsic::ID Id, const CallBase &CI,
224 bool TypeBasedOnly = false);
225
227 Intrinsic::ID Id, Type *RTy, ArrayRef<Type *> Tys,
228 FastMathFlags Flags = FastMathFlags(), const IntrinsicInst *I = nullptr,
230
233
237 const IntrinsicInst *I = nullptr,
240
241 Intrinsic::ID getID() const { return IID; }
242 const IntrinsicInst *getInst() const { return II; }
243 Type *getReturnType() const { return RetTy; }
244 FastMathFlags getFlags() const { return FMF; }
245 InstructionCost getScalarizationCost() const { return ScalarizationCost; }
247 const SmallVectorImpl<const Value *> &getArgs() const { return Arguments; }
248 const SmallVectorImpl<Type *> &getArgTypes() const { return ParamTys; }
249
250 bool isTypeBasedOnly() const {
251 return Arguments.empty();
252 }
253
254 bool skipScalarizationCost() const { return ScalarizationCost.isValid(); }
255};
256
258 /// Don't use tail folding
260 /// Use predicate only to mask operations on data in the loop.
261 /// When the VL is not known to be a power-of-2, this method requires a
262 /// runtime overflow check for the i + VL in the loop because it compares the
263 /// scalar induction variable against the tripcount rounded up by VL which may
264 /// overflow. When the VL is a power-of-2, both the increment and uprounded
265 /// tripcount will overflow to 0, which does not require a runtime check
266 /// since the loop is exited when the loop induction variable equals the
267 /// uprounded trip-count, which are both 0.
269 /// Same as Data, but avoids using the get.active.lane.mask intrinsic to
270 /// calculate the mask and instead implements this with a
271 /// splat/stepvector/cmp.
272 /// FIXME: Can this kind be removed now that SelectionDAGBuilder expands the
273 /// active.lane.mask intrinsic when it is not natively supported?
275 /// Use predicate to control both data and control flow.
276 /// This method always requires a runtime overflow check for the i + VL
277 /// increment inside the loop, because it uses the result direclty in the
278 /// active.lane.mask to calculate the mask for the next iteration. If the
279 /// increment overflows, the mask is no longer correct.
281 /// Use predicated EVL instructions for tail-folding.
282 /// Indicates that VP intrinsics should be used.
284};
285
294
295class TargetTransformInfo;
298
299/// This pass provides access to the codegen interfaces that are needed
300/// for IR-level transformations.
302public:
309
310 /// Get the kind of extension that an instruction represents.
313 /// Get the kind of extension that a cast opcode represents.
316 /// Get the cast opcode for an extension kind.
319
320 /// Construct a TTI object using a type implementing the \c Concept
321 /// API below.
322 ///
323 /// This is used by targets to construct a TTI wrapping their target-specific
324 /// implementation that encodes appropriate costs for their target.
326 std::unique_ptr<const TargetTransformInfoImplBase> Impl);
327
328 /// Construct a baseline TTI object using a minimal implementation of
329 /// the \c Concept API below.
330 ///
331 /// The TTI implementation will reflect the information in the DataLayout
332 /// provided if non-null.
333 LLVM_ABI explicit TargetTransformInfo(const DataLayout &DL);
334
335 // Provide move semantics.
338
339 // We need to define the destructor out-of-line to define our sub-classes
340 // out-of-line.
342
343 /// Handle the invalidation of this information.
344 ///
345 /// When used as a result of \c TargetIRAnalysis this method will be called
346 /// when the function this was computed for changes. When it returns false,
347 /// the information is preserved across those changes.
349 FunctionAnalysisManager::Invalidator &) {
350 // FIXME: We should probably in some way ensure that the subtarget
351 // information for a function hasn't changed.
352 return false;
353 }
354
355 /// \name Generic Target Information
356 /// @{
357
358 /// The kind of cost model.
359 ///
360 /// There are several different cost models that can be customized by the
361 /// target. The normalization of each cost model may be target specific.
362 /// e.g. TCK_SizeAndLatency should be comparable to target thresholds such as
363 /// those derived from MCSchedModel::LoopMicroOpBufferSize etc.
365 TCK_RecipThroughput, ///< Reciprocal throughput.
366 TCK_Latency, ///< The latency of instruction.
367 TCK_CodeSize, ///< Instruction code size.
368 TCK_SizeAndLatency ///< The weighted sum of size and latency.
369 };
370
371 /// Underlying constants for 'cost' values in this interface.
372 ///
373 /// Many APIs in this interface return a cost. This enum defines the
374 /// fundamental values that should be used to interpret (and produce) those
375 /// costs. The costs are returned as an int rather than a member of this
376 /// enumeration because it is expected that the cost of one IR instruction
377 /// may have a multiplicative factor to it or otherwise won't fit directly
378 /// into the enum. Moreover, it is common to sum or average costs which works
379 /// better as simple integral values. Thus this enum only provides constants.
380 /// Also note that the returned costs are signed integers to make it natural
381 /// to add, subtract, and test with zero (a common boundary condition). It is
382 /// not expected that 2^32 is a realistic cost to be modeling at any point.
383 ///
384 /// Note that these costs should usually reflect the intersection of code-size
385 /// cost and execution cost. A free instruction is typically one that folds
386 /// into another instruction. For example, reg-to-reg moves can often be
387 /// skipped by renaming the registers in the CPU, but they still are encoded
388 /// and thus wouldn't be considered 'free' here.
390 TCC_Free = 0, ///< Expected to fold away in lowering.
391 TCC_Basic = 1, ///< The cost of a typical 'add' instruction.
392 TCC_Expensive = 4 ///< The cost of a 'div' instruction on x86.
393 };
394
395 /// Estimate the cost of a GEP operation when lowered.
396 ///
397 /// \p PointeeType is the source element type of the GEP.
398 /// \p Ptr is the base pointer operand.
399 /// \p Operands is the list of indices following the base pointer.
400 ///
401 /// \p AccessType is a hint as to what type of memory might be accessed by
402 /// users of the GEP. getGEPCost will use it to determine if the GEP can be
403 /// folded into the addressing mode of a load/store. If AccessType is null,
404 /// then the resulting target type based off of PointeeType will be used as an
405 /// approximation.
406 LLVM_ABI InstructionCost getGEPCost(Type *PointeeType, const Value *Ptr,
408 TargetCostKind CostKind,
409 Type *AccessType = nullptr) const;
410
411 /// Describe known properties for a set of pointers.
413 /// All the GEPs in a set have same base address.
414 unsigned IsSameBaseAddress : 1;
415 /// These properties only valid if SameBaseAddress is set.
416 /// True if all pointers are separated by a unit stride.
417 unsigned IsUnitStride : 1;
418 /// True if distance between any two neigbouring pointers is a known value.
419 unsigned IsKnownStride : 1;
420 unsigned Reserved : 29;
421
422 bool isSameBase() const { return IsSameBaseAddress; }
423 bool isUnitStride() const { return IsSameBaseAddress && IsUnitStride; }
425
427 return {/*IsSameBaseAddress=*/1, /*IsUnitStride=*/1,
428 /*IsKnownStride=*/1, 0};
429 }
431 return {/*IsSameBaseAddress=*/1, /*IsUnitStride=*/0,
432 /*IsKnownStride=*/1, 0};
433 }
435 return {/*IsSameBaseAddress=*/1, /*IsUnitStride=*/0,
436 /*IsKnownStride=*/0, 0};
437 }
438 };
439 static_assert(sizeof(PointersChainInfo) == 4, "Was size increase justified?");
440
441 /// Estimate the cost of a chain of pointers (typically pointer operands of a
442 /// chain of loads or stores within same block) operations set when lowered.
443 /// \p AccessTy is the type of the loads/stores that will ultimately use the
444 /// \p Ptrs.
447 const PointersChainInfo &Info, Type *AccessTy,
448 const TargetCostKind CostKind) const;
449
450 /// \returns A value by which our inlining threshold should be multiplied.
451 /// This is primarily used to bump up the inlining threshold wholesale on
452 /// targets where calls are unusually expensive.
453 ///
454 /// TODO: This is a rather blunt instrument. Perhaps altering the costs of
455 /// individual classes of instructions would be better.
457
460
461 /// \returns The bonus of inlining the last call to a static function.
463
464 /// \returns A value to be added to the inlining threshold.
465 LLVM_ABI unsigned adjustInliningThreshold(const CallBase *CB) const;
466
467 /// \returns The cost of having an Alloca in the caller if not inlined, to be
468 /// added to the threshold
469 LLVM_ABI unsigned getCallerAllocaCost(const CallBase *CB,
470 const AllocaInst *AI) const;
471
472 /// \returns Vector bonus in percent.
473 ///
474 /// Vector bonuses: We want to more aggressively inline vector-dense kernels
475 /// and apply this bonus based on the percentage of vector instructions. A
476 /// bonus is applied if the vector instructions exceed 50% and half that
477 /// amount is applied if it exceeds 10%. Note that these bonuses are some what
478 /// arbitrary and evolved over time by accident as much as because they are
479 /// principled bonuses.
480 /// FIXME: It would be nice to base the bonus values on something more
481 /// scientific. A target may has no bonus on vector instructions.
483
484 /// \return the expected cost of a memcpy, which could e.g. depend on the
485 /// source/destination type and alignment and the number of bytes copied.
487
488 /// Returns the maximum memset / memcpy size in bytes that still makes it
489 /// profitable to inline the call.
491
492 /// Returns the maximum length in bytes for which a memset with a
493 /// non-constant length, known to be bounded by that value, is preferably
494 /// expanded inline instead of calling the library. Zero disables the
495 /// expansion.
497
498 /// \return The estimated number of case clusters when lowering \p 'SI'.
499 /// \p JTSize Set a jump table size only when \p SI is suitable for a jump
500 /// table.
501 LLVM_ABI unsigned
502 getEstimatedNumberOfCaseClusters(const SwitchInst &SI, unsigned &JTSize,
504 BlockFrequencyInfo *BFI) const;
505
506 /// Estimate the cost of a given IR user when lowered.
507 ///
508 /// This can estimate the cost of either a ConstantExpr or Instruction when
509 /// lowered.
510 ///
511 /// \p Operands is a list of operands which can be a result of transformations
512 /// of the current operands. The number of the operands on the list must equal
513 /// to the number of the current operands the IR user has. Their order on the
514 /// list must be the same as the order of the current operands the IR user
515 /// has.
516 ///
517 /// The returned cost is defined in terms of \c TargetCostConstants, see its
518 /// comments for a detailed explanation of the cost values.
521 TargetCostKind CostKind) const;
522
523 /// This is a helper function which calls the three-argument
524 /// getInstructionCost with \p Operands which are the current operands U has.
526 TargetCostKind CostKind) const {
527 SmallVector<const Value *, 4> Operands(U->operand_values());
528 return getInstructionCost(U, Operands, CostKind);
529 }
530
531 /// If a branch or a select condition is skewed in one direction by more than
532 /// this factor, it is very likely to be predicted correctly.
534
535 /// Returns estimated penalty of a branch misprediction in latency. Indicates
536 /// how aggressive the target wants for eliminating unpredictable branches. A
537 /// zero return value means extra optimization applied to them should be
538 /// minimal.
540
541 /// Return true if branch divergence exists.
542 ///
543 /// Branch divergence has a significantly negative impact on GPU performance
544 /// when threads in the same wavefront take different paths due to conditional
545 /// branches.
546 ///
547 /// If \p F is passed, provides a context function. If \p F is known to only
548 /// execute in a single threaded environment, the target may choose to skip
549 /// uniformity analysis and assume all values are uniform.
550 LLVM_ABI bool hasBranchDivergence(const Function *F = nullptr) const;
551
552 /// Get target-specific uniformity information for a value.
553 /// This allows targets to provide more fine-grained control over
554 /// uniformity analysis by specifying whether specific values
555 /// should always or never be considered uniform, or require custom
556 /// operand-based analysis.
557 /// \param V The value to query for uniformity information.
558 /// \return ValueUniformity.
560
561 /// Query the target whether the specified address space cast from FromAS to
562 /// ToAS is valid.
563 LLVM_ABI bool isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const;
564
565 /// Return false if a \p AS0 address cannot possibly alias a \p AS1 address.
566 LLVM_ABI bool addrspacesMayAlias(unsigned AS0, unsigned AS1) const;
567
568 /// Returns the address space ID for a target's 'flat' address space. Note
569 /// this is not necessarily the same as addrspace(0), which LLVM sometimes
570 /// refers to as the generic address space. The flat address space is a
571 /// generic address space that can be used access multiple segments of memory
572 /// with different address spaces. Access of a memory location through a
573 /// pointer with this address space is expected to be legal but slower
574 /// compared to the same memory location accessed through a pointer with a
575 /// different address space.
576 //
577 /// This is for targets with different pointer representations which can
578 /// be converted with the addrspacecast instruction. If a pointer is converted
579 /// to this address space, optimizations should attempt to replace the access
580 /// with the source address space.
581 ///
582 /// \returns ~0u if the target does not have such a flat address space to
583 /// optimize away.
584 LLVM_ABI unsigned getFlatAddressSpace() const;
585
586 /// Return the most specific common address space containing AS1 and AS2.
587 /// AS1 and AS2 must be distinct, and pointers from both spaces must be
588 /// convertible to the target's flat address space with addrspacecast.
589 /// Pointers from either input space must be convertible to the result with
590 /// addrspacecast. Return getFlatAddressSpace() if no more specific common
591 /// address space is available.
592 LLVM_ABI unsigned getAddressSpaceJoin(unsigned AS1, unsigned AS2) const;
593
594 /// Return any intrinsic address operand indexes which may be rewritten if
595 /// they use a flat address space pointer.
596 ///
597 /// \returns true if the intrinsic was handled.
599 Intrinsic::ID IID) const;
600
601 LLVM_ABI bool isNoopAddrSpaceCast(unsigned FromAS, unsigned ToAS) const;
602
603 // Given an address space cast of the given pointer value, calculate the known
604 // bits of the source pointer in the source addrspace and the destination
605 // pointer in the destination addrspace.
606 LLVM_ABI std::pair<KnownBits, KnownBits>
607 computeKnownBitsAddrSpaceCast(unsigned ToAS, const Value &PtrOp) const;
608
609 // Given an address space cast, calculate the known bits of the resulting ptr
610 // in the destination addrspace using the known bits of the source pointer in
611 // the source addrspace.
613 unsigned FromAS, unsigned ToAS, const KnownBits &FromPtrBits) const;
614
615 /// Returns a mask indicating which bits of a pointer remain unchanged when
616 /// casting between address spaces. The returned APInt has the same bit width
617 /// as the source address space pointer size.
618 ///
619 /// Some targets allow certain bits of a pointer to change (e.g., the low
620 /// bits within a page) while still preserving the address space. This mask
621 /// identifies those bits that are guaranteed to be preserved. If the mask is
622 /// all zeros, no bits are preserved and address space inference cannot be
623 /// performed safely.
624 ///
625 /// For example, given:
626 /// %gp = addrspacecast ptr addrspace(2) %sp to ptr
627 /// %a = ptrtoint ptr %gp to i64
628 /// %b = xor i64 7, %a
629 /// %gp2 = inttoptr i64 %b to ptr
630 /// store i16 0, ptr %gp2, align 2
631 /// if the target preserves the upper bits, `%gp2` can be safely replaced
632 /// with `inttoptr i64 %b to ptr addrspace(2)`.
634 unsigned DstAS) const;
635
636 /// Return true if globals in this address space can have initializers other
637 /// than `undef`.
638 LLVM_ABI bool
640
641 LLVM_ABI unsigned getAssumedAddrSpace(const Value *V) const;
642
643 LLVM_ABI std::pair<const Value *, unsigned>
644 getPredicatedAddrSpace(const Value *V) const;
645
646 /// Rewrite intrinsic call \p II such that \p OldV will be replaced with \p
647 /// NewV, which has a different address space. This should happen for every
648 /// operand index that collectFlatAddressOperands returned for the intrinsic.
649 /// \returns nullptr if the intrinsic was not handled. Otherwise, returns the
650 /// new value (which may be the original \p II with modified operands).
651 /// If a different value is returned, the caller replaces all uses of \p II
652 /// and erases it, including any side effects of the original call.
654 Value *OldV,
655 Value *NewV) const;
656
657 /// Test whether calls to a function lower to actual program function
658 /// calls.
659 ///
660 /// The idea is to test whether the program is likely to require a 'call'
661 /// instruction or equivalent in order to call the given function.
662 ///
663 /// FIXME: It's not clear that this is a good or useful query API. Client's
664 /// should probably move to simpler cost metrics using the above.
665 /// Alternatively, we could split the cost interface into distinct code-size
666 /// and execution-speed costs. This would allow modelling the core of this
667 /// query more accurately as a call is a single small instruction, but
668 /// incurs significant execution cost.
669 LLVM_ABI bool isLoweredToCall(const Function *F) const;
670
671 struct LSRCost {
672 /// TODO: Some of these could be merged. Also, a lexical ordering
673 /// isn't always optimal.
674 unsigned Insns;
675 unsigned NumRegs;
676 unsigned AddRecCost;
677 unsigned NumIVMuls;
678 unsigned NumBaseAdds;
679 unsigned ImmCost;
680 unsigned SetupCost;
681 unsigned ScaleCost;
683 };
684
685 /// Parameters that control the generic loop unrolling transformation.
687 /// The cost threshold for the unrolled loop. Should be relative to the
688 /// getInstructionCost values returned by this API, and the expectation is
689 /// that the unrolled loop's instructions when run through that interface
690 /// should not exceed this cost. However, this is only an estimate. Also,
691 /// specific loops may be unrolled even with a cost above this threshold if
692 /// deemed profitable. Set this to UINT_MAX to disable the loop body cost
693 /// restriction.
694 unsigned Threshold = 150;
695 /// If complete unrolling will reduce the cost of the loop, we will boost
696 /// the Threshold by a certain percent to allow more aggressive complete
697 /// unrolling. This value provides the maximum boost percentage that we
698 /// can apply to Threshold (The value should be no less than 100).
699 /// BoostedThreshold = Threshold * min(RolledCost / UnrolledCost,
700 /// MaxPercentThresholdBoost / 100)
701 /// E.g. if complete unrolling reduces the loop execution time by 50%
702 /// then we boost the threshold by the factor of 2x. If unrolling is not
703 /// expected to reduce the running time, then we do not increase the
704 /// threshold.
706 /// The cost threshold for the unrolled loop when optimizing for size (set
707 /// to UINT_MAX to disable).
708 unsigned OptSizeThreshold = 0;
709 /// The cost threshold for the unrolled loop, like Threshold, but used
710 /// for partial/runtime unrolling (set to UINT_MAX to disable).
711 unsigned PartialThreshold = 150;
712 /// The cost threshold for the unrolled loop when optimizing for size, like
713 /// OptSizeThreshold, but used for partial/runtime unrolling (set to
714 /// UINT_MAX to disable).
716 /// Default unroll count for loops with run-time trip count.
718 // Set the maximum unrolling factor. The unrolling factor may be selected
719 // using the appropriate cost threshold, but may not exceed this number
720 // (set to UINT_MAX to disable). This does not apply in cases where the
721 // loop is being fully unrolled.
722 unsigned MaxCount = std::numeric_limits<unsigned>::max();
723 /// Set the maximum upper bound of trip count. Allowing the MaxUpperBound
724 /// to be overrided by a target gives more flexiblity on certain cases.
725 /// By default, MaxUpperBound uses UnrollMaxUpperBound which value is 8.
726 unsigned MaxUpperBound = 8;
727 /// Set the maximum unrolling factor for full unrolling. Like MaxCount, but
728 /// applies even if full unrolling is selected. This allows a target to fall
729 /// back to Partial unrolling if full unrolling is above FullUnrollMaxCount.
730 unsigned FullUnrollMaxCount = std::numeric_limits<unsigned>::max();
731 // Represents number of instructions optimized when "back edge"
732 // becomes "fall through" in unrolled loop.
733 // For now we count a conditional branch on a backedge and a comparison
734 // feeding it.
735 unsigned BEInsns = 2;
736 /// Allow partial unrolling (unrolling of loops to expand the size of the
737 /// loop body, not only to eliminate small constant-trip-count loops).
738 bool Partial = false;
739 /// Allow runtime unrolling (unrolling of loops to expand the size of the
740 /// loop body even when the number of loop iterations is not known at
741 /// compile time).
742 bool Runtime = false;
743 /// Allow generation of a loop remainder (extra iterations after unroll).
744 bool AllowRemainder = true;
745 /// Allow emitting expensive instructions (such as divisions) when computing
746 /// the trip count of a loop for runtime unrolling.
748 /// Apply loop unroll on any kind of loop
749 /// (mainly to loops that fail runtime unrolling).
750 bool Force = false;
751 /// Allow using trip count upper bound to unroll loops.
752 bool UpperBound = false;
753 /// Allow unrolling of all the iterations of the runtime loop remainder.
754 bool UnrollRemainder = false;
755 /// Allow unroll and jam. Used to enable unroll and jam for the target.
756 bool UnrollAndJam = false;
757 /// Threshold for unroll and jam, for inner loop size. The 'Threshold'
758 /// value above is used during unroll and jam for the outer loop size.
759 /// This value is used in the same manner to limit the size of the inner
760 /// loop.
762 /// Don't allow loop unrolling to simulate more than this number of
763 /// iterations when checking full unroll profitability
765 /// Disable runtime unrolling by default for vectorized loops.
767 /// Don't allow runtime unrolling if expanding the trip count takes more
768 /// than SCEVExpansionBudget.
770 /// Allow runtime unrolling multi-exit loops. Should only be set if the
771 /// target determined that multi-exit unrolling is profitable for the loop.
772 /// Fall back to the generic logic to determine whether multi-exit unrolling
773 /// is profitable if set to false.
775 /// Allow unrolling to add parallel reduction phis.
777 };
778
779 /// Get target-customized preferences for the generic loop unrolling
780 /// transformation. The caller will initialize UP with the current
781 /// target-independent defaults.
784 OptimizationRemarkEmitter *ORE) const;
785
786 /// Query the target whether it would be profitable to convert the given loop
787 /// into a hardware loop.
789 AssumptionCache &AC,
790 TargetLibraryInfo *LibInfo,
791 HardwareLoopInfo &HWLoopInfo) const;
792
793 // Query the target for which minimum vectorization factor epilogue
794 // vectorization should be considered.
796
797 /// Query the target whether it would be preferred to create a tail-folded
798 /// vector loop, which can avoid the need to emit a scalar epilogue loop.
800
801 /// Query the target what the preferred style of tail folding is.
803
804 // Parameters that control the loop peeling transformation
806 /// A forced peeling factor (the number of bodied of the original loop
807 /// that should be peeled off before the loop body). When set to 0, the
808 /// a peeling factor based on profile information and other factors.
809 unsigned PeelCount;
810 /// Allow peeling off loop iterations.
812 /// Allow peeling off loop iterations for loop nests.
814 /// Allow peeling basing on profile. Uses to enable peeling off all
815 /// iterations basing on provided profile.
816 /// If the value is true the peeling cost model can decide to peel only
817 /// some iterations and in this case it will set this to false.
819
820 /// Peel off the last PeelCount loop iterations.
822 };
823
824 /// Get target-customized preferences for the generic loop peeling
825 /// transformation. The caller will initialize \p PP with the current
826 /// target-independent defaults with information from \p L and \p SE.
828 PeelingPreferences &PP) const;
829
830 /// Targets can implement their own combinations for target-specific
831 /// intrinsics. This function will be called from the InstCombine pass every
832 /// time a target-specific intrinsic is encountered.
833 ///
834 /// \returns std::nullopt to not do anything target specific or a value that
835 /// will be returned from the InstCombiner. It is possible to return null and
836 /// stop further processing of the intrinsic by returning nullptr.
837 LLVM_ABI std::optional<Instruction *>
839 /// Can be used to implement target-specific instruction combining.
840 /// \see instCombineIntrinsic
841 LLVM_ABI std::optional<Value *>
843 APInt DemandedMask, KnownBits &Known,
844 bool &KnownBitsComputed) const;
845 /// Can be used to implement target-specific instruction combining.
846 /// \see instCombineIntrinsic
847 LLVM_ABI std::optional<Value *> simplifyDemandedVectorEltsIntrinsic(
848 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
849 APInt &UndefElts2, APInt &UndefElts3,
850 std::function<void(Instruction *, unsigned, APInt, APInt &)>
851 SimplifyAndSetOp) const;
852 /// @}
853
854 /// \name Scalar Target Information
855 /// @{
856
857 /// Flags indicating the kind of support for population count.
858 ///
859 /// Compared to the SW implementation, HW support is supposed to
860 /// significantly boost the performance when the population is dense, and it
861 /// may or may not degrade performance if the population is sparse. A HW
862 /// support is considered as "Fast" if it can outperform, or is on a par
863 /// with, SW implementation when the population is sparse; otherwise, it is
864 /// considered as "Slow".
866
867 /// Return true if the specified immediate is legal add immediate, that
868 /// is the target has add instructions which can add a register with the
869 /// immediate without having to materialize the immediate into a register.
870 LLVM_ABI bool isLegalAddImmediate(int64_t Imm) const;
871
872 /// Return true if adding the specified scalable immediate is legal, that is
873 /// the target has add instructions which can add a register with the
874 /// immediate (multiplied by vscale) without having to materialize the
875 /// immediate into a register.
876 LLVM_ABI bool isLegalAddScalableImmediate(int64_t Imm) const;
877
878 /// Return true if the specified immediate is legal icmp immediate,
879 /// that is the target has icmp instructions which can compare a register
880 /// against the immediate without having to materialize the immediate into a
881 /// register.
882 LLVM_ABI bool isLegalICmpImmediate(int64_t Imm) const;
883
884 /// Return true if replacing the relational loop exit comparison \p Cond
885 /// with an equality comparison against \p PostIncExitValue is profitable.
886 LLVM_ABI bool
888 const APInt &PostIncExitValue) const;
889
890 /// Return true if the addressing mode represented by AM is legal for
891 /// this target, for a load/store of the specified type.
892 /// The type may be VoidTy, in which case only return true if the addressing
893 /// mode is legal for a load/store of any legal type.
894 /// If target returns true in LSRWithInstrQueries(), I may be valid.
895 /// \param ScalableOffset represents a quantity of bytes multiplied by vscale,
896 /// an invariant value known only at runtime. Most targets should not accept
897 /// a scalable offset.
898 ///
899 /// TODO: Handle pre/postinc as well.
901 int64_t BaseOffset, bool HasBaseReg,
902 int64_t Scale, unsigned AddrSpace = 0,
903 Instruction *I = nullptr,
904 int64_t ScalableOffset = 0) const;
905
906 /// Return true if LSR cost of C1 is lower than C2.
908 const TargetTransformInfo::LSRCost &C2) const;
909
910 /// Return true if LSR major cost is number of registers. Targets which
911 /// implement their own isLSRCostLess and unset number of registers as major
912 /// cost should return false, otherwise return true.
914
915 /// Return true if LSR should drop a found solution if it's calculated to be
916 /// less profitable than the baseline.
918
919 /// \returns true if LSR should not optimize a chain that includes \p I.
921
922 /// Return true if the target can fuse a compare and branch.
923 /// Loop-strength-reduction (LSR) uses that knowledge to adjust its cost
924 /// calculation for the instructions in a loop.
925 LLVM_ABI bool canMacroFuseCmp() const;
926
927 /// Return true if the target can save a compare for loop count, for example
928 /// hardware loop saves a compare.
931 TargetLibraryInfo *LibInfo) const;
932
933 /// Which addressing mode Loop Strength Reduction will try to generate.
935 AMK_None = 0x0, ///< Don't prefer any addressing mode
936 AMK_PreIndexed = 0x1, ///< Prefer pre-indexed addressing mode
937 AMK_PostIndexed = 0x2, ///< Prefer post-indexed addressing mode
938 AMK_All = 0x3, ///< Consider all addressing modes
939 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/AMK_All)
940 };
941
942 /// Return the preferred addressing mode LSR should make efforts to generate.
945
946 /// Some targets only support masked load/store with a constant mask.
951
952 /// Return true if the target supports masked store.
953 LLVM_ABI bool
954 isLegalMaskedStore(Type *DataType, Align Alignment, unsigned AddressSpace,
956 /// Return true if the target supports masked load.
957 LLVM_ABI bool
958 isLegalMaskedLoad(Type *DataType, Align Alignment, unsigned AddressSpace,
960
961 /// Return true if the target supports speculatively loading \p DataType from
962 /// address space \p AddressSpace, i.e. @llvm.can.load.speculatively can
963 /// return true for the store size of \p DataType.
964 LLVM_ABI bool isLegalSpeculativeLoad(Type *DataType,
965 unsigned AddressSpace) const;
966
967 /// Return true if the target supports nontemporal store.
968 LLVM_ABI bool isLegalNTStore(Type *DataType, Align Alignment) const;
969 /// Return true if the target supports nontemporal load.
970 LLVM_ABI bool isLegalNTLoad(Type *DataType, Align Alignment) const;
971
972 /// \Returns true if the target supports broadcasting a load to a vector of
973 /// type <NumElements x ElementTy>.
974 LLVM_ABI bool isLegalBroadcastLoad(Type *ElementTy,
975 ElementCount NumElements) const;
976
977 /// Return true if the target supports masked scatter.
978 LLVM_ABI bool isLegalMaskedScatter(Type *DataType, Align Alignment) const;
979 /// Return true if the target supports masked gather.
980 LLVM_ABI bool isLegalMaskedGather(Type *DataType, Align Alignment) const;
981 /// Return true if the target forces scalarizing of llvm.masked.gather
982 /// intrinsics.
984 Align Alignment) const;
985 /// Return true if the target forces scalarizing of llvm.masked.scatter
986 /// intrinsics.
988 Align Alignment) const;
989
990 /// Return true if the target supports masked compress store.
992 Align Alignment) const;
993 /// Return true if the target supports masked expand load.
994 LLVM_ABI bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) const;
995
996 /// Return true if the target supports strided load.
997 LLVM_ABI bool isLegalStridedLoadStore(Type *DataType, Align Alignment) const;
998
999 /// Return true is the target supports interleaved access for the given vector
1000 /// type \p VTy, interleave factor \p Factor, alignment \p Alignment and
1001 /// address space \p AddrSpace.
1002 LLVM_ABI bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
1003 Align Alignment,
1004 unsigned AddrSpace) const;
1005
1006 // Return true if the target supports masked vector histograms.
1008 Type *DataType) const;
1009
1010 /// Return true if this is an alternating opcode pattern that can be lowered
1011 /// to a single instruction on the target. In X86 this is for the addsub
1012 /// instruction which corrsponds to a Shuffle + Fadd + FSub pattern in IR.
1013 /// This function expectes two opcodes: \p Opcode1 and \p Opcode2 being
1014 /// selected by \p OpcodeMask. The mask contains one bit per lane and is a `0`
1015 /// when \p Opcode0 is selected and `1` when Opcode1 is selected.
1016 /// \p VecTy is the vector type of the instruction to be generated.
1017 /// \p Scalars are the scalar instructions of the lanes, if known. The target
1018 /// may use them to find the patterns, which depend on the operands, like
1019 /// the alternating fused multiply-add.
1020 LLVM_ABI bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0,
1021 unsigned Opcode1,
1022 const SmallBitVector &OpcodeMask,
1023 ArrayRef<const Value *> Scalars = {}) const;
1024
1025 /// Enum describing the source/producer of a mask.
1026 enum class MaskSource {
1027 /// The operation is unmasked.
1029 /// The operation is masked with an arbitrary predicate.
1031 /// The operation is masked with a contiguous active lane mask.
1033 };
1034
1035 /// Return true if the target supports loading or storing \p NumVectors
1036 /// contiguous vectors of type \p VectorTy as a single operation. A null
1037 /// \p VectorTy queries whether the target supports this operation in
1038 /// general.
1039 ///
1040 /// \p CastHint is non-null if a stored value is produced by a cast or a
1041 /// loaded value is consumed by one.
1043 unsigned NumVectors, MaskSource Mask, VectorType *VectorTy = nullptr,
1044 bool IsStore = false,
1045 std::optional<Instruction::CastOps> CastHint = std::nullopt) const;
1046
1047 /// Return true if we should be enabling ordered reductions for the target.
1048 LLVM_ABI bool enableOrderedReductions() const;
1049
1050 /// Return true if the target has a unified operation to calculate division
1051 /// and remainder. If so, the additional implicit multiplication and
1052 /// subtraction required to calculate a remainder from division are free. This
1053 /// can enable more aggressive transformations for division and remainder than
1054 /// would typically be allowed using throughput or size cost models.
1055 LLVM_ABI bool hasDivRemOp(Type *DataType, bool IsSigned) const;
1056
1057 /// Return true if the given instruction (assumed to be a memory access
1058 /// instruction) has a volatile variant. If that's the case then we can avoid
1059 /// addrspacecast to generic AS for volatile loads/stores. Default
1060 /// implementation returns false, which prevents address space inference for
1061 /// volatile loads/stores.
1062 LLVM_ABI bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) const;
1063
1064 /// Return true if target doesn't mind addresses in vectors.
1066
1067 /// Return the cost of the scaling factor used in the addressing
1068 /// mode represented by AM for this target, for a load/store
1069 /// of the specified type.
1070 /// If the AM is supported, the return value must be >= 0.
1071 /// If the AM is not supported, it returns a negative value.
1072 /// TODO: Handle pre/postinc as well.
1074 StackOffset BaseOffset,
1075 bool HasBaseReg, int64_t Scale,
1076 unsigned AddrSpace = 0) const;
1077
1078 /// Return true if the loop strength reduce pass should make
1079 /// Instruction* based TTI queries to isLegalAddressingMode(). This is
1080 /// needed on SystemZ, where e.g. a memcpy can only have a 12 bit unsigned
1081 /// immediate offset and no index register.
1082 LLVM_ABI bool LSRWithInstrQueries() const;
1083
1084 /// Return true if it's free to truncate a value of type Ty1 to type
1085 /// Ty2. e.g. On x86 it's free to truncate a i32 value in register EAX to i16
1086 /// by referencing its sub-register AX.
1087 LLVM_ABI bool isTruncateFree(Type *Ty1, Type *Ty2) const;
1088
1089 /// Return true if it is profitable to hoist instruction in the
1090 /// then/else to before if.
1092
1093 LLVM_ABI bool useAA() const;
1094
1095 /// Return true if this type is legal.
1096 LLVM_ABI bool isTypeLegal(Type *Ty) const;
1097
1098 /// Returns the estimated number of registers required to represent \p Ty,
1099 /// or zero if \p Ty cannot be legalized.
1100 LLVM_ABI unsigned getRegUsageForType(Type *Ty) const;
1101
1102 /// Return true if switches should be turned into lookup tables for the
1103 /// target.
1104 LLVM_ABI bool shouldBuildLookupTables() const;
1105
1106 /// Return true if switches should be turned into lookup tables
1107 /// containing this constant value for the target.
1109
1110 /// Return the minimum bit width to use for integer switch lookup table
1111 /// elements on this target.
1113
1114 /// Return true if lookup tables should be turned into relative lookup tables.
1116
1117 /// Return true if the input function which is cold at all call sites,
1118 /// should use coldcc calling convention.
1120
1121 /// Return true if the input function is internal, should use fastcc calling
1122 /// convention.
1124
1125 /// Identifies if the vector form of the intrinsic has a scalar operand.
1127 unsigned ScalarOpdIdx) const;
1128
1129 /// Identifies if the vector form of the intrinsic is overloaded on the type
1130 /// of the operand at index \p OpdIdx, or on the return type if \p OpdIdx is
1131 /// -1.
1133 int OpdIdx) const;
1134
1135 /// Identifies if the vector form of the intrinsic that returns a struct is
1136 /// overloaded at the struct element index \p RetIdx.
1137 LLVM_ABI bool
1139 int RetIdx) const;
1140
1142
1143 /// Combines 2 context hints into a single value. If both are equal, keep the
1144 /// shared context, otherwise fall back to no specific context.
1148
1149 /// Stores information about the uses of a build vector
1159
1160 /// Calculates a VectorInstrContext from \p I.
1163
1164 /// Calculates a VectorInstrContext from the values in \p VL at indices in
1165 /// \p DemandedElts.
1167 getVectorInstrContextHint(ArrayRef<Value *> VL, const APInt &DemandedElts);
1168
1169 /// Calculates a VectorInstrContext for buildvector-like gather sequences.
1170 ///
1171 /// \p GatherUserOps must collect all users of \p Scalars relevant for
1172 /// determining whether a splat can be folded as a scalar operand. It returns
1173 /// false if those users cannot be gathered in the required form.
1175 ArrayRef<int> Mask, ArrayRef<Value *> Scalars,
1176 function_ref<bool(SmallVectorImpl<BuildVectorUseOp> &)> GatherUseOps)
1177 const;
1178
1179 /// Estimate the overhead of scalarizing an instruction. Insert and Extract
1180 /// are set if the demanded result elements need to be inserted and/or
1181 /// extracted from vectors. The involved values may be passed in VL if
1182 /// Insert is true.
1184 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
1185 TTI::TargetCostKind CostKind, bool ForPoisonSrc = true,
1186 ArrayRef<Value *> VL = {},
1188
1189 /// Estimate the overhead of scalarizing operands with the given types. The
1190 /// (potentially vector) types to use for each of argument are passes via Tys.
1194
1195 /// If target has efficient vector element load/store instructions, it can
1196 /// return true here so that insertion/extraction costs are not added to
1197 /// the scalarization cost of a load/store.
1199
1200 /// If the target supports tail calls.
1201 LLVM_ABI bool supportsTailCalls() const;
1202
1203 /// If target supports tail call on \p CB
1204 LLVM_ABI bool supportsTailCallFor(const CallBase *CB) const;
1205
1206 /// Don't restrict interleaved unrolling to small loops.
1207 LLVM_ABI bool enableAggressiveInterleaving(bool LoopHasReductions) const;
1208
1209 /// Returns options for expansion of memcmp. IsZeroCmp is
1210 // true if this is the expansion of memcmp(p1, p2, s) == 0.
1212 // Return true if memcmp expansion is enabled.
1213 operator bool() const { return MaxNumLoads > 0; }
1214
1215 // Maximum number of load operations.
1216 unsigned MaxNumLoads = 0;
1217
1218 // The list of available load sizes (in bytes), sorted in decreasing order.
1220
1221 // For memcmp expansion, allow up to this number of load pairs per block.
1222 // As an example, this may allow 'memcmp(a, b, 3) == 0' in a single block:
1223 // a0 = load2bytes &a[0]
1224 // b0 = load2bytes &b[0]
1225 // a2 = load1byte &a[2]
1226 // b2 = load1byte &b[2]
1227 // r = cmp eq (a0 ^ b0 | a2 ^ b2), 0
1228 // Equality comparisons combine the differences with xor/or. Ordering
1229 // comparisons pack the loads in memory order into a wider integer before
1230 // comparing, without exceeding the target's preferred load width.
1231 unsigned NumLoadsPerBlock = 1;
1232
1233 // Set to true to allow overlapping loads. For example, 7-byte compares can
1234 // be done with two 4-byte compares instead of 4+2+1-byte compares. This
1235 // requires all loads in LoadSizes to be doable in an unaligned way.
1237
1238 // Sometimes, the amount of data that needs to be compared is smaller than
1239 // the standard register size, but it cannot be loaded with just one load
1240 // instruction. For example, if the size of the memory comparison is 6
1241 // bytes, we can handle it more efficiently by loading all 6 bytes in a
1242 // single block and generating an 8-byte number, instead of generating two
1243 // separate blocks with conditional jumps for 4 and 2 byte loads. This
1244 // approach simplifies the process and produces the comparison result as
1245 // normal. This array lists the allowed sizes of memcmp tails that can be
1246 // merged into one block
1248 };
1250 bool IsZeroCmp) const;
1251
1252 /// Should the Select Optimization pass be enabled and ran.
1253 LLVM_ABI bool enableSelectOptimize() const;
1254
1255 /// Should the Select Optimization pass treat the given instruction like a
1256 /// select, potentially converting it to a conditional branch. This can
1257 /// include select-like instructions like or(zext(c), x) that can be converted
1258 /// to selects.
1260
1261 /// Enable matching of interleaved access groups.
1263
1264 /// Enable matching of interleaved access groups that contain predicated
1265 /// accesses or gaps and therefore vectorized using masked
1266 /// vector loads/stores.
1268
1269 /// Indicate that it is potentially unsafe to automatically vectorize
1270 /// floating-point operations because the semantics of vector and scalar
1271 /// floating-point semantics may differ. For example, ARM NEON v7 SIMD math
1272 /// does not support IEEE-754 denormal numbers, while depending on the
1273 /// platform, scalar floating-point math does.
1274 /// This applies to floating-point math operations and calls, not memory
1275 /// operations, shuffles, or casts.
1277
1278 /// Determine if the target supports unaligned memory accesses.
1280 unsigned BitWidth,
1281 unsigned AddressSpace = 0,
1282 Align Alignment = Align(1),
1283 unsigned *Fast = nullptr) const;
1284
1285 /// Return hardware support for population count.
1286 LLVM_ABI PopcntSupportKind getPopcntSupport(unsigned IntTyWidthInBit) const;
1287
1288 /// Return true if the hardware has a fast square-root instruction.
1289 LLVM_ABI bool haveFastSqrt(Type *Ty) const;
1290
1291 /// Return true if the cost of the instruction is too high to speculatively
1292 /// execute and should be kept behind a branch.
1293 /// This normally just wraps around a getInstructionCost() call, but some
1294 /// targets might report a low TCK_SizeAndLatency value that is incompatible
1295 /// with the fixed TCC_Expensive value.
1296 /// NOTE: This assumes the instruction passes isSafeToSpeculativelyExecute().
1298
1299 /// Return true if it is faster to check if a floating-point value is NaN
1300 /// (or not-NaN) versus a comparison against a constant FP zero value.
1301 /// Targets should override this if materializing a 0.0 for comparison is
1302 /// generally as cheap as checking for ordered/unordered.
1304
1305 /// Return the expected cost of supporting the floating point operation
1306 /// of the specified type.
1308
1309 /// Return the expected cost of materializing for the given integer
1310 /// immediate of the specified type.
1312 TargetCostKind CostKind) const;
1313
1314 /// Return the expected cost of materialization for the given integer
1315 /// immediate of the specified type for a given instruction. The cost can be
1316 /// zero if the immediate can be folded into the specified instruction.
1317 LLVM_ABI InstructionCost getIntImmCostInst(unsigned Opc, unsigned Idx,
1318 const APInt &Imm, Type *Ty,
1319 TargetCostKind CostKind,
1320 Instruction *Inst = nullptr) const;
1322 const APInt &Imm, Type *Ty,
1323 TargetCostKind CostKind) const;
1324
1325 /// Return the expected cost for the given integer when optimising
1326 /// for size. This is different than the other integer immediate cost
1327 /// functions in that it is subtarget agnostic. This is useful when you e.g.
1328 /// target one ISA such as Aarch32 but smaller encodings could be possible
1329 /// with another such as Thumb. This return value is used as a penalty when
1330 /// the total costs for a constant is calculated (the bigger the cost, the
1331 /// more beneficial constant hoisting is).
1332 LLVM_ABI InstructionCost getIntImmCodeSizeCost(unsigned Opc, unsigned Idx,
1333 const APInt &Imm,
1334 Type *Ty) const;
1335
1336 /// It can be advantageous to detach complex constants from their uses to make
1337 /// their generation cheaper. This hook allows targets to report when such
1338 /// transformations might negatively effect the code generation of the
1339 /// underlying operation. The motivating example is divides whereby hoisting
1340 /// constants prevents the code generator's ability to transform them into
1341 /// combinations of simpler operations.
1343 const Function &Fn) const;
1344
1345 /// @}
1346
1347 /// \name Vector Target Information
1348 /// @{
1349
1350 /// The various kinds of shuffle patterns for vector queries.
1352 SK_Broadcast, ///< Broadcast element 0 to all other elements.
1353 SK_Reverse, ///< Reverse the order of the vector.
1354 SK_Select, ///< Selects elements from the corresponding lane of
1355 ///< either source operand. This is equivalent to a
1356 ///< vector select with a constant condition operand.
1357 SK_Transpose, ///< Transpose two vectors.
1358 SK_InsertSubvector, ///< InsertSubvector. Index indicates start offset.
1359 SK_ExtractSubvector, ///< ExtractSubvector Index indicates start offset.
1360 SK_PermuteTwoSrc, ///< Merge elements from two source vectors into one
1361 ///< with any shuffle mask.
1362 SK_PermuteSingleSrc, ///< Shuffle elements of single source vector with any
1363 ///< shuffle mask.
1364 SK_Splice ///< Concatenates elements from the first input vector
1365 ///< with elements of the second input vector. Returning
1366 ///< a vector of the same type as the input vectors.
1367 ///< Index indicates start offset in first input vector.
1368 };
1369
1370 /// Additional information about an operand's possible values.
1372 OK_AnyValue, // Operand can have any value.
1373 OK_UniformValue, // Operand is uniform (splat of a value).
1374 OK_UniformConstantValue, // Operand is uniform constant.
1375 OK_NonUniformConstantValue // Operand is a non uniform constant value.
1376 };
1377
1378 /// Additional properties of an operand's values.
1384
1385 // Describe the values an operand can take. We're in the process
1386 // of migrating uses of OperandValueKind and OperandValueProperties
1387 // to use this class, and then will change the internal representation.
1391
1392 bool isConstant() const {
1394 }
1395 bool isUniform() const {
1397 }
1398 bool isPowerOf2() const {
1399 return Properties == OP_PowerOf2;
1400 }
1401 bool isNegatedPowerOf2() const {
1403 }
1404
1406 return {Kind, OP_None};
1407 }
1408
1410 OperandValueKind MergeKind = OK_AnyValue;
1411 if (isConstant() && OpInfoY.isConstant())
1412 MergeKind = OK_NonUniformConstantValue;
1413
1414 OperandValueProperties MergeProp = OP_None;
1415 if (Properties == OpInfoY.Properties)
1416 MergeProp = Properties;
1417 return {MergeKind, MergeProp};
1418 }
1419 };
1420
1421 /// \return the number of registers in the target-provided register class.
1422 LLVM_ABI unsigned getNumberOfRegisters(unsigned ClassID) const;
1423
1424 /// \return true if the target supports load/store that enables fault
1425 /// suppression of memory operands when the source condition is false.
1426 LLVM_ABI bool hasConditionalLoadStoreForType(Type *Ty, bool IsStore) const;
1427
1428 /// \return the target-provided register class ID for the provided type,
1429 /// accounting for type promotion and other type-legalization techniques that
1430 /// the target might apply. However, it specifically does not account for the
1431 /// scalarization or splitting of vector types. Should a vector type require
1432 /// scalarization or splitting into multiple underlying vector registers, that
1433 /// type should be mapped to a register class containing no registers.
1434 /// Specifically, this is designed to provide a simple, high-level view of the
1435 /// register allocation later performed by the backend. These register classes
1436 /// don't necessarily map onto the register classes used by the backend.
1437 /// FIXME: It's not currently possible to determine how many registers
1438 /// are used by the provided type.
1440 Type *Ty = nullptr) const;
1441
1442 /// \return the target-provided register class name
1443 LLVM_ABI const char *getRegisterClassName(unsigned ClassID) const;
1444
1445 /// \return the cost of spilling a register in the target-provided register
1446 /// class to the stack.
1448 getRegisterClassSpillCost(unsigned ClassID, TargetCostKind CostKind) const;
1449
1450 /// \return the cost of reloading a register in the target-provided register
1451 /// class from the stack.
1453 getRegisterClassReloadCost(unsigned ClassID, TargetCostKind CostKind) const;
1454
1456
1457 /// \return The width of the largest scalar or vector register type.
1458 LLVM_ABI TypeSize getRegisterBitWidth(RegisterKind K) const;
1459
1460 /// \return The width of the smallest vector register type.
1461 LLVM_ABI unsigned getMinVectorRegisterBitWidth() const;
1462
1463 /// \return the value of vscale to tune the cost model for.
1464 LLVM_ABI std::optional<unsigned> getVScaleForTuning() const;
1465
1467
1468 /// \return True if the vectorization factor should be chosen to
1469 /// make the vector of the smallest element type match the size of a
1470 /// vector register. For wider element types, this could result in
1471 /// creating vectors that span multiple vector registers.
1472 /// If false, the vectorization factor will be chosen based on the
1473 /// size of the widest element type. The optional \p ElementTy parameter
1474 /// allows targets to avoid maximizing bandwidth for specific element types
1475 /// (e.g. `i1s`). \p K VectorKind for vectorization.
1477 Type *ElementTy = nullptr) const;
1478
1479 /// \return The minimum vectorization factor for types of given element
1480 /// bit width, or 0 if there is no minimum VF. The returned value only
1481 /// applies when shouldMaximizeVectorBandwidth returns true.
1482 /// If IsScalable is true, the returned ElementCount must be a scalable VF.
1483 LLVM_ABI ElementCount getMinimumVF(unsigned ElemWidth, bool IsScalable) const;
1484
1485 /// \return The maximum vectorization factor for types of given element
1486 /// bit width and opcode, or 0 if there is no maximum VF.
1487 /// Currently only used by the SLP vectorizer.
1488 LLVM_ABI unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const;
1489
1490 /// \return The minimum vectorization factor for the store instruction. Given
1491 /// the initial estimation of the minimum vector factor and store value type,
1492 /// it tries to find possible lowest VF, which still might be profitable for
1493 /// the vectorization.
1494 /// \param VF Initial estimation of the minimum vector factor.
1495 /// \param ScalarMemTy Scalar memory type of the store operation.
1496 /// \param ScalarValTy Scalar type of the stored value.
1497 /// \param Alignment Alignment of the store
1498 /// \param AddrSpace Address space of the store
1499 /// Currently only used by the SLP vectorizer.
1500 LLVM_ABI unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
1501 Type *ScalarValTy, Align Alignment,
1502 unsigned AddrSpace) const;
1503
1504 /// \return True if it should be considered for address type promotion.
1505 /// \p AllowPromotionWithoutCommonHeader Set true if promoting \p I is
1506 /// profitable without finding other extensions fed by the same input.
1508 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const;
1509
1510 /// \return The size of a cache line in bytes.
1511 LLVM_ABI unsigned getCacheLineSize() const;
1512
1513 /// The possible cache levels
1514 enum class CacheLevel {
1515 L1D, // The L1 data cache
1516 L2D, // The L2 data cache
1517
1518 // We currently do not model L3 caches, as their sizes differ widely between
1519 // microarchitectures. Also, we currently do not have a use for L3 cache
1520 // size modeling yet.
1521 };
1522
1523 /// \return The size of the cache level in bytes, if available.
1524 LLVM_ABI std::optional<unsigned> getCacheSize(CacheLevel Level) const;
1525
1526 /// \return The associativity of the cache level, if available.
1527 LLVM_ABI std::optional<unsigned>
1528 getCacheAssociativity(CacheLevel Level) const;
1529
1530 /// \return The minimum architectural page size for the target.
1531 LLVM_ABI std::optional<unsigned> getMinPageSize() const;
1532
1533 /// \return How much before a load we should place the prefetch
1534 /// instruction. This is currently measured in number of
1535 /// instructions.
1536 LLVM_ABI unsigned getPrefetchDistance() const;
1537
1538 /// Some HW prefetchers can handle accesses up to a certain constant stride.
1539 /// Sometimes prefetching is beneficial even below the HW prefetcher limit,
1540 /// and the arguments provided are meant to serve as a basis for deciding this
1541 /// for a particular loop.
1542 ///
1543 /// \param NumMemAccesses Number of memory accesses in the loop.
1544 /// \param NumStridedMemAccesses Number of the memory accesses that
1545 /// ScalarEvolution could find a known stride
1546 /// for.
1547 /// \param NumPrefetches Number of software prefetches that will be
1548 /// emitted as determined by the addresses
1549 /// involved and the cache line size.
1550 /// \param HasCall True if the loop contains a call.
1551 ///
1552 /// \return This is the minimum stride in bytes where it makes sense to start
1553 /// adding SW prefetches. The default is 1, i.e. prefetch with any
1554 /// stride.
1555 LLVM_ABI unsigned getMinPrefetchStride(unsigned NumMemAccesses,
1556 unsigned NumStridedMemAccesses,
1557 unsigned NumPrefetches,
1558 bool HasCall) const;
1559
1560 /// \return The maximum number of iterations to prefetch ahead. If
1561 /// the required number of iterations is more than this number, no
1562 /// prefetching is performed.
1563 LLVM_ABI unsigned getMaxPrefetchIterationsAhead() const;
1564
1565 /// \return True if prefetching should also be done for writes.
1566 LLVM_ABI bool enableWritePrefetching() const;
1567
1568 /// \return if target want to issue a prefetch in address space \p AS.
1569 LLVM_ABI bool shouldPrefetchAddressSpace(unsigned AS) const;
1570
1571 /// \return The cost of a partial reduction, which is a reduction from a
1572 /// vector to another vector with fewer elements of larger size. They are
1573 /// represented by the llvm.vector.partial.reduce.add and
1574 /// llvm.vector.partial.reduce.fadd intrinsics, which take an accumulator of
1575 /// type \p AccumType and a second vector operand to be accumulated, whose
1576 /// element count is specified by \p VF. The type of reduction is specified by
1577 /// \p Opcode. The second operand passed to the intrinsic could be the result
1578 /// of an extend, such as sext or zext. In this case \p BinOp is nullopt,
1579 /// \p InputTypeA represents the type being extended and \p OpAExtend the
1580 /// operation, i.e. sign- or zero-extend.
1581 /// For floating-point partial reductions, any fast math flags (FMF) should be
1582 /// provided to govern which reductions are valid to perform (depending on
1583 /// reassoc or contract, for example), whereas this must be nullopt for
1584 /// integer partial reductions.
1585 /// Also, \p InputTypeB should be nullptr and OpBExtend should be None.
1586 /// Alternatively, the second operand could be the result of a binary
1587 /// operation performed on two extends, i.e.
1588 /// mul(zext i8 %a -> i32, zext i8 %b -> i32).
1589 /// In this case \p BinOp may specify the opcode of the binary operation,
1590 /// \p InputTypeA and \p InputTypeB the types being extended, and
1591 /// \p OpAExtend, \p OpBExtend the form of extensions. An example of an
1592 /// operation that uses a partial reduction is a dot product, which reduces
1593 /// two vectors in binary mul operation to another of 4 times fewer and 4
1594 /// times larger elements.
1596 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
1598 PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
1599 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const;
1600
1601 /// \return The maximum interleave factor that any transform should try to
1602 /// perform for this target. This number depends on the level of parallelism
1603 /// and the number of execution units in the CPU. HasUnorderedReductions
1604 /// specifies whether (unordered) reductions are present in the loop being
1605 /// vectorized.
1607 bool HasUnorderedReductions) const;
1608
1609 /// Collect properties of V used in cost analysis, e.g. OP_PowerOf2.
1610 LLVM_ABI static OperandValueInfo getOperandInfo(const Value *V);
1611
1612 /// Return information about the vector formed for the specified index
1613 /// of a vector of (the same) instruction.
1614 LLVM_ABI static OperandValueInfo getOperandInfo(ArrayRef<Value *> Ops);
1615
1616 /// Collect common data between two OperandValueInfo inputs
1617 LLVM_ABI static OperandValueInfo commonOperandInfo(const Value *X,
1618 const Value *Y);
1619
1620 /// This is an approximation of reciprocal throughput of a math/logic op.
1621 /// A higher cost indicates less expected throughput.
1622 /// From Agner Fog's guides, reciprocal throughput is "the average number of
1623 /// clock cycles per instruction when the instructions are not part of a
1624 /// limiting dependency chain."
1625 /// Therefore, costs should be scaled to account for multiple execution units
1626 /// on the target that can process this type of instruction. For example, if
1627 /// there are 5 scalar integer units and 2 vector integer units that can
1628 /// calculate an 'add' in a single cycle, this model should indicate that the
1629 /// cost of the vector add instruction is 2.5 times the cost of the scalar
1630 /// add instruction.
1631 /// \p Args is an optional argument which holds the instruction operands
1632 /// values so the TTI can analyze those values searching for special
1633 /// cases or optimizations based on those values.
1634 /// \p CtxI is the optional original context instruction, if one exists, to
1635 /// provide even more information.
1636 /// \p TLibInfo is used to search for platform specific vector library
1637 /// functions for instructions that might be converted to calls (e.g. frem).
1639 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
1642 ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
1643 const TargetLibraryInfo *TLibInfo = nullptr) const;
1644
1645 /// Returns the cost estimation for alternating opcode pattern that can be
1646 /// lowered to a single instruction on the target. In X86 this is for the
1647 /// addsub instruction which corrsponds to a Shuffle + Fadd + FSub pattern in
1648 /// IR. This function expects two opcodes: \p Opcode1 and \p Opcode2 being
1649 /// selected by \p OpcodeMask. The mask contains one bit per lane and is a `0`
1650 /// when \p Opcode0 is selected and `1` when Opcode1 is selected.
1651 /// \p VecTy is the vector type of the instruction to be generated.
1652 /// \p Scalars are the scalar instructions of the lanes, if known. The target
1653 /// may use them to find the patterns, which depend on the operands, like
1654 /// the alternating fused multiply-add.
1656 VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
1657 const SmallBitVector &OpcodeMask, TTI::TargetCostKind CostKind,
1658 ArrayRef<const Value *> Scalars = {}) const;
1659
1660 /// \return The cost of a shuffle instruction of kind Kind with inputs of type
1661 /// SrcTy, producing a vector of type DstTy. The exact mask may be passed as
1662 /// Mask, or else the array will be empty. The Index and SubTp parameters
1663 /// are used by the subvector insertions shuffle kinds to show the insert
1664 /// point and the type of the subvector being inserted. The operands of the
1665 /// shuffle can be passed through \p Args, which helps improve the cost
1666 /// estimation in some cases, like in broadcast loads.
1668 ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
1669 TTI::TargetCostKind CostKind, ArrayRef<int> Mask = {}, int Index = 0,
1670 VectorType *SubTp = nullptr, ArrayRef<const Value *> Args = {},
1671 const Instruction *CtxI = nullptr,
1673
1674 /// Represents a hint about the context in which a cast is used.
1675 ///
1676 /// For zext/sext, the context of the cast is the operand, which must be a
1677 /// load of some kind. For trunc, the context is of the cast is the single
1678 /// user of the instruction, which must be a store of some kind.
1679 ///
1680 /// This enum allows the vectorizer to give getCastInstrCost an idea of the
1681 /// type of cast it's dealing with, as not every cast is equal. For instance,
1682 /// the zext of a load may be free, but the zext of an interleaving load can
1683 //// be (very) expensive!
1684 ///
1685 /// See \c getCastContextHint to compute a CastContextHint from a cast
1686 /// Instruction*. Callers can use it if they don't need to override the
1687 /// context and just want it to be calculated from the instruction.
1688 ///
1689 /// FIXME: This handles the types of load/store that the vectorizer can
1690 /// produce, which are the cases where the context instruction is most
1691 /// likely to be incorrect. There are other situations where that can happen
1692 /// too, which might be handled here but in the long run a more general
1693 /// solution of costing multiple instructions at the same times may be better.
1695 None, ///< The cast is not used with a load/store of any kind.
1696 Normal, ///< The cast is used with a normal load/store.
1697 Masked, ///< The cast is used with a masked load/store.
1698 GatherScatter, ///< The cast is used with a gather/scatter.
1699 Interleave, ///< The cast is used with an interleaved load/store.
1700 Reversed, ///< The cast is used with a reversed load/store.
1701 };
1702
1703 /// Calculates a CastContextHint from \p I.
1704 /// This should be used by callers of getCastInstrCost if they wish to
1705 /// determine the context from some instruction.
1706 /// \returns the CastContextHint for ZExt/SExt/Trunc, None if \p I is nullptr,
1707 /// or if it's another type of cast.
1709
1710 /// \return The expected cost of cast instructions, such as bitcast, trunc,
1711 /// zext, etc. If there is an existing instruction that holds Opcode, it
1712 /// may be passed in the 'I' parameter.
1714 unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH,
1715 TTI::TargetCostKind CostKind, const Instruction *I = nullptr) const;
1716
1717 /// \return The expected cost of a sign- or zero-extended vector extract. Use
1718 /// Index = -1 to indicate that there is no information about the index value.
1720 getExtractWithExtendCost(unsigned Opcode, Type *Dst, VectorType *VecTy,
1721 unsigned Index, TTI::TargetCostKind CostKind) const;
1722
1723 /// \return The expected cost of control-flow related instructions such as
1724 /// Phi, Ret, Br, Switch.
1725 LLVM_ABI InstructionCost getCFInstrCost(unsigned Opcode,
1726 TTI::TargetCostKind CostKind,
1727 const Instruction *I = nullptr) const;
1728
1729 /// \returns The expected cost of compare and select instructions. If there
1730 /// is an existing instruction that holds Opcode, it may be passed in the
1731 /// 'I' parameter. The \p VecPred parameter can be used to indicate the select
1732 /// is using a compare with the specified predicate as condition. When vector
1733 /// types are passed, \p VecPred must be used for all lanes. For a
1734 /// comparison, the two operands are the natural values. For a select, the
1735 /// two operands are the *value* operands, not the condition operand.
1737 getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
1738 CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind,
1739 OperandValueInfo Op1Info = {OK_AnyValue, OP_None},
1740 OperandValueInfo Op2Info = {OK_AnyValue, OP_None},
1741 const Instruction *I = nullptr) const;
1742
1743 /// \return The expected cost of vector Insert and Extract.
1744 /// Use -1 to indicate that there is no information on the index value.
1745 /// This is used when the instruction is not available; a typical use
1746 /// case is to provision the cost of vectorization/scalarization in
1747 /// vectorizer passes.
1749 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
1750 unsigned Index = -1, const Value *Op0 = nullptr,
1751 const Value *Op1 = nullptr,
1753
1754 /// \return The expected cost of vector Insert and Extract.
1755 /// Use -1 to indicate that there is no information on the index value.
1756 /// This is used when the instruction is not available; a typical use
1757 /// case is to provision the cost of vectorization/scalarization in
1758 /// vectorizer passes.
1759 /// \param ScalarUserAndIdx encodes the information about extracts from a
1760 /// vector with 'Scalar' being the value being extracted,'User' being the user
1761 /// of the extract(nullptr if user is not known before vectorization) and
1762 /// 'Idx' being the extract lane.
1764 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
1765 Value *Scalar,
1766 ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
1768
1769 /// \return The expected cost of vector Insert and Extract.
1770 /// This is used when instruction is available, and implementation
1771 /// asserts 'I' is not nullptr.
1772 ///
1773 /// A typical suitable use case is cost estimation when vector instruction
1774 /// exists (e.g., from basic blocks during transformation).
1776 const Instruction &I, Type *Val, TTI::TargetCostKind CostKind,
1777 unsigned Index = -1,
1779
1780 /// \return The expected cost of inserting or extracting a lane that is \p
1781 /// Index elements from the end of a vector, i.e. the mathematical expression
1782 /// for the lane is (VF - 1 - Index). This is required for scalable vectors
1783 /// where the exact lane index is unknown at compile time.
1785 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
1786 unsigned Index) const;
1787
1788 /// \return The expected cost of aggregate inserts and extracts. This is
1789 /// used when the instruction is not available; a typical use case is to
1790 /// provision the cost of vectorization/scalarization in vectorizer passes.
1792 unsigned Opcode, TTI::TargetCostKind CostKind) const;
1793
1794 /// \return The cost of replication shuffle of \p VF elements typed \p EltTy
1795 /// \p ReplicationFactor times.
1796 ///
1797 /// For example, the mask for \p ReplicationFactor=3 and \p VF=4 is:
1798 /// <0,0,0,1,1,1,2,2,2,3,3,3>
1800 Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts,
1801 TTI::TargetCostKind CostKind) const;
1802
1803 /// \return The cost of Load and Store instructions. The operand info
1804 /// \p OpdInfo should refer to the stored value for stores and the address
1805 /// for loads.
1807 getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
1808 unsigned AddressSpace, TTI::TargetCostKind CostKind,
1810 const Instruction *I = nullptr) const;
1811
1812 /// \return The cost of the interleaved memory operation.
1813 /// \p Opcode is the memory operation code
1814 /// \p VecTy is the vector type of the interleaved access.
1815 /// \p Factor is the interleave factor
1816 /// \p Indices is the indices for interleaved load members (as interleaved
1817 /// load allows gaps)
1818 /// \p Alignment is the alignment of the memory operation
1819 /// \p AddressSpace is address space of the pointer.
1820 /// \p UseMaskForCond indicates if the memory access is predicated.
1821 /// \p UseMaskForGaps indicates if gaps should be masked.
1823 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1824 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1825 bool UseMaskForCond = false, bool UseMaskForGaps = false) const;
1826
1827 /// A helper function to determine the type of reduction algorithm used
1828 /// for a given \p Opcode and set of FastMathFlags \p FMF.
1829 static bool requiresOrderedReduction(std::optional<FastMathFlags> FMF) {
1830 return FMF && !(*FMF).allowReassoc();
1831 }
1832
1833 /// Calculate the cost of vector reduction intrinsics.
1834 ///
1835 /// This is the cost of reducing the vector value of type \p Ty to a scalar
1836 /// value using the operation denoted by \p Opcode. The FastMathFlags
1837 /// parameter \p FMF indicates what type of reduction we are performing:
1838 /// 1. Tree-wise. This is the typical 'fast' reduction performed that
1839 /// involves successively splitting a vector into half and doing the
1840 /// operation on the pair of halves until you have a scalar value. For
1841 /// example:
1842 /// (v0, v1, v2, v3)
1843 /// ((v0+v2), (v1+v3), undef, undef)
1844 /// ((v0+v2+v1+v3), undef, undef, undef)
1845 /// This is the default behaviour for integer operations, whereas for
1846 /// floating point we only do this if \p FMF indicates that
1847 /// reassociation is allowed.
1848 /// 2. Ordered. For a vector with N elements this involves performing N
1849 /// operations in lane order, starting with an initial scalar value, i.e.
1850 /// result = InitVal + v0
1851 /// result = result + v1
1852 /// result = result + v2
1853 /// result = result + v3
1854 /// This is only the case for FP operations and when reassociation is not
1855 /// allowed.
1856 ///
1858 unsigned Opcode, VectorType *Ty, std::optional<FastMathFlags> FMF,
1859 TTI::TargetCostKind CostKind) const;
1860
1863 TTI::TargetCostKind CostKind) const;
1864
1865 /// Calculate the cost of an extended reduction pattern, similar to
1866 /// getArithmeticReductionCost of an Add/Sub reduction with multiply and
1867 /// optional extensions. This is the cost of as:
1868 /// * ResTy vecreduce.add/sub(mul (A, B)) or,
1869 /// * ResTy vecreduce.add/sub(mul(ext(Ty A), ext(Ty B)).
1871 getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode, Type *ResTy,
1872 VectorType *Ty, TTI::TargetCostKind CostKind) const;
1873
1874 /// Calculate the cost of an extended reduction pattern, similar to
1875 /// getArithmeticReductionCost of a reduction with an extension.
1876 /// This is the cost of as:
1877 /// ResTy vecreduce.opcode(ext(Ty A)).
1879 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *Ty,
1880 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const;
1881
1882 /// \returns The cost of Intrinsic instructions. Analyses the real arguments.
1883 /// Three cases are handled: 1. scalar instruction 2. vector instruction
1884 /// 3. scalar instruction which is to be vectorized.
1886 const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) const;
1887
1888 /// \returns The cost of memory intrinsic instructions.
1889 /// Used when IntrinsicInst is not materialized.
1892 TTI::TargetCostKind CostKind) const;
1893
1894 /// \returns The cost of Call instructions.
1896 ArrayRef<Type *> Tys,
1897 TTI::TargetCostKind CostKind) const;
1898
1899 /// \returns The number of pieces into which the provided type must be
1900 /// split during legalization. Zero is returned when the answer is unknown.
1901 LLVM_ABI unsigned getNumberOfParts(Type *Tp) const;
1902
1903 /// \returns The cost of the address computation. For most targets this can be
1904 /// merged into the instruction indexing mode. Some targets might want to
1905 /// distinguish between address computation for memory operations with vector
1906 /// pointer types and scalar pointer types. Such targets should override this
1907 /// function. \p SE holds the pointer for the scalar evolution object which
1908 /// was used in order to get the Ptr step value. \p Ptr holds the SCEV of the
1909 /// access pointer.
1911 getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr,
1912 TTI::TargetCostKind CostKind) const;
1913
1914 /// \returns The cost, if any, of keeping values of the given types alive
1915 /// over a callsite.
1916 ///
1917 /// Some types may require the use of register classes that do not have
1918 /// any callee-saved registers, so would require a spill and fill.
1921
1922 /// \returns True if the intrinsic is a supported memory intrinsic. Info
1923 /// will contain additional information - whether the intrinsic may write
1924 /// or read to memory, volatility and the pointer. Info is undefined
1925 /// if false is returned.
1927 MemIntrinsicInfo &Info) const;
1928
1929 /// \returns The maximum element size, in bytes, for an element
1930 /// unordered-atomic memory intrinsic.
1932
1933 /// \returns A value which is the result of the given memory intrinsic. If \p
1934 /// CanCreate is true, new instructions may be created to extract the result
1935 /// from the given intrinsic memory operation. Returns nullptr if the target
1936 /// cannot create a result from the given intrinsic.
1937 LLVM_ABI Value *
1939 bool CanCreate = true) const;
1940
1941 /// \returns The type to use in a loop expansion of a memcpy call.
1943 LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
1944 unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
1945 std::optional<uint32_t> AtomicElementSize = std::nullopt) const;
1946
1947 /// \param[out] OpsOut The operand types to copy RemainingBytes of memory.
1948 /// \param RemainingBytes The number of bytes to copy.
1949 ///
1950 /// Calculates the operand types to use when copying \p RemainingBytes of
1951 /// memory, where source and destination alignments are \p SrcAlign and
1952 /// \p DestAlign respectively.
1954 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
1955 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
1956 Align SrcAlign, Align DestAlign,
1957 std::optional<uint32_t> AtomicCpySize = std::nullopt) const;
1958
1959 /// \returns True if the two functions have compatible attributes for inlining
1960 /// purposes.
1961 LLVM_ABI bool areInlineCompatible(const Function *Caller,
1962 const Function *Callee) const;
1963
1964 /// Returns a penalty for invoking call \p Call in \p F.
1965 /// For example, if a function F calls a function G, which in turn calls
1966 /// function H, then getInlineCallPenalty(F, H()) would return the
1967 /// penalty of calling H from F, e.g. after inlining G into F.
1968 /// \p DefaultCallPenalty is passed to give a default penalty that
1969 /// the target can amend or override.
1970 LLVM_ABI unsigned getInlineCallPenalty(const Function *F,
1971 const CallBase &Call,
1972 unsigned DefaultCallPenalty) const;
1973
1974 /// \returns true if `Caller`'s `Attr` should be added to the new function
1975 /// created by outlining part of `Caller`.
1976 LLVM_ABI bool
1978 const Attribute &Attr) const;
1979
1980 /// \returns True if the caller and callee agree on how \p Types will be
1981 /// passed to or returned from the callee.
1982 /// to the callee.
1983 /// \param Types List of types to check.
1984 LLVM_ABI bool areTypesABICompatible(const Function *Caller,
1985 const Function *Callee,
1986 ArrayRef<Type *> Types) const;
1987
1988 /// The type of load/store indexing.
1990 MIM_Unindexed, ///< No indexing.
1991 MIM_PreInc, ///< Pre-incrementing.
1992 MIM_PreDec, ///< Pre-decrementing.
1993 MIM_PostInc, ///< Post-incrementing.
1994 MIM_PostDec ///< Post-decrementing.
1995 };
1996
1997 /// \returns True if the specified indexed load for the given type is legal.
1998 LLVM_ABI bool isIndexedLoadLegal(enum MemIndexedMode Mode, Type *Ty) const;
1999
2000 /// \returns True if the specified indexed store for the given type is legal.
2001 LLVM_ABI bool isIndexedStoreLegal(enum MemIndexedMode Mode, Type *Ty) const;
2002
2003 /// \returns The bitwidth of the largest vector type that should be used to
2004 /// load/store in the given address space.
2005 LLVM_ABI unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const;
2006
2007 /// \returns True if the load instruction is legal to vectorize.
2009
2010 /// \returns True if the store instruction is legal to vectorize.
2012
2013 /// \returns True if it is legal to vectorize the given load chain.
2014 LLVM_ABI bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
2015 Align Alignment,
2016 unsigned AddrSpace) const;
2017
2018 /// \returns True if it is legal to vectorize the given store chain.
2019 LLVM_ABI bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
2020 Align Alignment,
2021 unsigned AddrSpace) const;
2022
2023 /// \returns True if it is legal to vectorize the given reduction kind.
2025 ElementCount VF) const;
2026
2027 /// \returns True if the given type is supported for scalable vectors
2029
2030 /// \returns The new vector factor value if the target doesn't support \p
2031 /// SizeInBytes loads or has a better vector factor.
2032 LLVM_ABI unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize,
2033 unsigned ChainSizeInBytes,
2034 VectorType *VecTy) const;
2035
2036 /// \returns The new vector factor value if the target doesn't support \p
2037 /// SizeInBytes stores or has a better vector factor.
2038 LLVM_ABI unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize,
2039 unsigned ChainSizeInBytes,
2040 VectorType *VecTy) const;
2041
2042 /// \returns True if the target prefers fixed width vectorization if the
2043 /// loop vectorizer's cost-model assigns an equal cost to the fixed and
2044 /// scalable version of the vectorized loop.
2046
2047 /// \returns True if target prefers SLP vectorizer with altermate opcode
2048 /// vectorization, false - otherwise.
2050
2051 /// \returns True if the SLP vectorizer should apply the instruction-count
2052 /// check that rejects 2-element vector trees when the vector instruction
2053 /// count exceeds the scalar instruction count, false if the target opts out
2054 /// of this heuristic.
2055 LLVM_ABI bool preferSLPInstCountCheck() const;
2056
2057 /// \returns True if the target prefers reductions of \p Kind to be performed
2058 /// in the loop.
2059 LLVM_ABI bool preferInLoopReduction(RecurKind Kind, Type *Ty) const;
2060
2061 /// \returns True if the target prefers reductions select kept in the loop
2062 /// when tail folding. i.e.
2063 /// loop:
2064 /// p = phi (0, s)
2065 /// a = add (p, x)
2066 /// s = select (mask, a, p)
2067 /// vecreduce.add(s)
2068 ///
2069 /// As opposed to the normal scheme of p = phi (0, a) which allows the select
2070 /// to be pulled out of the loop. If the select(.., add, ..) can be predicated
2071 /// by the target, this can lead to cleaner code generation.
2073
2074 /// Return true if the loop vectorizer should consider vectorizing an
2075 /// otherwise scalar epilogue loop if the loop already has been vectorized
2076 /// processing \p Iters scalar iterations per vector iteration.
2078
2079 /// \returns True if the loop vectorizer should discard any VFs where the
2080 /// maximum register pressure exceeds getNumberOfRegisters.
2082
2083 /// \returns True if the target wants to expand the given reduction intrinsic
2084 /// into a shuffle sequence.
2086
2088
2089 /// \returns The shuffle sequence pattern used to expand the given reduction
2090 /// intrinsic.
2093
2094 /// \returns the size cost of rematerializing a GlobalValue address relative
2095 /// to a stack reload.
2096 LLVM_ABI unsigned getGISelRematGlobalCost() const;
2097
2098 /// \returns the lower bound of a trip count to decide on vectorization
2099 /// while tail-folding.
2101
2102 /// \returns True if the target supports scalable vectors.
2103 LLVM_ABI bool supportsScalableVectors() const;
2104
2105 /// \return true when scalable vectorization is preferred.
2107
2108 /// \name Vector Predication Information
2109 /// @{
2110 /// Whether the target supports the %evl parameter of VP intrinsic efficiently
2111 /// in hardware. (see LLVM Language Reference - "Vector Predication
2112 /// Intrinsics"). Use of %evl is discouraged when that is not the case.
2113 LLVM_ABI bool hasActiveVectorLength() const;
2114
2115 /// Return true if sinking I's operands to the same basic block as I is
2116 /// profitable, e.g. because the operands can be folded into a target
2117 /// instruction during instruction selection. After calling the function
2118 /// \p Ops contains the Uses to sink ordered by dominance (dominating users
2119 /// come first). \p LI, when available, lets targets account for loop
2120 /// boundaries when deciding profitability.
2123 const LoopInfo *LI = nullptr) const;
2124
2125 /// Return true if it's significantly cheaper to shift a vector by a uniform
2126 /// scalar than by an amount which will vary across each lane. On x86 before
2127 /// AVX2 for example, there is a "psllw" instruction for the former case, but
2128 /// no simple instruction for a general "a << b" operation on vectors.
2129 /// This should also apply to lowering for vector funnel shifts (rotates).
2131
2134 // keep the predicating parameter
2136 // where legal, discard the predicate parameter
2138 // transform into something else that is also predicating
2140 };
2141
2142 // How to transform the EVL parameter.
2143 // Legal: keep the EVL parameter as it is.
2144 // Discard: Ignore the EVL parameter where it is safe to do so.
2145 // Convert: Fold the EVL into the mask parameter.
2147
2148 // How to transform the operator.
2149 // Legal: The target supports this operator.
2150 // Convert: Convert this to a non-VP operation.
2151 // The 'Discard' strategy is invalid.
2153
2154 bool shouldDoNothing() const {
2155 return (EVLParamStrategy == Legal) && (OpStrategy == Legal);
2156 }
2159 };
2160
2161 /// \returns How the target needs this vector-predicated operation to be
2162 /// transformed.
2164 getVPLegalizationStrategy(const VPIntrinsic &PI) const;
2165 /// @}
2166
2167 /// \returns Whether a 32-bit branch instruction is available in Arm or Thumb
2168 /// state.
2169 ///
2170 /// Used by the LowerTypeTests pass, which constructs an IR inline assembler
2171 /// node containing a jump table in a format suitable for the target, so it
2172 /// needs to know what format of jump table it can legally use.
2173 ///
2174 /// For non-Arm targets, this function isn't used. It defaults to returning
2175 /// false, but it shouldn't matter what it returns anyway.
2176 LLVM_ABI bool hasArmWideBranch(bool Thumb) const;
2177
2178 /// Returns a bitmask constructed from the target-features or fmv-features
2179 /// metadata of a function corresponding to its Arch Extensions.
2180 LLVM_ABI APInt getFeatureMask(const Function &F) const;
2181
2182 /// Returns a bitmask constructed from the target-features or fmv-features
2183 /// metadata of a function corresponding to its FMV priority.
2184 LLVM_ABI APInt getPriorityMask(const Function &F) const;
2185
2186 /// Returns true if this is an instance of a function with multiple versions.
2187 LLVM_ABI bool isMultiversionedFunction(const Function &F) const;
2188
2189 /// \return The maximum number of function arguments the target supports.
2190 LLVM_ABI unsigned getMaxNumArgs() const;
2191
2192 /// @}
2193
2194 /// Collect kernel launch bounds for \p F into \p LB.
2196 const Function &F,
2197 SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const;
2198
2199 /// Returns true if GEP should not be used to index into vectors for this
2200 /// target.
2202
2203 /// Determine if an instruction with Custom uniformity can be proven uniform
2204 /// based on which operands are uniform.
2205 ///
2206 /// \param I The instruction to check.
2207 /// \param UniformArgs A bitvector indicating which operands are known to be
2208 /// uniform (bit N corresponds to operand N).
2209 /// \returns true if the instruction result can be proven uniform given the
2210 /// uniform operands, false otherwise.
2211 LLVM_ABI bool isUniform(const Instruction *I,
2212 const SmallBitVector &UniformArgs) const;
2213
2214private:
2215 std::unique_ptr<const TargetTransformInfoImplBase> TTIImpl;
2216};
2217
2218/// Costs for each cost kind in a target cost table. Field order matches
2219/// TargetCostKind, since table entries initialize these fields positionally.
2220/// A cost of ~0U means the table provides no cost for that kind.
2222 unsigned RecipThroughputCost = ~0U;
2223 unsigned LatencyCost = ~0U;
2224 unsigned CodeSizeCost = ~0U;
2225 unsigned SizeAndLatencyCost = ~0U;
2226
2227 std::optional<unsigned>
2229 unsigned Cost = ~0U;
2230 switch (Kind) {
2233 break;
2235 Cost = LatencyCost;
2236 break;
2239 break;
2242 break;
2243 }
2244 if (Cost == ~0U)
2245 return std::nullopt;
2246 return Cost;
2247 }
2248};
2249
2250/// Analysis pass providing the \c TargetTransformInfo.
2251///
2252/// The core idea of the TargetIRAnalysis is to expose an interface through
2253/// which LLVM targets can analyze and provide information about the middle
2254/// end's target-independent IR. This supports use cases such as target-aware
2255/// cost modeling of IR constructs.
2256///
2257/// This is a function analysis because much of the cost modeling for targets
2258/// is done in a subtarget specific way and LLVM supports compiling different
2259/// functions targeting different subtargets in order to support runtime
2260/// dispatch according to the observed subtarget.
2261class TargetIRAnalysis : public AnalysisInfoMixin<TargetIRAnalysis> {
2262public:
2264
2265 /// Default construct a target IR analysis.
2266 ///
2267 /// This will use the module's datalayout to construct a baseline
2268 /// conservative TTI result.
2270
2271 /// Construct an IR analysis pass around a target-provide callback.
2272 ///
2273 /// The callback will be called with a particular function for which the TTI
2274 /// is needed and must return a TTI object for that function.
2275 LLVM_ABI
2276 TargetIRAnalysis(std::function<Result(const Function &)> TTICallback);
2277
2278 // Value semantics. We spell out the constructors for MSVC.
2280 : TTICallback(Arg.TTICallback) {}
2282 : TTICallback(std::move(Arg.TTICallback)) {}
2284 TTICallback = RHS.TTICallback;
2285 return *this;
2286 }
2288 TTICallback = std::move(RHS.TTICallback);
2289 return *this;
2290 }
2291
2293
2294private:
2296 LLVM_ABI static AnalysisKey Key;
2297
2298 /// The callback used to produce a result.
2299 ///
2300 /// We use a completely opaque callback so that targets can provide whatever
2301 /// mechanism they desire for constructing the TTI for a given function.
2302 ///
2303 /// FIXME: Should we really use std::function? It's relatively inefficient.
2304 /// It might be possible to arrange for even stateful callbacks to outlive
2305 /// the analysis and thus use a function_ref which would be lighter weight.
2306 /// This may also be less error prone as the callback is likely to reference
2307 /// the external TargetMachine, and that reference needs to never dangle.
2308 std::function<Result(const Function &)> TTICallback;
2309
2310 /// Helper function used as the callback in the default constructor.
2311 static Result getDefaultTTI(const Function &F);
2312};
2313
2314/// Wrapper pass for TargetTransformInfo.
2315///
2316/// This pass can be constructed from a TTI object which it stores internally
2317/// and is queried by passes.
2319 TargetIRAnalysis TIRA;
2320 std::optional<TargetTransformInfo> TTI;
2321
2322 virtual void anchor();
2323
2324public:
2325 static char ID;
2326
2327 /// We must provide a default constructor for the pass but it should
2328 /// never be used.
2329 ///
2330 /// Use the constructor below or call one of the creation routines.
2332
2334
2336};
2337
2338/// Create an analysis pass wrapper around a TTI object.
2339///
2340/// This analysis pass just holds the TTI instance and makes it available to
2341/// clients.
2344
2345} // namespace llvm
2346
2347#endif
bool IsScalable
uint64_t uint64_t
uint64_t Imm
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Atomic ordering constants.
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
#define LLVM_ABI
Definition Compiler.h:211
TargetTransformInfo::VPLegalization VPLegalization
This header defines various interfaces for pass management in LLVM.
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
uint64_t IntrinsicInst * II
SmallVector< MachineOperand, 4 > Cond
int64_t C1
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
SI Fold Operands
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
Value * RHS
Class for arbitrary precision integers.
Definition APInt.h:78
an instruction to allocate memory on the stack
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
A cache of @llvm.assume calls within a function.
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:107
LLVM Basic Block Representation.
Definition BasicBlock.h:62
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
Conditional Branch instruction.
This is an important base class in LLVM.
Definition Constant.h:44
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
This instruction compares its operands according to the predicate given to the constructor.
ImmutablePass class - This class is used to provide information that does not need to be run.
Definition Pass.h:283
ImmutablePass(char &pid)
Definition Pass.h:285
The core instruction combiner logic.
static InstructionCost getInvalid(CostType Val=0)
Class to represent integer types.
Drive the analysis of interleaved memory accesses in the loop.
const SmallVectorImpl< Type * > & getArgTypes() const
const SmallVectorImpl< const Value * > & getArgs() const
LLVM_ABI IntrinsicCostAttributes(Intrinsic::ID Id, const CallBase &CI, InstructionCost ScalarCost=InstructionCost::getInvalid(), bool TypeBasedOnly=false)
VectorInstrContext getVectorInstrContext() const
InstructionCost getScalarizationCost() const
const IntrinsicInst * getInst() const
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
An instruction for reading from memory.
LoopVectorizationLegality checks if it is legal to vectorize a loop, and to what vectorization factor...
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
Information for memory intrinsic cost model.
MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy, const Value *Ptr, bool VariableMask, Align Alignment, const Instruction *I=nullptr, const Value *StrideVal=nullptr)
MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy, bool VariableMask, Align Alignment, const Instruction *I=nullptr, const Value *StrideVal=nullptr)
MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy, Align Alignment, unsigned AddressSpace=0, const Value *StrideVal=nullptr)
const Instruction * getInst() const
The optimization diagnostic interface.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
Analysis providing profile information.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
This class represents an analyzed expression in the program.
The main scalar evolution driver.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
Definition TypeSize.h:31
An instruction for storing to memory.
Multiway switch.
Analysis pass providing the TargetTransformInfo.
TargetIRAnalysis(const TargetIRAnalysis &Arg)
TargetIRAnalysis & operator=(const TargetIRAnalysis &RHS)
LLVM_ABI Result run(const Function &F, FunctionAnalysisManager &)
LLVM_ABI TargetIRAnalysis()
Default construct a target IR analysis.
TargetIRAnalysis & operator=(TargetIRAnalysis &&RHS)
TargetIRAnalysis(TargetIRAnalysis &&Arg)
Provides information about what library functions are available for the current target.
Base class for use as a mix-in that aids implementing a TargetTransformInfo-compatible class.
TargetTransformInfoWrapperPass()
We must provide a default constructor for the pass but it should never be used.
TargetTransformInfo & getTTI(const Function &F)
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
LLVM_ABI bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const
LLVM_ABI Value * getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, Type *ExpectedType, bool CanCreate=true) const
LLVM_ABI bool isLegalToVectorizeLoad(LoadInst *LI) const
LLVM_ABI std::optional< unsigned > getVScaleForTuning() const
LLVM_ABI uint64_t getMaxBoundedMemSetInlineSize() const
Returns the maximum length in bytes for which a memset with a non-constant length,...
static LLVM_ABI CastContextHint getCastContextHint(const Instruction *I)
Calculates a CastContextHint from I.
LLVM_ABI unsigned getMaxNumArgs() const
LLVM_ABI bool addrspacesMayAlias(unsigned AS0, unsigned AS1) const
Return false if a AS0 address cannot possibly alias a AS1 address.
LLVM_ABI bool isLegalMaskedScatter(Type *DataType, Align Alignment) const
Return true if the target supports masked scatter.
LLVM_ABI bool shouldBuildLookupTables() const
Return true if switches should be turned into lookup tables for the target.
LLVM_ABI VectorInstrContext getBuildVectorContextHint(ArrayRef< int > Mask, ArrayRef< Value * > Scalars, function_ref< bool(SmallVectorImpl< BuildVectorUseOp > &)> GatherUseOps) const
Calculates a VectorInstrContext for buildvector-like gather sequences.
LLVM_ABI bool isLegalToVectorizeStore(StoreInst *SI) const
LLVM_ABI bool areTypesABICompatible(const Function *Caller, const Function *Callee, ArrayRef< Type * > Types) const
LLVM_ABI bool enableAggressiveInterleaving(bool LoopHasReductions) const
Don't restrict interleaved unrolling to small loops.
LLVM_ABI bool isLegalSpeculativeLoad(Type *DataType, unsigned AddressSpace) const
Return true if the target supports speculatively loading DataType from address space AddressSpace,...
LLVM_ABI bool isMultiversionedFunction(const Function &F) const
Returns true if this is an instance of a function with multiple versions.
LLVM_ABI unsigned getMaxInterleaveFactor(ElementCount VF, bool HasUnorderedReductions) const
LLVM_ABI bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) const
Return true if it is faster to check if a floating-point value is NaN (or not-NaN) versus a compariso...
LLVM_ABI bool isLegalMaskedStore(Type *DataType, Align Alignment, unsigned AddressSpace, MaskKind MaskKind=VariableOrConstantMask) const
Return true if the target supports masked store.
LLVM_ABI unsigned getMinimumLookupTableEntryBitWidth() const
Return the minimum bit width to use for integer switch lookup table elements on this target.
LLVM_ABI bool supportsEfficientVectorElementLoadStore() const
If target has efficient vector element load/store instructions, it can return true here so that inser...
LLVM_ABI unsigned getAssumedAddrSpace(const Value *V) const
LLVM_ABI bool preferAlternateOpcodeVectorization() const
LLVM_ABI bool shouldDropLSRSolutionIfLessProfitable() const
Return true if LSR should drop a found solution if it's calculated to be less profitable than the bas...
LLVM_ABI bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1, const TargetTransformInfo::LSRCost &C2) const
Return true if LSR cost of C1 is lower than C2.
LLVM_ABI unsigned getPrefetchDistance() const
LLVM_ABI Type * getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, unsigned SrcAddrSpace, unsigned DestAddrSpace, Align SrcAlign, Align DestAlign, std::optional< uint32_t > AtomicElementSize=std::nullopt) const
LLVM_ABI bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) const
Return true if the target supports masked expand load.
LLVM_ABI bool prefersVectorizedAddressing() const
Return true if target doesn't mind addresses in vectors.
LLVM_ABI InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I=nullptr) const
LLVM_ABI bool hasBranchDivergence(const Function *F=nullptr) const
Return true if branch divergence exists.
LLVM_ABI bool preferEpilogueVectorization(ElementCount Iters) const
Return true if the loop vectorizer should consider vectorizing an otherwise scalar epilogue loop if t...
LLVM_ABI MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const
bool invalidate(Function &, const PreservedAnalyses &, FunctionAnalysisManager::Invalidator &)
Handle the invalidation of this information.
LLVM_ABI void getUnrollingPreferences(Loop *L, ScalarEvolution &, UnrollingPreferences &UP, OptimizationRemarkEmitter *ORE) const
Get target-customized preferences for the generic loop unrolling transformation.
LLVM_ABI bool shouldBuildLookupTablesForConstant(Constant *C) const
Return true if switches should be turned into lookup tables containing this constant value for the ta...
LLVM_ABI InstructionCost getGEPCost(Type *PointeeType, const Value *Ptr, ArrayRef< const Value * > Operands, TargetCostKind CostKind, Type *AccessType=nullptr) const
Estimate the cost of a GEP operation when lowered.
LLVM_ABI TailFoldingStyle getPreferredTailFoldingStyle() const
Query the target what the preferred style of tail folding is.
LLVM_ABI bool supportsTailCallFor(const CallBase *CB) const
If target supports tail call on CB.
LLVM_ABI std::optional< Instruction * > instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const
Targets can implement their own combinations for target-specific intrinsics.
LLVM_ABI bool isProfitableLSRChainElement(Instruction *I) const
LLVM_ABI TypeSize getRegisterBitWidth(RegisterKind K) const
MaskKind
Some targets only support masked load/store with a constant mask.
LLVM_ABI unsigned getInlineCallPenalty(const Function *F, const CallBase &Call, unsigned DefaultCallPenalty) const
Returns a penalty for invoking call Call in F.
LLVM_ABI InstructionCost getOperandsScalarizationOverhead(ArrayRef< Type * > Tys, TTI::TargetCostKind CostKind, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const
Estimate the overhead of scalarizing operands with the given types.
LLVM_ABI bool hasActiveVectorLength() const
LLVM_ABI bool isExpensiveToSpeculativelyExecute(const Instruction *I) const
Return true if the cost of the instruction is too high to speculatively execute and should be kept be...
LLVM_ABI InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, OperandValueInfo OpdInfo={OK_AnyValue, OP_None}, const Instruction *I=nullptr) const
LLVM_ABI bool isLegalMaskedGather(Type *DataType, Align Alignment) const
Return true if the target supports masked gather.
LLVM_ABI ValueUniformity getValueUniformity(const Value *V) const
Get target-specific uniformity information for a value.
static LLVM_ABI OperandValueInfo commonOperandInfo(const Value *X, const Value *Y)
Collect common data between two OperandValueInfo inputs.
LLVM_ABI InstructionCost getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts, TTI::TargetCostKind CostKind) const
LLVM_ABI bool allowVectorElementIndexingUsingGEP() const
Returns true if GEP should not be used to index into vectors for this target.
LLVM_ABI bool preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const
Query the target whether it would be preferred to create a tail-folded vector loop,...
LLVM_ABI std::optional< Value * > simplifyDemandedVectorEltsIntrinsic(InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3, std::function< void(Instruction *, unsigned, APInt, APInt &)> SimplifyAndSetOp) const
Can be used to implement target-specific instruction combining.
LLVM_ABI bool enableOrderedReductions() const
Return true if we should be enabling ordered reductions for the target.
InstructionCost getInstructionCost(const User *U, TargetCostKind CostKind) const
This is a helper function which calls the three-argument getInstructionCost with Operands which are t...
LLVM_ABI unsigned getInliningCostBenefitAnalysisProfitableMultiplier() const
LLVM_ABI bool hasMultiVectorLoadStore(unsigned NumVectors, MaskSource Mask, VectorType *VectorTy=nullptr, bool IsStore=false, std::optional< Instruction::CastOps > CastHint=std::nullopt) const
Return true if the target supports loading or storing NumVectors contiguous vectors of type VectorTy ...
LLVM_ABI InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) const
LLVM_ABI unsigned getAtomicMemIntrinsicMaxElementSize() const
LLVM_ABI InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index=-1, const Value *Op0=nullptr, const Value *Op1=nullptr, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const
LLVM_ABI std::pair< KnownBits, KnownBits > computeKnownBitsAddrSpaceCast(unsigned ToAS, const Value &PtrOp) const
LLVM_ABI bool LSRWithInstrQueries() const
Return true if the loop strength reduce pass should make Instruction* based TTI queries to isLegalAdd...
LLVM_ABI unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize, unsigned ChainSizeInBytes, VectorType *VecTy) const
LLVM_ABI VPLegalization getVPLegalizationStrategy(const VPIntrinsic &PI) const
static LLVM_ABI PartialReductionExtendKind getPartialReductionExtendKind(Instruction *I)
Get the kind of extension that an instruction represents.
LLVM_ABI InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind, OperandValueInfo Op1Info={OK_AnyValue, OP_None}, OperandValueInfo Op2Info={OK_AnyValue, OP_None}, const Instruction *I=nullptr) const
LLVM_ABI bool shouldConsiderVectorizationRegPressure() const
LLVM_ABI bool enableWritePrefetching() const
LLVM_ABI bool shouldTreatInstructionLikeSelect(const Instruction *I) const
Should the Select Optimization pass treat the given instruction like a select, potentially converting...
LLVM_ABI bool isNoopAddrSpaceCast(unsigned FromAS, unsigned ToAS) const
LLVM_ABI InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef< int > Mask={}, int Index=0, VectorType *SubTp=nullptr, ArrayRef< const Value * > Args={}, const Instruction *CtxI=nullptr, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const
LLVM_ABI bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const
LLVM_ABI bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace) const
Return true is the target supports interleaved access for the given vector type VTy,...
LLVM_ABI unsigned getRegUsageForType(Type *Ty) const
Returns the estimated number of registers required to represent Ty, or zero if Ty cannot be legalized...
LLVM_ABI bool isLegalBroadcastLoad(Type *ElementTy, ElementCount NumElements) const
\Returns true if the target supports broadcasting a load to a vector of type <NumElements x ElementTy...
LLVM_ABI bool isIndexedStoreLegal(enum MemIndexedMode Mode, Type *Ty) const
LLVM_ABI std::pair< const Value *, unsigned > getPredicatedAddrSpace(const Value *V) const
static LLVM_ABI TargetTransformInfo::VectorInstrContext combineVectorInstrContexts(TargetTransformInfo::VectorInstrContext Ctx1, TargetTransformInfo::VectorInstrContext Ctx2)
Combines 2 context hints into a single value.
LLVM_ABI unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const
LLVM_ABI InstructionCost getRegisterClassReloadCost(unsigned ClassID, TargetCostKind CostKind) const
LLVM_ABI ReductionShuffle getPreferredExpandedReductionShuffle(const IntrinsicInst *II) const
static LLVM_ABI OperandValueInfo getOperandInfo(const Value *V)
Collect properties of V used in cost analysis, e.g. OP_PowerOf2.
LLVM_ABI InstructionCost getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef< unsigned > Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, bool UseMaskForCond=false, bool UseMaskForGaps=false) const
LLVM_ABI InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, const Instruction *I=nullptr) const
LLVM_ABI unsigned getAddressSpaceJoin(unsigned AS1, unsigned AS2) const
Return the most specific common address space containing AS1 and AS2.
LLVM_ABI unsigned getRegisterClassForType(bool Vector, Type *Ty=nullptr) const
LLVM_ABI bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace=0, Instruction *I=nullptr, int64_t ScalableOffset=0) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
LLVM_ABI PopcntSupportKind getPopcntSupport(unsigned IntTyWidthInBit) const
Return hardware support for population count.
LLVM_ABI unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI, unsigned &JTSize, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) const
LLVM_ABI bool isElementTypeLegalForScalableVector(Type *Ty) const
LLVM_ABI bool forceScalarizeMaskedGather(VectorType *Type, Align Alignment) const
Return true if the target forces scalarizing of llvm.masked.gather intrinsics.
LLVM_ABI InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty, std::optional< FastMathFlags > FMF, TTI::TargetCostKind CostKind) const
Calculate the cost of vector reduction intrinsics.
LLVM_ABI unsigned getMaxPrefetchIterationsAhead() const
LLVM_ABI bool canHaveNonUndefGlobalInitializerInAddressSpace(unsigned AS) const
Return true if globals in this address space can have initializers other than undef.
LLVM_ABI ElementCount getMinimumVF(unsigned ElemWidth, bool IsScalable) const
LLVM_ABI InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, Type *Ty, TargetCostKind CostKind) const
LLVM_ABI bool enableMaskedInterleavedAccessVectorization() const
Enable matching of interleaved access groups that contain predicated accesses or gaps and therefore v...
LLVM_ABI InstructionCost getIntImmCostInst(unsigned Opc, unsigned Idx, const APInt &Imm, Type *Ty, TargetCostKind CostKind, Instruction *Inst=nullptr) const
Return the expected cost of materialization for the given integer immediate of the specified type for...
LLVM_ABI bool isLegalStridedLoadStore(Type *DataType, Align Alignment) const
Return true if the target supports strided load.
LLVM_ABI TargetTransformInfo & operator=(TargetTransformInfo &&RHS)
LLVM_ABI InstructionCost getExtendedReductionCost(unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *Ty, std::optional< FastMathFlags > FMF, TTI::TargetCostKind CostKind) const
Calculate the cost of an extended reduction pattern, similar to getArithmeticReductionCost of a reduc...
TargetCostKind
The kind of cost model.
@ TCK_RecipThroughput
Reciprocal throughput.
@ TCK_CodeSize
Instruction code size.
@ TCK_SizeAndLatency
The weighted sum of size and latency.
@ TCK_Latency
The latency of instruction.
LLVM_ABI bool enableSelectOptimize() const
Should the Select Optimization pass be enabled and ran.
LLVM_ABI bool collectFlatAddressOperands(SmallVectorImpl< int > &OpIndexes, Intrinsic::ID IID) const
Return any intrinsic address operand indexes which may be rewritten if they use a flat address space ...
OperandValueProperties
Additional properties of an operand's values.
LLVM_ABI int getInliningLastCallToStaticBonus() const
LLVM_ABI bool isIndexedLoadLegal(enum MemIndexedMode Mode, Type *Ty) const
LLVM_ABI InstructionCost getCallInstrCost(Function *F, Type *RetTy, ArrayRef< Type * > Tys, TTI::TargetCostKind CostKind) const
LLVM_ABI unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const
LLVM_ABI unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, Type *ScalarValTy, Align Alignment, unsigned AddrSpace) const
LLVM_ABI bool isLegalICmpImmediate(int64_t Imm) const
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
LLVM_ABI bool isTypeLegal(Type *Ty) const
Return true if this type is legal.
static bool requiresOrderedReduction(std::optional< FastMathFlags > FMF)
A helper function to determine the type of reduction algorithm used for a given Opcode and set of Fas...
LLVM_ABI bool isLegalToVectorizeReduction(const RecurrenceDescriptor &RdxDesc, ElementCount VF) const
LLVM_ABI std::optional< unsigned > getCacheAssociativity(CacheLevel Level) const
LLVM_ABI bool isLegalNTLoad(Type *DataType, Align Alignment) const
Return true if the target supports nontemporal load.
LLVM_ABI bool isUniform(const Instruction *I, const SmallBitVector &UniformArgs) const
Determine if an instruction with Custom uniformity can be proven uniform based on which operands are ...
LLVM_ABI InstructionCost getMemcpyCost(const Instruction *I) const
LLVM_ABI unsigned adjustInliningThreshold(const CallBase *CB) const
LLVM_ABI bool isLegalAddImmediate(int64_t Imm) const
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
LLVM_ABI bool isTargetIntrinsicWithStructReturnOverloadAtField(Intrinsic::ID ID, int RetIdx) const
Identifies if the vector form of the intrinsic that returns a struct is overloaded at the struct elem...
LLVM_ABI unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize, unsigned ChainSizeInBytes, VectorType *VecTy) const
LLVM_ABI InstructionCost getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const
LLVM_ABI InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF, TTI::TargetCostKind CostKind) const
LLVM_ABI Value * rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, Value *OldV, Value *NewV) const
Rewrite intrinsic call II such that OldV will be replaced with NewV, which has a different address sp...
LLVM_ABI InstructionCost getCostOfKeepingLiveOverCall(ArrayRef< Type * > Tys) const
LLVM_ABI bool canSaveCmp(Loop *L, CondBrInst **BI, ScalarEvolution *SE, LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC, TargetLibraryInfo *LibInfo) const
Return true if the target can save a compare for loop count, for example hardware loop saves a compar...
LLVM_ABI unsigned getMinPrefetchStride(unsigned NumMemAccesses, unsigned NumStridedMemAccesses, unsigned NumPrefetches, bool HasCall) const
Some HW prefetchers can handle accesses up to a certain constant stride.
LLVM_ABI bool shouldPrefetchAddressSpace(unsigned AS) const
LLVM_ABI InstructionCost getIntImmCost(const APInt &Imm, Type *Ty, TargetCostKind CostKind) const
Return the expected cost of materializing for the given integer immediate of the specified type.
LLVM_ABI unsigned getMinVectorRegisterBitWidth() const
LLVM_ABI InstructionCost getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr, TTI::TargetCostKind CostKind) const
LLVM_ABI bool isLegalNTStore(Type *DataType, Align Alignment) const
Return true if the target supports nontemporal store.
LLVM_ABI unsigned getFlatAddressSpace() const
Returns the address space ID for a target's 'flat' address space.
LLVM_ABI bool preferToKeepConstantsAttached(const Instruction &Inst, const Function &Fn) const
It can be advantageous to detach complex constants from their uses to make their generation cheaper.
LLVM_ABI bool hasArmWideBranch(bool Thumb) const
LLVM_ABI const char * getRegisterClassName(unsigned ClassID) const
MaskSource
Enum describing the source/producer of a mask.
@ ArbitraryPredicate
The operation is masked with an arbitrary predicate.
@ ActiveLaneMask
The operation is masked with a contiguous active lane mask.
LLVM_ABI bool shouldConsiderAddressTypePromotion(const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const
LLVM_ABI APInt getPriorityMask(const Function &F) const
Returns a bitmask constructed from the target-features or fmv-features metadata of a function corresp...
LLVM_ABI BranchProbability getPredictableBranchThreshold() const
If a branch or a select condition is skewed in one direction by more than this factor,...
LLVM_ABI TargetTransformInfo(std::unique_ptr< const TargetTransformInfoImplBase > Impl)
Construct a TTI object using a type implementing the Concept API below.
LLVM_ABI bool preferInLoopReduction(RecurKind Kind, Type *Ty) const
LLVM_ABI unsigned getCallerAllocaCost(const CallBase *CB, const AllocaInst *AI) const
LLVM_ABI bool hasConditionalLoadStoreForType(Type *Ty, bool IsStore) const
LLVM_ABI InstructionCost getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode, Type *ResTy, VectorType *Ty, TTI::TargetCostKind CostKind) const
Calculate the cost of an extended reduction pattern, similar to getArithmeticReductionCost of an Add/...
LLVM_ABI unsigned getCacheLineSize() const
LLVM_ABI bool allowsMisalignedMemoryAccesses(LLVMContext &Context, unsigned BitWidth, unsigned AddressSpace=0, Align Alignment=Align(1), unsigned *Fast=nullptr) const
Determine if the target supports unaligned memory accesses.
LLVM_ABI InstructionCost getAltInstrCost(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, const SmallBitVector &OpcodeMask, TTI::TargetCostKind CostKind, ArrayRef< const Value * > Scalars={}) const
Returns the cost estimation for alternating opcode pattern that can be lowered to a single instructio...
LLVM_ABI bool shouldCopyAttributeWhenOutliningFrom(const Function *Caller, const Attribute &Attr) const
LLVM_ABI APInt getAddrSpaceCastPreservedPtrMask(unsigned SrcAS, unsigned DstAS) const
Returns a mask indicating which bits of a pointer remain unchanged when casting between address space...
LLVM_ABI int getInlinerVectorBonusPercent() const
LLVM_ABI unsigned getEpilogueVectorizationMinVF() const
LLVM_ABI void collectKernelLaunchBounds(const Function &F, SmallVectorImpl< std::pair< StringRef, int64_t > > &LB) const
Collect kernel launch bounds for F into LB.
PopcntSupportKind
Flags indicating the kind of support for population count.
LLVM_ABI bool preferPredicatedReductionSelect() const
LLVM_ABI InstructionCost getIntImmCodeSizeCost(unsigned Opc, unsigned Idx, const APInt &Imm, Type *Ty) const
Return the expected cost for the given integer when optimising for size.
LLVM_ABI AddressingModeKind getPreferredAddressingMode(const Loop *L, ScalarEvolution *SE) const
Return the preferred addressing mode LSR should make efforts to generate.
LLVM_ABI bool isLoweredToCall(const Function *F) const
Test whether calls to a function lower to actual program function calls.
llvm::VectorInstrContext VectorInstrContext
LLVM_ABI bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const
LLVM_ABI bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE, AssumptionCache &AC, TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const
Query the target whether it would be profitable to convert the given loop into a hardware loop.
LLVM_ABI unsigned getInliningThresholdMultiplier() const
LLVM_ABI InstructionCost getBranchMispredictPenalty() const
Returns estimated penalty of a branch misprediction in latency.
LLVM_ABI unsigned getNumberOfRegisters(unsigned ClassID) const
LLVM_ABI bool isProfitableToHoist(Instruction *I) const
Return true if it is profitable to hoist instruction in the then/else to before if.
LLVM_ABI bool supportsScalableVectors() const
LLVM_ABI bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) const
Return true if the given instruction (assumed to be a memory access instruction) has a volatile varia...
LLVM_ABI bool isLegalMaskedCompressStore(Type *DataType, Align Alignment) const
Return true if the target supports masked compress store.
LLVM_ABI std::optional< unsigned > getMinPageSize() const
LLVM_ABI bool preferSLPInstCountCheck() const
LLVM_ABI bool isFPVectorizationPotentiallyUnsafe() const
Indicate that it is potentially unsafe to automatically vectorize floating-point operations because t...
LLVM_ABI InstructionCost getInsertExtractValueCost(unsigned Opcode, TTI::TargetCostKind CostKind) const
LLVM_ABI bool shouldBuildRelLookupTables() const
Return true if lookup tables should be turned into relative lookup tables.
LLVM_ABI std::optional< unsigned > getCacheSize(CacheLevel Level) const
LLVM_ABI bool shouldMaximizeVectorBandwidth(VectorKind K, Type *ElementTy=nullptr) const
LLVM_ABI std::optional< Value * > simplifyDemandedUseBitsIntrinsic(InstCombiner &IC, IntrinsicInst &II, APInt DemandedMask, KnownBits &Known, bool &KnownBitsComputed) const
Can be used to implement target-specific instruction combining.
LLVM_ABI InstructionCost getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Opd1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Opd2Info={TTI::OK_AnyValue, TTI::OP_None}, ArrayRef< const Value * > Args={}, const Instruction *CtxI=nullptr, const TargetLibraryInfo *TLibInfo=nullptr) const
This is an approximation of reciprocal throughput of a math/logic op.
LLVM_ABI bool isLegalAddScalableImmediate(int64_t Imm) const
Return true if adding the specified scalable immediate is legal, that is the target has add instructi...
LLVM_ABI bool isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx) const
Identifies if the vector form of the intrinsic has a scalar operand.
LLVM_ABI bool hasDivRemOp(Type *DataType, bool IsSigned) const
Return true if the target has a unified operation to calculate division and remainder.
TargetCostConstants
Underlying constants for 'cost' values in this interface.
@ TCC_Expensive
The cost of a 'div' instruction on x86.
@ TCC_Free
Expected to fold away in lowering.
@ TCC_Basic
The cost of a typical 'add' instruction.
LLVM_ABI bool enableInterleavedAccessVectorization() const
Enable matching of interleaved access groups.
LLVM_ABI unsigned getMinTripCountTailFoldingThreshold() const
LLVM_ABI InstructionCost getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType, ElementCount VF, PartialReductionExtendKind OpAExtend, PartialReductionExtendKind OpBExtend, std::optional< unsigned > BinOp, TTI::TargetCostKind CostKind, std::optional< FastMathFlags > FMF) const
LLVM_ABI InstructionCost getInstructionCost(const User *U, ArrayRef< const Value * > Operands, TargetCostKind CostKind) const
Estimate the cost of a given IR user when lowered.
LLVM_ABI bool enableScalableVectorization() const
LLVM_ABI bool useFastCCForInternalCall(Function &F) const
Return true if the input function is internal, should use fastcc calling convention.
LLVM_ABI bool isVectorShiftByScalarCheap(Type *Ty) const
Return true if it's significantly cheaper to shift a vector by a uniform scalar than by an amount whi...
LLVM_ABI bool isNumRegsMajorCostOfLSR() const
Return true if LSR major cost is number of registers.
LLVM_ABI unsigned getInliningCostBenefitAnalysisSavingsMultiplier() const
LLVM_ABI bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) const
LLVM_ABI unsigned getGISelRematGlobalCost() const
static LLVM_ABI Instruction::CastOps getOpcodeForPartialReductionExtendKind(PartialReductionExtendKind Kind)
Get the cast opcode for an extension kind.
MemIndexedMode
The type of load/store indexing.
LLVM_ABI bool isLegalMaskedLoad(Type *DataType, Align Alignment, unsigned AddressSpace, MaskKind MaskKind=VariableOrConstantMask) const
Return true if the target supports masked load.
LLVM_ABI InstructionCost getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index) const
LLVM_ABI bool isProfitableToSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops, const LoopInfo *LI=nullptr) const
Return true if sinking I's operands to the same basic block as I is profitable, e....
LLVM_ABI bool areInlineCompatible(const Function *Caller, const Function *Callee) const
LLVM_ABI bool useColdCCForColdCall(Function &F) const
Return true if the input function which is cold at all call sites, should use coldcc calling conventi...
LLVM_ABI InstructionCost getFPOpCost(Type *Ty) const
Return the expected cost of supporting the floating point operation of the specified type.
LLVM_ABI bool supportsTailCalls() const
If the target supports tail calls.
LLVM_ABI bool canMacroFuseCmp() const
Return true if the target can fuse a compare and branch.
LLVM_ABI bool isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const
Query the target whether the specified address space cast from FromAS to ToAS is valid.
LLVM_ABI unsigned getNumberOfParts(Type *Tp) const
AddressingModeKind
Which addressing mode Loop Strength Reduction will try to generate.
@ AMK_PostIndexed
Prefer post-indexed addressing mode.
@ AMK_All
Consider all addressing modes.
@ AMK_PreIndexed
Prefer pre-indexed addressing mode.
@ AMK_None
Don't prefer any addressing mode.
LLVM_ABI bool preferLoopExitWithEqualityComparison(const ICmpInst &Cond, const APInt &PostIncExitValue) const
Return true if replacing the relational loop exit comparison Cond with an equality comparison against...
LLVM_ABI InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace=0) const
Return the cost of the scaling factor used in the addressing mode represented by AM for this target,...
LLVM_ABI bool isTruncateFree(Type *Ty1, Type *Ty2) const
Return true if it's free to truncate a value of type Ty1 to type Ty2.
LLVM_ABI void getMemcpyLoopResidualLoweringType(SmallVectorImpl< Type * > &OpsOut, LLVMContext &Context, unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace, Align SrcAlign, Align DestAlign, std::optional< uint32_t > AtomicCpySize=std::nullopt) const
LLVM_ABI bool forceScalarizeMaskedScatter(VectorType *Type, Align Alignment) const
Return true if the target forces scalarizing of llvm.masked.scatter intrinsics.
LLVM_ABI bool isTargetIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx) const
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
static LLVM_ABI VectorInstrContext getVectorInstrContextHint(const Instruction *I)
Calculates a VectorInstrContext from I.
LLVM_ABI InstructionCost getPointersChainCost(ArrayRef< const Value * > Ptrs, const Value *Base, const PointersChainInfo &Info, Type *AccessTy, const TargetCostKind CostKind) const
Estimate the cost of a chain of pointers (typically pointer operands of a chain of loads or stores wi...
LLVM_ABI bool haveFastSqrt(Type *Ty) const
Return true if the hardware has a fast square-root instruction.
LLVM_ABI bool shouldExpandReduction(const IntrinsicInst *II) const
LLVM_ABI InstructionCost getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract, TTI::TargetCostKind CostKind, bool ForPoisonSrc=true, ArrayRef< Value * > VL={}, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const
Estimate the overhead of scalarizing an instruction.
LLVM_ABI uint64_t getMaxMemIntrinsicInlineSizeThreshold() const
Returns the maximum memset / memcpy size in bytes that still makes it profitable to inline the call.
LLVM_ABI bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, const SmallBitVector &OpcodeMask, ArrayRef< const Value * > Scalars={}) const
Return true if this is an alternating opcode pattern that can be lowered to a single instruction on t...
ShuffleKind
The various kinds of shuffle patterns for vector queries.
@ SK_InsertSubvector
InsertSubvector. Index indicates start offset.
@ SK_Select
Selects elements from the corresponding lane of either source operand.
@ SK_PermuteSingleSrc
Shuffle elements of single source vector with any shuffle mask.
@ SK_Transpose
Transpose two vectors.
@ SK_Splice
Concatenates elements from the first input vector with elements of the second input vector.
@ SK_Broadcast
Broadcast element 0 to all other elements.
@ SK_PermuteTwoSrc
Merge elements from two source vectors into one with any shuffle mask.
@ SK_Reverse
Reverse the order of the vector.
@ SK_ExtractSubvector
ExtractSubvector Index indicates start offset.
LLVM_ABI APInt getFeatureMask(const Function &F) const
Returns a bitmask constructed from the target-features or fmv-features metadata of a function corresp...
LLVM_ABI void getPeelingPreferences(Loop *L, ScalarEvolution &SE, PeelingPreferences &PP) const
Get target-customized preferences for the generic loop peeling transformation.
CastContextHint
Represents a hint about the context in which a cast is used.
@ Reversed
The cast is used with a reversed load/store.
@ Masked
The cast is used with a masked load/store.
@ Normal
The cast is used with a normal load/store.
@ Interleave
The cast is used with an interleaved load/store.
@ GatherScatter
The cast is used with a gather/scatter.
LLVM_ABI InstructionCost getExtractWithExtendCost(unsigned Opcode, Type *Dst, VectorType *VecTy, unsigned Index, TTI::TargetCostKind CostKind) const
LLVM_ABI InstructionCost getRegisterClassSpillCost(unsigned ClassID, TargetCostKind CostKind) const
OperandValueKind
Additional information about an operand's possible values.
CacheLevel
The possible cache levels.
LLVM_ABI bool preferFixedOverScalableIfEqualCost() const
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
This is the common base class for vector predication intrinsics.
LLVM Value Representation.
Definition Value.h:75
Base class of all SIMD vector types.
An efficient, type-erasing, non-owning reference to a callable.
CallInst * Call
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
@ Length
Definition DWP.cpp:577
InstructionCost Cost
@ Known
Known to have no common set bits.
VectorInstrContext
Represents a hint about the context in which a vector instruction or intrinsic is used.
@ None
The instruction is not folded.
@ BinaryOp
One of the operands is a binary op.
@ SplatOpFolded
All of the value's users support splatting the value.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
AtomicOrdering
Atomic ordering for LLVM's memory model.
TargetTransformInfo TTI
LLVM_ABI ImmutablePass * createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA)
Create an analysis pass wrapper around a TTI object.
RecurKind
These are the kinds of recurrences that we support.
@ Fast
Assign the register banks as fast as possible (default).
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr unsigned BitWidth
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1933
@ DataWithEVL
Use predicated EVL instructions for tail-folding.
@ DataAndControlFlow
Use predicate to control both data and control flow.
@ DataWithoutLaneMask
Same as Data, but avoids using the get.active.lane.mask intrinsic to calculate the mask and instead i...
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
ValueUniformity
Enum describing how values behave with respect to uniformity and divergence, to answer the question: ...
Definition Uniformity.h:18
Implement std::hash so that hash_code can be used in STL containers.
Definition BitVector.h:879
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
A CRTP mix-in that provides informational APIs needed for analysis passes.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition Analysis.h:29
Costs for each cost kind in a target cost table.
std::optional< unsigned > operator[](TargetTransformInfo::TargetCostKind Kind) const
Attributes of a target dependent hardware loop.
LLVM_ABI bool canAnalyze(LoopInfo &LI)
LLVM_ABI bool isHardwareLoopCandidate(ScalarEvolution &SE, LoopInfo &LI, DominatorTree &DT, bool ForceNestedLoop=false, bool ForceHardwareLoopPHI=false)
Information about a load/store intrinsic defined by the target.
SmallVector< InterestingMemoryOperand, 1 > InterestingOperands
Value * PtrVal
This is the pointer that the intrinsic is loading from or storing to.
InterleavedAccessInfo * IAI
TailFoldingInfo(TargetLibraryInfo *TLI, LoopVectorizationLegality *LVL, InterleavedAccessInfo *IAI)
TargetLibraryInfo * TLI
LoopVectorizationLegality * LVL
BuildVectorUseOp(unsigned Opcode, Intrinsic::ID IntrinsicID, int OperandIndex)
unsigned Insns
TODO: Some of these could be merged.
Returns options for expansion of memcmp. IsZeroCmp is.
OperandValueInfo mergeWith(const OperandValueInfo OpInfoY)
bool AllowPeeling
Allow peeling off loop iterations.
bool AllowLoopNestsPeeling
Allow peeling off loop iterations for loop nests.
bool PeelLast
Peel off the last PeelCount loop iterations.
bool PeelProfiledIterations
Allow peeling basing on profile.
unsigned PeelCount
A forced peeling factor (the number of bodied of the original loop that should be peeled off before t...
Describe known properties for a set of pointers.
unsigned IsKnownStride
True if distance between any two neigbouring pointers is a known value.
unsigned IsUnitStride
These properties only valid if SameBaseAddress is set.
unsigned IsSameBaseAddress
All the GEPs in a set have same base address.
Parameters that control the generic loop unrolling transformation.
bool UpperBound
Allow using trip count upper bound to unroll loops.
unsigned Threshold
The cost threshold for the unrolled loop.
bool Force
Apply loop unroll on any kind of loop (mainly to loops that fail runtime unrolling).
unsigned PartialOptSizeThreshold
The cost threshold for the unrolled loop when optimizing for size, like OptSizeThreshold,...
bool UnrollVectorizedLoop
Disable runtime unrolling by default for vectorized loops.
unsigned DefaultUnrollRuntimeCount
Default unroll count for loops with run-time trip count.
unsigned MaxPercentThresholdBoost
If complete unrolling will reduce the cost of the loop, we will boost the Threshold by a certain perc...
bool RuntimeUnrollMultiExit
Allow runtime unrolling multi-exit loops.
unsigned SCEVExpansionBudget
Don't allow runtime unrolling if expanding the trip count takes more than SCEVExpansionBudget.
bool AddAdditionalAccumulators
Allow unrolling to add parallel reduction phis.
unsigned UnrollAndJamInnerLoopThreshold
Threshold for unroll and jam, for inner loop size.
unsigned MaxIterationsCountToAnalyze
Don't allow loop unrolling to simulate more than this number of iterations when checking full unroll ...
bool AllowRemainder
Allow generation of a loop remainder (extra iterations after unroll).
bool UnrollAndJam
Allow unroll and jam. Used to enable unroll and jam for the target.
bool UnrollRemainder
Allow unrolling of all the iterations of the runtime loop remainder.
unsigned FullUnrollMaxCount
Set the maximum unrolling factor for full unrolling.
unsigned PartialThreshold
The cost threshold for the unrolled loop, like Threshold, but used for partial/runtime unrolling (set...
bool Runtime
Allow runtime unrolling (unrolling of loops to expand the size of the loop body even when the number ...
bool Partial
Allow partial unrolling (unrolling of loops to expand the size of the loop body, not only to eliminat...
unsigned OptSizeThreshold
The cost threshold for the unrolled loop when optimizing for size (set to UINT_MAX to disable).
bool AllowExpensiveTripCount
Allow emitting expensive instructions (such as divisions) when computing the trip count of a loop for...
unsigned MaxUpperBound
Set the maximum upper bound of trip count.
VPLegalization(VPTransform EVLParamStrategy, VPTransform OpStrategy)