LLVM 24.0.0git
VectorUtils.h
Go to the documentation of this file.
1//===- llvm/Analysis/VectorUtils.h - Vector utilities -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines some vectorizer utilities.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_ANALYSIS_VECTORUTILS_H
14#define LLVM_ANALYSIS_VECTORUTILS_H
15
16#include "llvm/ADT/MapVector.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/Sequence.h"
21#include "llvm/IR/Module.h"
26
27namespace llvm {
29class IntrinsicInst;
30
31/// The Vector Function Database.
32///
33/// Helper class used to find the vector functions associated to a
34/// scalar CallInst.
36 /// The Module of the CallInst CI.
37 const Module *M;
38 /// The CallInst instance being queried for scalar to vector mappings.
39 const CallInst &CI;
40 /// List of vector functions descriptors associated to the call
41 /// instruction.
42 const SmallVector<VFInfo, 8> ScalarToVectorMappings;
43
44 /// Retrieve the scalar-to-vector mappings associated to the rule of
45 /// a vector Function ABI.
46 static void getVFABIMappings(const CallInst &CI,
47 SmallVectorImpl<VFInfo> &Mappings) {
48 if (!CI.getCalledFunction())
49 return;
50
51 const StringRef ScalarName = CI.getCalledFunction()->getName();
52
53 SmallVector<std::string, 8> ListOfStrings;
54 // The check for the vector-function-abi-variant attribute is done when
55 // retrieving the vector variant names here.
56 VFABI::getVectorVariantNames(CI, ListOfStrings);
57 if (ListOfStrings.empty())
58 return;
59 for (const auto &MangledName : ListOfStrings) {
60 const std::optional<VFInfo> Shape =
61 VFABI::tryDemangleForVFABI(MangledName, CI.getFunctionType());
62 // A match is found via scalar and vector names, and also by
63 // ensuring that the variant described in the attribute has a
64 // corresponding definition or declaration of the vector
65 // function in the Module M.
66 if (Shape && (Shape->ScalarName == ScalarName)) {
67 assert(CI.getModule()->getFunction(Shape->VectorName) &&
68 "Vector function is missing.");
69 Mappings.push_back(*Shape);
70 }
71 }
72 }
73
74public:
75 /// Retrieve all the VFInfo instances associated to the CallInst CI.
78
79 // Get mappings from the Vector Function ABI variants.
80 getVFABIMappings(CI, Ret);
81
82 // Other non-VFABI variants should be retrieved here.
83
84 return Ret;
85 }
86
87 static bool hasMaskedVariant(const CallInst &CI,
88 std::optional<ElementCount> VF = std::nullopt) {
89 // Check whether we have at least one masked vector version of a scalar
90 // function. If no VF is specified then we check for any masked variant,
91 // otherwise we look for one that matches the supplied VF.
92 auto Mappings = VFDatabase::getMappings(CI);
93 for (VFInfo Info : Mappings)
94 if (!VF || Info.Shape.VF == *VF)
95 if (Info.isMasked())
96 return true;
97
98 return false;
99 }
100
101 /// Constructor, requires a CallInst instance.
103 : M(CI.getModule()), CI(CI),
104 ScalarToVectorMappings(VFDatabase::getMappings(CI)) {}
105
106 /// \defgroup VFDatabase query interface.
107 ///
108 /// @{
109 /// Retrieve the Function with VFShape \p Shape.
111 if (Shape == VFShape::getScalarShape(CI.getFunctionType()))
112 return CI.getCalledFunction();
113
114 for (const auto &Info : ScalarToVectorMappings)
115 if (Info.Shape == Shape)
116 return M->getFunction(Info.VectorName);
117
118 return nullptr;
119 }
120 /// @}
121};
122
123struct AnalysisOptions;
124template <typename T> class ArrayRef;
125class DemandedBits;
126template <typename InstTy> class InterleaveGroup;
127class IRBuilderBase;
128class Loop;
129class TargetTransformInfo;
130class Value;
131
132namespace Intrinsic {
133typedef unsigned ID;
134}
135
136/// The kind of update operation performed on the buckets of a histogram, i.e.
137/// buckets[indices[i]] = UpdateOp(buckets[indices[i]], Val).
145
146/// Identify if the intrinsic is trivially vectorizable.
147/// This method returns true if the intrinsic's argument types are all scalars
148/// for the scalar form of the intrinsic and all vectors (or scalars handled by
149/// isVectorIntrinsicWithScalarOpAtArg) for the vector form of the intrinsic.
150///
151/// Note: isTriviallyVectorizable implies isTriviallyScalarizable.
153
154/// Identify if the intrinsic is trivially scalarizable.
155/// This method returns true following the same predicates of
156/// isTriviallyVectorizable.
157
158/// Note: There are intrinsics where implementing vectorization for the
159/// intrinsic is redundant, but we want to implement scalarization of the
160/// vector. To prevent the requirement that an intrinsic also implements
161/// vectorization we provide this separate function.
163
164/// Identifies if the vector form of the intrinsic has a scalar operand.
165/// \p TTI is used to consider target specific intrinsics, if no target specific
166/// intrinsics will be considered then it is appropriate to pass in nullptr.
167LLVM_ABI bool
168isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx,
169 const TargetTransformInfo *TTI);
170
171/// Identifies if the vector form of the intrinsic is overloaded on the type of
172/// the operand at index \p OpdIdx, or on the return type if \p OpdIdx is -1.
173/// \p TTI is used to consider target specific intrinsics, if no target specific
174/// intrinsics will be considered then it is appropriate to pass in nullptr.
175LLVM_ABI bool
177 const TargetTransformInfo *TTI);
178
179/// Returns the argument index of the pointer parameter for the vector memory
180/// intrinsic \p ID, or `std::nullopt` if the intrinsic does not have a pointer
181/// operand.
182LLVM_ABI std::optional<unsigned>
184
185/// Returns the argument index of the data value of the vector store intrinsic
186/// \p ID, or `std::nullopt` if the intrinsic does not have a data operand.
187LLVM_ABI std::optional<unsigned>
189
190/// Returns the argument index of the mask for the vector intrinsic \p ID, or
191/// `std::nullopt` if the intrinsic does not have a mask operand.
192LLVM_ABI std::optional<unsigned> getVectorIntrinsicMaskArgIdx(Intrinsic::ID ID);
193
194/// Identifies if the vector form of the intrinsic that returns a struct is
195/// overloaded at the struct element index \p RetIdx. /// \p TTI is used to
196/// consider target specific intrinsics, if no target specific intrinsics
197/// will be considered then it is appropriate to pass in nullptr.
199 Intrinsic::ID ID, int RetIdx, const TargetTransformInfo *TTI);
200
201/// Returns intrinsic ID for call.
202/// For the input call instruction it finds mapping intrinsic and returns
203/// its intrinsic ID, in case it does not found it return not_intrinsic.
205getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI);
206
207/// Returns true if the vector llvm.sincos with \p ScalarTy elements and \p EC
208/// element count has no vector library mapping, but both llvm.sin and llvm.cos
209/// have, so it is split into the separate vector sin and cos library calls.
210LLVM_ABI bool shouldSplitVectorSinCos(const TargetLibraryInfo &TLI,
211 Type *ScalarTy, ElementCount EC,
212 Module *M);
213
214/// Returns the corresponding factor of llvm.vector.interleaveN intrinsics.
216
217/// Returns the corresponding factor of llvm.vector.deinterleaveN intrinsics.
219
220/// Given a deinterleaveN intrinsic, return the (narrow) vector type of each
221/// factor.
223
224/// Given a vector and an element number, see if the scalar value is
225/// already around as a register, for example if it were inserted then extracted
226/// from the vector.
227LLVM_ABI Value *findScalarElement(Value *V, unsigned EltNo);
228
229/// If all non-negative \p Mask elements are the same value, return that value.
230/// If all elements are negative (undefined) or \p Mask contains different
231/// non-negative values, return -1.
232LLVM_ABI int getSplatIndex(ArrayRef<int> Mask);
233
234/// Get splat value if the input is a splat vector or return nullptr.
235/// The value may be extracted from a splat constants vector or from
236/// a sequence of instructions that broadcast a single value into a vector.
237LLVM_ABI Value *getSplatValue(const Value *V);
238
239/// Return true if each element of the vector value \p V is poisoned or equal to
240/// every other non-poisoned element. If an index element is specified, either
241/// every element of the vector is poisoned or the element at that index is not
242/// poisoned and equal to every other non-poisoned element.
243/// This may be more powerful than the related getSplatValue() because it is
244/// not limited by finding a scalar source value to a splatted vector.
245LLVM_ABI bool isSplatValue(const Value *V, int Index = -1, unsigned Depth = 0);
246
247/// Transform a shuffle mask's output demanded element mask into demanded
248/// element masks for the 2 operands, returns false if the mask isn't valid.
249/// Both \p DemandedLHS and \p DemandedRHS are initialised to [SrcWidth].
250/// \p AllowUndefElts permits "-1" indices to be treated as undef.
251LLVM_ABI bool getShuffleDemandedElts(int SrcWidth, ArrayRef<int> Mask,
252 const APInt &DemandedElts,
253 APInt &DemandedLHS, APInt &DemandedRHS,
254 bool AllowUndefElts = false);
255
256/// Does this shuffle mask represent either one slide shuffle or a pair of
257/// two slide shuffles, combined with a select on some constant vector mask?
258/// A slide is a shuffle mask which shifts some set of elements up or down
259/// the vector, with all other elements being undefined. An identity shuffle
260/// will be matched a slide by 0. The output parameter provides the source
261/// (-1 means no source), and slide direction for each slide.
262LLVM_ABI bool isMaskedSlidePair(ArrayRef<int> Mask, int NumElts,
263 std::array<std::pair<int, int>, 2> &SrcInfo);
264
265/// Replace each shuffle mask index with the scaled sequential indices for an
266/// equivalent mask of narrowed elements. Mask elements that are less than 0
267/// (sentinel values) are repeated in the output mask.
268///
269/// Example with Scale = 4:
270/// <4 x i32> <3, 2, 0, -1> -->
271/// <16 x i8> <12, 13, 14, 15, 8, 9, 10, 11, 0, 1, 2, 3, -1, -1, -1, -1>
272///
273/// This is the reverse process of widening shuffle mask elements, but it always
274/// succeeds because the indexes can always be multiplied (scaled up) to map to
275/// narrower vector elements.
276LLVM_ABI void narrowShuffleMaskElts(int Scale, ArrayRef<int> Mask,
277 SmallVectorImpl<int> &ScaledMask);
278
279/// Try to transform a shuffle mask by replacing elements with the scaled index
280/// for an equivalent mask of widened elements. If all mask elements that would
281/// map to a wider element of the new mask are the same negative number
282/// (sentinel value), that element of the new mask is the same value. If any
283/// element in a given slice is negative and some other element in that slice is
284/// not the same value, return false (partial matches with sentinel values are
285/// not allowed).
286///
287/// Example with Scale = 4:
288/// <16 x i8> <12, 13, 14, 15, 8, 9, 10, 11, 0, 1, 2, 3, -1, -1, -1, -1> -->
289/// <4 x i32> <3, 2, 0, -1>
290///
291/// This is the reverse process of narrowing shuffle mask elements if it
292/// succeeds. This transform is not always possible because indexes may not
293/// divide evenly (scale down) to map to wider vector elements.
294LLVM_ABI bool widenShuffleMaskElts(int Scale, ArrayRef<int> Mask,
295 SmallVectorImpl<int> &ScaledMask);
296
297/// A variant of the previous method which is specialized for Scale=2, and
298/// treats -1 as undef and allows widening when a wider element is partially
299/// undef in the narrow form of the mask. This transformation discards
300/// information about which bytes in the original shuffle were undef.
301LLVM_ABI bool widenShuffleMaskElts(ArrayRef<int> M,
302 SmallVectorImpl<int> &NewMask);
303
304/// Attempt to narrow/widen the \p Mask shuffle mask to the \p NumDstElts target
305/// width. Internally this will call narrowShuffleMaskElts/widenShuffleMaskElts.
306/// This will assert unless NumDstElts is a multiple of Mask.size (or
307/// vice-versa). Returns false on failure, and ScaledMask will be in an
308/// undefined state.
309LLVM_ABI bool scaleShuffleMaskElts(unsigned NumDstElts, ArrayRef<int> Mask,
310 SmallVectorImpl<int> &ScaledMask);
311
312/// Repetitively apply `widenShuffleMaskElts()` for as long as it succeeds,
313/// to get the shuffle mask with widest possible elements.
314LLVM_ABI void getShuffleMaskWithWidestElts(ArrayRef<int> Mask,
315 SmallVectorImpl<int> &ScaledMask);
316
317/// Splits and processes shuffle mask depending on the number of input and
318/// output registers. The function does 2 main things: 1) splits the
319/// source/destination vectors into real registers; 2) do the mask analysis to
320/// identify which real registers are permuted. Then the function processes
321/// resulting registers mask using provided action items. If no input register
322/// is defined, \p NoInputAction action is used. If only 1 input register is
323/// used, \p SingleInputAction is used, otherwise \p ManyInputsAction is used to
324/// process > 2 input registers and masks.
325/// \param Mask Original shuffle mask.
326/// \param NumOfSrcRegs Number of source registers.
327/// \param NumOfDestRegs Number of destination registers.
328/// \param NumOfUsedRegs Number of actually used destination registers.
330 ArrayRef<int> Mask, unsigned NumOfSrcRegs, unsigned NumOfDestRegs,
331 unsigned NumOfUsedRegs, function_ref<void()> NoInputAction,
332 function_ref<void(ArrayRef<int>, unsigned, unsigned)> SingleInputAction,
333 function_ref<void(ArrayRef<int>, unsigned, unsigned, bool)>
334 ManyInputsAction);
335
336/// Compute the demanded elements mask of horizontal binary operations. A
337/// horizontal operation combines two adjacent elements in a vector operand.
338/// This function returns a mask for the elements that correspond to the first
339/// operand of this horizontal combination. For example, for two vectors
340/// [X1, X2, X3, X4] and [Y1, Y2, Y3, Y4], the resulting mask can include the
341/// elements X1, X3, Y1, and Y3. To get the other operands, simply shift the
342/// result of this function to the left by 1.
343///
344/// \param VectorBitWidth the total bit width of the vector
345/// \param DemandedElts the demanded elements mask for the operation
346/// \param DemandedLHS the demanded elements mask for the left operand
347/// \param DemandedRHS the demanded elements mask for the right operand
348LLVM_ABI void getHorizDemandedEltsForFirstOperand(unsigned VectorBitWidth,
349 const APInt &DemandedElts,
350 APInt &DemandedLHS,
351 APInt &DemandedRHS);
352
353/// Compute a map of integer instructions to their minimum legal type
354/// size.
355///
356/// C semantics force sub-int-sized values (e.g. i8, i16) to be promoted to int
357/// type (e.g. i32) whenever arithmetic is performed on them.
358///
359/// For targets with native i8 or i16 operations, usually InstCombine can shrink
360/// the arithmetic type down again. However InstCombine refuses to create
361/// illegal types, so for targets without i8 or i16 registers, the lengthening
362/// and shrinking remains.
363///
364/// Most SIMD ISAs (e.g. NEON) however support vectors of i8 or i16 even when
365/// their scalar equivalents do not, so during vectorization it is important to
366/// remove these lengthens and truncates when deciding the profitability of
367/// vectorization.
368///
369/// This function analyzes the given range of instructions and determines the
370/// minimum type size each can be converted to. It attempts to remove or
371/// minimize type size changes across each def-use chain, so for example in the
372/// following code:
373///
374/// %1 = load i8, i8*
375/// %2 = add i8 %1, 2
376/// %3 = load i16, i16*
377/// %4 = zext i8 %2 to i32
378/// %5 = zext i16 %3 to i32
379/// %6 = add i32 %4, %5
380/// %7 = trunc i32 %6 to i16
381///
382/// Instruction %6 must be done at least in i16, so computeMinimumValueSizes
383/// will return: {%1: 16, %2: 16, %3: 16, %4: 16, %5: 16, %6: 16, %7: 16}.
384///
385/// If the optional TargetTransformInfo is provided, this function tries harder
386/// to do less work by only looking at illegal types.
387LLVM_ABI MapVector<Instruction *, uint64_t>
388computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB,
389 const TargetTransformInfo *TTI = nullptr);
390
391/// Compute the union of two access-group lists.
392///
393/// If the list contains just one access group, it is returned directly. If the
394/// list is empty, returns nullptr.
395LLVM_ABI MDNode *uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2);
396
397/// Compute the access-group list of access groups that @p Inst1 and @p Inst2
398/// are both in. If either instruction does not access memory at all, it is
399/// considered to be in every list.
400///
401/// If the list contains just one access group, it is returned directly. If the
402/// list is empty, returns nullptr.
403LLVM_ABI MDNode *intersectAccessGroups(const Instruction *Inst1,
404 const Instruction *Inst2);
405
406/// Add metadata from \p Inst to \p Metadata, if it can be preserved after
407/// vectorization. It can be preserved after vectorization if the kind is one of
408/// [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,
409/// MD_access_group, MD_mmra].
411 Instruction *Inst,
412 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata);
413
414/// Propagate metadata that can be preserved after vectorization from \p VL to
415/// \p I, retaining only information valid for all sources and the destination.
416/// Return \p I.
417LLVM_ABI Instruction *propagateMetadata(Instruction *I, ArrayRef<Value *> VL);
418
419/// Create a mask that filters the members of an interleave group where there
420/// are gaps.
421///
422/// For example, the mask for \p Group with interleave-factor 3
423/// and \p VF 4, that has only its first member present is:
424///
425/// <1,0,0,1,0,0,1,0,0,1,0,0>
426///
427/// Note: The result is a mask of 0's and 1's, as opposed to the other
428/// create[*]Mask() utilities which create a shuffle mask (mask that
429/// consists of indices).
430LLVM_ABI Constant *
431createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF,
432 const InterleaveGroup<Instruction> &Group);
433
434/// Create a mask with replicated elements.
435///
436/// This function creates a shuffle mask for replicating each of the \p VF
437/// elements in a vector \p ReplicationFactor times. It can be used to
438/// transform a mask of \p VF elements into a mask of
439/// \p VF * \p ReplicationFactor elements used by a predicated
440/// interleaved-group of loads/stores whose Interleaved-factor ==
441/// \p ReplicationFactor.
442///
443/// For example, the mask for \p ReplicationFactor=3 and \p VF=4 is:
444///
445/// <0,0,0,1,1,1,2,2,2,3,3,3>
447createReplicatedMask(unsigned ReplicationFactor, unsigned VF);
448
449/// Create an interleave shuffle mask.
450///
451/// This function creates a shuffle mask for interleaving \p NumVecs vectors of
452/// vectorization factor \p VF into a single wide vector. The mask is of the
453/// form:
454///
455/// <0, VF, VF * 2, ..., VF * (NumVecs - 1), 1, VF + 1, VF * 2 + 1, ...>
456///
457/// For example, the mask for VF = 4 and NumVecs = 2 is:
458///
459/// <0, 4, 1, 5, 2, 6, 3, 7>.
461 unsigned NumVecs);
462
463/// Create a stride shuffle mask.
464///
465/// This function creates a shuffle mask whose elements begin at \p Start and
466/// are incremented by \p Stride. The mask can be used to deinterleave an
467/// interleaved vector into separate vectors of vectorization factor \p VF. The
468/// mask is of the form:
469///
470/// <Start, Start + Stride, ..., Start + Stride * (VF - 1)>
471///
472/// For example, the mask for Start = 0, Stride = 2, and VF = 4 is:
473///
474/// <0, 2, 4, 6>
476createStrideMask(unsigned Start, unsigned Stride, unsigned VF);
477
478/// Create a sequential shuffle mask.
479///
480/// This function creates shuffle mask whose elements are sequential and begin
481/// at \p Start. The mask contains \p NumInts integers and is padded with \p
482/// NumUndefs undef values. The mask is of the form:
483///
484/// <Start, Start + 1, ... Start + NumInts - 1, undef_1, ... undef_NumUndefs>
485///
486/// For example, the mask for Start = 0, NumInsts = 4, and NumUndefs = 4 is:
487///
488/// <0, 1, 2, 3, undef, undef, undef, undef>
490createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs);
491
492/// Given a shuffle mask for a binary shuffle, create the equivalent shuffle
493/// mask assuming both operands are identical. This assumes that the unary
494/// shuffle will use elements from operand 0 (operand 1 will be unused).
496 unsigned NumElts);
497
498/// Concatenate a list of vectors.
499///
500/// This function generates code that concatenate the vectors in \p Vecs into a
501/// single large vector. The number of vectors should be greater than one, and
502/// their element types should be the same. The number of elements in the
503/// vectors should also be the same; however, if the last vector has fewer
504/// elements, it will be padded with undefs.
505LLVM_ABI Value *concatenateVectors(IRBuilderBase &Builder,
506 ArrayRef<Value *> Vecs);
507
508/// Given a mask vector of i1, Return true if any of the elements of this
509/// predicate mask are known to be true or undef. That is, return true if at
510/// least one lane can be assumed active.
511LLVM_ABI bool maskContainsAllOneOrUndef(Value *Mask);
512
513/// Given a mask vector of the form <Y x i1>, return an APInt (of bitwidth Y)
514/// for each lane which may be active.
515LLVM_ABI APInt possiblyDemandedEltsInMask(Value *Mask);
516
517/// The group of interleaved loads/stores sharing the same stride and
518/// close to each other.
519///
520/// Each member in this group has an index starting from 0, and the largest
521/// index should be less than interleaved factor, which is equal to the absolute
522/// value of the access's stride.
523///
524/// E.g. An interleaved load group of factor 4:
525/// for (unsigned i = 0; i < 1024; i+=4) {
526/// a = A[i]; // Member of index 0
527/// b = A[i+1]; // Member of index 1
528/// d = A[i+3]; // Member of index 3
529/// ...
530/// }
531///
532/// An interleaved store group of factor 4:
533/// for (unsigned i = 0; i < 1024; i+=4) {
534/// ...
535/// A[i] = a; // Member of index 0
536/// A[i+1] = b; // Member of index 1
537/// A[i+2] = c; // Member of index 2
538/// A[i+3] = d; // Member of index 3
539/// }
540///
541/// Note: the interleaved load group could have gaps (missing members), but
542/// the interleaved store group doesn't allow gaps.
543template <typename InstTy> class InterleaveGroup {
544public:
545 InterleaveGroup(uint32_t Factor, bool Reverse, Align Alignment)
546 : Factor(Factor), Reverse(Reverse), Alignment(Alignment),
547 InsertPos(nullptr) {}
548
549 InterleaveGroup(InstTy *Instr, int32_t Stride, Align Alignment)
550 : Alignment(Alignment), InsertPos(Instr) {
551 Factor = std::abs(Stride);
552 assert(Factor > 1 && "Invalid interleave factor");
553
554 Reverse = Stride < 0;
555 Members[0] = Instr;
556 }
557
558 bool isReverse() const { return Reverse; }
559 uint32_t getFactor() const { return Factor; }
560 Align getAlign() const { return Alignment; }
561 uint32_t getNumMembers() const { return Members.size(); }
562
563 /// Try to insert a new member \p Instr with index \p Index and
564 /// alignment \p NewAlign. The index is related to the leader and it could be
565 /// negative if it is the new leader.
566 ///
567 /// \returns false if the instruction doesn't belong to the group.
568 bool insertMember(InstTy *Instr, int32_t Index, Align NewAlign) {
569 // Make sure the key fits in an int32_t.
570 std::optional<int32_t> MaybeKey = checkedAdd(Index, SmallestKey);
571 if (!MaybeKey)
572 return false;
573 int32_t Key = *MaybeKey;
574
575 // Skip if there is already a member with the same index.
576 if (Members.contains(Key))
577 return false;
578
579 if (Key > LargestKey) {
580 // The largest index is always less than the interleave factor.
581 if (Index >= static_cast<int32_t>(Factor))
582 return false;
583
584 LargestKey = Key;
585 } else if (Key < SmallestKey) {
586
587 // Make sure the largest index fits in an int32_t.
588 std::optional<int32_t> MaybeLargestIndex = checkedSub(LargestKey, Key);
589 if (!MaybeLargestIndex)
590 return false;
591
592 // The largest index is always less than the interleave factor.
593 if (*MaybeLargestIndex >= static_cast<int64_t>(Factor))
594 return false;
595
596 SmallestKey = Key;
597 }
598
599 // It's always safe to select the minimum alignment.
600 Alignment = std::min(Alignment, NewAlign);
601 Members[Key] = Instr;
602 return true;
603 }
604
605 /// Get the member with the given index \p Index
606 ///
607 /// \returns nullptr if contains no such member.
608 InstTy *getMember(uint32_t Index) const {
609 int32_t Key = SmallestKey + Index;
610 return Members.lookup(Key);
611 }
612
613 /// Return an iterator range over the non-null members of this group, in
614 /// index order.
615 auto members() const {
616 return make_filter_range(
617 map_range(seq<uint32_t>(0, Factor),
618 [this](uint32_t I) { return getMember(I); }),
619 [](InstTy *I) { return I != nullptr; });
620 }
621
622 /// Get the index for the given member. Unlike the key in the member
623 /// map, the index starts from 0.
624 uint32_t getIndex(const InstTy *Instr) const {
625 for (auto I : Members) {
626 if (I.second == Instr)
627 return I.first - SmallestKey;
628 }
629
630 llvm_unreachable("InterleaveGroup contains no such member");
631 }
632
633 InstTy *getInsertPos() const { return InsertPos; }
634 void setInsertPos(InstTy *Inst) { InsertPos = Inst; }
635
636 /// Add metadata (e.g. alias info) from the instructions in this group to \p
637 /// NewInst.
638 ///
639 /// FIXME: this function currently does not add noalias metadata a'la
640 /// addNewMedata. To do that we need to compute the intersection of the
641 /// noalias info from all members.
642 void addMetadata(InstTy *NewInst) const;
643
644 /// Returns true if this Group requires a scalar iteration to handle gaps.
646 // If the last member of the Group exists, then a scalar epilog is not
647 // needed for this group.
648 if (getMember(getFactor() - 1))
649 return false;
650
651 // We have a group with gaps. It therefore can't be a reversed access,
652 // because such groups get invalidated (TODO).
653 assert(!isReverse() && "Group should have been invalidated");
654
655 // This is a group of loads, with gaps, and without a last-member
656 return true;
657 }
658
659 /// Return true if this group is full, i.e. it has no gaps.
660 bool isFull() const { return getNumMembers() == getFactor(); }
661
662private:
663 uint32_t Factor; // Interleave Factor.
664 bool Reverse;
665 Align Alignment;
667 int32_t SmallestKey = 0;
668 int32_t LargestKey = 0;
669
670 // To avoid breaking dependences, vectorized instructions of an interleave
671 // group should be inserted at either the first load or the last store in
672 // program order.
673 //
674 // E.g. %even = load i32 // Insert Position
675 // %add = add i32 %even // Use of %even
676 // %odd = load i32
677 //
678 // store i32 %even
679 // %odd = add i32 // Def of %odd
680 // store i32 %odd // Insert Position
681 InstTy *InsertPos;
682};
683
684/// Drive the analysis of interleaved memory accesses in the loop.
685///
686/// Use this class to analyze interleaved accesses only when we can vectorize
687/// a loop. Otherwise it's meaningless to do analysis as the vectorization
688/// on interleaved accesses is unsafe.
689///
690/// The analysis collects interleave groups and records the relationships
691/// between the member and the group in a map.
693public:
695 DominatorTree *DT, LoopInfo *LI,
696 const LoopAccessInfo *LAI, bool OptForSize);
697
699
700 /// Analyze the interleaved accesses and collect them in interleave
701 /// groups. Substitute symbolic strides using \p Strides.
702 /// Consider also predicated loads/stores in the analysis if
703 /// \p EnableMaskedInterleavedGroup is true.
704 LLVM_ABI void analyzeInterleaving(bool EnableMaskedInterleavedGroup);
705
706 /// Invalidate groups, e.g., in case all blocks in loop will be predicated
707 /// contrary to original assumption. Although we currently prevent group
708 /// formation for predicated accesses, we may be able to relax this limitation
709 /// in the future once we handle more complicated blocks. Returns true if any
710 /// groups were invalidated.
712 if (InterleaveGroups.empty()) {
713 assert(
714 !RequiresScalarEpilogue &&
715 "RequiresScalarEpilog should not be set without interleave groups");
716 return false;
717 }
718
719 InterleaveGroupMap.clear();
720 for (auto *Ptr : InterleaveGroups)
721 delete Ptr;
722 InterleaveGroups.clear();
723 RequiresScalarEpilogue = false;
724 return true;
725 }
726
727 /// Check if \p Instr belongs to any interleave group.
728 bool isInterleaved(Instruction *Instr) const {
729 return InterleaveGroupMap.contains(Instr);
730 }
731
732 /// Get the interleave group that \p Instr belongs to.
733 ///
734 /// \returns nullptr if doesn't have such group.
736 getInterleaveGroup(const Instruction *Instr) const {
737 return InterleaveGroupMap.lookup(Instr);
738 }
739
742 return make_range(InterleaveGroups.begin(), InterleaveGroups.end());
743 }
744
745 /// Returns true if an interleaved group that may access memory
746 /// out-of-bounds requires a scalar epilogue iteration for correctness.
747 bool requiresScalarEpilogue() const { return RequiresScalarEpilogue; }
748
749 /// Invalidate groups that require a scalar epilogue (due to gaps). This can
750 /// happen when optimizing for size forbids a scalar epilogue, and the gap
751 /// cannot be filtered by masking the load/store.
753
754 /// Returns true if we have any interleave groups.
755 bool hasGroups() const { return !InterleaveGroups.empty(); }
756
757private:
758 const AnalysisOptions &Opts;
759
760 /// A wrapper around ScalarEvolution, used to add runtime SCEV checks.
761 /// Simplifies SCEV expressions in the context of existing SCEV assumptions.
762 /// The interleaved access analysis can also add new predicates (for example
763 /// by versioning strides of pointers).
765
766 Loop *TheLoop;
767 DominatorTree *DT;
768 LoopInfo *LI;
769 const LoopAccessInfo *LAI;
770 bool OptForSize;
771
772 /// True if the loop may contain non-reversed interleaved groups with
773 /// out-of-bounds accesses. We ensure we don't speculatively access memory
774 /// out-of-bounds by executing at least one scalar epilogue iteration.
775 bool RequiresScalarEpilogue = false;
776
777 /// Holds the relationships between the members and the interleave group.
779
780 SmallPtrSet<InterleaveGroup<Instruction> *, 4> InterleaveGroups;
781
782 /// Holds dependences among the memory accesses in the loop. It maps a source
783 /// access to a set of dependent sink accesses.
785
786 /// The descriptor for a strided memory access.
787 struct StrideDescriptor {
788 StrideDescriptor() = default;
789 StrideDescriptor(int64_t Stride, const SCEV *Scev, uint64_t Size,
790 Align Alignment)
791 : Stride(Stride), Scev(Scev), Size(Size), Alignment(Alignment) {}
792
793 // The access's stride. It is negative for a reverse access.
794 int64_t Stride = 0;
795
796 // The scalar expression of this access.
797 const SCEV *Scev = nullptr;
798
799 // The size of the memory object.
800 uint64_t Size = 0;
801
802 // The alignment of this access.
803 Align Alignment;
804 };
805
806 /// A type for holding instructions and their stride descriptors.
807 using StrideEntry = std::pair<Instruction *, StrideDescriptor>;
808
809 /// Create a new interleave group with the given instruction \p Instr,
810 /// stride \p Stride and alignment \p Align.
811 ///
812 /// \returns the newly created interleave group.
813 InterleaveGroup<Instruction> *
814 createInterleaveGroup(Instruction *Instr, int Stride, Align Alignment) {
815 auto [It, Inserted] = InterleaveGroupMap.try_emplace(Instr);
816 assert(Inserted && "Already in an interleaved access group");
817 It->second = new InterleaveGroup<Instruction>(Instr, Stride, Alignment);
818 InterleaveGroups.insert(It->second);
819 return It->second;
820 }
821
822 /// Release the group and remove all the relationships.
823 void releaseGroup(InterleaveGroup<Instruction> *Group) {
824 InterleaveGroups.erase(Group);
825 releaseGroupWithoutRemovingFromSet(Group);
826 }
827
828 /// Do everything necessary to release the group, apart from removing it from
829 /// the InterleaveGroups set.
830 void releaseGroupWithoutRemovingFromSet(InterleaveGroup<Instruction> *Group) {
831 for (unsigned i = 0; i < Group->getFactor(); i++)
832 if (Instruction *Member = Group->getMember(i))
833 InterleaveGroupMap.erase(Member);
834
835 delete Group;
836 }
837
838 /// Collect all the accesses with a constant stride in program order. Any
839 /// SCEV predicates needed to compute the strides are added to \p
840 /// Predicates if it is not nullptr.
841 void collectConstStrideAccesses(
842 MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo,
843 const SymbolicStrideMap &Strides,
844 SmallVectorImpl<const SCEVPredicate *> *Predicates);
845
846 /// Returns true if \p Stride is allowed in an interleaved group.
847 LLVM_ABI bool isStrided(int Stride) const;
848
849 /// Returns true if \p BB is a predicated block.
850 bool isPredicated(BasicBlock *BB) const {
851 return LoopAccessInfo::blockNeedsPredication(BB, TheLoop, DT);
852 }
853
854 /// Returns true if LoopAccessInfo can be used for dependence queries.
855 bool areDependencesValid() const {
856 return LAI && LAI->getDepChecker().getDependences();
857 }
858
859 /// Returns true if memory accesses \p A and \p B can be reordered, if
860 /// necessary, when constructing interleaved groups.
861 ///
862 /// \p A must precede \p B in program order. We return false if reordering is
863 /// not necessary or is prevented because \p A and \p B may be dependent.
864 bool canReorderMemAccessesForInterleavedGroups(StrideEntry *A,
865 StrideEntry *B) const {
866 // Code motion for interleaved accesses can potentially hoist strided loads
867 // and sink strided stores. The code below checks the legality of the
868 // following two conditions:
869 //
870 // 1. Potentially moving a strided load (B) before any store (A) that
871 // precedes B, or
872 //
873 // 2. Potentially moving a strided store (A) after any load or store (B)
874 // that A precedes.
875 //
876 // It's legal to reorder A and B if we know there isn't a dependence from A
877 // to B. Note that this determination is conservative since some
878 // dependences could potentially be reordered safely.
879
880 // A is potentially the source of a dependence.
881 auto *Src = A->first;
882 auto SrcDes = A->second;
883
884 // B is potentially the sink of a dependence.
885 auto *Sink = B->first;
886 auto SinkDes = B->second;
887
888 // Code motion for interleaved accesses can't violate WAR dependences.
889 // Thus, reordering is legal if the source isn't a write.
890 if (!Src->mayWriteToMemory())
891 return true;
892
893 // At least one of the accesses must be strided.
894 if (!isStrided(SrcDes.Stride) && !isStrided(SinkDes.Stride))
895 return true;
896
897 // If dependence information is not available from LoopAccessInfo,
898 // conservatively assume the instructions can't be reordered.
899 if (!areDependencesValid())
900 return false;
901
902 // If we know there is a dependence from source to sink, assume the
903 // instructions can't be reordered. Otherwise, reordering is legal.
904 return !Dependences.contains(Src) || !Dependences.lookup(Src).count(Sink);
905 }
906
907 /// Collect the dependences from LoopAccessInfo.
908 ///
909 /// We process the dependences once during the interleaved access analysis to
910 /// enable constant-time dependence queries.
911 void collectDependences() {
912 if (!areDependencesValid())
913 return;
914 const auto &DepChecker = LAI->getDepChecker();
915 auto *Deps = DepChecker.getDependences();
916 for (auto Dep : *Deps)
917 Dependences[Dep.getSource(DepChecker)].insert(
918 Dep.getDestination(DepChecker));
919 }
920};
921
922} // llvm namespace
923
924#endif
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_ABI
Definition Compiler.h:211
dxil translate DXIL Translate Metadata
Module.h This file contains the declarations for the Module class.
#define I(x, y, z)
Definition MD5.cpp:57
Machine Check Debug Module
This file implements a map that provides insertion order iteration.
This file contains some templates that are useful if you are working with the STL at all.
Provides some synthesis utilities to produce sequences of values.
This file defines the SmallVector class.
This class represents a function call, abstracting a target machine's calling convention.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
const Function & getFunction() const
Definition Function.h:167
The group of interleaved loads/stores sharing the same stride and close to each other.
auto members() const
Return an iterator range over the non-null members of this group, in index order.
bool requiresScalarEpilogue() const
Returns true if this Group requires a scalar iteration to handle gaps.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
InterleaveGroup(uint32_t Factor, bool Reverse, Align Alignment)
bool isFull() const
Return true if this group is full, i.e. it has no gaps.
uint32_t getIndex(const InstTy *Instr) const
Get the index for the given member.
void setInsertPos(InstTy *Inst)
bool isReverse() const
InstTy * getInsertPos() const
void addMetadata(InstTy *NewInst) const
Add metadata (e.g.
Align getAlign() const
InterleaveGroup(InstTy *Instr, int32_t Stride, Align Alignment)
bool insertMember(InstTy *Instr, int32_t Index, Align NewAlign)
Try to insert a new member Instr with index Index and alignment NewAlign.
uint32_t getNumMembers() const
InterleaveGroup< Instruction > * getInterleaveGroup(const Instruction *Instr) const
Get the interleave group that Instr belongs to.
bool requiresScalarEpilogue() const
Returns true if an interleaved group that may access memory out-of-bounds requires a scalar epilogue ...
bool hasGroups() const
Returns true if we have any interleave groups.
bool isInterleaved(Instruction *Instr) const
Check if Instr belongs to any interleave group.
bool invalidateGroups()
Invalidate groups, e.g., in case all blocks in loop will be predicated contrary to original assumptio...
iterator_range< SmallPtrSetIterator< llvm::InterleaveGroup< Instruction > * > > getInterleaveGroups()
LLVM_ABI InterleavedAccessInfo(PredicatedScalarEvolution &PSE, Loop *L, DominatorTree *DT, LoopInfo *LI, const LoopAccessInfo *LAI, bool OptForSize)
LLVM_ABI void analyzeInterleaving(bool EnableMaskedInterleavedGroup)
Analyze the interleaved accesses and collect them in interleave groups.
LLVM_ABI void invalidateGroupsRequiringScalarEpilogue()
Invalidate groups that require a scalar epilogue (due to gaps).
A wrapper class for inspecting calls to intrinsic functions.
Drive the analysis of memory accesses in the loop.
static LLVM_ABI bool blockNeedsPredication(const BasicBlock *BB, const Loop *TheLoop, const DominatorTree *DT)
Return true if the block BB needs to be predicated in order for the loop to be vectorized.
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
This class represents an analyzed expression in the program.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
Provides information about what library functions are available for the current target.
VFDatabase(CallInst &CI)
Constructor, requires a CallInst instance.
static bool hasMaskedVariant(const CallInst &CI, std::optional< ElementCount > VF=std::nullopt)
Definition VectorUtils.h:87
static SmallVector< VFInfo, 8 > getMappings(const CallInst &CI)
Retrieve all the VFInfo instances associated to the CallInst CI.
Definition VectorUtils.h:76
A range adaptor for a pair of iterators.
Function * getVectorizedFunction(const VFShape &Shape) const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_ABI std::optional< VFInfo > tryDemangleForVFABI(StringRef MangledName, const FunctionType *FTy)
Function to construct a VFInfo out of a mangled names in the following format:
LLVM_ABI void getVectorVariantNames(const CallInst &CI, SmallVectorImpl< std::string > &VariantMappings)
Populates a set of strings representing the Vector Function ABI variants associated to the CallInst C...
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
LLVM_ABI APInt possiblyDemandedEltsInMask(Value *Mask)
Given a mask vector of the form <Y x i1>, return an APInt (of bitwidth Y) for each lane which may be ...
LLVM_ABI std::optional< unsigned > getVectorStoreIntrinsicDataArgIdx(Intrinsic::ID ID)
Returns the argument index of the data value of the vector store intrinsic ID, or std::nullopt if the...
LLVM_ABI llvm::SmallVector< int, 16 > createUnaryMask(ArrayRef< int > Mask, unsigned NumElts)
Given a shuffle mask for a binary shuffle, create the equivalent shuffle mask assuming both operands ...
LLVM_ABI void getMetadataToPropagate(Instruction *Inst, SmallVectorImpl< std::pair< unsigned, MDNode * > > &Metadata)
Add metadata from Inst to Metadata, if it can be preserved after vectorization.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
HistogramUpdateKind
The kind of update operation performed on the buckets of a histogram, i.e.
LLVM_ABI Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
LLVM_ABI bool widenShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Try to transform a shuffle mask by replacing elements with the scaled index for an equivalent mask of...
LLVM_ABI Instruction * propagateMetadata(Instruction *I, ArrayRef< Value * > VL)
Propagate metadata that can be preserved after vectorization from VL to I, retaining only information...
LLVM_ABI Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
auto map_range(ContainerTy &&C, FuncTy F)
Return a range that applies F to the elements of C.
Definition STLExtras.h:366
LLVM_ABI MDNode * intersectAccessGroups(const Instruction *Inst1, const Instruction *Inst2)
Compute the access-group list of access groups that Inst1 and Inst2 are both in.
LLVM_ABI bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
LLVM_ABI bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
LLVM_ABI Constant * createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, const InterleaveGroup< Instruction > &Group)
Create a mask that filters the members of an interleave group where there are gaps.
LLVM_ABI llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
LLVM_ABI void getHorizDemandedEltsForFirstOperand(unsigned VectorBitWidth, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS)
Compute the demanded elements mask of horizontal binary operations.
LLVM_ABI llvm::SmallVector< int, 16 > createReplicatedMask(unsigned ReplicationFactor, unsigned VF)
Create a mask with replicated elements.
DenseMap< Value *, const SCEVUnknown * > SymbolicStrideMap
Maps a pointer to its symbolic (non-constant) stride.
LLVM_ABI unsigned getDeinterleaveIntrinsicFactor(Intrinsic::ID ID)
Returns the corresponding factor of llvm.vector.deinterleaveN intrinsics.
LLVM_ABI unsigned getInterleaveIntrinsicFactor(Intrinsic::ID ID)
Returns the corresponding factor of llvm.vector.interleaveN intrinsics.
iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)
Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...
Definition STLExtras.h:552
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI bool isTriviallyScalarizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially scalarizable.
LLVM_ABI bool isVectorIntrinsicWithStructReturnOverloadAtField(Intrinsic::ID ID, int RetIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic that returns a struct is overloaded at the struct elem...
TargetTransformInfo TTI
LLVM_ABI void narrowShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Replace each shuffle mask index with the scaled sequential indices for an equivalent mask of narrowed...
LLVM_ABI bool isMaskedSlidePair(ArrayRef< int > Mask, int NumElts, std::array< std::pair< int, int >, 2 > &SrcInfo)
Does this shuffle mask represent either one slide shuffle or a pair of two slide shuffles,...
LLVM_ABI VectorType * getDeinterleavedVectorType(IntrinsicInst *DI)
Given a deinterleaveN intrinsic, return the (narrow) vector type of each factor.
LLVM_ABI llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
LLVM_ABI std::optional< unsigned > getVectorMemoryIntrinsicPointerArgIdx(Intrinsic::ID ID)
Returns the argument index of the pointer parameter for the vector memory intrinsic ID,...
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
@ UMax
Unsigned integer max implemented in terms of select(cmp()).
LLVM_ABI bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic has a scalar operand.
LLVM_ABI Value * findScalarElement(Value *V, unsigned EltNo)
Given a vector and an element number, see if the scalar value is already around as a register,...
LLVM_ABI MDNode * uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2)
Compute the union of two access-group lists.
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI std::optional< unsigned > getVectorIntrinsicMaskArgIdx(Intrinsic::ID ID)
Returns the argument index of the mask for the vector intrinsic ID, or std::nullopt if the intrinsic ...
std::enable_if_t< std::is_signed_v< T >, std::optional< T > > checkedSub(T LHS, T RHS)
Subtract two signed integers LHS and RHS.
LLVM_ABI bool shouldSplitVectorSinCos(const TargetLibraryInfo &TLI, Type *ScalarTy, ElementCount EC, Module *M)
Returns true if the vector llvm.sincos with ScalarTy elements and EC element count has no vector libr...
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:341
LLVM_ABI void getShuffleMaskWithWidestElts(ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Repetitively apply widenShuffleMaskElts() for as long as it succeeds, to get the shuffle mask with wi...
std::enable_if_t< std::is_signed_v< T >, std::optional< T > > checkedAdd(T LHS, T RHS)
Add two signed integers LHS and RHS.
LLVM_ABI void processShuffleMasks(ArrayRef< int > Mask, unsigned NumOfSrcRegs, unsigned NumOfDestRegs, unsigned NumOfUsedRegs, function_ref< void()> NoInputAction, function_ref< void(ArrayRef< int >, unsigned, unsigned)> SingleInputAction, function_ref< void(ArrayRef< int >, unsigned, unsigned, bool)> ManyInputsAction)
Splits and processes shuffle mask depending on the number of input and output registers.
LLVM_ABI bool maskContainsAllOneOrUndef(Value *Mask)
Given a mask vector of i1, Return true if any of the elements of this predicate mask are known to be ...
LLVM_ABI bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
LLVM_ABI llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
LLVM_ABI bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
LLVM_ABI MapVector< Instruction *, uint64_t > computeMinimumValueSizes(ArrayRef< BasicBlock * > Blocks, DemandedBits &DB, const TargetTransformInfo *TTI=nullptr)
Compute a map of integer instructions to their minimum legal type size.
LLVM_ABI bool scaleShuffleMaskElts(unsigned NumDstElts, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Attempt to narrow/widen the Mask shuffle mask to the NumDstElts target width.
LLVM_ABI int getSplatIndex(ArrayRef< int > Mask)
If all non-negative Mask elements are the same value, return that value.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Holds the VFShape for a specific scalar to vector function mapping.
Contains the information about the kind of vectorization available.
static VFShape getScalarShape(const FunctionType *FTy)
Retrieve the VFShape that can be used to map a scalar function to itself, with VF = 1.