LLVM 24.0.0git
AArch64TargetTransformInfo.cpp
Go to the documentation of this file.
1//===-- AArch64TargetTransformInfo.cpp - AArch64 specific TTI -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
10#include "AArch64ExpandImm.h"
14#include "llvm/ADT/DenseMap.h"
15#include "llvm/ADT/bit.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsAArch64.h"
27#include "llvm/Support/Debug.h"
32#include <algorithm>
33#include <optional>
34using namespace llvm;
35using namespace llvm::PatternMatch;
36
37#define DEBUG_TYPE "aarch64tti"
38
39namespace {
40class TailFoldingOption {
41 // These bitfields will only ever be set to something non-zero in operator=,
42 // when setting the -sve-tail-folding option. This option should always be of
43 // the form (default|simple|all|disable)[+(Flag1|Flag2|etc)], where here
44 // InitialBits is one of (disabled|all|simple). EnableBits represents
45 // additional flags we're enabling, and DisableBits for those flags we're
46 // disabling. The default flag is tracked in the variable NeedsDefault, since
47 // at the time of setting the option we may not know what the default value
48 // for the CPU is.
52
53 // This value needs to be initialised to true in case the user does not
54 // explicitly set the -sve-tail-folding option.
55 bool NeedsDefault = true;
56
57 void setInitialBits(TailFoldingOpts Bits) { InitialBits = Bits; }
58
59 void setNeedsDefault(bool V) { NeedsDefault = V; }
60
61 void setEnableBit(TailFoldingOpts Bit) {
62 EnableBits |= Bit;
63 DisableBits &= ~Bit;
64 }
65
66 void setDisableBit(TailFoldingOpts Bit) {
67 EnableBits &= ~Bit;
68 DisableBits |= Bit;
69 }
70
71 TailFoldingOpts getBits(TailFoldingOpts DefaultBits) const {
73
74 assert((InitialBits == TailFoldingOpts::Disabled || !NeedsDefault) &&
75 "Initial bits should only include one of "
76 "(disabled|all|simple|default)");
77 Bits = NeedsDefault ? DefaultBits : InitialBits;
78 Bits |= EnableBits;
79 Bits &= ~DisableBits;
80
81 return Bits;
82 }
83
84 void reportError(std::string Opt) {
85 errs() << "invalid argument '" << Opt
86 << "' to -sve-tail-folding=; the option should be of the form\n"
87 " (disabled|all|default|simple)[+(reductions|recurrences"
88 "|reverse|noreductions|norecurrences|noreverse)]\n";
89 report_fatal_error("Unrecognised tail-folding option");
90 }
91
92public:
93
94 void operator=(const std::string &Val) {
95 // If the user explicitly sets -sve-tail-folding= then treat as an error.
96 if (Val.empty()) {
97 reportError("");
98 return;
99 }
100
101 // Since the user is explicitly setting the option we don't automatically
102 // need the default unless they require it.
103 setNeedsDefault(false);
104
105 SmallVector<StringRef, 4> TailFoldTypes;
106 StringRef(Val).split(TailFoldTypes, '+', -1, false);
107
108 unsigned StartIdx = 1;
109 if (TailFoldTypes[0] == "disabled")
110 setInitialBits(TailFoldingOpts::Disabled);
111 else if (TailFoldTypes[0] == "all")
112 setInitialBits(TailFoldingOpts::All);
113 else if (TailFoldTypes[0] == "default")
114 setNeedsDefault(true);
115 else if (TailFoldTypes[0] == "simple")
116 setInitialBits(TailFoldingOpts::Simple);
117 else {
118 StartIdx = 0;
119 setInitialBits(TailFoldingOpts::Disabled);
120 }
121
122 for (unsigned I = StartIdx; I < TailFoldTypes.size(); I++) {
123 if (TailFoldTypes[I] == "reductions")
124 setEnableBit(TailFoldingOpts::Reductions);
125 else if (TailFoldTypes[I] == "recurrences")
126 setEnableBit(TailFoldingOpts::Recurrences);
127 else if (TailFoldTypes[I] == "reverse")
128 setEnableBit(TailFoldingOpts::Reverse);
129 else if (TailFoldTypes[I] == "noreductions")
130 setDisableBit(TailFoldingOpts::Reductions);
131 else if (TailFoldTypes[I] == "norecurrences")
132 setDisableBit(TailFoldingOpts::Recurrences);
133 else if (TailFoldTypes[I] == "noreverse")
134 setDisableBit(TailFoldingOpts::Reverse);
135 else
136 reportError(Val);
137 }
138 }
139
140 bool isDisabled(TailFoldingOpts DefaultBits) const {
141 return getBits(DefaultBits) == TailFoldingOpts::Disabled;
142 }
143
144 bool satisfies(TailFoldingOpts DefaultBits, TailFoldingOpts Required) const {
145 return (getBits(DefaultBits) & Required) == Required;
146 }
147};
148} // namespace
149
150TailFoldingOption TailFoldingOptionLoc;
151
153 "sve-tail-folding",
154 cl::desc(
155 "Control the use of vectorisation using tail-folding for SVE where the"
156 " option is specified in the form (Initial)[+(Flag1|Flag2|...)]:"
157 "\ndisabled (Initial) No loop types will vectorize using "
158 "tail-folding"
159 "\ndefault (Initial) Uses the default tail-folding settings for "
160 "the target CPU"
161 "\nall (Initial) All legal loop types will vectorize using "
162 "tail-folding"
163 "\nsimple (Initial) Use tail-folding for simple loops (not "
164 "reductions or recurrences)"
165 "\nreductions Use tail-folding for loops containing reductions"
166 "\nnoreductions Inverse of above"
167 "\nrecurrences Use tail-folding for loops containing fixed order "
168 "recurrences"
169 "\nnorecurrences Inverse of above"
170 "\nreverse Use tail-folding for loops requiring reversed "
171 "predicates"
172 "\nnoreverse Inverse of above"),
174
175/// Returns true if the function has explicit operations that can only be
176/// lowered using incompatible instructions for the selected mode. This also
177/// returns true if the function F may use or modify ZA state.
179 bool ConsiderZA,
180 bool ConsiderZT,
181 bool ConsiderSM) const {
182 if (!ConsiderZA && !ConsiderZT && !ConsiderSM)
183 return false;
184
185 bool IsAlwaysInline = F->hasFnAttribute(Attribute::AlwaysInline);
186 // If the interface has vscale-dependent arguments/return value, then the ACLE
187 // describes that the function can only have defined behaviour if vscale is
188 // the same for both streaming/non-streaming mode, meaning that we can assume
189 // vscale is equivalent.
190 // FIXME: Handle the case where vscale_range is known and equivalent for
191 // caller and callee.
192 bool AssumeVScaleIsEquivalent =
193 F->getReturnType()->isScalableTy() ||
194 any_of(F->getFunctionType()->params(),
195 [](const Type *T) { return T->isScalableTy(); });
196
197 for (const BasicBlock &BB : *F) {
198 for (const Instruction &I : BB) {
199 if (auto *CB = dyn_cast<CallBase>(&I)) {
200 SMECallAttrs CallAttrs(*CB, &TLI->getRuntimeLibcallsInfo());
201
202 // Inline asm must be rejected as it could use SME state.
203 if (CB->isInlineAsm())
204 return true;
205
206 if (CallAttrs.callee().isSMEABIRoutine())
207 return true;
208
209 // If the callee has calls to streaming compatible functions, then those
210 // may have statements which depend on SME state (e.g. vscale or
211 // explicit reads of PSTATE.SM). If we were to inline those calls,
212 // behaviour may change after inlining because the streaming-compatible
213 // function would be executed in a different streaming mode.
214 // Conservatively avoid inlining for now.
215 if (ConsiderSM && CallAttrs.callee().hasStreamingCompatibleInterface())
216 return true;
217 }
218
219 if (!ConsiderSM)
220 continue;
221
222 auto *II = dyn_cast<IntrinsicInst>(&I);
223
224 // FIXME: Be less conservative about target intrinsics.
225 if (II && II->getCalledFunction()->isTargetIntrinsic())
226 return true;
227
228 // Determine if the operation is vscale dependent by looking at it's
229 // operand/result types.
230 if (!AssumeVScaleIsEquivalent &&
231 ((II && II->getIntrinsicID() == Intrinsic::vscale) ||
232 I.getType()->isScalableTy() ||
233 any_of(
234 I.operand_values(),
235 [](const Value *V) { return V->getType()->isScalableTy(); }) ||
237 .getSourceElementType()
238 ->isScalableTy()) ||
239 (isa<AllocaInst>(I) && cast<AllocaInst>(I).isScalable())))
240 return true;
241
242 // Inlining operations on fixed-length vectors when the streaming
243 // mode does not match is rejected because performance may be impacted.
244 // This decision should eventually be moved to the cost-model.
245 if (!IsAlwaysInline && (isa<FixedVectorType>(I.getType()) ||
246 any_of(I.operand_values(), [](const Value *V) {
247 return isa<FixedVectorType>(V->getType());
248 })))
249 return true;
250
251 // If the operation is invalid in the current mode, it should not be
252 // inlined.
253 SmallVector<Value *> Opnds(I.op_begin(), I.op_end());
255 return true;
256 }
257 }
258
259 return false;
260}
261
263 SmallVectorImpl<StringRef> &Features) {
264 StringRef AttributeStr =
265 TTI->isMultiversionedFunction(F) ? "fmv-features" : "target-features";
266 StringRef FeatureStr = F.getFnAttribute(AttributeStr).getValueAsString();
267 FeatureStr.split(Features, ",");
268}
269
272 extractAttrFeatures(F, this, Features);
273 return AArch64::getCpuSupportsMask(Features);
274}
275
278 extractAttrFeatures(F, this, Features);
279 return AArch64::getFMVPriority(Features);
280}
281
283 return F.hasFnAttribute("fmv-features");
284}
285
286/// The compiler must not inline when that may alter the behavior of the
287/// program. This is especially relevant around SME which implements different
288/// runtime modes and maintains external state through attributes.
289///
290/// The compiler must not inline when:
291//
292/// * The module is compiled with 'strict-fp' and the called function
293/// implements a different FP environment than the caller.
294///
295/// * The called function has operations that are incompatible in the mode
296/// of the caller, e.g.:
297/// * inlining non-streaming-only instructions into a streaming function.
298/// * inlining streaming-only instructions into a non-streaming function.
299///
300/// Inline asm blocks must be entered and exited in the [streaming] mode of
301/// the parent function. There is no language-level mechanism to inform the
302/// compiler that a particular inline asm block is streaming compatible, so
303/// the compiler must reject this as a candidate for inlining.
304///
305/// * The called function contains vscale-dependent operations but otherwise
306/// does not take/return VL-dependent arguments (see definition in the
307/// ACLE (Arm C/C++ Language Extensions)).
308///
309/// * The called function sets up new ZA/ZT state into a function that already
310/// has ZA or ZT state, as that is not valid as per the ACLE.
311///
312/// The compiler should not inline when:
313//
314/// * The called function has fixed-length vectors and the caller is in
315/// streaming mode, as this may cause performance regressions. This should
316/// never result in an error to be reported.
317///
318/// * The called function sets up new ZA/ZT state into a function that has no
319/// ZA and no ZT state, as the compiler currently cannot transfer the
320/// attribute to the caller. It may also impact performance, but that should
321/// be covered by AArch64TTIImpl::getInlineCallPenalty().
322///
323/// If a callee is incompatible with its caller, then the inliner should prevent
324/// inlining. LLVM's `alwaysinline` attribute has no impact on whether a callee
325/// is inline compatible. This is especially relevant with streaming-mode
326/// attributes, where `alwaysinline`ing a function into a function with a
327/// different streaming mode may otherwise impact runtime behaviour.
329 const Function *Callee) const {
330 SMECallAttrs CallAttrs(*Caller, *Callee);
331
332 // Never inline a function explicitly marked as being streaming,
333 // into a non-streaming function. Assume it was marked as streaming
334 // for a reason.
335 if (CallAttrs.caller().hasNonStreamingInterfaceAndBody() &&
336 CallAttrs.callee().hasStreamingInterfaceOrBody())
337 return false;
338
339 if (CallAttrs.callee().isNewZA() || CallAttrs.callee().isNewZT0())
340 return false;
341
342 // When inlining, we should consider the body of the function, not the
343 // interface.
344 if (CallAttrs.callee().hasStreamingBody()) {
345 CallAttrs.callee().set(SMEAttrs::SM_Compatible, false);
346 CallAttrs.callee().set(SMEAttrs::SM_Enabled, true);
347 }
348
349 bool ConsiderZA = CallAttrs.requiresZASave();
350 bool ConsiderZT = CallAttrs.requiresPreservingZT0() ||
351 CallAttrs.requiresPreservingAllZAState();
352 bool ConsiderSM = CallAttrs.requiresSMChange();
353
354 // FP environment is interpreted differently between streaming mode and
355 // non-streaming mode, so are not inline-compatible.
356 // FIXME: Analyze whether the callee actually has any FP operations.
357 if (ConsiderSM && (Caller->isStrictFP() || Callee->isStrictFP()))
358 return false;
359
360 if (hasPossibleIncompatibleOps(Callee, ConsiderZA, ConsiderZT, ConsiderSM))
361 return false;
362
363 return BaseT::areInlineCompatible(Caller, Callee);
364}
365
367 const Function *Callee,
368 ArrayRef<Type *> Types) const {
369 if (!BaseT::areTypesABICompatible(Caller, Callee, Types))
370 return false;
371
372 // We need to ensure that argument promotion does not attempt to promote
373 // pointers to fixed-length vector types larger than 128 bits like
374 // <8 x float> (and pointers to aggregate types which have such fixed-length
375 // vector type members) into the values of the pointees. Such vector types
376 // are used for SVE VLS but there is no ABI for SVE VLS arguments and the
377 // backend cannot lower such value arguments. The 128-bit fixed-length SVE
378 // types can be safely treated as 128-bit NEON types and they cannot be
379 // distinguished in IR.
380 if (ST->useSVEForFixedLengthVectors() && llvm::any_of(Types, [](Type *Ty) {
381 auto FVTy = dyn_cast<FixedVectorType>(Ty);
382 return FVTy &&
383 FVTy->getScalarSizeInBits() * FVTy->getNumElements() > 128;
384 }))
385 return false;
386
387 return true;
388}
389
390unsigned
392 unsigned DefaultCallPenalty) const {
393 // This function calculates a penalty for executing Call in F.
394 //
395 // There are two ways this function can be called:
396 // (1) F:
397 // call from F -> G (the call here is Call)
398 //
399 // For (1), Call.getCaller() == F, so it will always return a high cost if
400 // a streaming-mode change is required (thus promoting the need to inline the
401 // function)
402 //
403 // (2) F:
404 // call from F -> G (the call here is not Call)
405 // G:
406 // call from G -> H (the call here is Call)
407 //
408 // For (2), if after inlining the body of G into F the call to H requires a
409 // streaming-mode change, and the call to G from F would also require a
410 // streaming-mode change, then there is benefit to do the streaming-mode
411 // change only once and avoid inlining of G into F.
412
413 SMEAttrs FAttrs(*F);
414 SMECallAttrs CallAttrs(Call, &getTLI()->getRuntimeLibcallsInfo());
415
416 if (SMECallAttrs(FAttrs, CallAttrs.callee()).requiresSMChange()) {
417 if (F == Call.getCaller()) // (1)
418 return ST->getCLOpts().call_penalty_sm_change * DefaultCallPenalty;
419 if (SMECallAttrs(FAttrs, CallAttrs.caller()).requiresSMChange()) // (2)
420 return ST->getCLOpts().inline_call_penalty_sm_change * DefaultCallPenalty;
421 }
422
423 return DefaultCallPenalty;
424}
425
427 TargetTransformInfo::VectorKind K, Type *ElementTy) const {
428 // Don't maximize bandwidth for i1s. The legal width of predicates is derived
429 // from the data vectors they are used with. Maximizing the predicate width
430 // will just result in excessive VFs with a lot of unpacking.
431 if (ElementTy && ElementTy->isIntegerTy(1))
432 return false;
433
434 // If we don't know the specific class of vector just return true. Later VF
435 // selection will ask us about fixed or scalable vectors.
437 return true;
438
439 if (K == TargetTransformInfo::VectorKind::FixedWidth && ST->isNeonAvailable())
440 return true;
441
443 ST->isSVEorStreamingSVEAvailable() &&
444 !ST->disableMaximizeScalableBandwidth();
445}
446
447/// Calculate the cost of materializing a 64-bit value. This helper
448/// method might only calculate a fraction of a larger immediate. Therefore it
449/// is valid to return a cost of ZERO.
451 // Check if the immediate can be encoded within an instruction.
452 if (Val == 0 || AArch64_AM::isLogicalImmediate(Val, 64))
453 return 0;
454
455 if (Val < 0)
456 Val = ~Val;
457
458 // Calculate how many moves we will need to materialize this constant.
460 AArch64_IMM::expandMOVImm(Val, 64, Insn);
461 return Insn.size();
462}
463
464/// Calculate the cost of materializing the given constant.
467 TTI::TargetCostKind CostKind) const {
468 assert(Ty->isIntegerTy());
469
470 unsigned BitSize = Ty->getPrimitiveSizeInBits();
471 if (BitSize == 0)
472 return ~0U;
473
474 // Sign-extend all constants to a multiple of 64-bit.
475 APInt ImmVal = Imm;
476 if (BitSize & 0x3f)
477 ImmVal = Imm.sext((BitSize + 63) & ~0x3fU);
478
479 // Split the constant into 64-bit chunks and calculate the cost for each
480 // chunk.
482 for (unsigned ShiftVal = 0; ShiftVal < BitSize; ShiftVal += 64) {
483 APInt Tmp = ImmVal.ashr(ShiftVal).sextOrTrunc(64);
484 int64_t Val = Tmp.getSExtValue();
485 Cost += getIntImmCost(Val);
486 }
487 // We need at least one instruction to materialze the constant.
488 return std::max<InstructionCost>(1, Cost);
489}
490
492 const APInt &Imm, Type *Ty,
493 TTI::TargetCostKind CostKind,
494 Instruction *Inst) const {
495 assert(Ty->isIntegerTy());
496
497 unsigned BitSize = Ty->getPrimitiveSizeInBits();
498 // There is no cost model for constants with a bit size of 0. Return TCC_Free
499 // here, so that constant hoisting will ignore this constant.
500 if (BitSize == 0)
501 return TTI::TCC_Free;
502
503 unsigned ImmIdx = ~0U;
504 switch (Opcode) {
505 default:
506 return TTI::TCC_Free;
507 case Instruction::GetElementPtr:
508 // Always hoist the base address of a GetElementPtr.
509 if (Idx == 0)
510 return 2 * TTI::TCC_Basic;
511 return TTI::TCC_Free;
512 case Instruction::Store:
513 ImmIdx = 0;
514 break;
515 case Instruction::Add:
516 case Instruction::Sub:
517 case Instruction::Mul:
518 case Instruction::UDiv:
519 case Instruction::SDiv:
520 case Instruction::URem:
521 case Instruction::SRem:
522 case Instruction::And:
523 case Instruction::Or:
524 case Instruction::Xor:
525 case Instruction::ICmp:
526 ImmIdx = 1;
527 break;
528 // Always return TCC_Free for the shift value of a shift instruction.
529 case Instruction::Shl:
530 case Instruction::LShr:
531 case Instruction::AShr:
532 if (Idx == 1)
533 return TTI::TCC_Free;
534 break;
535 case Instruction::Trunc:
536 case Instruction::ZExt:
537 case Instruction::SExt:
538 case Instruction::IntToPtr:
539 case Instruction::PtrToInt:
540 case Instruction::BitCast:
541 case Instruction::PHI:
542 case Instruction::Call:
543 case Instruction::Select:
544 case Instruction::Ret:
545 case Instruction::Load:
546 break;
547 }
548
549 if (Idx == ImmIdx) {
550 int NumConstants = (BitSize + 63) / 64;
552 return (Cost <= NumConstants * TTI::TCC_Basic)
553 ? static_cast<int>(TTI::TCC_Free)
554 : Cost;
555 }
556 return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
557}
558
561 const APInt &Imm, Type *Ty,
562 TTI::TargetCostKind CostKind) const {
563 assert(Ty->isIntegerTy());
564
565 unsigned BitSize = Ty->getPrimitiveSizeInBits();
566 // There is no cost model for constants with a bit size of 0. Return TCC_Free
567 // here, so that constant hoisting will ignore this constant.
568 if (BitSize == 0)
569 return TTI::TCC_Free;
570
571 // Most (all?) AArch64 intrinsics do not support folding immediates into the
572 // selected instruction, so we compute the materialization cost for the
573 // immediate directly.
574 if (IID >= Intrinsic::aarch64_addg && IID <= Intrinsic::aarch64_udiv)
575 return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
576
577 switch (IID) {
578 default:
579 return TTI::TCC_Free;
580 case Intrinsic::sadd_with_overflow:
581 case Intrinsic::uadd_with_overflow:
582 case Intrinsic::ssub_with_overflow:
583 case Intrinsic::usub_with_overflow:
584 case Intrinsic::smul_with_overflow:
585 case Intrinsic::umul_with_overflow:
586 if (Idx == 1) {
587 int NumConstants = (BitSize + 63) / 64;
589 return (Cost <= NumConstants * TTI::TCC_Basic)
590 ? static_cast<int>(TTI::TCC_Free)
591 : Cost;
592 }
593 break;
594 case Intrinsic::experimental_stackmap:
595 if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
596 return TTI::TCC_Free;
597 break;
598 case Intrinsic::experimental_patchpoint_void:
599 case Intrinsic::experimental_patchpoint:
600 if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
601 return TTI::TCC_Free;
602 break;
603 case Intrinsic::experimental_gc_statepoint:
604 if ((Idx < 5) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
605 return TTI::TCC_Free;
606 break;
607 }
608 return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
609}
610
612AArch64TTIImpl::getPopcntSupport(unsigned TyWidth) const {
613 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
614 if (TyWidth == 32 || TyWidth == 64)
616 // TODO: AArch64TargetLowering::LowerCTPOP() supports 128bit popcount.
617 return TTI::PSK_Software;
618}
619
621 // MispredictPenalty is defined per-CPU in AArch64Sched*.td (e.g.,
622 // AArch64SchedNeoverseV2.td).
623 return ST->getMispredictionPenalty();
624}
625
626static bool isUnpackedVectorVT(EVT VecVT) {
627 return VecVT.isScalableVector() &&
629}
630
632 const IntrinsicCostAttributes &ICA) {
633 // We need to know at least the number of elements in the vector of buckets
634 // and the size of each element to update.
635 if (ICA.getArgTypes().size() < 2)
637
638 // Only interested in costing for the hardware instruction from SVE2.
639 if (!ST->isSVEAvailable() || !ST->hasSVE2())
641
642 Type *BucketPtrsTy = ICA.getArgTypes()[0]; // Type of vector of pointers
643 Type *EltTy = ICA.getArgTypes()[1]; // Type of bucket elements
644 unsigned TotalHistCnts = 1;
645
646 unsigned EltSize = EltTy->getScalarSizeInBits();
647 // Only allow (up to 64b) integers or pointers
648 if ((!EltTy->isIntegerTy() && !EltTy->isPointerTy()) || EltSize > 64)
650
651 // FIXME: We should be able to generate histcnt for fixed-length vectors
652 // using ptrue with a specific VL.
653 if (VectorType *VTy = dyn_cast<VectorType>(BucketPtrsTy)) {
654 unsigned EC = VTy->getElementCount().getKnownMinValue();
655 if (!isPowerOf2_64(EC) || !VTy->isScalableTy() || EC == 1)
657
658 // HistCnt only supports 32b and 64b element types
659 unsigned LegalEltSize = EltSize <= 32 ? 32 : 64;
660
661 if (EC == 2 || (LegalEltSize == 32 && EC == 4))
662 return InstructionCost(ST->getCLOpts().base_histcnt_cost);
663
664 unsigned NaturalVectorWidth = AArch64::SVEBitsPerBlock / LegalEltSize;
665 TotalHistCnts = EC / NaturalVectorWidth;
666
667 return InstructionCost(ST->getCLOpts().base_histcnt_cost * TotalHistCnts);
668 }
669
671}
672
675 TTI::TargetCostKind CostKind) const {
676 // The code-generator is currently not able to handle scalable vectors
677 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
678 // it. This change will be removed when code-generation for these types is
679 // sufficiently reliable.
680 // Only allow masked ld/st to pass through to getMemIntrinsicInstrCost().
681 auto *RetTy = ICA.getReturnType();
682 if (auto *VTy = dyn_cast<ScalableVectorType>(RetTy))
683 if (VTy->getElementCount() == ElementCount::getScalable(1) &&
684 !is_contained({Intrinsic::masked_load, Intrinsic::masked_store},
685 ICA.getID()))
687
688 switch (ICA.getID()) {
689 case Intrinsic::experimental_vector_histogram_add: {
690 InstructionCost HistCost = getHistogramCost(ST, ICA);
691 // If the cost isn't valid, we may still be able to scalarize
692 if (HistCost.isValid())
693 return HistCost;
694 break;
695 }
696 case Intrinsic::experimental_vector_histogram_uadd_sat:
697 case Intrinsic::experimental_vector_histogram_umax:
698 case Intrinsic::experimental_vector_histogram_umin:
699 // No hardware support for these histogram variants on AArch64.
701 case Intrinsic::clmul: {
702 auto LT = getTypeLegalizationCost(RetTy);
703
704 // PMUL v8i8/v16i8 is always available on AArch64
705 if (ST->hasNEON()) {
706 if (LT.second == MVT::v8i8 || LT.second == MVT::v16i8)
707 return LT.first;
708
709 // Scalar i8 lowers through scalar/vector moves around PMUL.
710 if (TLI->getValueType(DL, RetTy, true) == MVT::i8) {
711 auto *VecTy =
712 FixedVectorType::get(Type::getInt8Ty(RetTy->getContext()), 8);
713 return 1 +
714 getVectorInstrCost(Instruction::ExtractElement, VecTy, CostKind,
715 -1, nullptr, nullptr) *
716 2 +
717 getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind,
718 -1, nullptr, nullptr);
719 }
720 }
721
722 if (LT.second.SimpleTy == MVT::nxv2i64)
723 if (ST->hasSVEAES() && (ST->isSVEAvailable() || ST->hasSSVE_AES()))
724 return LT.first * 3;
725
726 if (ST->hasSVE2() || ST->hasSME()) {
727 switch (LT.second.SimpleTy) {
728 case MVT::nxv16i8:
729 return LT.first;
730 case MVT::nxv8i16:
731 return LT.first * 6;
732 case MVT::nxv4i32:
733 return LT.first * 3;
734 case MVT::nxv2i64:
735 return LT.first * 8;
736 default:
737 break;
738 }
739 }
740
741 // Avoid +sve giving this cost 2 due to custom lowering: It's very slow
742 if (LT.second.SimpleTy == MVT::nxv2i64)
743 return 192;
744
745 if (ST->hasAES()) {
746 switch (LT.second.SimpleTy) {
747 case MVT::i16:
748 case MVT::i32:
749 case MVT::i64:
750 case MVT::i128: {
751 auto *VecTy =
752 FixedVectorType::get(Type::getInt64Ty(RetTy->getContext()), 1);
753 return LT.first *
754 (1 +
755 getVectorInstrCost(Instruction::ExtractElement, VecTy, CostKind,
756 -1, nullptr, nullptr) *
757 2 +
758 getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind,
759 -1, nullptr, nullptr));
760 }
761 case MVT::v1i64:
762 return LT.first;
763 case MVT::v2i64:
764 return LT.first * 3;
765 case MVT::v2i32:
766 return LT.first * 6;
767 case MVT::v4i32:
768 return LT.first * 11;
769 case MVT::v4i16:
770 return LT.first * 14;
771 default:
772 break;
773 }
774 }
775 break;
776 }
777 case Intrinsic::smulh:
778 case Intrinsic::umulh: {
779 InstructionCost MulCost =
780 getArithmeticInstrCost(Instruction::Mul, RetTy, CostKind);
781
782 // Wide types like i128 need every partial product of a wide multiply
783 // and the carry chain between them. This is a crude cost approximation.
784 if (RetTy->getScalarSizeInBits() > 64)
785 return 2 * MulCost;
786
787 auto LT = getTypeLegalizationCost(RetTy);
788 MVT MTy = LT.second;
789
790 // Per-register cost to extend the operands.
791 InstructionCost ExtraCost =
792 LT.first *
793 (MTy.getScalarSizeInBits() != RetTy->getScalarSizeInBits() ? 2 : 0);
794
795 // MULH and MUL lower similarly for 64-bit products. For vectors, they
796 // also share the reduced vector multiply bandwidth seen on some cores.
797 if (MTy.getScalarSizeInBits() == 64)
798 return MulCost + ExtraCost;
799
800 // Scalable vectors (and fixed-length vectors, when possible) use SVE.
801 if (MTy.isScalableVector() ||
802 (MTy.isFixedLengthVector() && ST->isSVEorStreamingSVEAvailable()))
803 return LT.first + ExtraCost;
804
805 static const CostTblEntry MulHighCostTbl[] = {
806 {ISD::MULHU, MVT::i32, 2}, // [su]mull + lsr
807
808 {ISD::MULHU, MVT::v8i8, 2}, // [su]mull + shrn
809 {ISD::MULHU, MVT::v4i16, 2}, // "
810 {ISD::MULHU, MVT::v2i32, 2}, // "
811
812 {ISD::MULHU, MVT::v16i8, 3}, // [su]mull + [su]mull2 + uzp2
813 {ISD::MULHU, MVT::v8i16, 3}, // "
814 {ISD::MULHU, MVT::v4i32, 3}, // "
815 };
816
817 // Currently we assume SMULH has the same cost as UMULH.
818 if (const auto *Entry = CostTableLookup(MulHighCostTbl, ISD::MULHU, MTy))
819 return LT.first * Entry->Cost + ExtraCost;
820
821 break;
822 }
823 case Intrinsic::umin:
824 case Intrinsic::umax:
825 case Intrinsic::smin:
826 case Intrinsic::smax: {
827 static const auto ValidMinMaxTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
828 MVT::v8i16, MVT::v2i32, MVT::v4i32,
829 MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32,
830 MVT::nxv2i64};
831 auto LT = getTypeLegalizationCost(RetTy);
832 // Type promotion for v2i8 and v2i16 types have a heavy cost when
833 // vectorising. Account for this cost to avoid vectorising unprofitable
834 // examples when vectorising loops with low trip counts.
835 bool IsSigned =
836 ICA.getID() == Intrinsic::smin || ICA.getID() == Intrinsic::smax;
837 EVT VT = TLI->getValueType(DL, RetTy, /*AllowUnknown=*/true);
838 if (VT == MVT::v2i8 || VT == MVT::v2i16 || VT == MVT::v4i8)
839 return LT.first * (IsSigned ? 5 : 3);
840 // v2i64 types get converted to cmp+bif hence the cost of 2
841 if (LT.second == MVT::v2i64)
842 return LT.first * 2;
843 if (any_of(ValidMinMaxTys, equal_to(LT.second)))
844 return LT.first;
845 break;
846 }
847 case Intrinsic::scmp:
848 case Intrinsic::ucmp: {
849 static const CostTblEntry BitreverseTbl[] = {
850 {Intrinsic::scmp, MVT::i32, 3}, // cmp+cset+csinv
851 {Intrinsic::scmp, MVT::i64, 3}, // cmp+cset+csinv
852 {Intrinsic::scmp, MVT::v8i8, 3}, // cmgt+cmgt+sub
853 {Intrinsic::scmp, MVT::v16i8, 3}, // cmgt+cmgt+sub
854 {Intrinsic::scmp, MVT::v4i16, 3}, // cmgt+cmgt+sub
855 {Intrinsic::scmp, MVT::v8i16, 3}, // cmgt+cmgt+sub
856 {Intrinsic::scmp, MVT::v2i32, 3}, // cmgt+cmgt+sub
857 {Intrinsic::scmp, MVT::v4i32, 3}, // cmgt+cmgt+sub
858 {Intrinsic::scmp, MVT::v1i64, 3}, // cmgt+cmgt+sub
859 {Intrinsic::scmp, MVT::v2i64, 3}, // cmgt+cmgt+sub
860 };
861 const auto LT = getTypeLegalizationCost(RetTy);
862 const auto *Entry =
863 CostTableLookup(BitreverseTbl, Intrinsic::scmp, LT.second);
864 if (Entry)
865 return Entry->Cost * LT.first;
866 break;
867 }
868 case Intrinsic::sadd_sat:
869 case Intrinsic::ssub_sat:
870 case Intrinsic::uadd_sat:
871 case Intrinsic::usub_sat: {
872 static const auto ValidSatTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
873 MVT::v8i16, MVT::v2i32, MVT::v4i32,
874 MVT::v2i64};
875 auto LT = getTypeLegalizationCost(RetTy);
876 // This is a base cost of 1 for the vadd, plus 3 extract shifts if we
877 // need to extend the type, as it uses shr(qadd(shl, shl)).
878 unsigned Instrs =
879 LT.second.getScalarSizeInBits() == RetTy->getScalarSizeInBits() ? 1 : 4;
880 if (any_of(ValidSatTys, equal_to(LT.second)))
881 return LT.first * Instrs;
882
884 uint64_t VectorSize = TS.getKnownMinValue();
885
886 if (ST->isSVEAvailable() && VectorSize >= 128 && isPowerOf2_64(VectorSize))
887 return LT.first * Instrs;
888
889 break;
890 }
891 case Intrinsic::abs: {
892 static const auto ValidAbsTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
893 MVT::v8i16, MVT::v2i32, MVT::v4i32,
894 MVT::v2i64, MVT::nxv16i8, MVT::nxv8i16,
895 MVT::nxv4i32, MVT::nxv2i64};
896 auto LT = getTypeLegalizationCost(RetTy);
897 if (any_of(ValidAbsTys, equal_to(LT.second)))
898 return LT.first;
899 break;
900 }
901 case Intrinsic::bswap: {
902 static const auto ValidAbsTys = {MVT::v4i16, MVT::v8i16, MVT::v2i32,
903 MVT::v4i32, MVT::v2i64};
904 auto LT = getTypeLegalizationCost(RetTy);
905 if (any_of(ValidAbsTys, equal_to(LT.second)) &&
906 LT.second.getScalarSizeInBits() == RetTy->getScalarSizeInBits())
907 return LT.first;
908 break;
909 }
910 case Intrinsic::fma:
911 case Intrinsic::fmuladd: {
912 // Given a fma or fmuladd, cost it the same as a fmul instruction which are
913 // usually the same for costs. TODO: Add fp16 and bf16 expansion costs.
914 Type *EltTy = RetTy->getScalarType();
915 if (EltTy->isFloatTy() || EltTy->isDoubleTy() ||
916 (EltTy->isHalfTy() && ST->hasFullFP16()))
917 return getArithmeticInstrCost(Instruction::FMul, RetTy, CostKind);
918 break;
919 }
920 case Intrinsic::stepvector: {
921 InstructionCost Cost = 1; // Cost of the `index' instruction
922 auto LT = getTypeLegalizationCost(RetTy);
923 // Legalisation of illegal vectors involves an `index' instruction plus
924 // (LT.first - 1) vector adds.
925 if (LT.first > 1) {
926 Type *LegalVTy = EVT(LT.second).getTypeForEVT(RetTy->getContext());
927 InstructionCost AddCost =
928 getArithmeticInstrCost(Instruction::Add, LegalVTy, CostKind);
929 Cost += AddCost * (LT.first - 1);
930 }
931 return Cost;
932 }
933 case Intrinsic::vector_extract:
934 case Intrinsic::vector_insert: {
935 // If both the vector and subvector types are legal types and the index
936 // is 0, then this should be a no-op or simple operation; return a
937 // relatively low cost.
938
939 // If arguments aren't actually supplied, then we cannot determine the
940 // value of the index. We also want to skip predicate types.
941 if (ICA.getArgs().size() != ICA.getArgTypes().size() ||
943 break;
944
945 LLVMContext &C = RetTy->getContext();
946 EVT VecVT = getTLI()->getValueType(DL, ICA.getArgTypes()[0]);
947 bool IsExtract = ICA.getID() == Intrinsic::vector_extract;
948 EVT SubVecVT = IsExtract ? getTLI()->getValueType(DL, RetTy)
949 : getTLI()->getValueType(DL, ICA.getArgTypes()[1]);
950 // Skip this if either the vector or subvector types are unpacked
951 // SVE types; they may get lowered to stack stores and loads.
952 if (isUnpackedVectorVT(VecVT) || isUnpackedVectorVT(SubVecVT))
953 break;
954
956 getTLI()->getTypeConversion(C, SubVecVT);
958 getTLI()->getTypeConversion(C, VecVT);
959 const Value *Idx = IsExtract ? ICA.getArgs()[1] : ICA.getArgs()[2];
960 const ConstantInt *CIdx = cast<ConstantInt>(Idx);
961 if (SubVecLK.first == TargetLoweringBase::TypeLegal &&
962 VecLK.first == TargetLoweringBase::TypeLegal && CIdx->isZero())
963 return TTI::TCC_Free;
964 break;
965 }
966 case Intrinsic::bitreverse: {
967 static const CostTblEntry BitreverseTbl[] = {
968 {Intrinsic::bitreverse, MVT::i32, 1},
969 {Intrinsic::bitreverse, MVT::i64, 1},
970 {Intrinsic::bitreverse, MVT::v8i8, 1},
971 {Intrinsic::bitreverse, MVT::v16i8, 1},
972 {Intrinsic::bitreverse, MVT::v4i16, 2},
973 {Intrinsic::bitreverse, MVT::v8i16, 2},
974 {Intrinsic::bitreverse, MVT::v2i32, 2},
975 {Intrinsic::bitreverse, MVT::v4i32, 2},
976 {Intrinsic::bitreverse, MVT::v1i64, 2},
977 {Intrinsic::bitreverse, MVT::v2i64, 2},
978 };
979 const auto LegalisationCost = getTypeLegalizationCost(RetTy);
980 const auto *Entry =
981 CostTableLookup(BitreverseTbl, ICA.getID(), LegalisationCost.second);
982 if (Entry) {
983 // Cost Model is using the legal type(i32) that i8 and i16 will be
984 // converted to +1 so that we match the actual lowering cost
985 if (TLI->getValueType(DL, RetTy, true) == MVT::i8 ||
986 TLI->getValueType(DL, RetTy, true) == MVT::i16)
987 return LegalisationCost.first * Entry->Cost + 1;
988
989 return LegalisationCost.first * Entry->Cost;
990 }
991 break;
992 }
993 case Intrinsic::pdep:
994 case Intrinsic::pext: {
995 unsigned BW = RetTy->getScalarSizeInBits();
996 if (!RetTy->isVectorTy() && ST->isSVEBitPermAvailable() && BW <= 64) {
997 Type *VecTy = VectorType::get(RetTy, 64 / BW, false);
999 getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, 0,
1000 nullptr, nullptr) +
1001 getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, 0,
1002 nullptr, nullptr) +
1003 1 +
1004 getVectorInstrCost(Instruction::ExtractElement, VecTy, CostKind, 0,
1005 nullptr, nullptr);
1006 return Cost;
1007 }
1008 break;
1009 }
1010 case Intrinsic::ctpop: {
1011 auto LT = getTypeLegalizationCost(RetTy);
1012 MVT MTy = LT.second;
1013
1014 if (ST->hasCSSC() && !RetTy->isVectorTy()) {
1015 int ExtraCost =
1016 MTy.getScalarSizeInBits() != RetTy->getScalarSizeInBits() ? 1 : 0;
1017 return LT.first + ExtraCost;
1018 }
1019 if (!ST->hasNEON()) {
1020 // 32-bit or 64-bit ctpop without NEON is 12 instructions.
1021 return getTypeLegalizationCost(RetTy).first * 12;
1022 }
1023 static const CostTblEntry CtpopCostTbl[] = {
1024 {ISD::CTPOP, MVT::v2i64, 4},
1025 {ISD::CTPOP, MVT::v4i32, 3},
1026 {ISD::CTPOP, MVT::v8i16, 2},
1027 {ISD::CTPOP, MVT::v16i8, 1},
1028 {ISD::CTPOP, MVT::i64, 4},
1029 {ISD::CTPOP, MVT::v2i32, 3},
1030 {ISD::CTPOP, MVT::v4i16, 2},
1031 {ISD::CTPOP, MVT::v8i8, 1},
1032 {ISD::CTPOP, MVT::i32, 5},
1033 // SVE types (For targets that override NEON for fixed length vectors)
1034 {ISD::CTPOP, MVT::nxv2i64, 1},
1035 {ISD::CTPOP, MVT::nxv4i32, 1},
1036 {ISD::CTPOP, MVT::nxv8i16, 1},
1037 {ISD::CTPOP, MVT::nxv16i8, 1},
1038 };
1039
1040 // When SVE is available CNT will be used for fixed and scalable vectors.
1041 if (ST->isSVEorStreamingSVEAvailable() && MTy.isFixedLengthVector())
1043 128 / MTy.getScalarSizeInBits());
1044
1045 if (const auto *Entry = CostTableLookup(CtpopCostTbl, ISD::CTPOP, MTy)) {
1046 // Extra cost of +1 when illegal vector types are legalized by promoting
1047 // the integer type.
1048 int ExtraCost = MTy.isVector() && MTy.getScalarSizeInBits() !=
1049 RetTy->getScalarSizeInBits()
1050 ? 1
1051 : 0;
1052 return LT.first * Entry->Cost + ExtraCost;
1053 }
1054 break;
1055 }
1056 case Intrinsic::sadd_with_overflow:
1057 case Intrinsic::uadd_with_overflow:
1058 case Intrinsic::ssub_with_overflow:
1059 case Intrinsic::usub_with_overflow:
1060 case Intrinsic::smul_with_overflow:
1061 case Intrinsic::umul_with_overflow: {
1062 static const CostTblEntry WithOverflowCostTbl[] = {
1063 {Intrinsic::sadd_with_overflow, MVT::i8, 3},
1064 {Intrinsic::uadd_with_overflow, MVT::i8, 3},
1065 {Intrinsic::sadd_with_overflow, MVT::i16, 3},
1066 {Intrinsic::uadd_with_overflow, MVT::i16, 3},
1067 {Intrinsic::sadd_with_overflow, MVT::i32, 1},
1068 {Intrinsic::uadd_with_overflow, MVT::i32, 1},
1069 {Intrinsic::sadd_with_overflow, MVT::i64, 1},
1070 {Intrinsic::uadd_with_overflow, MVT::i64, 1},
1071 {Intrinsic::ssub_with_overflow, MVT::i8, 3},
1072 {Intrinsic::usub_with_overflow, MVT::i8, 3},
1073 {Intrinsic::ssub_with_overflow, MVT::i16, 3},
1074 {Intrinsic::usub_with_overflow, MVT::i16, 3},
1075 {Intrinsic::ssub_with_overflow, MVT::i32, 1},
1076 {Intrinsic::usub_with_overflow, MVT::i32, 1},
1077 {Intrinsic::ssub_with_overflow, MVT::i64, 1},
1078 {Intrinsic::usub_with_overflow, MVT::i64, 1},
1079 {Intrinsic::smul_with_overflow, MVT::i8, 5},
1080 {Intrinsic::umul_with_overflow, MVT::i8, 4},
1081 {Intrinsic::smul_with_overflow, MVT::i16, 5},
1082 {Intrinsic::umul_with_overflow, MVT::i16, 4},
1083 {Intrinsic::smul_with_overflow, MVT::i32, 2}, // eg umull;tst
1084 {Intrinsic::umul_with_overflow, MVT::i32, 2}, // eg umull;cmp sxtw
1085 {Intrinsic::smul_with_overflow, MVT::i64, 3}, // eg mul;smulh;cmp
1086 {Intrinsic::umul_with_overflow, MVT::i64, 3}, // eg mul;umulh;cmp asr
1087 };
1088 EVT MTy = TLI->getValueType(DL, RetTy->getContainedType(0), true);
1089 if (MTy.isSimple())
1090 if (const auto *Entry = CostTableLookup(WithOverflowCostTbl, ICA.getID(),
1091 MTy.getSimpleVT()))
1092 return Entry->Cost;
1093 break;
1094 }
1095 case Intrinsic::fptosi_sat:
1096 case Intrinsic::fptoui_sat: {
1097 if (ICA.getArgTypes().empty())
1098 break;
1099 bool IsSigned = ICA.getID() == Intrinsic::fptosi_sat;
1100 auto LT = getTypeLegalizationCost(ICA.getArgTypes()[0]);
1101 EVT MTy = TLI->getValueType(DL, RetTy);
1102 // Check for the legal types, which are where the size of the input and the
1103 // output are the same, or we are using cvt f64->i32 or f32->i64.
1104 if ((LT.second == MVT::f32 || LT.second == MVT::f64 ||
1105 LT.second == MVT::v2f32 || LT.second == MVT::v4f32 ||
1106 LT.second == MVT::v2f64)) {
1107 if ((LT.second.getScalarSizeInBits() == MTy.getScalarSizeInBits() ||
1108 (LT.second == MVT::f64 && MTy == MVT::i32) ||
1109 (LT.second == MVT::f32 && MTy == MVT::i64)))
1110 return LT.first;
1111 // Extending vector types v2f32->v2i64, fcvtl*2 + fcvt*2
1112 if (LT.second.getScalarType() == MVT::f32 && MTy.isFixedLengthVector() &&
1113 MTy.getScalarSizeInBits() == 64)
1114 return LT.first * (MTy.getVectorNumElements() > 2 ? 4 : 2);
1115 }
1116 // Similarly for fp16 sizes. Without FullFP16 we generally need to fcvt to
1117 // f32.
1118 if (LT.second.getScalarType() == MVT::f16 && !ST->hasFullFP16())
1119 return LT.first + getIntrinsicInstrCost(
1120 {ICA.getID(),
1121 RetTy,
1122 {ICA.getArgTypes()[0]->getWithNewType(
1123 Type::getFloatTy(RetTy->getContext()))}},
1124 CostKind);
1125 if ((LT.second == MVT::f16 && MTy == MVT::i32) ||
1126 (LT.second == MVT::f16 && MTy == MVT::i64) ||
1127 ((LT.second == MVT::v4f16 || LT.second == MVT::v8f16) &&
1128 (LT.second.getScalarSizeInBits() == MTy.getScalarSizeInBits())))
1129 return LT.first;
1130 // Extending vector types v8f16->v8i32, fcvtl*2 + fcvt*2
1131 if (LT.second.getScalarType() == MVT::f16 && MTy.isFixedLengthVector() &&
1132 MTy.getScalarSizeInBits() == 32)
1133 return LT.first * (MTy.getVectorNumElements() > 4 ? 4 : 2);
1134 // Extending vector types v8f16->v8i32. These current scalarize but the
1135 // codegen could be better.
1136 if (LT.second.getScalarType() == MVT::f16 && MTy.isFixedLengthVector() &&
1137 MTy.getScalarSizeInBits() == 64)
1138 return MTy.getVectorNumElements() * 3;
1139
1140 // If we can we use a legal convert followed by a min+max
1141 if ((LT.second.getScalarType() == MVT::f32 ||
1142 LT.second.getScalarType() == MVT::f64 ||
1143 LT.second.getScalarType() == MVT::f16) &&
1144 LT.second.getScalarSizeInBits() >= MTy.getScalarSizeInBits()) {
1145 Type *LegalTy =
1146 Type::getIntNTy(RetTy->getContext(), LT.second.getScalarSizeInBits());
1147 if (LT.second.isVector())
1148 LegalTy = VectorType::get(LegalTy, LT.second.getVectorElementCount());
1150 IntrinsicCostAttributes Attrs1(IsSigned ? Intrinsic::smin
1151 : Intrinsic::umin,
1152 LegalTy, {LegalTy, LegalTy});
1153 Cost += getIntrinsicInstrCost(Attrs1, CostKind);
1154 IntrinsicCostAttributes Attrs2(IsSigned ? Intrinsic::smax
1155 : Intrinsic::umax,
1156 LegalTy, {LegalTy, LegalTy});
1157 Cost += getIntrinsicInstrCost(Attrs2, CostKind);
1158 return LT.first * Cost +
1159 ((LT.second.getScalarType() != MVT::f16 || ST->hasFullFP16()) ? 0
1160 : 1);
1161 }
1162 // Otherwise we need to follow the default expansion that clamps the value
1163 // using a float min/max with a fcmp+sel for nan handling when signed.
1164 Type *FPTy = ICA.getArgTypes()[0]->getScalarType();
1165 RetTy = RetTy->getScalarType();
1166 if (LT.second.isVector()) {
1167 FPTy = VectorType::get(FPTy, LT.second.getVectorElementCount());
1168 RetTy = VectorType::get(RetTy, LT.second.getVectorElementCount());
1169 }
1170 IntrinsicCostAttributes Attrs1(Intrinsic::minnum, FPTy, {FPTy, FPTy});
1171 InstructionCost Cost = getIntrinsicInstrCost(Attrs1, CostKind);
1172 IntrinsicCostAttributes Attrs2(Intrinsic::maxnum, FPTy, {FPTy, FPTy});
1173 Cost += getIntrinsicInstrCost(Attrs2, CostKind);
1174 Cost +=
1175 getCastInstrCost(IsSigned ? Instruction::FPToSI : Instruction::FPToUI,
1176 RetTy, FPTy, TTI::CastContextHint::None, CostKind);
1177 if (IsSigned) {
1178 Type *CondTy = RetTy->getWithNewBitWidth(1);
1179 Cost += getCmpSelInstrCost(BinaryOperator::FCmp, FPTy, CondTy,
1180 CmpInst::FCMP_UNO, CostKind);
1181 Cost += getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,
1182 CmpInst::FCMP_UNO, CostKind);
1183 }
1184 return LT.first * Cost;
1185 }
1186 case Intrinsic::fshl:
1187 case Intrinsic::fshr: {
1188 if (ICA.getArgs().empty())
1189 break;
1190
1191 const TTI::OperandValueInfo OpInfoZ = TTI::getOperandInfo(ICA.getArgs()[2]);
1192
1193 // ROTR / ROTL is a funnel shift with equal first and second operand. For
1194 // ROTR on integer registers (i32/i64) this can be done in a single ror
1195 // instruction. A fshl with a non-constant shift uses a neg + ror.
1196 if (RetTy->isIntegerTy() && ICA.getArgs()[0] == ICA.getArgs()[1] &&
1197 (RetTy->getPrimitiveSizeInBits() == 32 ||
1198 RetTy->getPrimitiveSizeInBits() == 64)) {
1199 InstructionCost NegCost =
1200 (ICA.getID() == Intrinsic::fshl && !OpInfoZ.isConstant()) ? 1 : 0;
1201 return 1 + NegCost;
1202 }
1203
1204 // TODO: Add handling for fshl where third argument is not a constant.
1205 if (!OpInfoZ.isConstant())
1206 break;
1207
1208 const auto LegalisationCost = getTypeLegalizationCost(RetTy);
1209 if (OpInfoZ.isUniform()) {
1210 static const CostTblEntry FshlTbl[] = {
1211 {Intrinsic::fshl, MVT::v4i32, 2}, // shl + usra
1212 {Intrinsic::fshl, MVT::v2i64, 2}, {Intrinsic::fshl, MVT::v16i8, 2},
1213 {Intrinsic::fshl, MVT::v8i16, 2}, {Intrinsic::fshl, MVT::v2i32, 2},
1214 {Intrinsic::fshl, MVT::v8i8, 2}, {Intrinsic::fshl, MVT::v4i16, 2}};
1215 // Costs for both fshl & fshr are the same, so just pass Intrinsic::fshl
1216 // to avoid having to duplicate the costs.
1217 const auto *Entry =
1218 CostTableLookup(FshlTbl, Intrinsic::fshl, LegalisationCost.second);
1219 if (Entry)
1220 return LegalisationCost.first * Entry->Cost;
1221 }
1222
1223 auto TyL = getTypeLegalizationCost(RetTy);
1224 if (!RetTy->isIntegerTy())
1225 break;
1226
1227 // Estimate cost manually, as types like i8 and i16 will get promoted to
1228 // i32 and CostTableLookup will ignore the extra conversion cost.
1229 bool HigherCost = (RetTy->getScalarSizeInBits() != 32 &&
1230 RetTy->getScalarSizeInBits() < 64) ||
1231 (RetTy->getScalarSizeInBits() % 64 != 0);
1232 unsigned ExtraCost = HigherCost ? 1 : 0;
1233 if (RetTy->getScalarSizeInBits() == 32 ||
1234 RetTy->getScalarSizeInBits() == 64)
1235 ExtraCost = 0; // fhsl/fshr for i32 and i64 can be lowered to a single
1236 // extr instruction.
1237 else if (HigherCost)
1238 ExtraCost = 1;
1239 else
1240 break;
1241 return TyL.first + ExtraCost;
1242 }
1243 case Intrinsic::get_active_lane_mask: {
1244 auto RetTy = cast<VectorType>(ICA.getReturnType());
1245 EVT RetVT = getTLI()->getValueType(DL, RetTy);
1246 EVT OpVT = getTLI()->getValueType(DL, ICA.getArgTypes()[0]);
1247 if (getTLI()->shouldExpandGetActiveLaneMask(RetVT, OpVT))
1248 break;
1249
1250 if (RetTy->isScalableTy()) {
1251 if (TLI->getTypeAction(RetTy->getContext(), RetVT) !=
1253 break;
1254
1255 auto LT = getTypeLegalizationCost(RetTy);
1256 InstructionCost Cost = LT.first;
1257 // When SVE2p1 or SME2 is available, we can halve getTypeLegalizationCost
1258 // as get_active_lane_mask may lower to the sve_whilelo_x2 intrinsic, e.g.
1259 // nxv32i1 = get_active_lane_mask(base, idx) ->
1260 // {nxv16i1, nxv16i1} = sve_whilelo_x2(base, idx)
1261 if (ST->hasSVE2p1() || ST->hasSME2()) {
1262 Cost /= 2;
1263 if (Cost == 1)
1264 return Cost;
1265 }
1266
1267 // If more than one whilelo intrinsic is required, include the extra cost
1268 // required by the saturating add & select required to increment the
1269 // start value after the first intrinsic call.
1270 Type *OpTy = ICA.getArgTypes()[0];
1271 IntrinsicCostAttributes AddAttrs(Intrinsic::uadd_sat, OpTy, {OpTy, OpTy});
1272 InstructionCost SplitCost = getIntrinsicInstrCost(AddAttrs, CostKind);
1273 Type *CondTy = OpTy->getWithNewBitWidth(1);
1274 SplitCost += getCmpSelInstrCost(Instruction::Select, OpTy, CondTy,
1275 CmpInst::ICMP_UGT, CostKind);
1276 return Cost + (SplitCost * (Cost - 1));
1277 } else if (!getTLI()->isTypeLegal(RetVT)) {
1278 // We don't have enough context at this point to determine if the mask
1279 // is going to be kept live after the block, which will force the vXi1
1280 // type to be expanded to legal vectors of integers, e.g. v4i1->v4i32.
1281 // For now, we just assume the vectorizer created this intrinsic and
1282 // the result will be the input for a PHI. In this case the cost will
1283 // be extremely high for fixed-width vectors.
1284 // NOTE: getScalarizationOverhead returns a cost that's far too
1285 // pessimistic for the actual generated codegen. In reality there are
1286 // two instructions generated per lane.
1287 return cast<FixedVectorType>(RetTy)->getNumElements() * 2;
1288 }
1289 break;
1290 }
1291 case Intrinsic::experimental_vector_match: {
1292 if (!ST->hasSVE2() || !ST->isSVEAvailable())
1293 break;
1294
1295 auto *NeedleTy = cast<FixedVectorType>(ICA.getArgTypes()[1]);
1296
1297 // We expand vector.matches with <= 2 elements to a chain of compares.
1298 unsigned SearchSize = NeedleTy->getNumElements();
1299 if (SearchSize <= 2)
1300 break;
1301
1302 auto [LegalParts, SearchVT] = getTypeLegalizationCost(ICA.getArgTypes()[0]);
1303 if (!is_contained(
1304 {MVT::nxv8i16, MVT::nxv16i8, MVT::v8i16, MVT::v16i8, MVT::v8i8},
1305 SearchVT.SimpleTy))
1306 break;
1307
1308 unsigned ElementSizeInBits = SearchVT.getScalarSizeInBits();
1309
1310 // Number of needle elements we can compare per `match` instruction.
1311 unsigned NeedleEltsPerMatch = AArch64::SVEBitsPerBlock / ElementSizeInBits;
1312
1313 // How many `match` instructions we need to match `SearchSize` elements.
1314 unsigned MatchesRequiredForNeedle =
1315 llvm::divideCeil(SearchSize, NeedleEltsPerMatch);
1316
1317 // Base cost for MATCH instructions. At least on the Neoverse V2 and
1318 // Neoverse V3, these are cheap operations with the same latency as a
1319 // vector ADD. In most cases, however, we also need to do an extra DUP.
1320 // For fixed-length vectors we currently need an extra five--six
1321 // instructions besides the MATCH.
1323 if (isa<FixedVectorType>(RetTy))
1324 Cost += 10;
1325
1326 return Cost * LegalParts * MatchesRequiredForNeedle;
1327 }
1328 case Intrinsic::cttz: {
1329 auto LT = getTypeLegalizationCost(ICA.getArgTypes()[0]);
1330 if (LT.second == MVT::v8i8 || LT.second == MVT::v16i8)
1331 return LT.first * 2;
1332 if (LT.second == MVT::v4i16 || LT.second == MVT::v8i16 ||
1333 LT.second == MVT::v2i32 || LT.second == MVT::v4i32)
1334 return LT.first * 3;
1335 break;
1336 }
1337 case Intrinsic::experimental_cttz_elts: {
1338 EVT ArgVT = getTLI()->getValueType(DL, ICA.getArgTypes()[0]);
1339 if (getTLI()->isOperationCustom(ISD::CTTZ_ELTS, ArgVT)) {
1340 // This will consist of a SVE brkb and a cntp instruction. These
1341 // typically have the same latency and half the throughput as a vector
1342 // add instruction.
1343 return 4;
1344 }
1345 break;
1346 }
1347 case Intrinsic::mask_beforefirst: {
1348 if (!ST->isSVEorStreamingSVEAvailable() ||
1349 !isa<ScalableVectorType>(RetTy) ||
1350 !TLI->isTypeLegal(getTLI()->getValueType(DL, RetTy)))
1351 break;
1352
1353 // mask_beforefirst(mask) -> brkb_z(ptrue, mask) but we ignore ptrue
1354 // because it's typically used by other instructions in the function.
1355 return 1;
1356 }
1357 case Intrinsic::loop_dependence_raw_mask:
1358 case Intrinsic::loop_dependence_war_mask: {
1359 // The whilewr/rw instructions require SVE2 or SME.
1360 if (ST->hasSVE2() || ST->hasSME()) {
1361 EVT VecVT = getTLI()->getValueType(DL, RetTy);
1362 unsigned EltSizeInBytes =
1363 cast<ConstantInt>(ICA.getArgs()[2])->getZExtValue();
1364 if (!is_contained({1u, 2u, 4u, 8u}, EltSizeInBytes) ||
1365 VecVT.getVectorMinNumElements() != (16 / EltSizeInBytes))
1366 break;
1367 // For fixed-vector types we need to AND the mask with a ptrue vl<N>.
1368 return isa<FixedVectorType>(RetTy) ? 2 : 1;
1369 }
1370 break;
1371 }
1372 case Intrinsic::experimental_vector_extract_last_active:
1373 if (ST->isSVEorStreamingSVEAvailable()) {
1374 auto [LegalCost, _] = getTypeLegalizationCost(ICA.getArgTypes()[0]);
1375 // This should turn into chained clastb instructions.
1376 return LegalCost;
1377 }
1378 break;
1379 case Intrinsic::pow: {
1380 // For scalar calls we know the target has the libcall, and for fixed-width
1381 // vectors we know for the worst case it can be scalarised.
1382 EVT VT = getTLI()->getValueType(DL, RetTy);
1383 RTLIB::Libcall LC = RTLIB::getPOW(VT);
1384 bool HasLibcall =
1385 getTLI()->getLibcallLoweringInfo().getVectorLibcallImplOrMasked(LC) !=
1386 RTLIB::Unsupported;
1387 bool CanLowerWithLibcalls = !isa<ScalableVectorType>(RetTy) || HasLibcall;
1388
1389 // If we know that the call can be lowered with libcalls then it's safe to
1390 // reduce the costs in some cases. This is important for scalable vectors,
1391 // since we cannot scalarize the call in the absence of a vector math
1392 // library.
1393 if (CanLowerWithLibcalls && ICA.getInst() && !ICA.getArgs().empty()) {
1394 // If we know the fast math flags and the exponent is a constant then the
1395 // cost may be less for some exponents like 0.25 and 0.75.
1396 const Constant *ExpC = dyn_cast<Constant>(ICA.getArgs()[1]);
1397 if (ExpC && isa<VectorType>(ExpC->getType()))
1398 ExpC = ExpC->getSplatValue();
1399 if (auto *ExpF = dyn_cast_or_null<ConstantFP>(ExpC)) {
1400 // The argument must be a FP constant.
1401 bool Is025 = ExpF->getValueAPF().isExactlyValue(0.25);
1402 bool Is075 = ExpF->getValueAPF().isExactlyValue(0.75);
1403 FastMathFlags FMF = ICA.getInst()->getFastMathFlags();
1404 if ((Is025 || Is075) && FMF.noInfs() && FMF.approxFunc() &&
1405 (!Is025 || FMF.noSignedZeros())) {
1406 IntrinsicCostAttributes Attrs(Intrinsic::sqrt, RetTy, {RetTy}, FMF);
1407 InstructionCost Sqrt = getIntrinsicInstrCost(Attrs, CostKind);
1408 if (Is025)
1409 return 2 * Sqrt;
1411 getArithmeticInstrCost(Instruction::FMul, RetTy, CostKind);
1412 return (Sqrt * 2) + FMul;
1413 }
1414 // TODO: For 1/3 exponents we expect the cbrt call to be slightly
1415 // cheaper than pow.
1416 }
1417 }
1418
1419 if (HasLibcall)
1420 return getCallInstrCost(nullptr, RetTy, ICA.getArgTypes(), CostKind);
1421 break;
1422 }
1423 case Intrinsic::sqrt:
1424 case Intrinsic::fabs:
1425 case Intrinsic::ceil:
1426 case Intrinsic::floor:
1427 case Intrinsic::nearbyint:
1428 case Intrinsic::round:
1429 case Intrinsic::rint:
1430 case Intrinsic::roundeven:
1431 case Intrinsic::trunc:
1432 case Intrinsic::minnum:
1433 case Intrinsic::maxnum:
1434 case Intrinsic::minimum:
1435 case Intrinsic::maximum: {
1436 if (isa<ScalableVectorType>(RetTy) && ST->isSVEorStreamingSVEAvailable()) {
1437 auto LT = getTypeLegalizationCost(RetTy);
1438 return LT.first;
1439 }
1440 break;
1441 }
1442 default:
1443 break;
1444 }
1445 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
1446}
1447
1448/// The function will remove redundant reinterprets casting in the presence
1449/// of the control flow
1450static std::optional<Instruction *> processPhiNode(InstCombiner &IC,
1451 IntrinsicInst &II) {
1453 auto RequiredType = II.getType();
1454
1455 auto *PN = dyn_cast<PHINode>(II.getArgOperand(0));
1456 assert(PN && "Expected Phi Node!");
1457
1458 // Don't create a new Phi unless we can remove the old one.
1459 if (!PN->hasOneUse())
1460 return std::nullopt;
1461
1462 for (Value *IncValPhi : PN->incoming_values()) {
1463 auto *Reinterpret = dyn_cast<IntrinsicInst>(IncValPhi);
1464 if (!Reinterpret ||
1465 Reinterpret->getIntrinsicID() !=
1466 Intrinsic::aarch64_sve_convert_to_svbool ||
1467 RequiredType != Reinterpret->getArgOperand(0)->getType())
1468 return std::nullopt;
1469 }
1470
1471 // Create the new Phi
1472 IC.Builder.SetInsertPoint(PN);
1473 PHINode *NPN = IC.Builder.CreatePHI(RequiredType, PN->getNumIncomingValues());
1474 Worklist.push_back(PN);
1475
1476 for (unsigned I = 0; I < PN->getNumIncomingValues(); I++) {
1477 auto *Reinterpret = cast<Instruction>(PN->getIncomingValue(I));
1478 NPN->addIncoming(Reinterpret->getOperand(0), PN->getIncomingBlock(I));
1479 Worklist.push_back(Reinterpret);
1480 }
1481
1482 // Cleanup Phi Node and reinterprets
1483 return IC.replaceInstUsesWith(II, NPN);
1484}
1485
1486// A collection of properties common to SVE intrinsics that allow for combines
1487// to be written without needing to know the specific intrinsic.
1489 //
1490 // Helper routines for common intrinsic definitions.
1491 //
1492
1493 // e.g. llvm.aarch64.sve.add pg, op1, op2
1494 // with IID ==> llvm.aarch64.sve.add_u
1495 static SVEIntrinsicInfo
1502
1503 // e.g. llvm.aarch64.sve.neg inactive, pg, op
1510
1511 // e.g. llvm.aarch64.sve.fcvtnt inactive, pg, op
1517
1518 // e.g. llvm.aarch64.sve.add_u pg, op1, op2
1524
1525 // e.g. llvm.aarch64.sve.prf pg, ptr (GPIndex = 0)
1526 // llvm.aarch64.sve.st1 data, pg, ptr (GPIndex = 1)
1527 static SVEIntrinsicInfo defaultVoidOp(unsigned GPIndex) {
1528 return SVEIntrinsicInfo()
1531 }
1532
1533 // e.g. llvm.aarch64.sve.cmpeq pg, op1, op2
1534 // llvm.aarch64.sve.ld1 pg, ptr
1541
1542 // All properties relate to predication and thus having a general predicate
1543 // is the minimum requirement to say there is intrinsic info to act on.
1544 explicit operator bool() const { return hasGoverningPredicate(); }
1545
1546 //
1547 // Properties relating to the governing predicate.
1548 //
1549
1551 return GoverningPredicateIdx != std::numeric_limits<unsigned>::max();
1552 }
1553
1555 assert(hasGoverningPredicate() && "Property not set!");
1556 return GoverningPredicateIdx;
1557 }
1558
1560 assert(!hasGoverningPredicate() && "Cannot set property twice!");
1561 GoverningPredicateIdx = Index;
1562 return *this;
1563 }
1564
1565 //
1566 // Properties relating to operations the intrinsic could be transformed into.
1567 // NOTE: This does not mean such a transformation is always possible, but the
1568 // knowledge makes it possible to reuse existing optimisations without needing
1569 // to embed specific handling for each intrinsic. For example, instruction
1570 // simplification can be used to optimise an intrinsic's active lanes.
1571 //
1572
1573 //
1574 // Intrinsic that produces the same result for active lanes.
1575 //
1576
1578 return UndefIntrinsic != Intrinsic::not_intrinsic;
1579 }
1580
1582 assert(hasMatchingUndefIntrinsic() && "Property not set!");
1583 return UndefIntrinsic;
1584 }
1585
1587 assert(!hasMatchingUndefIntrinsic() && "Cannot set property twice!");
1588 UndefIntrinsic = IID;
1589 return *this;
1590 }
1591
1592 //
1593 // Instruction where active lanes produce the same result.
1594 //
1595
1596 bool hasMatchingIROpode() const { return IROpcode != 0; }
1597
1598 unsigned getMatchingIROpode() const {
1599 assert(hasMatchingIROpode() && "Property not set!");
1600 return IROpcode;
1601 }
1602
1604 assert(!hasMatchingIROpode() && "Cannot set property twice!");
1605 IROpcode = Opcode;
1606 return *this;
1607 }
1608
1609 bool hasCmpPredicate() const {
1610 return CmpPredicate != CmpInst::BAD_ICMP_PREDICATE;
1611 }
1612
1614 assert(hasCmpPredicate() && "Property not set!");
1615 return CmpPredicate;
1616 }
1617
1619 assert(!hasCmpPredicate() && "Cannot set property twice!");
1620 CmpPredicate = Pred;
1621
1622 if (CmpInst::isFPPredicate(Pred))
1623 return setMatchingIROpcode(Instruction::FCmp);
1624
1625 if (CmpInst::isIntPredicate(Pred))
1626 return setMatchingIROpcode(Instruction::ICmp);
1627
1628 llvm_unreachable("Unsupported compare predicate!");
1629 }
1630
1631 //
1632 // Properties relating to the result of inactive lanes.
1633 //
1634
1636 return ResultLanes == InactiveLanesTakenFromOperand;
1637 }
1638
1640 assert(inactiveLanesTakenFromOperand() && "Property not set!");
1641 return OperandIdxForInactiveLanes;
1642 }
1643
1645 assert(ResultLanes == Uninitialized && "Cannot set property twice!");
1646 ResultLanes = InactiveLanesTakenFromOperand;
1647 OperandIdxForInactiveLanes = Index;
1648 return *this;
1649 }
1650
1652 return ResultLanes == InactiveLanesAreNotDefined;
1653 }
1654
1656 assert(ResultLanes == Uninitialized && "Cannot set property twice!");
1657 ResultLanes = InactiveLanesAreNotDefined;
1658 return *this;
1659 }
1660
1662 return ResultLanes == InactiveLanesAreUnused;
1663 }
1664
1666 assert(ResultLanes == Uninitialized && "Cannot set property twice!");
1667 ResultLanes = InactiveLanesAreUnused;
1668 return *this;
1669 }
1670
1671 // NOTE: Whilst not limited to only inactive lanes, the common use case is:
1672 // inactiveLanesAreZeroed =
1673 // resultIsZeroInitialized() && inactiveLanesAreUnused()
1674 bool resultIsZeroInitialized() const { return ResultIsZeroInitialized; }
1675
1677 ResultIsZeroInitialized = true;
1678 return *this;
1679 }
1680
1681 //
1682 // The first operand of unary merging operations is typically only used to
1683 // set the result for inactive lanes. Knowing this allows us to deadcode the
1684 // operand when we can prove there are no inactive lanes.
1685 //
1686
1688 return OperandIdxWithNoActiveLanes != std::numeric_limits<unsigned>::max();
1689 }
1690
1692 assert(hasOperandWithNoActiveLanes() && "Property not set!");
1693 return OperandIdxWithNoActiveLanes;
1694 }
1695
1697 assert(!hasOperandWithNoActiveLanes() && "Cannot set property twice!");
1698 OperandIdxWithNoActiveLanes = Index;
1699 return *this;
1700 }
1701
1702private:
1703 unsigned GoverningPredicateIdx = std::numeric_limits<unsigned>::max();
1704
1705 Intrinsic::ID UndefIntrinsic = Intrinsic::not_intrinsic;
1706 unsigned IROpcode = 0;
1708
1709 enum PredicationStyle {
1711 InactiveLanesTakenFromOperand,
1712 InactiveLanesAreNotDefined,
1713 InactiveLanesAreUnused
1714 } ResultLanes = Uninitialized;
1715
1716 bool ResultIsZeroInitialized = false;
1717 unsigned OperandIdxForInactiveLanes = std::numeric_limits<unsigned>::max();
1718 unsigned OperandIdxWithNoActiveLanes = std::numeric_limits<unsigned>::max();
1719};
1720
1722 // Some SVE intrinsics do not use scalable vector types, but since they are
1723 // not relevant from an SVEIntrinsicInfo perspective, they are also ignored.
1724 if (!isa<ScalableVectorType>(II.getType()) &&
1725 all_of(II.args(), [&](const Value *V) {
1726 return !isa<ScalableVectorType>(V->getType());
1727 }))
1728 return SVEIntrinsicInfo();
1729
1730 Intrinsic::ID IID = II.getIntrinsicID();
1731 switch (IID) {
1732 default:
1733 break;
1734 case Intrinsic::aarch64_sve_fcvt_bf16f32_v2:
1735 case Intrinsic::aarch64_sve_fcvt_f16f32:
1736 case Intrinsic::aarch64_sve_fcvt_f16f64:
1737 case Intrinsic::aarch64_sve_fcvt_f32f16:
1738 case Intrinsic::aarch64_sve_fcvt_f32f64:
1739 case Intrinsic::aarch64_sve_fcvt_f64f16:
1740 case Intrinsic::aarch64_sve_fcvt_f64f32:
1741 case Intrinsic::aarch64_sve_fcvtlt_f32f16:
1742 case Intrinsic::aarch64_sve_fcvtlt_f64f32:
1743 case Intrinsic::aarch64_sve_fcvtx_f32f64:
1744 case Intrinsic::aarch64_sve_fcvtzs:
1745 case Intrinsic::aarch64_sve_fcvtzs_i32f16:
1746 case Intrinsic::aarch64_sve_fcvtzs_i32f64:
1747 case Intrinsic::aarch64_sve_fcvtzs_i64f16:
1748 case Intrinsic::aarch64_sve_fcvtzs_i64f32:
1749 case Intrinsic::aarch64_sve_fcvtzu:
1750 case Intrinsic::aarch64_sve_fcvtzu_i32f16:
1751 case Intrinsic::aarch64_sve_fcvtzu_i32f64:
1752 case Intrinsic::aarch64_sve_fcvtzu_i64f16:
1753 case Intrinsic::aarch64_sve_fcvtzu_i64f32:
1754 case Intrinsic::aarch64_sve_revb:
1755 case Intrinsic::aarch64_sve_revh:
1756 case Intrinsic::aarch64_sve_revw:
1757 case Intrinsic::aarch64_sve_revd:
1758 case Intrinsic::aarch64_sve_scvtf:
1759 case Intrinsic::aarch64_sve_scvtf_f16i32:
1760 case Intrinsic::aarch64_sve_scvtf_f16i64:
1761 case Intrinsic::aarch64_sve_scvtf_f32i64:
1762 case Intrinsic::aarch64_sve_scvtf_f64i32:
1763 case Intrinsic::aarch64_sve_ucvtf:
1764 case Intrinsic::aarch64_sve_ucvtf_f16i32:
1765 case Intrinsic::aarch64_sve_ucvtf_f16i64:
1766 case Intrinsic::aarch64_sve_ucvtf_f32i64:
1767 case Intrinsic::aarch64_sve_ucvtf_f64i32:
1769
1770 case Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2:
1771 case Intrinsic::aarch64_sve_fcvtnt_f16f32:
1772 case Intrinsic::aarch64_sve_fcvtnt_f32f64:
1773 case Intrinsic::aarch64_sve_fcvtxnt_f32f64:
1775
1776 case Intrinsic::aarch64_sve_fabd:
1777 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fabd_u);
1778 case Intrinsic::aarch64_sve_fadd:
1779 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fadd_u)
1780 .setMatchingIROpcode(Instruction::FAdd);
1781 case Intrinsic::aarch64_sve_fdiv:
1782 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fdiv_u)
1783 .setMatchingIROpcode(Instruction::FDiv);
1784 case Intrinsic::aarch64_sve_fmax:
1785 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fmax_u);
1786 case Intrinsic::aarch64_sve_fmaxnm:
1787 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fmaxnm_u);
1788 case Intrinsic::aarch64_sve_fmin:
1789 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fmin_u);
1790 case Intrinsic::aarch64_sve_fminnm:
1791 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fminnm_u);
1792 case Intrinsic::aarch64_sve_fmla:
1793 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fmla_u);
1794 case Intrinsic::aarch64_sve_fmls:
1795 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fmls_u);
1796 case Intrinsic::aarch64_sve_fmul:
1797 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fmul_u)
1798 .setMatchingIROpcode(Instruction::FMul);
1799 case Intrinsic::aarch64_sve_fmulx:
1800 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fmulx_u);
1801 case Intrinsic::aarch64_sve_fnmla:
1802 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fnmla_u);
1803 case Intrinsic::aarch64_sve_fnmls:
1804 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fnmls_u);
1805 case Intrinsic::aarch64_sve_fsub:
1806 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fsub_u)
1807 .setMatchingIROpcode(Instruction::FSub);
1808 case Intrinsic::aarch64_sve_add:
1809 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_add_u)
1810 .setMatchingIROpcode(Instruction::Add);
1811 case Intrinsic::aarch64_sve_mla:
1812 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_mla_u);
1813 case Intrinsic::aarch64_sve_mls:
1814 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_mls_u);
1815 case Intrinsic::aarch64_sve_mul:
1816 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_mul_u)
1817 .setMatchingIROpcode(Instruction::Mul);
1818 case Intrinsic::aarch64_sve_sabd:
1819 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_sabd_u);
1820 case Intrinsic::aarch64_sve_sdiv:
1821 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_sdiv_u)
1822 .setMatchingIROpcode(Instruction::SDiv);
1823 case Intrinsic::aarch64_sve_smax:
1824 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_smax_u);
1825 case Intrinsic::aarch64_sve_smin:
1826 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_smin_u);
1827 case Intrinsic::aarch64_sve_smulh:
1828 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_smulh_u);
1829 case Intrinsic::aarch64_sve_sub:
1830 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_sub_u)
1831 .setMatchingIROpcode(Instruction::Sub);
1832 case Intrinsic::aarch64_sve_uabd:
1833 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_uabd_u);
1834 case Intrinsic::aarch64_sve_udiv:
1835 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_udiv_u)
1836 .setMatchingIROpcode(Instruction::UDiv);
1837 case Intrinsic::aarch64_sve_umax:
1838 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_umax_u);
1839 case Intrinsic::aarch64_sve_umin:
1840 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_umin_u);
1841 case Intrinsic::aarch64_sve_umulh:
1842 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_umulh_u);
1843 case Intrinsic::aarch64_sve_asr:
1844 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_asr_u)
1845 .setMatchingIROpcode(Instruction::AShr);
1846 case Intrinsic::aarch64_sve_lsl:
1847 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_lsl_u)
1848 .setMatchingIROpcode(Instruction::Shl);
1849 case Intrinsic::aarch64_sve_lsr:
1850 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_lsr_u)
1851 .setMatchingIROpcode(Instruction::LShr);
1852 case Intrinsic::aarch64_sve_and:
1853 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_and_u)
1854 .setMatchingIROpcode(Instruction::And);
1855 case Intrinsic::aarch64_sve_bic:
1856 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_bic_u);
1857 case Intrinsic::aarch64_sve_eor:
1858 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_eor_u)
1859 .setMatchingIROpcode(Instruction::Xor);
1860 case Intrinsic::aarch64_sve_orr:
1861 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_orr_u)
1862 .setMatchingIROpcode(Instruction::Or);
1863 case Intrinsic::aarch64_sve_shsub:
1864 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_shsub_u);
1865 case Intrinsic::aarch64_sve_shsubr:
1867 case Intrinsic::aarch64_sve_sqrshl:
1868 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_sqrshl_u);
1869 case Intrinsic::aarch64_sve_sqshl:
1870 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_sqshl_u);
1871 case Intrinsic::aarch64_sve_sqsub:
1872 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_sqsub_u);
1873 case Intrinsic::aarch64_sve_srshl:
1874 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_srshl_u);
1875 case Intrinsic::aarch64_sve_uhsub:
1876 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_uhsub_u);
1877 case Intrinsic::aarch64_sve_uhsubr:
1879 case Intrinsic::aarch64_sve_uqrshl:
1880 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_uqrshl_u);
1881 case Intrinsic::aarch64_sve_uqshl:
1882 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_uqshl_u);
1883 case Intrinsic::aarch64_sve_uqsub:
1884 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_uqsub_u);
1885 case Intrinsic::aarch64_sve_urshl:
1886 return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_urshl_u);
1887
1888 case Intrinsic::aarch64_sve_add_u:
1890 Instruction::Add);
1891 case Intrinsic::aarch64_sve_and_u:
1893 Instruction::And);
1894 case Intrinsic::aarch64_sve_asr_u:
1896 Instruction::AShr);
1897 case Intrinsic::aarch64_sve_eor_u:
1899 Instruction::Xor);
1900 case Intrinsic::aarch64_sve_fadd_u:
1902 Instruction::FAdd);
1903 case Intrinsic::aarch64_sve_fdiv_u:
1905 Instruction::FDiv);
1906 case Intrinsic::aarch64_sve_fmul_u:
1908 Instruction::FMul);
1909 case Intrinsic::aarch64_sve_fsub_u:
1911 Instruction::FSub);
1912 case Intrinsic::aarch64_sve_lsl_u:
1914 Instruction::Shl);
1915 case Intrinsic::aarch64_sve_lsr_u:
1917 Instruction::LShr);
1918 case Intrinsic::aarch64_sve_mul_u:
1920 Instruction::Mul);
1921 case Intrinsic::aarch64_sve_orr_u:
1923 Instruction::Or);
1924 case Intrinsic::aarch64_sve_sdiv_u:
1926 Instruction::SDiv);
1927 case Intrinsic::aarch64_sve_sub_u:
1929 Instruction::Sub);
1930 case Intrinsic::aarch64_sve_udiv_u:
1932 Instruction::UDiv);
1933
1934 case Intrinsic::aarch64_sve_addqv:
1935 case Intrinsic::aarch64_sve_bic_z:
1936 case Intrinsic::aarch64_sve_brka_z:
1937 case Intrinsic::aarch64_sve_brkb_z:
1938 case Intrinsic::aarch64_sve_brkn_z:
1939 case Intrinsic::aarch64_sve_brkpa_z:
1940 case Intrinsic::aarch64_sve_brkpb_z:
1941 case Intrinsic::aarch64_sve_cntp:
1942 case Intrinsic::aarch64_sve_compact:
1943 case Intrinsic::aarch64_sve_eorv:
1944 case Intrinsic::aarch64_sve_eorqv:
1945 case Intrinsic::aarch64_sve_nand_z:
1946 case Intrinsic::aarch64_sve_nor_z:
1947 case Intrinsic::aarch64_sve_orn_z:
1948 case Intrinsic::aarch64_sve_orv:
1949 case Intrinsic::aarch64_sve_orqv:
1950 case Intrinsic::aarch64_sve_pnext:
1951 case Intrinsic::aarch64_sve_rdffr_z:
1952 case Intrinsic::aarch64_sve_saddv:
1953 case Intrinsic::aarch64_sve_uaddv:
1954 case Intrinsic::aarch64_sve_umaxv:
1955 case Intrinsic::aarch64_sve_umaxqv:
1956 case Intrinsic::aarch64_sve_facge:
1957 case Intrinsic::aarch64_sve_facgt:
1958 case Intrinsic::aarch64_sve_ld1:
1959 case Intrinsic::aarch64_sve_ld1_gather:
1960 case Intrinsic::aarch64_sve_ld1_gather_index:
1961 case Intrinsic::aarch64_sve_ld1_gather_scalar_offset:
1962 case Intrinsic::aarch64_sve_ld1_gather_sxtw:
1963 case Intrinsic::aarch64_sve_ld1_gather_sxtw_index:
1964 case Intrinsic::aarch64_sve_ld1_gather_uxtw:
1965 case Intrinsic::aarch64_sve_ld1_gather_uxtw_index:
1966 case Intrinsic::aarch64_sve_ld1q_gather_index:
1967 case Intrinsic::aarch64_sve_ld1q_gather_scalar_offset:
1968 case Intrinsic::aarch64_sve_ld1q_gather_vector_offset:
1969 case Intrinsic::aarch64_sve_ld1ro:
1970 case Intrinsic::aarch64_sve_ld1rq:
1971 case Intrinsic::aarch64_sve_ld1udq:
1972 case Intrinsic::aarch64_sve_ld1uwq:
1973 case Intrinsic::aarch64_sve_ld2_sret:
1974 case Intrinsic::aarch64_sve_ld2q_sret:
1975 case Intrinsic::aarch64_sve_ld3_sret:
1976 case Intrinsic::aarch64_sve_ld3q_sret:
1977 case Intrinsic::aarch64_sve_ld4_sret:
1978 case Intrinsic::aarch64_sve_ld4q_sret:
1979 case Intrinsic::aarch64_sve_ldff1:
1980 case Intrinsic::aarch64_sve_ldff1_gather:
1981 case Intrinsic::aarch64_sve_ldff1_gather_index:
1982 case Intrinsic::aarch64_sve_ldff1_gather_scalar_offset:
1983 case Intrinsic::aarch64_sve_ldff1_gather_sxtw:
1984 case Intrinsic::aarch64_sve_ldff1_gather_sxtw_index:
1985 case Intrinsic::aarch64_sve_ldff1_gather_uxtw:
1986 case Intrinsic::aarch64_sve_ldff1_gather_uxtw_index:
1987 case Intrinsic::aarch64_sve_ldnf1:
1988 case Intrinsic::aarch64_sve_ldnt1:
1989 case Intrinsic::aarch64_sve_ldnt1_gather:
1990 case Intrinsic::aarch64_sve_ldnt1_gather_index:
1991 case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
1992 case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
1994
1995 case Intrinsic::aarch64_sve_and_z:
1997 Instruction::And);
1998 case Intrinsic::aarch64_sve_orr_z:
2000 Instruction::Or);
2001 case Intrinsic::aarch64_sve_eor_z:
2003 Instruction::Xor);
2004
2005 case Intrinsic::aarch64_sve_cmpeq:
2006 case Intrinsic::aarch64_sve_cmpeq_wide:
2009 case Intrinsic::aarch64_sve_cmpge:
2010 case Intrinsic::aarch64_sve_cmpge_wide:
2013 case Intrinsic::aarch64_sve_cmpgt:
2014 case Intrinsic::aarch64_sve_cmpgt_wide:
2017 case Intrinsic::aarch64_sve_cmphi:
2018 case Intrinsic::aarch64_sve_cmphi_wide:
2021 case Intrinsic::aarch64_sve_cmphs:
2022 case Intrinsic::aarch64_sve_cmphs_wide:
2025 case Intrinsic::aarch64_sve_cmple_wide:
2028 case Intrinsic::aarch64_sve_cmplo_wide:
2031 case Intrinsic::aarch64_sve_cmpls_wide:
2034 case Intrinsic::aarch64_sve_cmplt_wide:
2037 case Intrinsic::aarch64_sve_cmpne:
2038 case Intrinsic::aarch64_sve_cmpne_wide:
2041 case Intrinsic::aarch64_sve_fcmpeq:
2044 case Intrinsic::aarch64_sve_fcmpge:
2047 case Intrinsic::aarch64_sve_fcmpgt:
2050 case Intrinsic::aarch64_sve_fcmpne:
2053 case Intrinsic::aarch64_sve_fcmpuo:
2056
2057 case Intrinsic::aarch64_sve_prf:
2058 case Intrinsic::aarch64_sve_prfb_gather_index:
2059 case Intrinsic::aarch64_sve_prfb_gather_scalar_offset:
2060 case Intrinsic::aarch64_sve_prfb_gather_sxtw_index:
2061 case Intrinsic::aarch64_sve_prfb_gather_uxtw_index:
2062 case Intrinsic::aarch64_sve_prfd_gather_index:
2063 case Intrinsic::aarch64_sve_prfd_gather_scalar_offset:
2064 case Intrinsic::aarch64_sve_prfd_gather_sxtw_index:
2065 case Intrinsic::aarch64_sve_prfd_gather_uxtw_index:
2066 case Intrinsic::aarch64_sve_prfh_gather_index:
2067 case Intrinsic::aarch64_sve_prfh_gather_scalar_offset:
2068 case Intrinsic::aarch64_sve_prfh_gather_sxtw_index:
2069 case Intrinsic::aarch64_sve_prfh_gather_uxtw_index:
2070 case Intrinsic::aarch64_sve_prfw_gather_index:
2071 case Intrinsic::aarch64_sve_prfw_gather_scalar_offset:
2072 case Intrinsic::aarch64_sve_prfw_gather_sxtw_index:
2073 case Intrinsic::aarch64_sve_prfw_gather_uxtw_index:
2075
2076 case Intrinsic::aarch64_sve_st1_scatter:
2077 case Intrinsic::aarch64_sve_st1_scatter_scalar_offset:
2078 case Intrinsic::aarch64_sve_st1_scatter_sxtw:
2079 case Intrinsic::aarch64_sve_st1_scatter_sxtw_index:
2080 case Intrinsic::aarch64_sve_st1_scatter_uxtw:
2081 case Intrinsic::aarch64_sve_st1_scatter_uxtw_index:
2082 case Intrinsic::aarch64_sve_st1dq:
2083 case Intrinsic::aarch64_sve_st1q_scatter_index:
2084 case Intrinsic::aarch64_sve_st1q_scatter_scalar_offset:
2085 case Intrinsic::aarch64_sve_st1q_scatter_vector_offset:
2086 case Intrinsic::aarch64_sve_st1wq:
2087 case Intrinsic::aarch64_sve_stnt1:
2088 case Intrinsic::aarch64_sve_stnt1_scatter:
2089 case Intrinsic::aarch64_sve_stnt1_scatter_index:
2090 case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset:
2091 case Intrinsic::aarch64_sve_stnt1_scatter_uxtw:
2093 case Intrinsic::aarch64_sve_st2:
2094 case Intrinsic::aarch64_sve_st2q:
2096 case Intrinsic::aarch64_sve_st3:
2097 case Intrinsic::aarch64_sve_st3q:
2099 case Intrinsic::aarch64_sve_st4:
2100 case Intrinsic::aarch64_sve_st4q:
2102 }
2103
2104 return SVEIntrinsicInfo();
2105}
2106
2108 Value *UncastedPred;
2109
2110 // Look through predicate casts that only remove lanes.
2112 m_Value(UncastedPred)))) {
2113 auto *OrigPredTy = cast<ScalableVectorType>(Pred->getType());
2114 Pred = UncastedPred;
2115
2117 m_Value(UncastedPred))))
2118 // If the predicate has the same or less lanes than the uncasted predicate
2119 // then we know the casting has no effect.
2120 if (OrigPredTy->getMinNumElements() <=
2121 cast<ScalableVectorType>(UncastedPred->getType())
2122 ->getMinNumElements())
2123 Pred = UncastedPred;
2124 }
2125
2126 auto *C = dyn_cast<Constant>(Pred);
2127 return C && C->isAllOnesValue();
2128}
2129
2130// Simplify `V` by only considering the operations that affect active lanes.
2131// This function should only return existing Values or newly created Constants.
2132static Value *stripInactiveLanes(Value *V, const Value *Pg) {
2133 auto *Dup = dyn_cast<IntrinsicInst>(V);
2134 if (Dup && Dup->getIntrinsicID() == Intrinsic::aarch64_sve_dup &&
2135 Dup->getOperand(1) == Pg && isa<Constant>(Dup->getOperand(2)))
2137 cast<VectorType>(V->getType())->getElementCount(),
2138 cast<Constant>(Dup->getOperand(2)));
2139
2140 return V;
2141}
2142
2143static std::optional<Instruction *>
2145 const SVEIntrinsicInfo &IInfo) {
2146 const unsigned Opc = IInfo.getMatchingIROpode();
2147 assert(Instruction::isBinaryOp(Opc) && "Expected a binary operation!");
2148
2149 Value *Pg = II.getOperand(0);
2150 Value *Op1 = II.getOperand(1);
2151 Value *Op2 = II.getOperand(2);
2152 const DataLayout &DL = II.getDataLayout();
2153
2154 // Canonicalise constants to the RHS.
2156 isa<Constant>(Op1) && !isa<Constant>(Op2)) {
2157 IC.replaceOperand(II, 1, Op2);
2158 IC.replaceOperand(II, 2, Op1);
2159 return &II;
2160 }
2161
2162 // Only active lanes matter when simplifying the operation.
2163 Op1 = stripInactiveLanes(Op1, Pg);
2164 Op2 = stripInactiveLanes(Op2, Pg);
2165
2166 Value *SimpleII;
2167 if (auto FII = dyn_cast<FPMathOperator>(&II))
2168 SimpleII = simplifyBinOp(Opc, Op1, Op2, FII->getFastMathFlags(), DL);
2169 else
2170 SimpleII = simplifyBinOp(Opc, Op1, Op2, DL);
2171
2172 // If both operands are convert.to.svbool from the same narrower predicate
2173 // type, try to simplify the operation at that narrower type. This is valid
2174 // because the conversions zero the lanes not represented by the narrower
2175 // type, so those lanes of the result are zero either way.
2176 Value *NarrowOp1, *NarrowOp2;
2177 if (!SimpleII &&
2179 m_Value(NarrowOp1))) &&
2181 m_Value(NarrowOp2))) &&
2182 NarrowOp1->getType() == NarrowOp2->getType()) {
2183 Value *SimpleNarrow = simplifyBinOp(Opc, NarrowOp1, NarrowOp2, DL);
2184 if (SimpleNarrow && !isa<UndefValue>(SimpleNarrow)) {
2185 if (match(SimpleNarrow, m_ZeroInt()))
2186 SimpleII = Constant::getNullValue(II.getType());
2187 else if (SimpleNarrow == NarrowOp1)
2188 SimpleII = Op1;
2189 else if (SimpleNarrow == NarrowOp2)
2190 SimpleII = Op2;
2191 else
2192 SimpleII =
2193 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
2194 SimpleNarrow->getType(), SimpleNarrow);
2195 }
2196 }
2197
2198 // An SVE intrinsic's result is always defined. However, this is not the case
2199 // for its equivalent IR instruction (e.g. when shifting by an amount more
2200 // than the data's bitwidth). Simplifications to an undefined result must be
2201 // ignored to preserve the intrinsic's expected behaviour.
2202 if (!SimpleII || isa<UndefValue>(SimpleII))
2203 return std::nullopt;
2204
2205 if (IInfo.inactiveLanesAreNotDefined())
2206 return IC.replaceInstUsesWith(II, SimpleII);
2207
2208 Value *Inactive =
2210 ? Constant::getNullValue(II.getType())
2211 : II.getOperand(IInfo.getOperandIdxInactiveLanesTakenFrom());
2212
2213 // The intrinsic does nothing (e.g. sve.mul(pg, A, 1.0)).
2214 if (SimpleII == Inactive)
2215 return IC.replaceInstUsesWith(II, SimpleII);
2216
2217 // Inactive lanes must be preserved.
2218 SimpleII = IC.Builder.CreateSelect(Pg, SimpleII, Inactive);
2219 return IC.replaceInstUsesWith(II, SimpleII);
2220}
2221
2222static std::optional<Instruction *>
2224 const SVEIntrinsicInfo &IInfo) {
2225 const unsigned Opc = IInfo.getMatchingIROpode();
2226 assert((Opc == Instruction::ICmp || Opc == Instruction::FCmp) &&
2227 "Expected a compare operation!");
2228
2229 Value *Pg = II.getOperand(0);
2230 Value *LHS = II.getOperand(1);
2231 Value *RHS = II.getOperand(2);
2232 CmpInst::Predicate CmpPred = IInfo.getCmpPredicate();
2233 bool IsWideICmp =
2234 Opc == Instruction::ICmp && LHS->getType() != RHS->getType();
2235 assert((IsWideICmp || LHS->getType() == RHS->getType()) &&
2236 "Unexpected wide compare!");
2237
2238 // Canonicalise constants to the RHS.
2239 if ((ICmpInst::isCommutative(CmpPred) || FCmpInst::isCommutative(CmpPred)) &&
2240 isa<Constant>(LHS) && !isa<Constant>(RHS) && !IsWideICmp) {
2241 IC.replaceOperand(II, 1, RHS);
2242 IC.replaceOperand(II, 2, LHS);
2243 return &II;
2244 }
2245
2246 // Only active lanes matter when simplifying the operation.
2247 LHS = stripInactiveLanes(LHS, Pg);
2248 RHS = stripInactiveLanes(RHS, Pg);
2249
2250 if (IsWideICmp) {
2251 // We can do more for wide compares, but not using simplifyCmpInst.
2252 const APInt *LHSVal, *RHSVal;
2253 if (!match(LHS, m_APInt(LHSVal)) || !match(RHS, m_APInt(RHSVal)))
2254 return std::nullopt;
2255
2256 // Consider cmpge.wide(..., <vscale x 4 x i32> LHS, <vscale x 2 x i64> RHS),
2257 // we must reconstruct the constants because LHS has the wrong element type,
2258 // and RHS the wrong element count.
2259 Type *WideVT = VectorType::get(RHS->getType()->getScalarType(),
2260 cast<VectorType>(LHS->getType()));
2261 // NOTE: Wide equality comparisons are signed.
2262 if (ICmpInst::isUnsigned(CmpPred)) {
2263 LHS = ConstantInt::get(WideVT, LHSVal->getZExtValue());
2264 RHS = ConstantInt::get(WideVT, RHSVal->getZExtValue());
2265 } else {
2266 LHS = ConstantInt::get(WideVT, LHSVal->getSExtValue());
2267 RHS = ConstantInt::get(WideVT, RHSVal->getSExtValue());
2268 }
2269 }
2270
2271 // TODO: Allow fast-math flags for calls to compare intrinsics.
2272 const DataLayout &DL = II.getDataLayout();
2273 Value *SimpleII = simplifyCmpInst(CmpPred, LHS, RHS, DL);
2274
2275 // No simplification happened.
2276 if (!SimpleII)
2277 return std::nullopt;
2278
2279 assert(IInfo.resultIsZeroInitialized() && "Expected a zeroing operation!");
2280
2281 if (match(SimpleII, m_ZeroInt()))
2282 return IC.replaceInstUsesWith(II, SimpleII);
2283
2284 // Inactive lanes must be zeroed.
2285 SimpleII = IC.Builder.CreateLogicalAnd(Pg, SimpleII);
2286 return IC.replaceInstUsesWith(II, SimpleII);
2287}
2288
2289// Use SVE intrinsic info to eliminate redundant operands and/or canonicalise
2290// to operations with less strict inactive lane requirements.
2291static std::optional<Instruction *>
2293 const SVEIntrinsicInfo &IInfo) {
2294 if (!IInfo.hasGoverningPredicate())
2295 return std::nullopt;
2296
2297 auto *OpPredicate = II.getOperand(IInfo.getGoverningPredicateOperandIdx());
2298
2299 // If there are no active lanes.
2300 if (match(OpPredicate, m_ZeroInt())) {
2302 return IC.replaceInstUsesWith(
2303 II, II.getOperand(IInfo.getOperandIdxInactiveLanesTakenFrom()));
2304
2305 if (IInfo.inactiveLanesAreUnused()) {
2306 if (IInfo.resultIsZeroInitialized())
2308
2309 return IC.eraseInstFromFunction(II);
2310 }
2311 }
2312
2313 // If there are no inactive lanes.
2314 if (isAllActivePredicate(OpPredicate)) {
2315 if (IInfo.hasOperandWithNoActiveLanes()) {
2316 unsigned OpIdx = IInfo.getOperandIdxWithNoActiveLanes();
2317 if (!isa<UndefValue>(II.getOperand(OpIdx)))
2318 return IC.replaceOperand(II, OpIdx, UndefValue::get(II.getType()));
2319 }
2320
2321 if (IInfo.hasMatchingUndefIntrinsic()) {
2322 auto *NewDecl = Intrinsic::getOrInsertDeclaration(
2323 II.getModule(), IInfo.getMatchingUndefIntrinsic(), {II.getType()});
2324 II.setCalledFunction(NewDecl);
2325 return &II;
2326 }
2327 }
2328
2329 if (!IInfo.hasMatchingIROpode())
2330 return std::nullopt;
2331
2332 //
2333 // Operation specific simplifications.
2334 //
2335
2336 unsigned Opc = IInfo.getMatchingIROpode();
2337
2339 return simplifySVEIntrinsicBinOp(IC, II, IInfo);
2340
2341 if (Opc == Instruction::FCmp || Opc == Instruction::ICmp)
2342 return simplifySVEIntrinsicCompare(IC, II, IInfo);
2343
2344 return std::nullopt;
2345}
2346
2347// (from_svbool (binop (to_svbool pred) (svbool_t _) (svbool_t _))))
2348// => (binop (pred) (from_svbool _) (from_svbool _))
2349//
2350// The above transformation eliminates a `to_svbool` in the predicate
2351// operand of bitwise operation `binop` by narrowing the vector width of
2352// the operation. For example, it would convert a `<vscale x 16 x i1>
2353// and` into a `<vscale x 4 x i1> and`. This is profitable because
2354// to_svbool must zero the new lanes during widening, whereas
2355// from_svbool is free.
2356static std::optional<Instruction *>
2358 auto m_ConvertToSVBool = [](auto P) {
2360 };
2361 constexpr Intrinsic::ID ConvertFromSVBool =
2362 Intrinsic::aarch64_sve_convert_from_svbool;
2363
2364 Type *Ty = II.getType();
2365 Value *LHS, *RHS, *NarrowLHS, *NarrowRHS;
2366
2367 if (match(II.getOperand(0),
2369 m_ConvertToSVBool(m_SpecificType(Ty, NarrowRHS))))) {
2370 NarrowLHS = IC.Builder.CreateIntrinsic(ConvertFromSVBool, Ty, LHS);
2371 Value *NarrowAnd = IC.Builder.CreateLogicalAnd(NarrowLHS, NarrowRHS);
2372 return IC.replaceInstUsesWith(II, NarrowAnd);
2373 }
2374
2375 if (match(II.getOperand(0),
2376 m_LogicalAnd(m_ConvertToSVBool(m_SpecificType(Ty, NarrowLHS)),
2377 m_Value(RHS)))) {
2378 NarrowRHS = IC.Builder.CreateIntrinsic(ConvertFromSVBool, Ty, RHS);
2379 Value *NarrowAnd = IC.Builder.CreateLogicalAnd(NarrowLHS, NarrowRHS);
2380 return IC.replaceInstUsesWith(II, NarrowAnd);
2381 }
2382
2383 auto BinOp = dyn_cast<IntrinsicInst>(II.getOperand(0));
2384 if (!BinOp)
2385 return std::nullopt;
2386
2387 Intrinsic::ID BinOpIID = BinOp->getIntrinsicID();
2388 switch (BinOpIID) {
2389 case Intrinsic::aarch64_sve_and_z:
2390 case Intrinsic::aarch64_sve_bic_z:
2391 case Intrinsic::aarch64_sve_eor_z:
2392 case Intrinsic::aarch64_sve_nand_z:
2393 case Intrinsic::aarch64_sve_nor_z:
2394 case Intrinsic::aarch64_sve_orn_z:
2395 case Intrinsic::aarch64_sve_orr_z:
2396 break;
2397 default:
2398 return std::nullopt;
2399 }
2400
2401 Value *BinOpPred = BinOp->getOperand(0);
2402 Value *BinOpOp1 = BinOp->getOperand(1);
2403 Value *BinOpOp2 = BinOp->getOperand(2);
2404
2405 Value *NarrowBinOpPred;
2406 if (!match(BinOpPred, m_ConvertToSVBool(m_SpecificType(Ty, NarrowBinOpPred))))
2407 return std::nullopt;
2408
2409 Value *NarrowBinOpOp1 =
2410 IC.Builder.CreateIntrinsic(ConvertFromSVBool, Ty, BinOpOp1);
2411 Value *NarrowBinOpOp2 = NarrowBinOpOp1;
2412 if (BinOpOp1 != BinOpOp2)
2413 NarrowBinOpOp2 =
2414 IC.Builder.CreateIntrinsic(ConvertFromSVBool, Ty, BinOpOp2);
2415 Value *NarrowedBinOp = IC.Builder.CreateIntrinsic(
2416 BinOpIID, Ty, {NarrowBinOpPred, NarrowBinOpOp1, NarrowBinOpOp2});
2417 return IC.replaceInstUsesWith(II, NarrowedBinOp);
2418}
2419
2420static std::optional<Instruction *>
2422 // If the reinterpret instruction operand is a PHI Node
2423 if (isa<PHINode>(II.getArgOperand(0)))
2424 return processPhiNode(IC, II);
2425
2426 if (auto BinOpCombine = tryCombineFromSVBoolBinOp(IC, II))
2427 return BinOpCombine;
2428
2429 // Ignore converts to/from svcount_t.
2430 if (isa<TargetExtType>(II.getArgOperand(0)->getType()) ||
2431 isa<TargetExtType>(II.getType()))
2432 return std::nullopt;
2433
2434 SmallVector<Instruction *, 32> CandidatesForRemoval;
2435 Value *Cursor = II.getOperand(0), *EarliestReplacement = nullptr;
2436
2437 const auto *IVTy = cast<VectorType>(II.getType());
2438
2439 // Walk the chain of conversions.
2440 while (Cursor) {
2441 // If the type of the cursor has fewer lanes than the final result, zeroing
2442 // must take place, which breaks the equivalence chain.
2443 const auto *CursorVTy = cast<VectorType>(Cursor->getType());
2444 if (CursorVTy->getElementCount().getKnownMinValue() <
2445 IVTy->getElementCount().getKnownMinValue())
2446 break;
2447
2448 // If the cursor has the same type as I, it is a viable replacement.
2449 if (Cursor->getType() == IVTy)
2450 EarliestReplacement = Cursor;
2451
2452 auto *IntrinsicCursor = dyn_cast<IntrinsicInst>(Cursor);
2453
2454 // If this is not an SVE conversion intrinsic, this is the end of the chain.
2455 if (!IntrinsicCursor || !(IntrinsicCursor->getIntrinsicID() ==
2456 Intrinsic::aarch64_sve_convert_to_svbool ||
2457 IntrinsicCursor->getIntrinsicID() ==
2458 Intrinsic::aarch64_sve_convert_from_svbool))
2459 break;
2460
2461 CandidatesForRemoval.insert(CandidatesForRemoval.begin(), IntrinsicCursor);
2462 Cursor = IntrinsicCursor->getOperand(0);
2463 }
2464
2465 // If no viable replacement in the conversion chain was found, there is
2466 // nothing to do.
2467 if (!EarliestReplacement)
2468 return std::nullopt;
2469
2470 return IC.replaceInstUsesWith(II, EarliestReplacement);
2471}
2472
2473static std::optional<Instruction *> instCombineSVESel(InstCombiner &IC,
2474 IntrinsicInst &II) {
2475 // svsel(ptrue, x, y) => x
2476 auto *OpPredicate = II.getOperand(0);
2477 if (isAllActivePredicate(OpPredicate))
2478 return IC.replaceInstUsesWith(II, II.getOperand(1));
2479
2480 auto Select =
2481 IC.Builder.CreateSelect(OpPredicate, II.getOperand(1), II.getOperand(2));
2482 return IC.replaceInstUsesWith(II, Select);
2483}
2484
2485static std::optional<Instruction *> instCombineSVEDup(InstCombiner &IC,
2486 IntrinsicInst &II) {
2487 Value *Pg = II.getOperand(1);
2488
2489 // sve.dup(V, all_active, X) ==> splat(X)
2490 if (isAllActivePredicate(Pg)) {
2491 auto *RetTy = cast<ScalableVectorType>(II.getType());
2492 Value *Splat = IC.Builder.CreateVectorSplat(RetTy->getElementCount(),
2493 II.getArgOperand(2));
2494 return IC.replaceInstUsesWith(II, Splat);
2495 }
2496
2498 m_SpecificInt(AArch64SVEPredPattern::vl1))))
2499 return std::nullopt;
2500
2501 // sve.dup(V, sve.ptrue(vl1), X) ==> insertelement V, X, 0
2502 Value *Insert = IC.Builder.CreateInsertElement(
2503 II.getArgOperand(0), II.getArgOperand(2), uint64_t(0));
2504 return IC.replaceInstUsesWith(II, Insert);
2505}
2506
2507static std::optional<Instruction *> instCombineSVEDupX(InstCombiner &IC,
2508 IntrinsicInst &II) {
2509 // Replace DupX with a regular IR splat.
2510 auto *RetTy = cast<ScalableVectorType>(II.getType());
2511 Value *Splat = IC.Builder.CreateVectorSplat(RetTy->getElementCount(),
2512 II.getArgOperand(0));
2513 Splat->takeName(&II);
2514 return IC.replaceInstUsesWith(II, Splat);
2515}
2516
2517// xor(cmpne(%pg, %lhs, %rhs), %pg)
2518// -> cmpeq(%pg, %lhs, %rhs)
2519static std::optional<Instruction *> instCombineXorSVECmpCC(InstCombiner &IC,
2520 IntrinsicInst &II) {
2521 if (!II.hasOneUse())
2522 return std::nullopt;
2523 auto *User = cast<Instruction>(*II.user_begin());
2524 if (!match(User, m_c_Xor(m_Specific(&II), m_Specific(II.getOperand(0)))))
2525 return std::nullopt;
2526
2527 Intrinsic::ID IID;
2528 switch (II.getIntrinsicID()) {
2529 case Intrinsic::aarch64_sve_cmpne:
2530 IID = Intrinsic::aarch64_sve_cmpeq;
2531 break;
2532 case Intrinsic::aarch64_sve_cmpne_wide:
2533 IID = Intrinsic::aarch64_sve_cmpeq_wide;
2534 break;
2535 case Intrinsic::aarch64_sve_cmpeq:
2536 IID = Intrinsic::aarch64_sve_cmpne;
2537 break;
2538 case Intrinsic::aarch64_sve_cmpeq_wide:
2539 IID = Intrinsic::aarch64_sve_cmpne_wide;
2540 break;
2541 default:
2542 return std::nullopt;
2543 }
2544
2546 Value *CMPCC = IC.Builder.CreateIntrinsic(
2547 IID, II.getOperand(1)->getType(),
2548 {II.getOperand(0), II.getOperand(1), II.getOperand(2)});
2549 IC.replaceInstUsesWith(*User, CMPCC);
2551 return &II;
2552}
2553
2554// zext(cmpne(ptrue, %v, 0))
2555// -> umin(%pg, %v, 1)
2556static std::optional<Instruction *> instCombineZExtSVECmpNE(InstCombiner &IC,
2557 IntrinsicInst &II) {
2558 if (!isAllActivePredicate(II.getOperand(0)) ||
2559 !match(II.getOperand(2), m_Zero()))
2560 return std::nullopt;
2561
2562 for (auto *U : II.users()) {
2563 if (match(U, m_ZExt(m_Specific(&II)))) {
2564 auto *User = cast<Instruction>(U);
2565 Type *Ty = II.getOperand(1)->getType();
2566 if (User->getType() != Ty)
2567 continue;
2570 Intrinsic::aarch64_sve_umin, Ty,
2571 {II.getOperand(0), II.getOperand(1), ConstantInt::get(Ty, 1)});
2574 return &II;
2575 }
2576 }
2577 return std::nullopt;
2578}
2579
2580static std::optional<Instruction *> instCombineSVECmpNE(InstCombiner &IC,
2581 IntrinsicInst &II) {
2582 LLVMContext &Ctx = II.getContext();
2583
2584 if (auto Res = instCombineXorSVECmpCC(IC, II))
2585 return Res;
2586
2587 if (auto Res = instCombineZExtSVECmpNE(IC, II))
2588 return Res;
2589
2590 if (!isAllActivePredicate(II.getArgOperand(0)))
2591 return std::nullopt;
2592
2593 // Check that we have a compare of zero..
2594 auto *SplatValue =
2596 if (!SplatValue || !SplatValue->isZero())
2597 return std::nullopt;
2598
2599 // ..against a dupq
2600 auto *DupQLane = dyn_cast<IntrinsicInst>(II.getArgOperand(1));
2601 if (!DupQLane ||
2602 DupQLane->getIntrinsicID() != Intrinsic::aarch64_sve_dupq_lane)
2603 return std::nullopt;
2604
2605 // Where the dupq is a lane 0 replicate of a vector insert
2606 auto *DupQLaneIdx = dyn_cast<ConstantInt>(DupQLane->getArgOperand(1));
2607 if (!DupQLaneIdx || !DupQLaneIdx->isZero())
2608 return std::nullopt;
2609
2610 auto *VecIns = dyn_cast<IntrinsicInst>(DupQLane->getArgOperand(0));
2611 if (!VecIns || VecIns->getIntrinsicID() != Intrinsic::vector_insert)
2612 return std::nullopt;
2613
2614 // Where the vector insert is a fixed constant vector insert into undef at
2615 // index zero
2616 if (!isa<UndefValue>(VecIns->getArgOperand(0)))
2617 return std::nullopt;
2618
2619 if (!cast<ConstantInt>(VecIns->getArgOperand(2))->isZero())
2620 return std::nullopt;
2621
2622 auto *ConstVec = dyn_cast<Constant>(VecIns->getArgOperand(1));
2623 if (!ConstVec)
2624 return std::nullopt;
2625
2626 auto *VecTy = dyn_cast<FixedVectorType>(ConstVec->getType());
2627 auto *OutTy = dyn_cast<ScalableVectorType>(II.getType());
2628 if (!VecTy || !OutTy || VecTy->getNumElements() != OutTy->getMinNumElements())
2629 return std::nullopt;
2630
2631 unsigned NumElts = VecTy->getNumElements();
2632 unsigned PredicateBits = 0;
2633
2634 // Expand intrinsic operands to a 16-bit byte level predicate
2635 for (unsigned I = 0; I < NumElts; ++I) {
2636 auto *Arg = dyn_cast<ConstantInt>(ConstVec->getAggregateElement(I));
2637 if (!Arg)
2638 return std::nullopt;
2639 if (!Arg->isZero())
2640 PredicateBits |= 1 << (I * (16 / NumElts));
2641 }
2642
2643 // If all bits are zero bail early with an empty predicate
2644 if (PredicateBits == 0) {
2645 auto *PFalse = Constant::getNullValue(II.getType());
2646 PFalse->takeName(&II);
2647 return IC.replaceInstUsesWith(II, PFalse);
2648 }
2649
2650 // Calculate largest predicate type used (where byte predicate is largest)
2651 unsigned Mask = 8;
2652 for (unsigned I = 0; I < 16; ++I)
2653 if ((PredicateBits & (1 << I)) != 0)
2654 Mask |= (I % 8);
2655
2656 unsigned PredSize = Mask & -Mask;
2657 auto *PredType = ScalableVectorType::get(
2658 Type::getInt1Ty(Ctx), AArch64::SVEBitsPerBlock / (PredSize * 8));
2659
2660 // Ensure all relevant bits are set
2661 for (unsigned I = 0; I < 16; I += PredSize)
2662 if ((PredicateBits & (1 << I)) == 0)
2663 return std::nullopt;
2664
2665 auto *ConvertToSVBool =
2666 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
2667 PredType, ConstantInt::getTrue(PredType));
2668 auto *ConvertFromSVBool =
2669 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool,
2670 II.getType(), ConvertToSVBool);
2671
2672 ConvertFromSVBool->takeName(&II);
2673 return IC.replaceInstUsesWith(II, ConvertFromSVBool);
2674}
2675
2676static std::optional<Instruction *> instCombineSVELast(InstCombiner &IC,
2677 IntrinsicInst &II) {
2678 Value *Pg = II.getArgOperand(0);
2679 Value *Vec = II.getArgOperand(1);
2680 auto IntrinsicID = II.getIntrinsicID();
2681 bool IsAfter = IntrinsicID == Intrinsic::aarch64_sve_lasta;
2682
2683 // lastX(splat(X)) --> X
2684 if (auto *SplatVal = getSplatValue(Vec))
2685 return IC.replaceInstUsesWith(II, SplatVal);
2686
2687 // If x and/or y is a splat value then:
2688 // lastX (binop (x, y)) --> binop(lastX(x), lastX(y))
2689 Value *LHS, *RHS;
2690 if (match(Vec, m_OneUse(m_BinOp(m_Value(LHS), m_Value(RHS))))) {
2691 if (isSplatValue(LHS) || isSplatValue(RHS)) {
2692 auto *OldBinOp = cast<BinaryOperator>(Vec);
2693 auto OpC = OldBinOp->getOpcode();
2694 auto *NewLHS =
2695 IC.Builder.CreateIntrinsic(IntrinsicID, {Vec->getType()}, {Pg, LHS});
2696 auto *NewRHS =
2697 IC.Builder.CreateIntrinsic(IntrinsicID, {Vec->getType()}, {Pg, RHS});
2699 OpC, NewLHS, NewRHS, OldBinOp, OldBinOp->getName(), II.getIterator());
2700 return IC.replaceInstUsesWith(II, NewBinOp);
2701 }
2702 }
2703
2704 auto *C = dyn_cast<Constant>(Pg);
2705 if (IsAfter && C && C->isNullValue()) {
2706 // The intrinsic is extracting lane 0 so use an extract instead.
2707 auto *IdxTy = Type::getInt64Ty(II.getContext());
2708 auto *Extract = ExtractElementInst::Create(Vec, ConstantInt::get(IdxTy, 0));
2709 Extract->insertBefore(II.getIterator());
2710 Extract->takeName(&II);
2711 return IC.replaceInstUsesWith(II, Extract);
2712 }
2713
2714 auto *IntrPG = dyn_cast<IntrinsicInst>(Pg);
2715 if (!IntrPG)
2716 return std::nullopt;
2717
2718 if (IntrPG->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue)
2719 return std::nullopt;
2720
2721 const auto PTruePattern =
2722 cast<ConstantInt>(IntrPG->getOperand(0))->getZExtValue();
2723
2724 // Can the intrinsic's predicate be converted to a known constant index?
2725 unsigned MinNumElts = getNumElementsFromSVEPredPattern(PTruePattern);
2726 if (!MinNumElts)
2727 return std::nullopt;
2728
2729 unsigned Idx = MinNumElts - 1;
2730 // Increment the index if extracting the element after the last active
2731 // predicate element.
2732 if (IsAfter)
2733 ++Idx;
2734
2735 // Ignore extracts whose index is larger than the known minimum vector
2736 // length. NOTE: This is an artificial constraint where we prefer to
2737 // maintain what the user asked for until an alternative is proven faster.
2738 auto *PgVTy = cast<ScalableVectorType>(Pg->getType());
2739 if (Idx >= PgVTy->getMinNumElements())
2740 return std::nullopt;
2741
2742 // The intrinsic is extracting a fixed lane so use an extract instead.
2743 auto *IdxTy = Type::getInt64Ty(II.getContext());
2744 auto *Extract = ExtractElementInst::Create(Vec, ConstantInt::get(IdxTy, Idx));
2745 Extract->insertBefore(II.getIterator());
2746 Extract->takeName(&II);
2747 return IC.replaceInstUsesWith(II, Extract);
2748}
2749
2750static std::optional<Instruction *> instCombineSVECondLast(InstCombiner &IC,
2751 IntrinsicInst &II) {
2752 // The SIMD&FP variant of CLAST[AB] is significantly faster than the scalar
2753 // integer variant across a variety of micro-architectures. Replace scalar
2754 // integer CLAST[AB] intrinsic with optimal SIMD&FP variant. A simple
2755 // bitcast-to-fp + clast[ab] + bitcast-to-int will cost a cycle or two more
2756 // depending on the micro-architecture, but has been observed as generally
2757 // being faster, particularly when the CLAST[AB] op is a loop-carried
2758 // dependency.
2759 Value *Pg = II.getArgOperand(0);
2760 Value *Fallback = II.getArgOperand(1);
2761 Value *Vec = II.getArgOperand(2);
2762 Type *Ty = II.getType();
2763
2764 if (!Ty->isIntegerTy())
2765 return std::nullopt;
2766
2767 Type *FPTy;
2768 switch (cast<IntegerType>(Ty)->getBitWidth()) {
2769 default:
2770 return std::nullopt;
2771 case 16:
2772 FPTy = IC.Builder.getHalfTy();
2773 break;
2774 case 32:
2775 FPTy = IC.Builder.getFloatTy();
2776 break;
2777 case 64:
2778 FPTy = IC.Builder.getDoubleTy();
2779 break;
2780 }
2781
2782 Value *FPFallBack = IC.Builder.CreateBitCast(Fallback, FPTy);
2783 auto *FPVTy = VectorType::get(
2784 FPTy, cast<VectorType>(Vec->getType())->getElementCount());
2785 Value *FPVec = IC.Builder.CreateBitCast(Vec, FPVTy);
2786 auto *FPII = IC.Builder.CreateIntrinsic(
2787 II.getIntrinsicID(), {FPVec->getType()}, {Pg, FPFallBack, FPVec});
2788 Value *FPIItoInt = IC.Builder.CreateBitCast(FPII, II.getType());
2789 return IC.replaceInstUsesWith(II, FPIItoInt);
2790}
2791
2792static std::optional<Instruction *> instCombineRDFFR(InstCombiner &IC,
2793 IntrinsicInst &II) {
2794 // Replace rdffr with predicated rdffr.z intrinsic, so that optimizePTestInstr
2795 // can work with RDFFR_PP for ptest elimination.
2796 auto *RDFFR = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_rdffr_z,
2797 ConstantInt::getTrue(II.getType()));
2798 RDFFR->takeName(&II);
2799 return IC.replaceInstUsesWith(II, RDFFR);
2800}
2801
2802static std::optional<Instruction *>
2804 const auto Pattern = cast<ConstantInt>(II.getArgOperand(0))->getZExtValue();
2805
2806 if (Pattern == AArch64SVEPredPattern::all) {
2808 II.getType(), ElementCount::getScalable(NumElts));
2809 Cnt->takeName(&II);
2810 return IC.replaceInstUsesWith(II, Cnt);
2811 }
2812
2813 unsigned MinNumElts = getNumElementsFromSVEPredPattern(Pattern);
2814
2815 return MinNumElts && NumElts >= MinNumElts
2816 ? std::optional<Instruction *>(IC.replaceInstUsesWith(
2817 II, ConstantInt::get(II.getType(), MinNumElts)))
2818 : std::nullopt;
2819}
2820
2821static std::optional<Instruction *>
2823 const AArch64Subtarget *ST) {
2824 if (!ST->isStreaming())
2825 return std::nullopt;
2826
2827 // In streaming-mode, aarch64_sme_cntds is equivalent to aarch64_sve_cntd
2828 // with SVEPredPattern::all
2829 Value *Cnt =
2831 Cnt->takeName(&II);
2832 return IC.replaceInstUsesWith(II, Cnt);
2833}
2834
2835static std::optional<Instruction *> instCombineSVEPTest(InstCombiner &IC,
2836 IntrinsicInst &II) {
2837 Value *PgVal = II.getArgOperand(0);
2838 Value *OpVal = II.getArgOperand(1);
2839
2840 // PTEST_<FIRST|LAST>(X, X) is equivalent to PTEST_ANY(X, X).
2841 // Later optimizations prefer this form.
2842 if (PgVal == OpVal &&
2843 (II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_first ||
2844 II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_last)) {
2845 Value *Ops[] = {PgVal, OpVal};
2846 Type *Tys[] = {PgVal->getType()};
2847
2848 auto *PTest =
2849 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptest_any, Tys, Ops);
2850 PTest->takeName(&II);
2851
2852 return IC.replaceInstUsesWith(II, PTest);
2853 }
2854
2857
2858 if (!Pg || !Op)
2859 return std::nullopt;
2860
2861 Intrinsic::ID OpIID = Op->getIntrinsicID();
2862
2863 if (Pg->getIntrinsicID() == Intrinsic::aarch64_sve_convert_to_svbool &&
2864 OpIID == Intrinsic::aarch64_sve_convert_to_svbool &&
2865 Pg->getArgOperand(0)->getType() == Op->getArgOperand(0)->getType()) {
2866 Value *Ops[] = {Pg->getArgOperand(0), Op->getArgOperand(0)};
2867 Type *Tys[] = {Pg->getArgOperand(0)->getType()};
2868
2869 auto *PTest = IC.Builder.CreateIntrinsic(II.getIntrinsicID(), Tys, Ops);
2870
2871 PTest->takeName(&II);
2872 return IC.replaceInstUsesWith(II, PTest);
2873 }
2874
2875 // Transform PTEST_ANY(X=OP(PG,...), X) -> PTEST_ANY(PG, X)).
2876 // Later optimizations may rewrite sequence to use the flag-setting variant
2877 // of instruction X to remove PTEST.
2878 if ((Pg == Op) && (II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_any) &&
2879 ((OpIID == Intrinsic::aarch64_sve_brka_z) ||
2880 (OpIID == Intrinsic::aarch64_sve_brkb_z) ||
2881 (OpIID == Intrinsic::aarch64_sve_brkpa_z) ||
2882 (OpIID == Intrinsic::aarch64_sve_brkpb_z) ||
2883 (OpIID == Intrinsic::aarch64_sve_rdffr_z) ||
2884 (OpIID == Intrinsic::aarch64_sve_and_z) ||
2885 (OpIID == Intrinsic::aarch64_sve_bic_z) ||
2886 (OpIID == Intrinsic::aarch64_sve_eor_z) ||
2887 (OpIID == Intrinsic::aarch64_sve_nand_z) ||
2888 (OpIID == Intrinsic::aarch64_sve_nor_z) ||
2889 (OpIID == Intrinsic::aarch64_sve_orn_z) ||
2890 (OpIID == Intrinsic::aarch64_sve_orr_z))) {
2891 Value *Ops[] = {Pg->getArgOperand(0), Pg};
2892 Type *Tys[] = {Pg->getType()};
2893
2894 auto *PTest = IC.Builder.CreateIntrinsic(II.getIntrinsicID(), Tys, Ops);
2895 PTest->takeName(&II);
2896
2897 return IC.replaceInstUsesWith(II, PTest);
2898 }
2899
2900 return std::nullopt;
2901}
2902
2903template <Intrinsic::ID MulOpc, Intrinsic::ID FuseOpc>
2904static std::optional<Instruction *>
2906 bool MergeIntoAddendOp) {
2907 Value *P = II.getOperand(0);
2908 Value *MulOp0, *MulOp1, *AddendOp, *Mul;
2909 if (MergeIntoAddendOp) {
2910 AddendOp = II.getOperand(1);
2911 Mul = II.getOperand(2);
2912 } else {
2913 AddendOp = II.getOperand(2);
2914 Mul = II.getOperand(1);
2915 }
2916
2918 m_Value(MulOp1))))
2919 return std::nullopt;
2920
2921 if (!Mul->hasOneUse())
2922 return std::nullopt;
2923
2924 Instruction *FMFSource = nullptr;
2925 if (II.getType()->isFPOrFPVectorTy()) {
2926 llvm::FastMathFlags FAddFlags = II.getFastMathFlags();
2927 // Stop the combine when the flags on the inputs differ in case dropping
2928 // flags would lead to us missing out on more beneficial optimizations.
2929 if (FAddFlags != cast<CallInst>(Mul)->getFastMathFlags())
2930 return std::nullopt;
2931 if (!FAddFlags.allowContract())
2932 return std::nullopt;
2933 FMFSource = &II;
2934 }
2935
2936 Value *Res;
2937 if (MergeIntoAddendOp)
2938 Res = IC.Builder.CreateIntrinsic(FuseOpc, {II.getType()},
2939 {P, AddendOp, MulOp0, MulOp1}, FMFSource);
2940 else
2941 Res = IC.Builder.CreateIntrinsic(FuseOpc, {II.getType()},
2942 {P, MulOp0, MulOp1, AddendOp}, FMFSource);
2943
2944 return IC.replaceInstUsesWith(II, Res);
2945}
2946
2947static std::optional<Instruction *>
2949 Value *Pred = II.getOperand(0);
2950 Value *PtrOp = II.getOperand(1);
2951 Type *VecTy = II.getType();
2952
2954 LoadInst *Load = IC.Builder.CreateLoad(VecTy, PtrOp);
2955 Load->copyMetadata(II);
2956 return IC.replaceInstUsesWith(II, Load);
2957 }
2958
2959 CallInst *MaskedLoad =
2960 IC.Builder.CreateMaskedLoad(VecTy, PtrOp, PtrOp->getPointerAlignment(DL),
2962 MaskedLoad->copyMetadata(II);
2963 return IC.replaceInstUsesWith(II, MaskedLoad);
2964}
2965
2966static std::optional<Instruction *>
2968 Value *VecOp = II.getOperand(0);
2969 Value *Pred = II.getOperand(1);
2970 Value *PtrOp = II.getOperand(2);
2971
2973 StoreInst *Store = IC.Builder.CreateStore(VecOp, PtrOp);
2974 Store->copyMetadata(II);
2975 return IC.eraseInstFromFunction(II);
2976 }
2977
2978 CallInst *MaskedStore = IC.Builder.CreateMaskedStore(
2979 VecOp, PtrOp, PtrOp->getPointerAlignment(DL), Pred);
2980 MaskedStore->copyMetadata(II);
2981 return IC.eraseInstFromFunction(II);
2982}
2983
2985 switch (Intrinsic) {
2986 case Intrinsic::aarch64_sve_fmul_u:
2987 return Instruction::BinaryOps::FMul;
2988 case Intrinsic::aarch64_sve_fadd_u:
2989 return Instruction::BinaryOps::FAdd;
2990 case Intrinsic::aarch64_sve_fsub_u:
2991 return Instruction::BinaryOps::FSub;
2992 default:
2993 return Instruction::BinaryOpsEnd;
2994 }
2995}
2996
2997static std::optional<Instruction *>
2999 // Bail due to missing support for ISD::STRICT_ scalable vector operations.
3000 if (II.isStrictFP())
3001 return std::nullopt;
3002
3003 auto *OpPredicate = II.getOperand(0);
3004 auto BinOpCode = intrinsicIDToBinOpCode(II.getIntrinsicID());
3005 if (BinOpCode == Instruction::BinaryOpsEnd ||
3006 !isAllActivePredicate(OpPredicate))
3007 return std::nullopt;
3008 auto BinOp = IC.Builder.CreateBinOpFMF(
3009 BinOpCode, II.getOperand(1), II.getOperand(2), II.getFastMathFlags());
3010 return IC.replaceInstUsesWith(II, BinOp);
3011}
3012
3013static std::optional<Instruction *>
3015 assert(II.getIntrinsicID() == Intrinsic::aarch64_sve_mla_u &&
3016 "Expected MLA_U intrinsic");
3017 Value *Acc = II.getArgOperand(1);
3018 Value *MulOp0 = II.getArgOperand(2);
3019 Value *MulOp1 = II.getArgOperand(3);
3020
3021 // For mla_u, inactive lanes are undefined, so it is valid to drop the
3022 // predicate when replacing mla_u(acc, x, 1) with add(acc, x) or
3023 // mla_u(acc, x, -1) with sub(acc, x).
3024 if (match(MulOp0, m_One()))
3025 return IC.replaceInstUsesWith(II, IC.Builder.CreateAdd(Acc, MulOp1));
3026 if (match(MulOp1, m_One()))
3027 return IC.replaceInstUsesWith(II, IC.Builder.CreateAdd(Acc, MulOp0));
3028 if (match(MulOp0, m_AllOnes()))
3029 return IC.replaceInstUsesWith(II, IC.Builder.CreateSub(Acc, MulOp1));
3030 if (match(MulOp1, m_AllOnes()))
3031 return IC.replaceInstUsesWith(II, IC.Builder.CreateSub(Acc, MulOp0));
3032
3033 if (isa<Constant>(MulOp0) && !isa<Constant>(MulOp1)) {
3034 II.setArgOperand(2, MulOp1);
3035 II.setArgOperand(3, MulOp0);
3036 return &II;
3037 }
3038
3039 return std::nullopt;
3040}
3041
3042static std::optional<Instruction *>
3044 assert((II.getIntrinsicID() == Intrinsic::aarch64_sve_sadalp ||
3045 II.getIntrinsicID() == Intrinsic::aarch64_sve_uadalp) &&
3046 "Expected SADALP or UADALP intrinsic");
3047
3048 // Simplify add(adalp(pg, zeroinitializer, in), wide_acc)
3049 // -> adalp(pg, wide_acc, in)
3050 auto *User = dyn_cast_or_null<Instruction>(II.getUniqueUndroppableUser());
3051 if (!User || !match(II.getArgOperand(1), m_Zero()))
3052 return std::nullopt;
3053
3054 Value *Acc;
3055 if (!match(User, m_c_Add(m_Specific(&II), m_Value(Acc))))
3056 return std::nullopt;
3057
3059 Value *PairwiseAddLong = IC.Builder.CreateIntrinsic(
3060 II.getIntrinsicID(), {II.getType()},
3061 {II.getArgOperand(0), Acc, II.getArgOperand(2)});
3062
3063 IC.replaceInstUsesWith(*User, PairwiseAddLong);
3065 return &II; // II is now trivially dead and will get erased.
3066}
3067
3068static std::optional<Instruction *> instCombineSVEVectorAdd(InstCombiner &IC,
3069 IntrinsicInst &II) {
3070 if (auto MLA = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
3071 Intrinsic::aarch64_sve_mla>(
3072 IC, II, true))
3073 return MLA;
3074 if (auto MAD = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
3075 Intrinsic::aarch64_sve_mad>(
3076 IC, II, false))
3077 return MAD;
3078 return std::nullopt;
3079}
3080
3081static std::optional<Instruction *>
3083 if (auto FMLA =
3084 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3085 Intrinsic::aarch64_sve_fmla>(IC, II,
3086 true))
3087 return FMLA;
3088 if (auto FMAD =
3089 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3090 Intrinsic::aarch64_sve_fmad>(IC, II,
3091 false))
3092 return FMAD;
3093 if (auto FMLA =
3094 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
3095 Intrinsic::aarch64_sve_fmla>(IC, II,
3096 true))
3097 return FMLA;
3098 return std::nullopt;
3099}
3100
3101static std::optional<Instruction *>
3103 if (auto FMLA =
3104 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3105 Intrinsic::aarch64_sve_fmla>(IC, II,
3106 true))
3107 return FMLA;
3108 if (auto FMAD =
3109 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3110 Intrinsic::aarch64_sve_fmad>(IC, II,
3111 false))
3112 return FMAD;
3113 if (auto FMLA_U =
3114 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
3115 Intrinsic::aarch64_sve_fmla_u>(
3116 IC, II, true))
3117 return FMLA_U;
3118 return instCombineSVEVectorBinOp(IC, II);
3119}
3120
3121static std::optional<Instruction *>
3123 if (auto FMLS =
3124 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3125 Intrinsic::aarch64_sve_fmls>(IC, II,
3126 true))
3127 return FMLS;
3128 if (auto FMSB =
3129 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3130 Intrinsic::aarch64_sve_fnmsb>(
3131 IC, II, false))
3132 return FMSB;
3133 if (auto FMLS =
3134 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
3135 Intrinsic::aarch64_sve_fmls>(IC, II,
3136 true))
3137 return FMLS;
3138 return std::nullopt;
3139}
3140
3141static std::optional<Instruction *>
3143 if (auto FMLS =
3144 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3145 Intrinsic::aarch64_sve_fmls>(IC, II,
3146 true))
3147 return FMLS;
3148 if (auto FMSB =
3149 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3150 Intrinsic::aarch64_sve_fnmsb>(
3151 IC, II, false))
3152 return FMSB;
3153 if (auto FMLS_U =
3154 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
3155 Intrinsic::aarch64_sve_fmls_u>(
3156 IC, II, true))
3157 return FMLS_U;
3158 return instCombineSVEVectorBinOp(IC, II);
3159}
3160
3161static std::optional<Instruction *> instCombineSVEVectorSub(InstCombiner &IC,
3162 IntrinsicInst &II) {
3163 if (auto MLS = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
3164 Intrinsic::aarch64_sve_mls>(
3165 IC, II, true))
3166 return MLS;
3167 return std::nullopt;
3168}
3169
3170static std::optional<Instruction *> instCombineSVEUnpack(InstCombiner &IC,
3171 IntrinsicInst &II) {
3172 Value *UnpackArg = II.getArgOperand(0);
3173 auto *RetTy = cast<ScalableVectorType>(II.getType());
3174 bool IsSigned = II.getIntrinsicID() == Intrinsic::aarch64_sve_sunpkhi ||
3175 II.getIntrinsicID() == Intrinsic::aarch64_sve_sunpklo;
3176
3177 // Hi = uunpkhi(splat(X)) --> Hi = splat(extend(X))
3178 // Lo = uunpklo(splat(X)) --> Lo = splat(extend(X))
3179 if (auto *ScalarArg = getSplatValue(UnpackArg)) {
3180 ScalarArg =
3181 IC.Builder.CreateIntCast(ScalarArg, RetTy->getScalarType(), IsSigned);
3182 Value *NewVal =
3183 IC.Builder.CreateVectorSplat(RetTy->getElementCount(), ScalarArg);
3184 NewVal->takeName(&II);
3185 return IC.replaceInstUsesWith(II, NewVal);
3186 }
3187
3188 return std::nullopt;
3189}
3190static std::optional<Instruction *> instCombineSVETBL(InstCombiner &IC,
3191 IntrinsicInst &II) {
3192 auto *OpVal = II.getOperand(0);
3193 auto *OpIndices = II.getOperand(1);
3194 VectorType *VTy = cast<VectorType>(II.getType());
3195
3196 // Check whether OpIndices is a constant splat value < minimal element count
3197 // of result.
3198 auto *SplatValue = dyn_cast_or_null<ConstantInt>(getSplatValue(OpIndices));
3199 if (!SplatValue ||
3200 SplatValue->getValue().uge(VTy->getElementCount().getKnownMinValue()))
3201 return std::nullopt;
3202
3203 // Convert sve_tbl(OpVal sve_dup_x(SplatValue)) to
3204 // splat_vector(extractelement(OpVal, SplatValue)) for further optimization.
3205 auto *Extract = IC.Builder.CreateExtractElement(OpVal, SplatValue);
3206 auto *VectorSplat =
3207 IC.Builder.CreateVectorSplat(VTy->getElementCount(), Extract);
3208
3209 VectorSplat->takeName(&II);
3210 return IC.replaceInstUsesWith(II, VectorSplat);
3211}
3212
3213static std::optional<Instruction *> instCombineSVEUzp1(InstCombiner &IC,
3214 IntrinsicInst &II) {
3215 Value *A, *B;
3216 Type *RetTy = II.getType();
3217 constexpr Intrinsic::ID FromSVB = Intrinsic::aarch64_sve_convert_from_svbool;
3218 constexpr Intrinsic::ID ToSVB = Intrinsic::aarch64_sve_convert_to_svbool;
3219
3220 // uzp1(to_svbool(A), to_svbool(B)) --> <A, B>
3221 // uzp1(from_svbool(to_svbool(A)), from_svbool(to_svbool(B))) --> <A, B>
3222 if ((match(II.getArgOperand(0),
3224 match(II.getArgOperand(1),
3226 (match(II.getArgOperand(0), m_Intrinsic<ToSVB>(m_Value(A))) &&
3227 match(II.getArgOperand(1), m_Intrinsic<ToSVB>(m_Value(B))))) {
3228 auto *TyA = cast<ScalableVectorType>(A->getType());
3229 if (TyA == B->getType() &&
3231 auto *SubVec = IC.Builder.CreateInsertVector(
3232 RetTy, PoisonValue::get(RetTy), A, uint64_t(0));
3233 auto *ConcatVec = IC.Builder.CreateInsertVector(RetTy, SubVec, B,
3234 TyA->getMinNumElements());
3235 ConcatVec->takeName(&II);
3236 return IC.replaceInstUsesWith(II, ConcatVec);
3237 }
3238 }
3239
3240 return std::nullopt;
3241}
3242
3243static std::optional<Instruction *> instCombineSVEZip(InstCombiner &IC,
3244 IntrinsicInst &II) {
3245 // zip1(uzp1(A, B), uzp2(A, B)) --> A
3246 // zip2(uzp1(A, B), uzp2(A, B)) --> B
3247 Value *A, *B;
3248 if (match(II.getArgOperand(0),
3251 m_Specific(A), m_Specific(B))))
3252 return IC.replaceInstUsesWith(
3253 II, (II.getIntrinsicID() == Intrinsic::aarch64_sve_zip1 ? A : B));
3254
3255 return std::nullopt;
3256}
3257
3258static std::optional<Instruction *>
3260 Value *Mask = II.getOperand(0);
3261 Value *BasePtr = II.getOperand(1);
3262 Value *Index = II.getOperand(2);
3263 Type *Ty = II.getType();
3264 Value *PassThru = ConstantAggregateZero::get(Ty);
3265
3266 // Contiguous gather => masked load.
3267 // (sve.ld1.gather.index Mask BasePtr (sve.index IndexBase 1))
3268 // => (masked.load (gep BasePtr IndexBase) Align Mask zeroinitializer)
3269 Value *IndexBase;
3271 m_One()))) {
3272 Align Alignment =
3273 BasePtr->getPointerAlignment(II.getDataLayout());
3274
3275 Value *Ptr = IC.Builder.CreateGEP(cast<VectorType>(Ty)->getElementType(),
3276 BasePtr, IndexBase);
3277 CallInst *MaskedLoad =
3278 IC.Builder.CreateMaskedLoad(Ty, Ptr, Alignment, Mask, PassThru);
3279 MaskedLoad->takeName(&II);
3280 return IC.replaceInstUsesWith(II, MaskedLoad);
3281 }
3282
3283 return std::nullopt;
3284}
3285
3286static std::optional<Instruction *>
3288 Value *Val = II.getOperand(0);
3289 Value *Mask = II.getOperand(1);
3290 Value *BasePtr = II.getOperand(2);
3291 Value *Index = II.getOperand(3);
3292 Type *Ty = Val->getType();
3293
3294 // Contiguous scatter => masked store.
3295 // (sve.st1.scatter.index Value Mask BasePtr (sve.index IndexBase 1))
3296 // => (masked.store Value (gep BasePtr IndexBase) Align Mask)
3297 Value *IndexBase;
3299 m_One()))) {
3300 Align Alignment =
3301 BasePtr->getPointerAlignment(II.getDataLayout());
3302
3303 Value *Ptr = IC.Builder.CreateGEP(cast<VectorType>(Ty)->getElementType(),
3304 BasePtr, IndexBase);
3305 (void)IC.Builder.CreateMaskedStore(Val, Ptr, Alignment, Mask);
3306
3307 return IC.eraseInstFromFunction(II);
3308 }
3309
3310 return std::nullopt;
3311}
3312
3313static std::optional<Instruction *> instCombineSVESDIV(InstCombiner &IC,
3314 IntrinsicInst &II) {
3315 Type *Int32Ty = IC.Builder.getInt32Ty();
3316 Value *Pred = II.getOperand(0);
3317 Value *Vec = II.getOperand(1);
3318 Value *DivVec = II.getOperand(2);
3319
3320 Value *SplatValue = getSplatValue(DivVec);
3321 ConstantInt *SplatConstantInt = dyn_cast_or_null<ConstantInt>(SplatValue);
3322 if (!SplatConstantInt)
3323 return std::nullopt;
3324
3325 APInt Divisor = SplatConstantInt->getValue();
3326 const int64_t DivisorValue = Divisor.getSExtValue();
3327 if (DivisorValue == -1)
3328 return std::nullopt;
3329 if (DivisorValue == 1)
3330 IC.replaceInstUsesWith(II, Vec);
3331
3332 if (Divisor.isPowerOf2()) {
3333 Constant *DivisorLog2 = ConstantInt::get(Int32Ty, Divisor.logBase2());
3334 auto ASRD = IC.Builder.CreateIntrinsic(
3335 Intrinsic::aarch64_sve_asrd, {II.getType()}, {Pred, Vec, DivisorLog2});
3336 return IC.replaceInstUsesWith(II, ASRD);
3337 }
3338 if (Divisor.isNegatedPowerOf2()) {
3339 Divisor.negate();
3340 Constant *DivisorLog2 = ConstantInt::get(Int32Ty, Divisor.logBase2());
3341 auto ASRD = IC.Builder.CreateIntrinsic(
3342 Intrinsic::aarch64_sve_asrd, {II.getType()}, {Pred, Vec, DivisorLog2});
3343 auto NEG = IC.Builder.CreateIntrinsic(
3344 Intrinsic::aarch64_sve_neg, {ASRD->getType()}, {ASRD, Pred, ASRD});
3345 return IC.replaceInstUsesWith(II, NEG);
3346 }
3347
3348 return std::nullopt;
3349}
3350
3351/// Match Or(And(MASK, A), And(Not(MASK), B)), which ISel selects as
3352/// BSL/BIT/BIF. Both Ands must have one use and be in the Or's block. MainAnd,
3353///
3354/// OtherAnd, A and B are out parameters with the corresponding instructions.
3355/// MainAnd refers to And that has Not as one of its operands.
3356static bool matchBSL(const Instruction *I, Instruction *&MainAnd,
3357 Instruction *&OtherAnd, Value *&A, Value *&B) {
3358 Value *MaskValue;
3359 if (!match(I,
3360 m_c_Or(m_OneUse(m_Instruction(OtherAnd)),
3361 m_OneUse(m_c_And(m_Not(m_Value(MaskValue)), m_Value(A))))))
3362 return false;
3363
3364 if (!match(OtherAnd, m_c_And(m_Specific(MaskValue), m_Value(B))))
3365 return false;
3366
3367 MainAnd = I->getOperand(0) == OtherAnd ? cast<Instruction>(I->getOperand(1))
3368 : cast<Instruction>(I->getOperand(0));
3369
3370 // Both Ands should be in same basic block as Or
3371 if (I->getParent() != MainAnd->getParent() ||
3372 I->getParent() != OtherAnd->getParent())
3373 return false;
3374
3375 return true;
3376}
3377
3378bool SimplifyValuePattern(SmallVector<Value *> &Vec, bool AllowPoison) {
3379 size_t VecSize = Vec.size();
3380 if (VecSize == 1)
3381 return true;
3382 if (!isPowerOf2_64(VecSize))
3383 return false;
3384 size_t HalfVecSize = VecSize / 2;
3385
3386 for (auto LHS = Vec.begin(), RHS = Vec.begin() + HalfVecSize;
3387 RHS != Vec.end(); LHS++, RHS++) {
3388 if (*LHS != nullptr && *RHS != nullptr) {
3389 if (*LHS == *RHS)
3390 continue;
3391 else
3392 return false;
3393 }
3394 if (!AllowPoison)
3395 return false;
3396 if (*LHS == nullptr && *RHS != nullptr)
3397 *LHS = *RHS;
3398 }
3399
3400 Vec.resize(HalfVecSize);
3401 SimplifyValuePattern(Vec, AllowPoison);
3402 return true;
3403}
3404
3405// Try to simplify dupqlane patterns like dupqlane(f32 A, f32 B, f32 A, f32 B)
3406// to dupqlane(f64(C)) where C is A concatenated with B
3407static std::optional<Instruction *> instCombineSVEDupqLane(InstCombiner &IC,
3408 IntrinsicInst &II) {
3409 // An out-of-range index generates an all zero result. This means only an
3410 // index of zero has the same defined result across all vector lengths.
3411 if (!match(II.getOperand(1), m_Zero()))
3412 return std::nullopt;
3413
3414 Value *CurrentInsertElt = nullptr, *Default = nullptr;
3415 if (!match(II.getOperand(0),
3417 m_Value(Default), m_Value(CurrentInsertElt), m_Zero())) ||
3418 !isa<FixedVectorType>(CurrentInsertElt->getType()))
3419 return std::nullopt;
3420 auto IIScalableTy = cast<ScalableVectorType>(II.getType());
3421
3422 // Insert the scalars into a container ordered by InsertElement index
3423 SmallVector<Value *> Elts(IIScalableTy->getMinNumElements(), nullptr);
3424 while (auto InsertElt = dyn_cast<InsertElementInst>(CurrentInsertElt)) {
3425 auto Idx = cast<ConstantInt>(InsertElt->getOperand(2));
3426 Elts[Idx->getValue().getZExtValue()] = InsertElt->getOperand(1);
3427 CurrentInsertElt = InsertElt->getOperand(0);
3428 }
3429
3430 bool AllowPoison =
3431 isa<PoisonValue>(CurrentInsertElt) && isa<PoisonValue>(Default);
3432 if (!SimplifyValuePattern(Elts, AllowPoison))
3433 return std::nullopt;
3434
3435 // Rebuild the simplified chain of InsertElements. e.g. (a, b, a, b) as (a, b)
3436 Value *InsertEltChain = PoisonValue::get(CurrentInsertElt->getType());
3437 for (size_t I = 0; I < Elts.size(); I++) {
3438 if (Elts[I] == nullptr)
3439 continue;
3440 InsertEltChain = IC.Builder.CreateInsertElement(InsertEltChain, Elts[I],
3441 IC.Builder.getInt64(I));
3442 }
3443 if (InsertEltChain == nullptr)
3444 return std::nullopt;
3445
3446 // Splat the simplified sequence, e.g. (f16 a, f16 b, f16 c, f16 d) as one i64
3447 // value or (f16 a, f16 b) as one i32 value. This requires an InsertSubvector
3448 // be bitcast to a type wide enough to fit the sequence, be splatted, and then
3449 // be narrowed back to the original type.
3450 unsigned PatternWidth = IIScalableTy->getScalarSizeInBits() * Elts.size();
3451 unsigned PatternElementCount = IIScalableTy->getScalarSizeInBits() *
3452 IIScalableTy->getMinNumElements() /
3453 PatternWidth;
3454
3455 IntegerType *WideTy = IC.Builder.getIntNTy(PatternWidth);
3456 auto *WideScalableTy = ScalableVectorType::get(WideTy, PatternElementCount);
3457 auto *WideShuffleMaskTy =
3458 ScalableVectorType::get(IC.Builder.getInt32Ty(), PatternElementCount);
3459
3460 auto InsertSubvector = IC.Builder.CreateInsertVector(
3461 II.getType(), PoisonValue::get(II.getType()), InsertEltChain,
3462 uint64_t(0));
3463 auto WideBitcast =
3464 IC.Builder.CreateBitOrPointerCast(InsertSubvector, WideScalableTy);
3465 auto WideShuffleMask = ConstantAggregateZero::get(WideShuffleMaskTy);
3466 auto WideShuffle = IC.Builder.CreateShuffleVector(
3467 WideBitcast, PoisonValue::get(WideScalableTy), WideShuffleMask);
3468 auto NarrowBitcast =
3469 IC.Builder.CreateBitOrPointerCast(WideShuffle, II.getType());
3470
3471 return IC.replaceInstUsesWith(II, NarrowBitcast);
3472}
3473
3474static std::optional<Instruction *> instCombineMaxMinNM(InstCombiner &IC,
3475 IntrinsicInst &II) {
3476 Value *A = II.getArgOperand(0);
3477 Value *B = II.getArgOperand(1);
3478 if (A == B)
3479 return IC.replaceInstUsesWith(II, A);
3480
3481 return std::nullopt;
3482}
3483
3484static std::optional<Instruction *> instCombineSVESrshl(InstCombiner &IC,
3485 IntrinsicInst &II) {
3486 Value *Pred = II.getOperand(0);
3487 Value *Vec = II.getOperand(1);
3488 Value *Shift = II.getOperand(2);
3489
3490 // Convert SRSHL into the simpler LSL intrinsic when fed by an ABS intrinsic.
3491 Value *AbsPred, *MergedValue;
3493 m_Value(MergedValue), m_Value(AbsPred), m_Value())) &&
3495 m_Value(MergedValue), m_Value(AbsPred), m_Value())))
3496
3497 return std::nullopt;
3498
3499 // Transform is valid if any of the following are true:
3500 // * The ABS merge value is an undef or non-negative
3501 // * The ABS predicate is all active
3502 // * The ABS predicate and the SRSHL predicates are the same
3503 if (!isa<UndefValue>(MergedValue) && !match(MergedValue, m_NonNegative()) &&
3504 AbsPred != Pred && !isAllActivePredicate(AbsPred))
3505 return std::nullopt;
3506
3507 // Only valid when the shift amount is non-negative, otherwise the rounding
3508 // behaviour of SRSHL cannot be ignored.
3509 if (!match(Shift, m_NonNegative()))
3510 return std::nullopt;
3511
3512 auto LSL = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_lsl,
3513 {II.getType()}, {Pred, Vec, Shift});
3514
3515 return IC.replaceInstUsesWith(II, LSL);
3516}
3517
3518static std::optional<Instruction *> instCombineSVEInsr(InstCombiner &IC,
3519 IntrinsicInst &II) {
3520 Value *Vec = II.getOperand(0);
3521
3522 if (getSplatValue(Vec) == II.getOperand(1))
3523 return IC.replaceInstUsesWith(II, Vec);
3524
3525 return std::nullopt;
3526}
3527
3528static std::optional<Instruction *>
3530 unsigned LookaheadThreshold) {
3531 // If this barrier is post-dominated by identical one we can remove it
3532 auto *NI = II.getNextNode();
3533 auto CanSkipOver = [](Instruction *I) {
3534 return !I->mayReadOrWriteMemory() && !I->mayHaveSideEffects();
3535 };
3536 while (LookaheadThreshold-- && CanSkipOver(NI)) {
3537 auto *NIBB = NI->getParent();
3538 NI = NI->getNextNode();
3539 if (!NI) {
3540 if (auto *SuccBB = NIBB->getUniqueSuccessor())
3541 NI = &*SuccBB->getFirstNonPHIOrLifetime();
3542 else
3543 break;
3544 }
3545 }
3546 auto *NextII = dyn_cast_or_null<IntrinsicInst>(NI);
3547 if (NextII && II.isIdenticalTo(NextII))
3548 return IC.eraseInstFromFunction(II);
3549
3550 return std::nullopt;
3551}
3552
3553static std::optional<Instruction *> instCombineWhilelo(InstCombiner &IC,
3554 IntrinsicInst &II) {
3555 return IC.replaceInstUsesWith(
3556 II,
3557 IC.Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask,
3558 {II.getType(), II.getOperand(0)->getType()},
3559 {II.getOperand(0), II.getOperand(1)}));
3560}
3561
3562static std::optional<Instruction *> instCombinePTrue(InstCombiner &IC,
3563 IntrinsicInst &II) {
3564 unsigned PredPattern = cast<ConstantInt>(II.getOperand(0))->getZExtValue();
3565 // SVE vector length is a power-of-two, thus pow2 is synonymous with all.
3566 if (PredPattern == AArch64SVEPredPattern::all ||
3567 PredPattern == AArch64SVEPredPattern::pow2)
3568 return IC.replaceInstUsesWith(II, ConstantInt::getTrue(II.getType()));
3569 return std::nullopt;
3570}
3571
3572static std::optional<Instruction *> instCombineSVEUxt(InstCombiner &IC,
3574 unsigned NumBits) {
3575 Value *Passthru = II.getOperand(0);
3576 Value *Pg = II.getOperand(1);
3577 Value *Op = II.getOperand(2);
3578
3579 // Convert UXT[BHW] to AND.
3580 if (isa<UndefValue>(Passthru) || isAllActivePredicate(Pg)) {
3581 auto *Ty = cast<VectorType>(II.getType());
3582 auto MaskValue = APInt::getLowBitsSet(Ty->getScalarSizeInBits(), NumBits);
3583 auto *Mask = ConstantInt::get(Ty, MaskValue);
3584 auto *And = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_and_u, {Ty},
3585 {Pg, Op, Mask});
3586 return IC.replaceInstUsesWith(II, And);
3587 }
3588
3589 return std::nullopt;
3590}
3591
3592static std::optional<Instruction *>
3594 SMEAttrs FnSMEAttrs(*II.getFunction());
3595 bool IsStreaming = FnSMEAttrs.hasStreamingInterfaceOrBody();
3596 if (IsStreaming || !FnSMEAttrs.hasStreamingCompatibleInterface())
3597 return IC.replaceInstUsesWith(
3598 II, ConstantInt::getBool(II.getType(), IsStreaming));
3599 return std::nullopt;
3600}
3601
3602static std::optional<Instruction *> instCombineSVEUMin(InstCombiner &IC,
3603 IntrinsicInst &II) {
3604 // umin(umin(A, 1), umin(B, 1)) -> umin(umin(A,B), 1)
3605 constexpr Intrinsic::ID UMinID = Intrinsic::aarch64_sve_umin_u;
3606 Value *A, *B;
3607 Value *Pg = II.getOperand(0);
3608 if (match(II.getOperand(1), m_OneUse(m_Intrinsic<UMinID>(
3609 m_Specific(Pg), m_Value(A), m_One()))) &&
3610 match(II.getOperand(2), m_OneUse(m_Intrinsic<UMinID>(
3611 m_Specific(Pg), m_Value(B), m_One())))) {
3612 Value *NewUMin =
3613 IC.Builder.CreateIntrinsic(UMinID, II.getType(), {Pg, A, B});
3614 Value *NewLogicalUMin = IC.Builder.CreateIntrinsic(
3615 UMinID, II.getType(), {Pg, NewUMin, ConstantInt::get(II.getType(), 1)});
3616 return IC.replaceInstUsesWith(II, NewLogicalUMin);
3617 }
3618
3619 // umin(umin(A, 1), 1) -> umin(A, 1)
3620 if (match(II.getOperand(1),
3622 match(II.getOperand(2), m_One()))
3623 return IC.replaceInstUsesWith(II, II.getOperand(1));
3624
3625 return std::nullopt;
3626}
3627
3628static std::optional<Instruction *> instCombineSVEOrr(InstCombiner &IC,
3629 IntrinsicInst &II) {
3630 // orr(umin(A, 1), umin(B, 1)) -> umin(orr(A, B), 1)
3631 constexpr Intrinsic::ID UMinID = Intrinsic::aarch64_sve_umin_u;
3632 Value *Pg = II.getOperand(0);
3633
3634 Value *A, *B;
3635 if (!match(II.getOperand(1), m_OneUse(m_Intrinsic<UMinID>(
3636 m_Specific(Pg), m_Value(A), m_One()))) ||
3637 !match(II.getOperand(2), m_OneUse(m_Intrinsic<UMinID>(
3638 m_Specific(Pg), m_Value(B), m_One()))))
3639 return std::nullopt;
3640
3641 Value *NewOrr = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_orr_u,
3642 II.getType(), {Pg, A, B});
3643 Value *NewUMin = IC.Builder.CreateIntrinsic(
3644 UMinID, II.getType(), {Pg, NewOrr, ConstantInt::get(II.getType(), 1)});
3645 return IC.replaceInstUsesWith(II, NewUMin);
3646}
3647
3648static std::optional<Instruction *> instCombineSVEAnd(InstCombiner &IC,
3649 IntrinsicInst &II) {
3650 // and(cmphs(pg, ConstA, A), cmphs(pg, A, ConstB))
3651 // ->
3652 // cmphs(pg, ConstA - ConstB, sub(pg, A, ConstB))
3653 constexpr Intrinsic::ID CmphsID = Intrinsic::aarch64_sve_cmphs;
3654 Value *Pg = II.getOperand(0);
3655 Value *LHS = II.getOperand(1);
3656 Value *RHS = II.getOperand(2);
3657
3658 Value *A, *PgLHS, *PgRHS;
3659 uint64_t ConstA, ConstB;
3660 if (!match(LHS, m_Intrinsic<CmphsID>(m_Value(PgLHS), m_ConstantInt(ConstA),
3661 m_Value(A))) ||
3663 m_ConstantInt(ConstB))) ||
3664 !LHS->hasOneUser() || !RHS->hasOneUser())
3665 return std::nullopt;
3666
3667 // Always false regardless of predication
3668 if (ConstB > ConstA)
3669 return IC.replaceInstUsesWith(II, Constant::getNullValue(II.getType()));
3670
3671 // The predicate for both CMPHSs must match.
3672 // The predicate for the AND can either be equal to the CMPHS predicates, or
3673 // either of the CMPHS values.
3674 if (PgLHS != PgRHS || (Pg != LHS && Pg != RHS && Pg != PgLHS))
3675 return std::nullopt;
3676
3677 Type *VecTy = A->getType();
3678 Constant *Base = ConstantInt::get(VecTy, ConstB);
3679 Value *Sub = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_sub_u, VecTy,
3680 {PgLHS, A, Base});
3681 Constant *Limit = ConstantInt::get(VecTy, ConstA - ConstB);
3682 Value *NewCmphs =
3683 IC.Builder.CreateIntrinsic(CmphsID, VecTy, {PgLHS, Limit, Sub});
3684
3685 return IC.replaceInstUsesWith(II, NewCmphs);
3686}
3687
3688std::optional<Instruction *>
3690 IntrinsicInst &II) const {
3692 if (std::optional<Instruction *> I = simplifySVEIntrinsic(IC, II, IInfo))
3693 return I;
3694
3695 Intrinsic::ID IID = II.getIntrinsicID();
3696 switch (IID) {
3697 default:
3698 break;
3699 case Intrinsic::aarch64_dmb:
3700 return instCombineDMB(IC, II, ST->getCLOpts().dmb_lookahead_threshold);
3701 case Intrinsic::aarch64_neon_fmaxnm:
3702 case Intrinsic::aarch64_neon_fminnm:
3703 return instCombineMaxMinNM(IC, II);
3704 case Intrinsic::aarch64_sve_convert_from_svbool:
3705 return instCombineConvertFromSVBool(IC, II);
3706 case Intrinsic::aarch64_sve_dup:
3707 return instCombineSVEDup(IC, II);
3708 case Intrinsic::aarch64_sve_dup_x:
3709 return instCombineSVEDupX(IC, II);
3710 case Intrinsic::aarch64_sve_cmpeq:
3711 case Intrinsic::aarch64_sve_cmpeq_wide:
3712 return instCombineXorSVECmpCC(IC, II);
3713 case Intrinsic::aarch64_sve_cmpne:
3714 case Intrinsic::aarch64_sve_cmpne_wide:
3715 return instCombineSVECmpNE(IC, II);
3716 case Intrinsic::aarch64_sve_rdffr:
3717 return instCombineRDFFR(IC, II);
3718 case Intrinsic::aarch64_sve_lasta:
3719 case Intrinsic::aarch64_sve_lastb:
3720 return instCombineSVELast(IC, II);
3721 case Intrinsic::aarch64_sve_clasta_n:
3722 case Intrinsic::aarch64_sve_clastb_n:
3723 return instCombineSVECondLast(IC, II);
3724 case Intrinsic::aarch64_sve_cntd:
3725 return instCombineSVECntElts(IC, II, 2);
3726 case Intrinsic::aarch64_sve_cntw:
3727 return instCombineSVECntElts(IC, II, 4);
3728 case Intrinsic::aarch64_sve_cnth:
3729 return instCombineSVECntElts(IC, II, 8);
3730 case Intrinsic::aarch64_sve_cntb:
3731 return instCombineSVECntElts(IC, II, 16);
3732 case Intrinsic::aarch64_sme_cntsd:
3733 return instCombineSMECntsd(IC, II, ST);
3734 case Intrinsic::aarch64_sve_ptest_any:
3735 case Intrinsic::aarch64_sve_ptest_first:
3736 case Intrinsic::aarch64_sve_ptest_last:
3737 return instCombineSVEPTest(IC, II);
3738 case Intrinsic::aarch64_sve_fadd:
3739 return instCombineSVEVectorFAdd(IC, II);
3740 case Intrinsic::aarch64_sve_fadd_u:
3741 return instCombineSVEVectorFAddU(IC, II);
3742 case Intrinsic::aarch64_sve_fmul_u:
3743 return instCombineSVEVectorBinOp(IC, II);
3744 case Intrinsic::aarch64_sve_fsub:
3745 return instCombineSVEVectorFSub(IC, II);
3746 case Intrinsic::aarch64_sve_fsub_u:
3747 return instCombineSVEVectorFSubU(IC, II);
3748 case Intrinsic::aarch64_sve_add:
3749 return instCombineSVEVectorAdd(IC, II);
3750 case Intrinsic::aarch64_sve_add_u:
3751 return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u,
3752 Intrinsic::aarch64_sve_mla_u>(
3753 IC, II, true);
3754 case Intrinsic::aarch64_sve_mla_u:
3755 return instCombineSVEVectorMlaU(IC, II);
3756 case Intrinsic::aarch64_sve_sadalp:
3757 case Intrinsic::aarch64_sve_uadalp:
3759 case Intrinsic::aarch64_sve_sub:
3760 return instCombineSVEVectorSub(IC, II);
3761 case Intrinsic::aarch64_sve_sub_u:
3762 return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u,
3763 Intrinsic::aarch64_sve_mls_u>(
3764 IC, II, true);
3765 case Intrinsic::aarch64_sve_tbl:
3766 return instCombineSVETBL(IC, II);
3767 case Intrinsic::aarch64_sve_uunpkhi:
3768 case Intrinsic::aarch64_sve_uunpklo:
3769 case Intrinsic::aarch64_sve_sunpkhi:
3770 case Intrinsic::aarch64_sve_sunpklo:
3771 return instCombineSVEUnpack(IC, II);
3772 case Intrinsic::aarch64_sve_uzp1:
3773 return instCombineSVEUzp1(IC, II);
3774 case Intrinsic::aarch64_sve_zip1:
3775 case Intrinsic::aarch64_sve_zip2:
3776 return instCombineSVEZip(IC, II);
3777 case Intrinsic::aarch64_sve_ld1_gather_index:
3778 return instCombineLD1GatherIndex(IC, II);
3779 case Intrinsic::aarch64_sve_st1_scatter_index:
3780 return instCombineST1ScatterIndex(IC, II);
3781 case Intrinsic::aarch64_sve_ld1:
3782 return instCombineSVELD1(IC, II, DL);
3783 case Intrinsic::aarch64_sve_st1:
3784 return instCombineSVEST1(IC, II, DL);
3785 case Intrinsic::aarch64_sve_sdiv:
3786 return instCombineSVESDIV(IC, II);
3787 case Intrinsic::aarch64_sve_sel:
3788 return instCombineSVESel(IC, II);
3789 case Intrinsic::aarch64_sve_srshl:
3790 return instCombineSVESrshl(IC, II);
3791 case Intrinsic::aarch64_sve_dupq_lane:
3792 return instCombineSVEDupqLane(IC, II);
3793 case Intrinsic::aarch64_sve_insr:
3794 return instCombineSVEInsr(IC, II);
3795 case Intrinsic::aarch64_sve_whilelo:
3796 return instCombineWhilelo(IC, II);
3797 case Intrinsic::aarch64_sve_ptrue:
3798 return instCombinePTrue(IC, II);
3799 case Intrinsic::aarch64_sve_uxtb:
3800 return instCombineSVEUxt(IC, II, 8);
3801 case Intrinsic::aarch64_sve_uxth:
3802 return instCombineSVEUxt(IC, II, 16);
3803 case Intrinsic::aarch64_sve_uxtw:
3804 return instCombineSVEUxt(IC, II, 32);
3805 case Intrinsic::aarch64_sme_in_streaming_mode:
3806 return instCombineInStreamingMode(IC, II);
3807 case Intrinsic::aarch64_sve_umin_u:
3808 return instCombineSVEUMin(IC, II);
3809 case Intrinsic::aarch64_sve_orr_u:
3810 return instCombineSVEOrr(IC, II);
3811 case Intrinsic::aarch64_sve_and_z:
3812 return instCombineSVEAnd(IC, II);
3813 }
3814
3815 return std::nullopt;
3816}
3817
3819 InstCombiner &IC, IntrinsicInst &II, APInt OrigDemandedElts,
3820 APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3,
3821 std::function<void(Instruction *, unsigned, APInt, APInt &)>
3822 SimplifyAndSetOp) const {
3823 switch (II.getIntrinsicID()) {
3824 default:
3825 break;
3826 case Intrinsic::aarch64_neon_fcvtxn:
3827 case Intrinsic::aarch64_neon_rshrn:
3828 case Intrinsic::aarch64_neon_sqrshrn:
3829 case Intrinsic::aarch64_neon_sqrshrun:
3830 case Intrinsic::aarch64_neon_sqshrn:
3831 case Intrinsic::aarch64_neon_sqshrun:
3832 case Intrinsic::aarch64_neon_sqxtn:
3833 case Intrinsic::aarch64_neon_sqxtun:
3834 case Intrinsic::aarch64_neon_uqrshrn:
3835 case Intrinsic::aarch64_neon_uqshrn:
3836 case Intrinsic::aarch64_neon_uqxtn:
3837 SimplifyAndSetOp(&II, 0, OrigDemandedElts, UndefElts);
3838 break;
3839 }
3840
3841 return std::nullopt;
3842}
3843
3845 return ST->isSVEAvailable() ||
3846 (ST->isSVEorStreamingSVEAvailable() &&
3847 ST->getCLOpts().enable_scalable_autovec_in_streaming_mode);
3848}
3849
3852 switch (K) {
3854 return TypeSize::getFixed(64);
3856 if (ST->useSVEForFixedLengthVectors() &&
3857 (ST->isSVEAvailable() ||
3858 ST->getCLOpts().enable_fixedwidth_autovec_in_streaming_mode))
3859 return TypeSize::getFixed(
3860 std::max(ST->getMinSVEVectorSizeInBits(), 128u));
3861 else if (ST->isNeonAvailable())
3862 return TypeSize::getFixed(128);
3863 else
3864 return TypeSize::getFixed(0);
3866 if (ST->isSVEAvailable() ||
3867 (ST->isSVEorStreamingSVEAvailable() &&
3868 ST->getCLOpts().enable_scalable_autovec_in_streaming_mode))
3869 return TypeSize::getScalable(128);
3870 else
3871 return TypeSize::getScalable(0);
3872 }
3873 llvm_unreachable("Unsupported register kind");
3874}
3875
3876bool AArch64TTIImpl::isSingleExtWideningInstruction(
3877 unsigned Opcode, Type *DstTy, ArrayRef<const Value *> Args,
3878 Type *SrcOverrideTy) const {
3879 // A helper that returns a vector type from the given type. The number of
3880 // elements in type Ty determines the vector width.
3881 auto toVectorTy = [&](Type *ArgTy) {
3882 return VectorType::get(ArgTy->getScalarType(),
3883 cast<VectorType>(DstTy)->getElementCount());
3884 };
3885
3886 // Exit early if DstTy is not a vector type whose elements are one of [i16,
3887 // i32, i64]. SVE doesn't generally have the same set of instructions to
3888 // perform an extend with the add/sub/mul. There are SMULLB style
3889 // instructions, but they operate on top/bottom, requiring some sort of lane
3890 // interleaving to be used with zext/sext.
3891 unsigned DstEltSize = DstTy->getScalarSizeInBits();
3892 if (!useNeonVector(DstTy) || Args.size() != 2 ||
3893 (DstEltSize != 16 && DstEltSize != 32 && DstEltSize != 64))
3894 return false;
3895
3896 Type *SrcTy = SrcOverrideTy;
3897 switch (Opcode) {
3898 case Instruction::Add: // UADDW(2), SADDW(2).
3899 case Instruction::Sub: { // USUBW(2), SSUBW(2).
3900 // The second operand needs to be an extend
3901 if (isa<SExtInst>(Args[1]) || isa<ZExtInst>(Args[1])) {
3902 if (!SrcTy)
3903 SrcTy =
3904 toVectorTy(cast<Instruction>(Args[1])->getOperand(0)->getType());
3905 break;
3906 }
3907
3908 if (Opcode == Instruction::Sub)
3909 return false;
3910
3911 // UADDW(2), SADDW(2) can be commutted.
3912 if (isa<SExtInst>(Args[0]) || isa<ZExtInst>(Args[0])) {
3913 if (!SrcTy)
3914 SrcTy =
3915 toVectorTy(cast<Instruction>(Args[0])->getOperand(0)->getType());
3916 break;
3917 }
3918 return false;
3919 }
3920 default:
3921 return false;
3922 }
3923
3924 // Legalize the destination type and ensure it can be used in a widening
3925 // operation.
3926 auto DstTyL = getTypeLegalizationCost(DstTy);
3927 if (!DstTyL.second.isVector() || DstEltSize != DstTy->getScalarSizeInBits())
3928 return false;
3929
3930 // Legalize the source type and ensure it can be used in a widening
3931 // operation.
3932 assert(SrcTy && "Expected some SrcTy");
3933 auto SrcTyL = getTypeLegalizationCost(SrcTy);
3934 unsigned SrcElTySize = SrcTyL.second.getScalarSizeInBits();
3935 if (!SrcTyL.second.isVector() || SrcElTySize != SrcTy->getScalarSizeInBits())
3936 return false;
3937
3938 // Get the total number of vector elements in the legalized types.
3939 InstructionCost NumDstEls =
3940 DstTyL.first * DstTyL.second.getVectorMinNumElements();
3941 InstructionCost NumSrcEls =
3942 SrcTyL.first * SrcTyL.second.getVectorMinNumElements();
3943
3944 // Return true if the legalized types have the same number of vector elements
3945 // and the destination element type size is twice that of the source type.
3946 return NumDstEls == NumSrcEls && 2 * SrcElTySize == DstEltSize;
3947}
3948
3949Type *AArch64TTIImpl::isBinExtWideningInstruction(unsigned Opcode, Type *DstTy,
3951 Type *SrcOverrideTy) const {
3952 if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
3953 Opcode != Instruction::Mul)
3954 return nullptr;
3955
3956 // Exit early if DstTy is not a vector type whose elements are one of [i16,
3957 // i32, i64]. SVE doesn't generally have the same set of instructions to
3958 // perform an extend with the add/sub/mul. There are SMULLB style
3959 // instructions, but they operate on top/bottom, requiring some sort of lane
3960 // interleaving to be used with zext/sext.
3961 unsigned DstEltSize = DstTy->getScalarSizeInBits();
3962 if (!useNeonVector(DstTy) || Args.size() != 2 ||
3963 (DstEltSize != 16 && DstEltSize != 32 && DstEltSize != 64))
3964 return nullptr;
3965
3966 auto getScalarSizeWithOverride = [&](const Value *V) {
3967 if (SrcOverrideTy)
3968 return SrcOverrideTy->getScalarSizeInBits();
3969 return cast<Instruction>(V)
3970 ->getOperand(0)
3971 ->getType()
3972 ->getScalarSizeInBits();
3973 };
3974
3975 unsigned MaxEltSize = 0;
3976 if ((isa<SExtInst>(Args[0]) && isa<SExtInst>(Args[1])) ||
3977 (isa<ZExtInst>(Args[0]) && isa<ZExtInst>(Args[1]))) {
3978 unsigned EltSize0 = getScalarSizeWithOverride(Args[0]);
3979 unsigned EltSize1 = getScalarSizeWithOverride(Args[1]);
3980 MaxEltSize = std::max(EltSize0, EltSize1);
3981 } else if (isa<SExtInst, ZExtInst>(Args[0]) &&
3982 isa<SExtInst, ZExtInst>(Args[1])) {
3983 unsigned EltSize0 = getScalarSizeWithOverride(Args[0]);
3984 unsigned EltSize1 = getScalarSizeWithOverride(Args[1]);
3985 // mul(sext, zext) will become smull(sext, zext) if the extends are large
3986 // enough.
3987 if (EltSize0 >= DstEltSize / 2 || EltSize1 >= DstEltSize / 2)
3988 return nullptr;
3989 MaxEltSize = DstEltSize / 2;
3990 } else if (Opcode == Instruction::Mul &&
3991 (isa<ZExtInst>(Args[0]) || isa<ZExtInst>(Args[1]))) {
3992 // If one of the operands is a Zext and the other has enough zero bits
3993 // to be treated as unsigned, we can still generate a umull, meaning the
3994 // zext is free.
3995 KnownBits Known =
3996 computeKnownBits(isa<ZExtInst>(Args[0]) ? Args[1] : Args[0], DL);
3997 if (Args[0]->getType()->getScalarSizeInBits() -
3998 Known.Zero.countLeadingOnes() >
3999 DstTy->getScalarSizeInBits() / 2)
4000 return nullptr;
4001
4002 MaxEltSize =
4003 getScalarSizeWithOverride(isa<ZExtInst>(Args[0]) ? Args[0] : Args[1]);
4004 } else
4005 return nullptr;
4006
4007 if (MaxEltSize * 2 > DstEltSize)
4008 return nullptr;
4009
4010 Type *ExtTy = DstTy->getWithNewBitWidth(MaxEltSize * 2);
4011 if (ExtTy->getPrimitiveSizeInBits() <= 64)
4012 return nullptr;
4013 return ExtTy;
4014}
4015
4016// s/urhadd instructions implement the following pattern, making the
4017// extends free:
4018// %x = add ((zext i8 -> i16), 1)
4019// %y = (zext i8 -> i16)
4020// trunc i16 (lshr (add %x, %y), 1) -> i8
4021//
4023 Type *Src) const {
4024 // The source should be a legal vector type.
4025 if (!Src->isVectorTy() || !TLI->isTypeLegal(TLI->getValueType(DL, Src)) ||
4026 (Src->isScalableTy() && !ST->hasSVE2()))
4027 return false;
4028
4029 if (ExtUser->getOpcode() != Instruction::Add || !ExtUser->hasOneUse())
4030 return false;
4031
4032 // Look for trunc/shl/add before trying to match the pattern.
4033 const Instruction *Add = ExtUser;
4034 auto *AddUser =
4035 dyn_cast_or_null<Instruction>(Add->getUniqueUndroppableUser());
4036 if (AddUser && AddUser->getOpcode() == Instruction::Add)
4037 Add = AddUser;
4038
4039 auto *Shr = dyn_cast_or_null<Instruction>(Add->getUniqueUndroppableUser());
4040 if (!Shr || Shr->getOpcode() != Instruction::LShr)
4041 return false;
4042
4043 auto *Trunc = dyn_cast_or_null<Instruction>(Shr->getUniqueUndroppableUser());
4044 if (!Trunc || Trunc->getOpcode() != Instruction::Trunc ||
4045 Src->getScalarSizeInBits() !=
4046 cast<CastInst>(Trunc)->getDestTy()->getScalarSizeInBits())
4047 return false;
4048
4049 // Try to match the whole pattern. Ext could be either the first or second
4050 // m_ZExtOrSExt matched.
4051 Instruction *Ex1, *Ex2;
4052 if (!(match(Add, m_c_Add(m_Instruction(Ex1),
4053 m_c_Add(m_Instruction(Ex2), m_One())))))
4054 return false;
4055
4056 // Ensure both extends are of the same type
4057 if (match(Ex1, m_ZExtOrSExt(m_Value())) &&
4058 Ex1->getOpcode() == Ex2->getOpcode())
4059 return true;
4060
4061 return false;
4062}
4063
4065 Type *Src,
4067 TTI::TargetCostKind CostKind,
4068 const Instruction *I) const {
4069 int ISD = TLI->InstructionOpcodeToISD(Opcode);
4070 assert(ISD && "Invalid opcode");
4071
4072 // Codegen is not able to select all <vscale x 1 x Ty> casts yet.
4073 if (auto *VTy = dyn_cast<ScalableVectorType>(Dst);
4074 VTy && VTy->getElementCount() == ElementCount::getScalable(1)) {
4077 ISD))
4079 }
4080
4081 // A non-power-of-2 i1 vector that fits in a single register has no bitmask
4082 // lowering, so its lanes are extracted and combined one by one.
4083 if (auto *SrcVTy = dyn_cast<FixedVectorType>(Src);
4084 ISD == ISD::BITCAST && SrcVTy && Dst->isIntegerTy() &&
4085 Src->isIntOrIntVectorTy(1) && !isPowerOf2_32(SrcVTy->getNumElements()) &&
4086 getTypeLegalizationCost(SrcVTy).first == 1) {
4087 unsigned NumElts = SrcVTy->getNumElements();
4088 return getScalarizationOverhead(SrcVTy, APInt::getAllOnes(NumElts),
4089 /*Insert=*/false, /*Extract=*/true,
4090 CostKind) +
4091 NumElts - 1;
4092 }
4093
4094 // If the cast is observable, and it is used by a widening instruction (e.g.,
4095 // uaddl, saddw, etc.), it may be free.
4096 if (I && !I->users().empty()) {
4097 // Determine whether Usr can absorb the cast from Src to Dst into a
4098 // widening instruction (e.g. uaddl, saddw, urhadd), making the cast free
4099 // with respect to that user.
4100 auto GetUserAbsorbedCastCost =
4101 [&](const Instruction *Usr) -> std::optional<InstructionCost> {
4102 SmallVector<const Value *, 4> Operands(Usr->operand_values());
4103
4104 if (Type *ExtTy = isBinExtWideningInstruction(
4105 Usr->getOpcode(), Dst, Operands,
4106 Src != I->getOperand(0)->getType() ? Src : nullptr)) {
4107 // The cost from Src->Src*2 needs to be added if required, the cost
4108 // from Src*2->ExtTy is free.
4109 if (ExtTy->getScalarSizeInBits() > Src->getScalarSizeInBits() * 2) {
4110 Type *DoubleSrcTy =
4111 Src->getWithNewBitWidth(Src->getScalarSizeInBits() * 2);
4112 return getCastInstrCost(Opcode, DoubleSrcTy, Src,
4113 TTI::CastContextHint::None, CostKind);
4114 }
4115
4116 return 0;
4117 }
4118
4119 if (isSingleExtWideningInstruction(
4120 Usr->getOpcode(), Dst, Operands,
4121 Src != I->getOperand(0)->getType() ? Src : nullptr)) {
4122 // For adds only count the second operand as free if both operands
4123 // are extends but not the same operation. (i.e both operands are
4124 // not free in add(sext, zext)).
4125 if (Usr->getOpcode() == Instruction::Add) {
4126 if (I == Usr->getOperand(1) ||
4127 (isa<CastInst>(Usr->getOperand(1)) &&
4128 cast<CastInst>(Usr->getOperand(1))->getOpcode() == Opcode))
4129 return 0;
4130 } else {
4131 // Others are free so long as isSingleExtWideningInstruction
4132 // returned true.
4133 return 0;
4134 }
4135 }
4136
4138 return std::nullopt;
4139
4140 // The cast will be free for the s/urhadd instructions
4141 if (isExtPartOfAvgExpr(Usr, Dst, Src))
4142 return 0;
4143
4144 // [US]SHLL/SHLL fold a doubling extend into a shift left by a splat
4145 // immediate of at most the source element width.
4146 const APInt *ShAmt;
4147 unsigned SrcBits = Src->getScalarSizeInBits();
4148 if (match(Usr, m_Shl(m_Specific(I), m_APInt(ShAmt))) &&
4149 ShAmt->ule(SrcBits) && useNeonVector(Dst) &&
4150 Dst->getScalarSizeInBits() == 2 * SrcBits &&
4151 getTypeLegalizationCost(Src).second.getScalarSizeInBits() == SrcBits)
4152 return 0;
4153
4154 return std::nullopt;
4155 };
4156
4157 InstructionCost MaxAbsorbedCost = 0;
4158 bool AllUsersAbsorbCast = true;
4159 for (const User *U : I->users()) {
4160 auto *Usr = cast<Instruction>(U);
4161 std::optional<InstructionCost> UserCost = GetUserAbsorbedCastCost(Usr);
4162 if (!UserCost) {
4163 AllUsersAbsorbCast = false;
4164 break;
4165 }
4166 MaxAbsorbedCost = std::max(MaxAbsorbedCost, *UserCost);
4167 }
4168
4169 if (AllUsersAbsorbCast)
4170 return MaxAbsorbedCost;
4171 }
4172
4173 // A vector compare already produces a lane mask (CMxx/FCMxx), so sign
4174 // extending it to the compare width is free. That only holds while the
4175 // result stays a vector; scalarized lanes still need a CSETM each.
4176 const Instruction *Cmp;
4177 const Value *CmpOp;
4178 if (I && useNeonVector(Dst) &&
4179 match(I, m_SExt(m_Instruction(Cmp, m_Cmp(m_Value(CmpOp), m_Value())))) &&
4180 Cmp->getParent() == I->getParent() &&
4181 CmpOp->getType()->getScalarSizeInBits() == Dst->getScalarSizeInBits() &&
4182 getTypeLegalizationCost(Dst).second.isVector())
4183 return 0;
4184
4185 EVT SrcTy = TLI->getValueType(DL, Src);
4186 EVT DstTy = TLI->getValueType(DL, Dst);
4187
4188 // SVE has no elements wider than 64 bits. The generic cost for a bitcast
4189 // between unsupported scalable types can otherwise appear to be free.
4190 if ((SrcTy.isScalableVector() && SrcTy.getScalarSizeInBits() > 64) ||
4191 (DstTy.isScalableVector() && DstTy.getScalarSizeInBits() > 64))
4193
4194 // A bitcast that crosses register banks is costed as a vector insert/extract.
4195 // Between the same register bank it is free. Note: Some costs are not handled
4196 // well by this, like i1 vectors that need to be packed.
4197 if (ISD == ISD::BITCAST && ST->isLittleEndian()) {
4198 std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Src);
4199 std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Dst);
4200 bool SrcIsFloatOrVec =
4201 SrcLT.second.isVector() || SrcLT.second.isFloatingPoint();
4202 bool DstIsFloatOrVec =
4203 DstLT.second.isVector() || DstLT.second.isFloatingPoint();
4204
4205 // Consider the bitcast free if they are both the same type and have the
4206 // same size after legalization. Scalable vectors are handled specially as
4207 // the type legalization cost of v2i16 is v2i64.
4208 if (SrcIsFloatOrVec == DstIsFloatOrVec &&
4209 (SrcLT.second.getSizeInBits() == DstLT.second.getSizeInBits() ||
4210 (SrcLT.second.isScalableVector() &&
4211 SrcLT.second.getScalarSizeInBits() != 1 &&
4212 DstLT.second.getScalarSizeInBits() != 1)))
4213 return 0;
4214
4215 return std::max(SrcLT.first, DstLT.first) *
4216 (CostKind == TTI::TCK_CodeSize
4217 ? 1
4218 : ST->getVectorInsertExtractBaseCost());
4219 }
4220
4221 // From a vector to a scalarized vector will be an series of extract-element
4222 // and extends.
4223 if ((ISD == ISD::ZERO_EXTEND || ISD == ISD::SIGN_EXTEND) &&
4224 DstTy.getScalarSizeInBits() > 64 && SrcTy.getScalarSizeInBits() <= 64 &&
4225 DstTy.isFixedLengthVector()) {
4227 Instruction::ExtractElement, Src, CostKind, -1, nullptr, nullptr);
4229 Opcode, Dst->getScalarType(), Src->getScalarType(), CCH, CostKind);
4230 return DstTy.getVectorNumElements() * (LaneCost + ExtCost);
4231 }
4232
4233 if (!SrcTy.isSimple() || !DstTy.isSimple())
4234 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
4235
4236 // For the moment we do not have lowering for SVE1-only fptrunc f64->bf16 as
4237 // we use fcvtx under SVE2. Give them invalid costs.
4238 if (!ST->hasSVE2() && !ST->isStreamingSVEAvailable() &&
4239 ISD == ISD::FP_ROUND && SrcTy.isScalableVector() &&
4240 DstTy.getScalarType() == MVT::bf16 && SrcTy.getScalarType() == MVT::f64)
4242
4243 static const TypeConversionCostTblEntry BF16Tbl[] = {
4244 {ISD::FP_ROUND, MVT::bf16, MVT::f32, 1}, // bfcvt
4245 {ISD::FP_ROUND, MVT::bf16, MVT::f64, 1}, // bfcvt
4246 {ISD::FP_ROUND, MVT::v4bf16, MVT::v4f32, 1}, // bfcvtn
4247 {ISD::FP_ROUND, MVT::v8bf16, MVT::v8f32, 2}, // bfcvtn+bfcvtn2
4248 {ISD::FP_ROUND, MVT::v2bf16, MVT::v2f64, 2}, // bfcvtn+fcvtn
4249 {ISD::FP_ROUND, MVT::v4bf16, MVT::v4f64, 3}, // fcvtn+fcvtl2+bfcvtn
4250 {ISD::FP_ROUND, MVT::v8bf16, MVT::v8f64, 6}, // 2 * fcvtn+fcvtn2+bfcvtn
4251 {ISD::FP_ROUND, MVT::nxv2bf16, MVT::nxv2f32, 1}, // bfcvt
4252 {ISD::FP_ROUND, MVT::nxv4bf16, MVT::nxv4f32, 1}, // bfcvt
4253 {ISD::FP_ROUND, MVT::nxv8bf16, MVT::nxv8f32, 3}, // bfcvt+bfcvt+uzp1
4254 {ISD::FP_ROUND, MVT::nxv2bf16, MVT::nxv2f64, 2}, // fcvtx+bfcvt
4255 {ISD::FP_ROUND, MVT::nxv4bf16, MVT::nxv4f64, 5}, // 2*fcvtx+2*bfcvt+uzp1
4256 {ISD::FP_ROUND, MVT::nxv8bf16, MVT::nxv8f64, 11}, // 4*fcvt+4*bfcvt+3*uzp
4257 };
4258
4259 if (ST->hasBF16())
4260 if (const auto *Entry = ConvertCostTableLookup(
4261 BF16Tbl, ISD, DstTy.getSimpleVT(), SrcTy.getSimpleVT()))
4262 return Entry->Cost;
4263
4264 // We have to estimate a cost of fixed length operation upon
4265 // SVE registers(operations) with the number of registers required
4266 // for a fixed type to be represented upon SVE registers.
4267 EVT WiderTy = SrcTy.bitsGT(DstTy) ? SrcTy : DstTy;
4268 if (SrcTy.isFixedLengthVector() && DstTy.isFixedLengthVector() &&
4269 SrcTy.getVectorNumElements() == DstTy.getVectorNumElements() &&
4270 ST->useSVEForFixedLengthVectors(WiderTy)) {
4271 std::pair<InstructionCost, MVT> LT =
4272 getTypeLegalizationCost(WiderTy.getTypeForEVT(Dst->getContext()));
4273 unsigned NumElements =
4274 AArch64::SVEBitsPerBlock / LT.second.getScalarSizeInBits();
4275 return LT.first *
4277 Opcode,
4278 ScalableVectorType::get(Dst->getScalarType(), NumElements),
4279 ScalableVectorType::get(Src->getScalarType(), NumElements), CCH,
4280 CostKind, I);
4281 }
4282
4283 // Symbolic constants for the SVE sitofp/uitofp entries in the table below
4284 // The cost of unpacking twice is artificially increased for now in order
4285 // to avoid regressions against NEON, which will use tbl instructions directly
4286 // instead of multiple layers of [s|u]unpk[lo|hi].
4287 // We use the unpacks in cases where the destination type is illegal and
4288 // requires splitting of the input, even if the input type itself is legal.
4289 const unsigned int SVE_EXT_COST = 1;
4290 const unsigned int SVE_FCVT_COST = 1;
4291 const unsigned int SVE_UNPACK_ONCE = 4;
4292 const unsigned int SVE_UNPACK_TWICE = 16;
4293
4294 static const TypeConversionCostTblEntry ConversionTbl[] = {
4295 {ISD::TRUNCATE, MVT::v2i8, MVT::v2i64, 1}, // xtn
4296 {ISD::TRUNCATE, MVT::v2i16, MVT::v2i64, 1}, // xtn
4297 {ISD::TRUNCATE, MVT::v2i32, MVT::v2i64, 1}, // xtn
4298 {ISD::TRUNCATE, MVT::v4i8, MVT::v4i32, 1}, // xtn
4299 {ISD::TRUNCATE, MVT::v4i8, MVT::v4i64, 3}, // 2 xtn + 1 uzp1
4300 {ISD::TRUNCATE, MVT::v4i16, MVT::v4i32, 1}, // xtn
4301 {ISD::TRUNCATE, MVT::v4i16, MVT::v4i64, 2}, // 1 uzp1 + 1 xtn
4302 {ISD::TRUNCATE, MVT::v4i32, MVT::v4i64, 1}, // 1 uzp1
4303 {ISD::TRUNCATE, MVT::v8i8, MVT::v8i16, 1}, // 1 xtn
4304 {ISD::TRUNCATE, MVT::v8i8, MVT::v8i32, 2}, // 1 uzp1 + 1 xtn
4305 {ISD::TRUNCATE, MVT::v8i8, MVT::v8i64, 4}, // 3 x uzp1 + xtn
4306 {ISD::TRUNCATE, MVT::v8i16, MVT::v8i32, 1}, // 1 uzp1
4307 {ISD::TRUNCATE, MVT::v8i16, MVT::v8i64, 3}, // 3 x uzp1
4308 {ISD::TRUNCATE, MVT::v8i32, MVT::v8i64, 2}, // 2 x uzp1
4309 {ISD::TRUNCATE, MVT::v16i8, MVT::v16i16, 1}, // uzp1
4310 {ISD::TRUNCATE, MVT::v16i8, MVT::v16i32, 3}, // (2 + 1) x uzp1
4311 {ISD::TRUNCATE, MVT::v16i8, MVT::v16i64, 7}, // (4 + 2 + 1) x uzp1
4312 {ISD::TRUNCATE, MVT::v16i16, MVT::v16i32, 2}, // 2 x uzp1
4313 {ISD::TRUNCATE, MVT::v16i16, MVT::v16i64, 6}, // (4 + 2) x uzp1
4314 {ISD::TRUNCATE, MVT::v16i32, MVT::v16i64, 4}, // 4 x uzp1
4315
4316 // Truncations on nxvmiN
4317 {ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i8, 2},
4318 {ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i16, 2},
4319 {ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i32, 2},
4320 {ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i64, 2},
4321 {ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i8, 2},
4322 {ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i16, 2},
4323 {ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i32, 2},
4324 {ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i64, 5},
4325 {ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i8, 2},
4326 {ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i16, 2},
4327 {ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i32, 5},
4328 {ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i64, 11},
4329 {ISD::TRUNCATE, MVT::nxv16i1, MVT::nxv16i8, 2},
4330 {ISD::TRUNCATE, MVT::nxv2i8, MVT::nxv2i16, 0},
4331 {ISD::TRUNCATE, MVT::nxv2i8, MVT::nxv2i32, 0},
4332 {ISD::TRUNCATE, MVT::nxv2i8, MVT::nxv2i64, 0},
4333 {ISD::TRUNCATE, MVT::nxv2i16, MVT::nxv2i32, 0},
4334 {ISD::TRUNCATE, MVT::nxv2i16, MVT::nxv2i64, 0},
4335 {ISD::TRUNCATE, MVT::nxv2i32, MVT::nxv2i64, 0},
4336 {ISD::TRUNCATE, MVT::nxv4i8, MVT::nxv4i16, 0},
4337 {ISD::TRUNCATE, MVT::nxv4i8, MVT::nxv4i32, 0},
4338 {ISD::TRUNCATE, MVT::nxv4i8, MVT::nxv4i64, 1},
4339 {ISD::TRUNCATE, MVT::nxv4i16, MVT::nxv4i32, 0},
4340 {ISD::TRUNCATE, MVT::nxv4i16, MVT::nxv4i64, 1},
4341 {ISD::TRUNCATE, MVT::nxv4i32, MVT::nxv4i64, 1},
4342 {ISD::TRUNCATE, MVT::nxv8i8, MVT::nxv8i16, 0},
4343 {ISD::TRUNCATE, MVT::nxv8i8, MVT::nxv8i32, 1},
4344 {ISD::TRUNCATE, MVT::nxv8i8, MVT::nxv8i64, 3},
4345 {ISD::TRUNCATE, MVT::nxv8i16, MVT::nxv8i32, 1},
4346 {ISD::TRUNCATE, MVT::nxv8i16, MVT::nxv8i64, 3},
4347 {ISD::TRUNCATE, MVT::nxv16i8, MVT::nxv16i16, 1},
4348 {ISD::TRUNCATE, MVT::nxv16i8, MVT::nxv16i32, 3},
4349 {ISD::TRUNCATE, MVT::nxv16i8, MVT::nxv16i64, 7},
4350
4351 // The number of shll instructions for the extension.
4352 {ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i16, 3},
4353 {ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i16, 3},
4354 {ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i32, 2},
4355 {ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i32, 2},
4356 {ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i8, 3},
4357 {ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i8, 3},
4358 {ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, 2},
4359 {ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, 2},
4360 {ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i8, 7},
4361 {ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i8, 7},
4362 {ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i16, 6},
4363 {ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i16, 6},
4364 {ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, 2},
4365 {ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, 2},
4366 {ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i8, 6},
4367 {ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i8, 6},
4368
4369 // FP Ext and trunc
4370 {ISD::FP_EXTEND, MVT::f64, MVT::f32, 1}, // fcvt
4371 {ISD::FP_EXTEND, MVT::v2f64, MVT::v2f32, 1}, // fcvtl
4372 {ISD::FP_EXTEND, MVT::v4f64, MVT::v4f32, 2}, // fcvtl+fcvtl2
4373 // FP16
4374 {ISD::FP_EXTEND, MVT::f32, MVT::f16, 1}, // fcvt
4375 {ISD::FP_EXTEND, MVT::f64, MVT::f16, 1}, // fcvt
4376 {ISD::FP_EXTEND, MVT::v4f32, MVT::v4f16, 1}, // fcvtl
4377 {ISD::FP_EXTEND, MVT::v8f32, MVT::v8f16, 2}, // fcvtl+fcvtl2
4378 {ISD::FP_EXTEND, MVT::v2f64, MVT::v2f16, 2}, // fcvtl+fcvtl
4379 {ISD::FP_EXTEND, MVT::v4f64, MVT::v4f16, 3}, // fcvtl+fcvtl2+fcvtl
4380 {ISD::FP_EXTEND, MVT::v8f64, MVT::v8f16, 6}, // 2 * fcvtl+fcvtl2+fcvtl
4381 // BF16 (uses shift)
4382 {ISD::FP_EXTEND, MVT::f32, MVT::bf16, 1}, // shl
4383 {ISD::FP_EXTEND, MVT::f64, MVT::bf16, 2}, // shl+fcvt
4384 {ISD::FP_EXTEND, MVT::v4f32, MVT::v4bf16, 1}, // shll
4385 {ISD::FP_EXTEND, MVT::v8f32, MVT::v8bf16, 2}, // shll+shll2
4386 {ISD::FP_EXTEND, MVT::v2f64, MVT::v2bf16, 2}, // shll+fcvtl
4387 {ISD::FP_EXTEND, MVT::v4f64, MVT::v4bf16, 3}, // shll+fcvtl+fcvtl2
4388 {ISD::FP_EXTEND, MVT::v8f64, MVT::v8bf16, 6}, // 2 * shll+fcvtl+fcvtl2
4389 // FP Ext and trunc
4390 {ISD::FP_ROUND, MVT::f32, MVT::f64, 1}, // fcvt
4391 {ISD::FP_ROUND, MVT::v2f32, MVT::v2f64, 1}, // fcvtn
4392 {ISD::FP_ROUND, MVT::v4f32, MVT::v4f64, 2}, // fcvtn+fcvtn2
4393 // FP16
4394 {ISD::FP_ROUND, MVT::f16, MVT::f32, 1}, // fcvt
4395 {ISD::FP_ROUND, MVT::f16, MVT::f64, 1}, // fcvt
4396 {ISD::FP_ROUND, MVT::v4f16, MVT::v4f32, 1}, // fcvtn
4397 {ISD::FP_ROUND, MVT::v8f16, MVT::v8f32, 2}, // fcvtn+fcvtn2
4398 {ISD::FP_ROUND, MVT::v2f16, MVT::v2f64, 2}, // fcvtn+fcvtn
4399 {ISD::FP_ROUND, MVT::v4f16, MVT::v4f64, 3}, // fcvtn+fcvtn2+fcvtn
4400 {ISD::FP_ROUND, MVT::v8f16, MVT::v8f64, 6}, // 2 * fcvtn+fcvtn2+fcvtn
4401 // BF16 (more complex, with +bf16 is handled above)
4402 {ISD::FP_ROUND, MVT::bf16, MVT::f32, 8}, // Expansion is ~8 insns
4403 {ISD::FP_ROUND, MVT::bf16, MVT::f64, 9}, // fcvtn + above
4404 {ISD::FP_ROUND, MVT::v2bf16, MVT::v2f32, 8},
4405 {ISD::FP_ROUND, MVT::v4bf16, MVT::v4f32, 8},
4406 {ISD::FP_ROUND, MVT::v8bf16, MVT::v8f32, 15},
4407 {ISD::FP_ROUND, MVT::v2bf16, MVT::v2f64, 9},
4408 {ISD::FP_ROUND, MVT::v4bf16, MVT::v4f64, 10},
4409 {ISD::FP_ROUND, MVT::v8bf16, MVT::v8f64, 19},
4410
4411 // LowerVectorINT_TO_FP:
4412 {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i32, 1},
4413 {ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i32, 1},
4414 {ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i64, 1},
4415 {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i32, 1},
4416 {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, 1},
4417 {ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i64, 1},
4418
4419 // SVE: to nxv2f16
4420 {ISD::SINT_TO_FP, MVT::nxv2f16, MVT::nxv2i8,
4421 SVE_EXT_COST + SVE_FCVT_COST},
4422 {ISD::SINT_TO_FP, MVT::nxv2f16, MVT::nxv2i16, SVE_FCVT_COST},
4423 {ISD::SINT_TO_FP, MVT::nxv2f16, MVT::nxv2i32, SVE_FCVT_COST},
4424 {ISD::SINT_TO_FP, MVT::nxv2f16, MVT::nxv2i64, SVE_FCVT_COST},
4425 {ISD::UINT_TO_FP, MVT::nxv2f16, MVT::nxv2i8,
4426 SVE_EXT_COST + SVE_FCVT_COST},
4427 {ISD::UINT_TO_FP, MVT::nxv2f16, MVT::nxv2i16, SVE_FCVT_COST},
4428 {ISD::UINT_TO_FP, MVT::nxv2f16, MVT::nxv2i32, SVE_FCVT_COST},
4429 {ISD::UINT_TO_FP, MVT::nxv2f16, MVT::nxv2i64, SVE_FCVT_COST},
4430
4431 // SVE: to nxv4f16
4432 {ISD::SINT_TO_FP, MVT::nxv4f16, MVT::nxv4i8,
4433 SVE_EXT_COST + SVE_FCVT_COST},
4434 {ISD::SINT_TO_FP, MVT::nxv4f16, MVT::nxv4i16, SVE_FCVT_COST},
4435 {ISD::SINT_TO_FP, MVT::nxv4f16, MVT::nxv4i32, SVE_FCVT_COST},
4436 {ISD::UINT_TO_FP, MVT::nxv4f16, MVT::nxv4i8,
4437 SVE_EXT_COST + SVE_FCVT_COST},
4438 {ISD::UINT_TO_FP, MVT::nxv4f16, MVT::nxv4i16, SVE_FCVT_COST},
4439 {ISD::UINT_TO_FP, MVT::nxv4f16, MVT::nxv4i32, SVE_FCVT_COST},
4440
4441 // SVE: to nxv8f16
4442 {ISD::SINT_TO_FP, MVT::nxv8f16, MVT::nxv8i8,
4443 SVE_EXT_COST + SVE_FCVT_COST},
4444 {ISD::SINT_TO_FP, MVT::nxv8f16, MVT::nxv8i16, SVE_FCVT_COST},
4445 {ISD::UINT_TO_FP, MVT::nxv8f16, MVT::nxv8i8,
4446 SVE_EXT_COST + SVE_FCVT_COST},
4447 {ISD::UINT_TO_FP, MVT::nxv8f16, MVT::nxv8i16, SVE_FCVT_COST},
4448
4449 // SVE: to nxv16f16
4450 {ISD::SINT_TO_FP, MVT::nxv16f16, MVT::nxv16i8,
4451 SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4452 {ISD::UINT_TO_FP, MVT::nxv16f16, MVT::nxv16i8,
4453 SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4454
4455 // Complex: to v2f32
4456 {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i8, 3},
4457 {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i16, 3},
4458 {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i8, 3},
4459 {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i16, 3},
4460
4461 // SVE: to nxv2f32
4462 {ISD::SINT_TO_FP, MVT::nxv2f32, MVT::nxv2i8,
4463 SVE_EXT_COST + SVE_FCVT_COST},
4464 {ISD::SINT_TO_FP, MVT::nxv2f32, MVT::nxv2i16, SVE_FCVT_COST},
4465 {ISD::SINT_TO_FP, MVT::nxv2f32, MVT::nxv2i32, SVE_FCVT_COST},
4466 {ISD::SINT_TO_FP, MVT::nxv2f32, MVT::nxv2i64, SVE_FCVT_COST},
4467 {ISD::UINT_TO_FP, MVT::nxv2f32, MVT::nxv2i8,
4468 SVE_EXT_COST + SVE_FCVT_COST},
4469 {ISD::UINT_TO_FP, MVT::nxv2f32, MVT::nxv2i16, SVE_FCVT_COST},
4470 {ISD::UINT_TO_FP, MVT::nxv2f32, MVT::nxv2i32, SVE_FCVT_COST},
4471 {ISD::UINT_TO_FP, MVT::nxv2f32, MVT::nxv2i64, SVE_FCVT_COST},
4472
4473 // Complex: to v4f32
4474 {ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i8, 4},
4475 {ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i16, 2},
4476 {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i8, 3},
4477 {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i16, 2},
4478
4479 // SVE: to nxv4f32
4480 {ISD::SINT_TO_FP, MVT::nxv4f32, MVT::nxv4i8,
4481 SVE_EXT_COST + SVE_FCVT_COST},
4482 {ISD::SINT_TO_FP, MVT::nxv4f32, MVT::nxv4i16, SVE_FCVT_COST},
4483 {ISD::SINT_TO_FP, MVT::nxv4f32, MVT::nxv4i32, SVE_FCVT_COST},
4484 {ISD::UINT_TO_FP, MVT::nxv4f32, MVT::nxv4i8,
4485 SVE_EXT_COST + SVE_FCVT_COST},
4486 {ISD::UINT_TO_FP, MVT::nxv4f32, MVT::nxv4i16, SVE_FCVT_COST},
4487 {ISD::SINT_TO_FP, MVT::nxv4f32, MVT::nxv4i32, SVE_FCVT_COST},
4488
4489 // Complex: to v8f32
4490 {ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i8, 10},
4491 {ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i16, 4},
4492 {ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i8, 10},
4493 {ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i16, 4},
4494
4495 // SVE: to nxv8f32
4496 {ISD::SINT_TO_FP, MVT::nxv8f32, MVT::nxv8i8,
4497 SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4498 {ISD::SINT_TO_FP, MVT::nxv8f32, MVT::nxv8i16,
4499 SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4500 {ISD::UINT_TO_FP, MVT::nxv8f32, MVT::nxv8i8,
4501 SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4502 {ISD::UINT_TO_FP, MVT::nxv8f32, MVT::nxv8i16,
4503 SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4504
4505 // SVE: to nxv16f32
4506 {ISD::SINT_TO_FP, MVT::nxv16f32, MVT::nxv16i8,
4507 SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
4508 {ISD::UINT_TO_FP, MVT::nxv16f32, MVT::nxv16i8,
4509 SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
4510
4511 // Complex: to v16f32
4512 {ISD::SINT_TO_FP, MVT::v16f32, MVT::v16i8, 21},
4513 {ISD::UINT_TO_FP, MVT::v16f32, MVT::v16i8, 21},
4514
4515 // Complex: to v2f64
4516 {ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i8, 4},
4517 {ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i16, 4},
4518 {ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i32, 2},
4519 {ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i8, 4},
4520 {ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i16, 4},
4521 {ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i32, 2},
4522
4523 // SVE: to nxv2f64
4524 {ISD::SINT_TO_FP, MVT::nxv2f64, MVT::nxv2i8,
4525 SVE_EXT_COST + SVE_FCVT_COST},
4526 {ISD::SINT_TO_FP, MVT::nxv2f64, MVT::nxv2i16, SVE_FCVT_COST},
4527 {ISD::SINT_TO_FP, MVT::nxv2f64, MVT::nxv2i32, SVE_FCVT_COST},
4528 {ISD::SINT_TO_FP, MVT::nxv2f64, MVT::nxv2i64, SVE_FCVT_COST},
4529 {ISD::UINT_TO_FP, MVT::nxv2f64, MVT::nxv2i8,
4530 SVE_EXT_COST + SVE_FCVT_COST},
4531 {ISD::UINT_TO_FP, MVT::nxv2f64, MVT::nxv2i16, SVE_FCVT_COST},
4532 {ISD::UINT_TO_FP, MVT::nxv2f64, MVT::nxv2i32, SVE_FCVT_COST},
4533 {ISD::UINT_TO_FP, MVT::nxv2f64, MVT::nxv2i64, SVE_FCVT_COST},
4534
4535 // Complex: to v4f64
4536 {ISD::SINT_TO_FP, MVT::v4f64, MVT::v4i32, 4},
4537 {ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i32, 4},
4538
4539 // SVE: to nxv4f64
4540 {ISD::SINT_TO_FP, MVT::nxv4f64, MVT::nxv4i8,
4541 SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4542 {ISD::SINT_TO_FP, MVT::nxv4f64, MVT::nxv4i16,
4543 SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4544 {ISD::SINT_TO_FP, MVT::nxv4f64, MVT::nxv4i32,
4545 SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4546 {ISD::UINT_TO_FP, MVT::nxv4f64, MVT::nxv4i8,
4547 SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4548 {ISD::UINT_TO_FP, MVT::nxv4f64, MVT::nxv4i16,
4549 SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4550 {ISD::UINT_TO_FP, MVT::nxv4f64, MVT::nxv4i32,
4551 SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4552
4553 // SVE: to nxv8f64
4554 {ISD::SINT_TO_FP, MVT::nxv8f64, MVT::nxv8i8,
4555 SVE_EXT_COST + SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
4556 {ISD::SINT_TO_FP, MVT::nxv8f64, MVT::nxv8i16,
4557 SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
4558 {ISD::UINT_TO_FP, MVT::nxv8f64, MVT::nxv8i8,
4559 SVE_EXT_COST + SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
4560 {ISD::UINT_TO_FP, MVT::nxv8f64, MVT::nxv8i16,
4561 SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
4562
4563 // LowerVectorFP_TO_INT
4564 {ISD::FP_TO_SINT, MVT::v2i32, MVT::v2f32, 1},
4565 {ISD::FP_TO_SINT, MVT::v4i32, MVT::v4f32, 1},
4566 {ISD::FP_TO_SINT, MVT::v2i64, MVT::v2f64, 1},
4567 {ISD::FP_TO_UINT, MVT::v2i32, MVT::v2f32, 1},
4568 {ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f32, 1},
4569 {ISD::FP_TO_UINT, MVT::v2i64, MVT::v2f64, 1},
4570
4571 // Complex, from v2f32: legal type is v2i32 (no cost) or v2i64 (1 ext).
4572 {ISD::FP_TO_SINT, MVT::v2i64, MVT::v2f32, 2},
4573 {ISD::FP_TO_SINT, MVT::v2i16, MVT::v2f32, 1},
4574 {ISD::FP_TO_SINT, MVT::v2i8, MVT::v2f32, 1},
4575 {ISD::FP_TO_UINT, MVT::v2i64, MVT::v2f32, 2},
4576 {ISD::FP_TO_UINT, MVT::v2i16, MVT::v2f32, 1},
4577 {ISD::FP_TO_UINT, MVT::v2i8, MVT::v2f32, 1},
4578
4579 // Complex, from v4f32: legal type is v4i16, 1 narrowing => ~2
4580 {ISD::FP_TO_SINT, MVT::v4i16, MVT::v4f32, 2},
4581 {ISD::FP_TO_SINT, MVT::v4i8, MVT::v4f32, 2},
4582 {ISD::FP_TO_UINT, MVT::v4i16, MVT::v4f32, 2},
4583 {ISD::FP_TO_UINT, MVT::v4i8, MVT::v4f32, 2},
4584
4585 // Complex, from v2f64: legal type is v2i32, 1 narrowing => ~2.
4586 {ISD::FP_TO_SINT, MVT::v2i32, MVT::v2f64, 2},
4587 {ISD::FP_TO_SINT, MVT::v2i16, MVT::v2f64, 2},
4588 {ISD::FP_TO_SINT, MVT::v2i8, MVT::v2f64, 2},
4589 {ISD::FP_TO_UINT, MVT::v2i32, MVT::v2f64, 2},
4590 {ISD::FP_TO_UINT, MVT::v2i16, MVT::v2f64, 2},
4591 {ISD::FP_TO_UINT, MVT::v2i8, MVT::v2f64, 2},
4592
4593 // Complex, from nxv2f32.
4594 {ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f32, 1},
4595 {ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f32, 1},
4596 {ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f32, 1},
4597 {ISD::FP_TO_SINT, MVT::nxv2i8, MVT::nxv2f32, 1},
4598 {ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f32, 1},
4599 {ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f32, 1},
4600 {ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f32, 1},
4601 {ISD::FP_TO_UINT, MVT::nxv2i8, MVT::nxv2f32, 1},
4602
4603 // Complex, from nxv2f64.
4604 {ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f64, 1},
4605 {ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f64, 1},
4606 {ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f64, 1},
4607 {ISD::FP_TO_SINT, MVT::nxv2i8, MVT::nxv2f64, 1},
4608 {ISD::FP_TO_SINT, MVT::nxv2i1, MVT::nxv2f64, 1},
4609 {ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f64, 1},
4610 {ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f64, 1},
4611 {ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f64, 1},
4612 {ISD::FP_TO_UINT, MVT::nxv2i8, MVT::nxv2f64, 1},
4613 {ISD::FP_TO_UINT, MVT::nxv2i1, MVT::nxv2f64, 1},
4614
4615 // Complex, from nxv4f32.
4616 {ISD::FP_TO_SINT, MVT::nxv4i64, MVT::nxv4f32, 4},
4617 {ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f32, 1},
4618 {ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f32, 1},
4619 {ISD::FP_TO_SINT, MVT::nxv4i8, MVT::nxv4f32, 1},
4620 {ISD::FP_TO_SINT, MVT::nxv4i1, MVT::nxv4f32, 1},
4621 {ISD::FP_TO_UINT, MVT::nxv4i64, MVT::nxv4f32, 4},
4622 {ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f32, 1},
4623 {ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f32, 1},
4624 {ISD::FP_TO_UINT, MVT::nxv4i8, MVT::nxv4f32, 1},
4625 {ISD::FP_TO_UINT, MVT::nxv4i1, MVT::nxv4f32, 1},
4626
4627 // Complex, from nxv8f64. Illegal -> illegal conversions not required.
4628 {ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f64, 7},
4629 {ISD::FP_TO_SINT, MVT::nxv8i8, MVT::nxv8f64, 7},
4630 {ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f64, 7},
4631 {ISD::FP_TO_UINT, MVT::nxv8i8, MVT::nxv8f64, 7},
4632
4633 // Complex, from nxv4f64. Illegal -> illegal conversions not required.
4634 {ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f64, 3},
4635 {ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f64, 3},
4636 {ISD::FP_TO_SINT, MVT::nxv4i8, MVT::nxv4f64, 3},
4637 {ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f64, 3},
4638 {ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f64, 3},
4639 {ISD::FP_TO_UINT, MVT::nxv4i8, MVT::nxv4f64, 3},
4640
4641 // Complex, from nxv8f32. Illegal -> illegal conversions not required.
4642 {ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f32, 3},
4643 {ISD::FP_TO_SINT, MVT::nxv8i8, MVT::nxv8f32, 3},
4644 {ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f32, 3},
4645 {ISD::FP_TO_UINT, MVT::nxv8i8, MVT::nxv8f32, 3},
4646
4647 // Complex, from nxv8f16.
4648 {ISD::FP_TO_SINT, MVT::nxv8i64, MVT::nxv8f16, 10},
4649 {ISD::FP_TO_SINT, MVT::nxv8i32, MVT::nxv8f16, 4},
4650 {ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f16, 1},
4651 {ISD::FP_TO_SINT, MVT::nxv8i8, MVT::nxv8f16, 1},
4652 {ISD::FP_TO_SINT, MVT::nxv8i1, MVT::nxv8f16, 1},
4653 {ISD::FP_TO_UINT, MVT::nxv8i64, MVT::nxv8f16, 10},
4654 {ISD::FP_TO_UINT, MVT::nxv8i32, MVT::nxv8f16, 4},
4655 {ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f16, 1},
4656 {ISD::FP_TO_UINT, MVT::nxv8i8, MVT::nxv8f16, 1},
4657 {ISD::FP_TO_UINT, MVT::nxv8i1, MVT::nxv8f16, 1},
4658
4659 // Complex, from nxv4f16.
4660 {ISD::FP_TO_SINT, MVT::nxv4i64, MVT::nxv4f16, 4},
4661 {ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f16, 1},
4662 {ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f16, 1},
4663 {ISD::FP_TO_SINT, MVT::nxv4i8, MVT::nxv4f16, 1},
4664 {ISD::FP_TO_UINT, MVT::nxv4i64, MVT::nxv4f16, 4},
4665 {ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f16, 1},
4666 {ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f16, 1},
4667 {ISD::FP_TO_UINT, MVT::nxv4i8, MVT::nxv4f16, 1},
4668
4669 // Complex, from nxv2f16.
4670 {ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f16, 1},
4671 {ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f16, 1},
4672 {ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f16, 1},
4673 {ISD::FP_TO_SINT, MVT::nxv2i8, MVT::nxv2f16, 1},
4674 {ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f16, 1},
4675 {ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f16, 1},
4676 {ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f16, 1},
4677 {ISD::FP_TO_UINT, MVT::nxv2i8, MVT::nxv2f16, 1},
4678
4679 // Truncate from nxvmf32 to nxvmf16.
4680 {ISD::FP_ROUND, MVT::nxv2f16, MVT::nxv2f32, 1},
4681 {ISD::FP_ROUND, MVT::nxv4f16, MVT::nxv4f32, 1},
4682 {ISD::FP_ROUND, MVT::nxv8f16, MVT::nxv8f32, 3},
4683
4684 // Truncate from nxvmf32 to nxvmbf16.
4685 {ISD::FP_ROUND, MVT::nxv2bf16, MVT::nxv2f32, 8},
4686 {ISD::FP_ROUND, MVT::nxv4bf16, MVT::nxv4f32, 8},
4687 {ISD::FP_ROUND, MVT::nxv8bf16, MVT::nxv8f32, 17},
4688
4689 // Truncate from nxvmf64 to nxvmf16.
4690 {ISD::FP_ROUND, MVT::nxv2f16, MVT::nxv2f64, 1},
4691 {ISD::FP_ROUND, MVT::nxv4f16, MVT::nxv4f64, 3},
4692 {ISD::FP_ROUND, MVT::nxv8f16, MVT::nxv8f64, 7},
4693
4694 // Truncate from nxvmf64 to nxvmbf16.
4695 {ISD::FP_ROUND, MVT::nxv2bf16, MVT::nxv2f64, 9},
4696 {ISD::FP_ROUND, MVT::nxv4bf16, MVT::nxv4f64, 19},
4697 {ISD::FP_ROUND, MVT::nxv8bf16, MVT::nxv8f64, 39},
4698
4699 // Truncate from nxvmf64 to nxvmf32.
4700 {ISD::FP_ROUND, MVT::nxv2f32, MVT::nxv2f64, 1},
4701 {ISD::FP_ROUND, MVT::nxv4f32, MVT::nxv4f64, 3},
4702 {ISD::FP_ROUND, MVT::nxv8f32, MVT::nxv8f64, 6},
4703
4704 // Extend from nxvmf16 to nxvmf32.
4705 {ISD::FP_EXTEND, MVT::nxv2f32, MVT::nxv2f16, 1},
4706 {ISD::FP_EXTEND, MVT::nxv4f32, MVT::nxv4f16, 1},
4707 {ISD::FP_EXTEND, MVT::nxv8f32, MVT::nxv8f16, 2},
4708
4709 // Extend from nxvmbf16 to nxvmf32.
4710 {ISD::FP_EXTEND, MVT::nxv2f32, MVT::nxv2bf16, 1}, // lsl
4711 {ISD::FP_EXTEND, MVT::nxv4f32, MVT::nxv4bf16, 1}, // lsl
4712 {ISD::FP_EXTEND, MVT::nxv8f32, MVT::nxv8bf16, 4}, // unpck+unpck+lsl+lsl
4713
4714 // Extend from nxvmf16 to nxvmf64.
4715 {ISD::FP_EXTEND, MVT::nxv2f64, MVT::nxv2f16, 1},
4716 {ISD::FP_EXTEND, MVT::nxv4f64, MVT::nxv4f16, 2},
4717 {ISD::FP_EXTEND, MVT::nxv8f64, MVT::nxv8f16, 4},
4718
4719 // Extend from nxvmbf16 to nxvmf64.
4720 {ISD::FP_EXTEND, MVT::nxv2f64, MVT::nxv2bf16, 2}, // lsl+fcvt
4721 {ISD::FP_EXTEND, MVT::nxv4f64, MVT::nxv4bf16, 6}, // 2*unpck+2*lsl+2*fcvt
4722 {ISD::FP_EXTEND, MVT::nxv8f64, MVT::nxv8bf16, 14}, // 6*unpck+4*lsl+4*fcvt
4723
4724 // Extend from nxvmf32 to nxvmf64.
4725 {ISD::FP_EXTEND, MVT::nxv2f64, MVT::nxv2f32, 1},
4726 {ISD::FP_EXTEND, MVT::nxv4f64, MVT::nxv4f32, 2},
4727 {ISD::FP_EXTEND, MVT::nxv8f64, MVT::nxv8f32, 6},
4728
4729 // Add cost for extending to illegal -too wide- scalable vectors.
4730 // zero/sign extend are implemented by multiple unpack operations,
4731 // where each operation has a cost of 1.
4732 {ISD::ZERO_EXTEND, MVT::nxv16i16, MVT::nxv16i8, 2},
4733 {ISD::ZERO_EXTEND, MVT::nxv16i32, MVT::nxv16i8, 6},
4734 {ISD::ZERO_EXTEND, MVT::nxv16i64, MVT::nxv16i8, 14},
4735 {ISD::ZERO_EXTEND, MVT::nxv8i32, MVT::nxv8i16, 2},
4736 {ISD::ZERO_EXTEND, MVT::nxv8i64, MVT::nxv8i16, 6},
4737 {ISD::ZERO_EXTEND, MVT::nxv4i64, MVT::nxv4i32, 2},
4738
4739 {ISD::SIGN_EXTEND, MVT::nxv16i16, MVT::nxv16i8, 2},
4740 {ISD::SIGN_EXTEND, MVT::nxv16i32, MVT::nxv16i8, 6},
4741 {ISD::SIGN_EXTEND, MVT::nxv16i64, MVT::nxv16i8, 14},
4742 {ISD::SIGN_EXTEND, MVT::nxv8i32, MVT::nxv8i16, 2},
4743 {ISD::SIGN_EXTEND, MVT::nxv8i64, MVT::nxv8i16, 6},
4744 {ISD::SIGN_EXTEND, MVT::nxv4i64, MVT::nxv4i32, 2},
4745 };
4746
4747 if (const auto *Entry = ConvertCostTableLookup(
4748 ConversionTbl, ISD, DstTy.getSimpleVT(), SrcTy.getSimpleVT()))
4749 return Entry->Cost;
4750
4751 static const TypeConversionCostTblEntry FP16Tbl[] = {
4752 {ISD::FP_TO_SINT, MVT::v4i8, MVT::v4f16, 1}, // fcvtzs
4753 {ISD::FP_TO_UINT, MVT::v4i8, MVT::v4f16, 1},
4754 {ISD::FP_TO_SINT, MVT::v4i16, MVT::v4f16, 1}, // fcvtzs
4755 {ISD::FP_TO_UINT, MVT::v4i16, MVT::v4f16, 1},
4756 {ISD::FP_TO_SINT, MVT::v4i32, MVT::v4f16, 2}, // fcvtl+fcvtzs
4757 {ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f16, 2},
4758 {ISD::FP_TO_SINT, MVT::v8i8, MVT::v8f16, 2}, // fcvtzs+xtn
4759 {ISD::FP_TO_UINT, MVT::v8i8, MVT::v8f16, 2},
4760 {ISD::FP_TO_SINT, MVT::v8i16, MVT::v8f16, 1}, // fcvtzs
4761 {ISD::FP_TO_UINT, MVT::v8i16, MVT::v8f16, 1},
4762 {ISD::FP_TO_SINT, MVT::v8i32, MVT::v8f16, 4}, // 2*fcvtl+2*fcvtzs
4763 {ISD::FP_TO_UINT, MVT::v8i32, MVT::v8f16, 4},
4764 {ISD::FP_TO_SINT, MVT::v16i8, MVT::v16f16, 3}, // 2*fcvtzs+xtn
4765 {ISD::FP_TO_UINT, MVT::v16i8, MVT::v16f16, 3},
4766 {ISD::FP_TO_SINT, MVT::v16i16, MVT::v16f16, 2}, // 2*fcvtzs
4767 {ISD::FP_TO_UINT, MVT::v16i16, MVT::v16f16, 2},
4768 {ISD::FP_TO_SINT, MVT::v16i32, MVT::v16f16, 8}, // 4*fcvtl+4*fcvtzs
4769 {ISD::FP_TO_UINT, MVT::v16i32, MVT::v16f16, 8},
4770 {ISD::UINT_TO_FP, MVT::v8f16, MVT::v8i8, 2}, // ushll + ucvtf
4771 {ISD::SINT_TO_FP, MVT::v8f16, MVT::v8i8, 2}, // sshll + scvtf
4772 {ISD::UINT_TO_FP, MVT::v16f16, MVT::v16i8, 4}, // 2 * ushl(2) + 2 * ucvtf
4773 {ISD::SINT_TO_FP, MVT::v16f16, MVT::v16i8, 4}, // 2 * sshl(2) + 2 * scvtf
4774 };
4775
4776 if (ST->hasFullFP16())
4777 if (const auto *Entry = ConvertCostTableLookup(
4778 FP16Tbl, ISD, DstTy.getSimpleVT(), SrcTy.getSimpleVT()))
4779 return Entry->Cost;
4780
4781 // INT_TO_FP of i64->f32 will scalarize, which is required to avoid
4782 // double-rounding issues.
4783 if ((ISD == ISD::SINT_TO_FP || ISD == ISD::UINT_TO_FP) &&
4784 DstTy.getScalarType() == MVT::f32 && SrcTy.getScalarSizeInBits() > 32 &&
4786 return cast<FixedVectorType>(Dst)->getNumElements() *
4787 getCastInstrCost(Opcode, Dst->getScalarType(),
4788 Src->getScalarType(), CCH, CostKind) +
4790 true, CostKind) +
4792 false, CostKind);
4793
4794 if ((ISD == ISD::ZERO_EXTEND || ISD == ISD::SIGN_EXTEND) &&
4796 ST->isSVEorStreamingSVEAvailable() &&
4797 TLI->getTypeAction(Src->getContext(), SrcTy) ==
4799 TLI->getTypeAction(Dst->getContext(), DstTy) ==
4801 // The standard behaviour in the backend for these cases is to split the
4802 // extend up into two parts:
4803 // 1. Perform an extending load or masked load up to the legal type.
4804 // 2. Extend the loaded data to the final type.
4805 std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Src);
4806 Type *LegalTy = EVT(SrcLT.second).getTypeForEVT(Src->getContext());
4808 Opcode, LegalTy, Src, CCH, CostKind, I);
4810 Opcode, Dst, LegalTy, TTI::CastContextHint::None, CostKind, I);
4811 return Part1 + Part2;
4812 }
4813
4814 // The BasicTTIImpl version only deals with CCH==TTI::CastContextHint::Normal,
4815 // but we also want to include the TTI::CastContextHint::Masked case too.
4816 if ((ISD == ISD::ZERO_EXTEND || ISD == ISD::SIGN_EXTEND) &&
4818 ST->isSVEorStreamingSVEAvailable() && TLI->isTypeLegal(DstTy))
4820
4821 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
4822}
4823
4826 VectorType *VecTy, unsigned Index,
4827 TTI::TargetCostKind CostKind) const {
4828
4829 // Make sure we were given a valid extend opcode.
4830 assert((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) &&
4831 "Invalid opcode");
4832
4833 // We are extending an element we extract from a vector, so the source type
4834 // of the extend is the element type of the vector.
4835 auto *Src = VecTy->getElementType();
4836
4837 // Sign- and zero-extends are for integer types only.
4838 assert(isa<IntegerType>(Dst) && isa<IntegerType>(Src) && "Invalid type");
4839
4840 // Get the cost for the extract. We compute the cost (if any) for the extend
4841 // below.
4842 InstructionCost Cost = getVectorInstrCost(Instruction::ExtractElement, VecTy,
4843 CostKind, Index, nullptr, nullptr);
4844
4845 // Legalize the types.
4846 auto VecLT = getTypeLegalizationCost(VecTy);
4847 auto DstVT = TLI->getValueType(DL, Dst);
4848 auto SrcVT = TLI->getValueType(DL, Src);
4849
4850 // If the resulting type is still a vector and the destination type is legal,
4851 // we may get the extension for free. If not, get the default cost for the
4852 // extend.
4853 if (!VecLT.second.isVector() || !TLI->isTypeLegal(DstVT))
4854 return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None,
4855 CostKind);
4856
4857 // The destination type should be larger than the element type. If not, get
4858 // the default cost for the extend.
4859 if (DstVT.getFixedSizeInBits() < SrcVT.getFixedSizeInBits())
4860 return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None,
4861 CostKind);
4862
4863 switch (Opcode) {
4864 default:
4865 llvm_unreachable("Opcode should be either SExt or ZExt");
4866
4867 // For sign-extends, we only need a smov, which performs the extension
4868 // automatically.
4869 case Instruction::SExt:
4870 return Cost;
4871
4872 // For zero-extends, the extend is performed automatically by a umov unless
4873 // the destination type is i64 and the element type is i8 or i16.
4874 case Instruction::ZExt:
4875 if (DstVT.getSizeInBits() != 64u || SrcVT.getSizeInBits() == 32u)
4876 return Cost;
4877 }
4878
4879 // If we are unable to perform the extend for free, get the default cost.
4880 return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None,
4881 CostKind);
4882}
4883
4885 TTI::TargetCostKind CostKind,
4886 const Instruction *I) const {
4887 if (CostKind != TTI::TCK_RecipThroughput)
4888 return Opcode == Instruction::PHI ? 0 : 1;
4889 assert(CostKind == TTI::TCK_RecipThroughput && "unexpected CostKind");
4890 // Branches are assumed to be predicted.
4891 return 0;
4892}
4893
4894InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
4895 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, unsigned Index,
4896 const Instruction *I, Value *Scalar,
4897 ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
4898 TTI::VectorInstrContext VIC) const {
4899 assert(Ty->isVectorTy() && "This must be a vector type");
4900
4901 if (Index != -1U) {
4902 // Legalize the type.
4903 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
4904
4905 // This type is legalized to a scalar type.
4906 if (!LT.second.isVector())
4907 return 0;
4908
4909 // The type may be split. For fixed-width vectors we can normalize the
4910 // index to the new type.
4911 if (LT.second.isFixedLengthVector()) {
4912 unsigned Width = LT.second.getVectorNumElements();
4913 Index = Index % Width;
4914 }
4915
4916 // The element at index zero is already inside the vector.
4917 // - For a insert-element or extract-element
4918 // instruction that extracts integers, an explicit FPR -> GPR move is
4919 // needed. So it has non-zero cost.
4920 if (Index == 0 && !Ty->getScalarType()->isIntegerTy())
4921 return 0;
4922
4923 // SVE has no scalar move to an arbitrary lane above the low 128-bit portion
4924 // of a Z register, e.g. there is no equivalent of "mov z0.d[9], d0".
4925 // Fixed-length vectors wider than 128 bits therefore need
4926 // [splice]/index/pred/splat/cmp/pred-mov when scalarizing inserts for those
4927 // lanes, so model them as more expensive than ordinary NEON lane accesses.
4928 if (ST->useSVEForFixedLengthVectors()) {
4929 InstructionCost Cost = CostKind == TTI::TCK_CodeSize
4930 ? 1
4932 if (Index * Ty->getScalarSizeInBits() < 128)
4933 return Cost;
4934 if (Index * Ty->getScalarSizeInBits() < 512 &&
4935 Opcode == Instruction::ExtractElement)
4936 // Integer extracts (>128b, <512b) require extra mov from FPR -> GPR.
4937 return Ty->getScalarType()->isIntegerTy() ? Cost + 1 : Cost;
4938 if (Opcode == Instruction::ExtractElement)
4939 return Cost + 2; // cost of mov imm + whilels + lastb
4940 if (Opcode == Instruction::InsertElement)
4941 return Cost + 3; // cost of insert with cmp/splice
4942 llvm_unreachable("unexpected opcode");
4943 }
4944
4945 // This is recognising a LD1 single-element structure to one lane of one
4946 // register instruction. I.e., if this is an `insertelement` instruction,
4947 // and its second operand is a load, then we will generate a LD1, which
4948 // are expensive instructions on some uArchs.
4949 if (VIC == TTI::VectorInstrContext::Load) {
4950 if (ST->hasFastLD1Single())
4951 return 0;
4952 return CostKind == TTI::TCK_CodeSize
4953 ? 0
4954 : ST->getVectorInsertExtractBaseCost() + 1;
4955 }
4956
4957 // i1 inserts and extract will include an extra cset or cmp of the vector
4958 // value. Increase the cost by 1 to account.
4959 if (Ty->getScalarSizeInBits() == 1)
4960 return CostKind == TTI::TCK_CodeSize
4961 ? 2
4962 : ST->getVectorInsertExtractBaseCost() + 1;
4963
4964 // FIXME:
4965 // If the extract-element and insert-element instructions could be
4966 // simplified away (e.g., could be combined into users by looking at use-def
4967 // context), they have no cost. This is not done in the first place for
4968 // compile-time considerations.
4969 }
4970
4971 // In case of Neon, if there exists extractelement from lane != 0 such that
4972 // 1. extractelement does not necessitate a move from vector_reg -> GPR.
4973 // 2. extractelement result feeds into fmul.
4974 // 3. Other operand of fmul is an extractelement from lane 0 or lane
4975 // equivalent to 0.
4976 // then the extractelement can be merged with fmul in the backend and it
4977 // incurs no cost.
4978 // e.g.
4979 // define double @foo(<2 x double> %a) {
4980 // %1 = extractelement <2 x double> %a, i32 0
4981 // %2 = extractelement <2 x double> %a, i32 1
4982 // %res = fmul double %1, %2
4983 // ret double %res
4984 // }
4985 // %2 and %res can be merged in the backend to generate fmul d0, d0, v1.d[1]
4986 auto ExtractCanFuseWithFmul = [&]() {
4987 // We bail out if the extract is from lane 0.
4988 if (Index == 0)
4989 return false;
4990
4991 // Check if the scalar element type of the vector operand of ExtractElement
4992 // instruction is one of the allowed types.
4993 auto IsAllowedScalarTy = [&](const Type *T) {
4994 return T->isFloatTy() || T->isDoubleTy() ||
4995 (T->isHalfTy() && ST->hasFullFP16());
4996 };
4997
4998 // Check if the extractelement user is scalar fmul.
4999 auto IsUserFMulScalarTy = [](const Value *EEUser) {
5000 // Check if the user is scalar fmul.
5001 const auto *BO = dyn_cast<BinaryOperator>(EEUser);
5002 return BO && BO->getOpcode() == BinaryOperator::FMul &&
5003 !BO->getType()->isVectorTy();
5004 };
5005
5006 // Check if the extract index is from lane 0 or lane equivalent to 0 for a
5007 // certain scalar type and a certain vector register width.
5008 auto IsExtractLaneEquivalentToZero = [&](unsigned Idx, unsigned EltSz) {
5009 auto RegWidth =
5011 .getFixedValue();
5012 return Idx == 0 || (RegWidth != 0 && (Idx * EltSz) % RegWidth == 0);
5013 };
5014
5015 // Check if the type constraints on input vector type and result scalar type
5016 // of extractelement instruction are satisfied.
5017 if (!isa<FixedVectorType>(Ty) || !IsAllowedScalarTy(Ty->getScalarType()))
5018 return false;
5019
5020 if (Scalar) {
5021 DenseMap<User *, unsigned> UserToExtractIdx;
5022 for (auto *U : Scalar->users()) {
5023 if (!IsUserFMulScalarTy(U))
5024 return false;
5025 // Recording entry for the user is important. Index value is not
5026 // important.
5027 UserToExtractIdx[U];
5028 }
5029 if (UserToExtractIdx.empty())
5030 return false;
5031 for (auto &[S, U, L] : ScalarUserAndIdx) {
5032 for (auto *U : S->users()) {
5033 if (UserToExtractIdx.contains(U)) {
5034 auto *FMul = cast<BinaryOperator>(U);
5035 auto *Op0 = FMul->getOperand(0);
5036 auto *Op1 = FMul->getOperand(1);
5037 if ((Op0 == S && Op1 == S) || Op0 != S || Op1 != S) {
5038 UserToExtractIdx[U] = L;
5039 break;
5040 }
5041 }
5042 }
5043 }
5044 for (auto &[U, L] : UserToExtractIdx) {
5045 if (!IsExtractLaneEquivalentToZero(Index, Ty->getScalarSizeInBits()) &&
5046 !IsExtractLaneEquivalentToZero(L, Ty->getScalarSizeInBits()))
5047 return false;
5048 }
5049 } else {
5050 const auto *EE = cast<ExtractElementInst>(I);
5051
5052 const auto *IdxOp = dyn_cast<ConstantInt>(EE->getIndexOperand());
5053 if (!IdxOp)
5054 return false;
5055
5056 return !EE->users().empty() && all_of(EE->users(), [&](const User *U) {
5057 if (!IsUserFMulScalarTy(U))
5058 return false;
5059
5060 // Check if the other operand of extractelement is also extractelement
5061 // from lane equivalent to 0.
5062 const auto *BO = cast<BinaryOperator>(U);
5063 const auto *OtherEE = dyn_cast<ExtractElementInst>(
5064 BO->getOperand(0) == EE ? BO->getOperand(1) : BO->getOperand(0));
5065 if (OtherEE) {
5066 const auto *IdxOp = dyn_cast<ConstantInt>(OtherEE->getIndexOperand());
5067 if (!IdxOp)
5068 return false;
5069 return IsExtractLaneEquivalentToZero(
5070 cast<ConstantInt>(OtherEE->getIndexOperand())
5071 ->getValue()
5072 .getZExtValue(),
5073 OtherEE->getType()->getScalarSizeInBits());
5074 }
5075 return true;
5076 });
5077 }
5078 return true;
5079 };
5080
5081 if (Opcode == Instruction::ExtractElement && (I || Scalar) &&
5082 ExtractCanFuseWithFmul())
5083 return 0;
5084
5085 // All other insert/extracts cost this much.
5086 return CostKind == TTI::TCK_CodeSize ? 1
5087 : ST->getVectorInsertExtractBaseCost();
5088}
5089
5091 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, unsigned Index,
5092 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
5093 // Treat insert at lane 0 into a poison vector as having zero cost. This
5094 // ensures vector broadcasts via an insert + shuffle (and will be lowered to a
5095 // single dup) are treated as cheap.
5096 if (Opcode == Instruction::InsertElement && Index == 0 && Op0 &&
5097 isa<PoisonValue>(Op0))
5098 return 0;
5099 return getVectorInstrCostHelper(Opcode, Ty, CostKind, Index, nullptr, nullptr,
5100 {}, VIC);
5101}
5102
5104 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, unsigned Index,
5105 Value *Scalar, ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
5106 TTI::VectorInstrContext VIC) const {
5107 return getVectorInstrCostHelper(Opcode, Ty, CostKind, Index, nullptr, Scalar,
5108 ScalarUserAndIdx, VIC);
5109}
5110
5113 TTI::TargetCostKind CostKind, unsigned Index,
5114 TTI::VectorInstrContext VIC) const {
5115 return getVectorInstrCostHelper(I.getOpcode(), Ty, CostKind, Index, &I,
5116 nullptr, {}, VIC);
5117}
5118
5121 TTI::TargetCostKind CostKind,
5122 unsigned Index) const {
5123 if (isa<FixedVectorType>(Ty))
5124 return BaseT::getIndexedVectorInstrCostFromEnd(Opcode, Ty, CostKind, Index);
5125
5126 // This typically requires both while and lastb instructions in order
5127 // to extract the last element. If this is in a loop the while
5128 // instruction can at least be hoisted out, although it will consume a
5129 // predicate register. The cost should be more expensive than the base
5130 // extract cost, which is 2 for most CPUs.
5131 return CostKind == TTI::TCK_CodeSize
5132 ? 2
5133 : ST->getVectorInsertExtractBaseCost() + 1;
5134}
5135
5137 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
5138 TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
5139 TTI::VectorInstrContext VIC) const {
5142
5143 // There's no scalarization overhead if ld1/st1 is cheap and the
5144 // insert/extracts can be folded into the load/stores.
5145 if (ST->hasFastLD1Single()) {
5146 if ((VIC == TTI::VectorInstrContext::Store && Extract) ||
5147 (VIC == TTI::VectorInstrContext::Load && Insert))
5148 return 0;
5149 }
5150
5151 if (Ty->getElementType()->isFloatingPointTy())
5152 return BaseT::getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
5153 CostKind);
5154 unsigned VecInstCost =
5155 CostKind == TTI::TCK_CodeSize ? 1 : ST->getVectorInsertExtractBaseCost();
5156 return DemandedElts.popcount() * (Insert + Extract) * VecInstCost;
5157}
5158
5159std::optional<InstructionCost> AArch64TTIImpl::getFP16BF16PromoteCost(
5160 Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
5161 TTI::OperandValueInfo Op2Info, bool IncludeTrunc, bool CanUseSVE,
5162 std::function<InstructionCost(Type *)> InstCost) const {
5163 if (!Ty->getScalarType()->isHalfTy() && !Ty->getScalarType()->isBFloatTy())
5164 return std::nullopt;
5165 if (Ty->getScalarType()->isHalfTy() && ST->hasFullFP16())
5166 return std::nullopt;
5167 // If we have +sve-b16b16 the operation can be promoted to SVE.
5168 if (CanUseSVE && ST->hasSVEB16B16() && ST->isNonStreamingSVEorSME2Available())
5169 return std::nullopt;
5170
5171 Type *PromotedTy = Ty->getWithNewType(Type::getFloatTy(Ty->getContext()));
5172 InstructionCost Cost = getCastInstrCost(Instruction::FPExt, PromotedTy, Ty,
5173 TTI::CastContextHint::None, CostKind);
5174 if (!Op1Info.isConstant() && !Op2Info.isConstant())
5175 Cost *= 2;
5176 Cost += InstCost(PromotedTy);
5177 if (IncludeTrunc)
5178 Cost += getCastInstrCost(Instruction::FPTrunc, Ty, PromotedTy,
5179 TTI::CastContextHint::None, CostKind);
5180 return Cost;
5181}
5182
5184 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
5186 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
5187
5188 // The code-generator is currently not able to handle scalable vectors
5189 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
5190 // it until all instructions are vetted.
5191 int ISD = TLI->InstructionOpcodeToISD(Opcode);
5192 if (auto *VTy = dyn_cast<ScalableVectorType>(Ty))
5193 if (VTy->getElementCount() == ElementCount::getScalable(1))
5196
5197 if (CtxI && CtxI->getOpcode() == Opcode && !CtxI->use_empty() &&
5198 useNeonVector(Ty)) {
5199 // The Ands of a bit select are folded into the BSL that the Or becomes.
5200 Instruction *MainAnd, *OtherAnd;
5201 Value *A, *B;
5202 if (Opcode == Instruction::And && CtxI->hasOneUse() &&
5203 matchBSL(CtxI->user_back(), MainAnd, OtherAnd, A, B))
5204 return 0;
5205
5206 // A Not is folded into the BIC, ORN or BSL of each And/Or using it, unless
5207 // that And/Or negates both of its operands.
5208 if (match(CtxI, m_Not(m_Value())) &&
5209 all_of(CtxI->users(), [&](const User *U) {
5210 const auto *UI = cast<Instruction>(U);
5211 return is_contained({Instruction::And, Instruction::Or},
5212 UI->getOpcode()) &&
5213 UI->getParent() == CtxI->getParent() &&
5214 !match(UI, m_BinOp(m_Not(m_Value()), m_Not(m_Value())));
5215 }))
5216 return 0;
5217 }
5218
5219 // Legalize the type.
5220 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
5221
5222 // TODO: Handle more cost kinds for floating point operations.
5223 if (ISD == ISD::FADD || ISD == ISD::FSUB || ISD == ISD::FMUL ||
5224 ISD == ISD::FDIV || ISD == ISD::FREM || ISD == ISD::FNEG)
5225 if (CostKind != TTI::TCK_RecipThroughput)
5226 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
5227 Op2Info, Args, CtxI);
5228
5229 if (ISD == ISD::FADD || ISD == ISD::FSUB || ISD == ISD::FMUL ||
5230 ISD == ISD::FDIV || ISD == ISD::FREM) {
5231 // Increase the cost for half and bfloat types if not architecturally
5232 // supported.
5233 if (auto PromotedCost = getFP16BF16PromoteCost(
5234 Ty, CostKind, Op1Info, Op2Info, /*IncludeTrunc=*/true,
5235 // There is not native support for fdiv/frem even with +sve-b16b16.
5236 /*CanUseSVE=*/ISD != ISD::FDIV && ISD != ISD::FREM,
5237 [&](Type *PromotedTy) {
5238 return getArithmeticInstrCost(Opcode, PromotedTy, CostKind,
5239 Op1Info, Op2Info);
5240 }))
5241 return *PromotedCost;
5242
5243 // fp128 all go via libcalls
5244 if (Ty->getScalarType()->isFP128Ty())
5245 return (CostKind == TTI::TCK_CodeSize ? 1 : 10) * LT.first;
5246 }
5247
5248 // If the operation is a widening instruction (smull or umull) and both
5249 // operands are extends the cost can be cheaper by considering that the
5250 // operation will operate on the narrowest type size possible (double the
5251 // largest input size) and a further extend.
5252 if (Type *ExtTy = isBinExtWideningInstruction(Opcode, Ty, Args)) {
5253 if (ExtTy != Ty)
5254 return getArithmeticInstrCost(Opcode, ExtTy, CostKind) +
5255 getCastInstrCost(Instruction::ZExt, Ty, ExtTy,
5256 TTI::CastContextHint::None, CostKind);
5257 return LT.first;
5258 }
5259
5260 switch (ISD) {
5261 default:
5262 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
5263 Op2Info);
5264 case ISD::ADD:
5265 case ISD::SUB:
5266 return LT.first; // Also works for i128
5267 case ISD::MUL: {
5268 // i128 multiply is umulh + 2*madd + mul and grows ~O(Bitwidth^2). For
5269 // scalable vectors the cost of LT.first will be invalid, leading to an
5270 // invalid cost overall.
5271 unsigned Mul64CostFactor = (CostKind == TTI::TCK_RecipThroughput &&
5272 ST->hasLimited64bitVectorMulBandwidth())
5273 ? 4
5274 : 1;
5275 if (Ty->getScalarSizeInBits() > 64) {
5276 unsigned NumLanes = isa<FixedVectorType>(Ty)
5277 ? cast<FixedVectorType>(Ty)->getNumElements()
5278 : 1;
5279 InstructionCost CostPerLane = LT.first / NumLanes;
5280 return CostPerLane * CostPerLane * NumLanes * Mul64CostFactor;
5281 }
5282
5283 if (LT.second == MVT::v2i64) {
5284 // When SVE is available, then we can lower the v2i64 operation using
5285 // the SVE mul instruction, which has a lower cost.
5286 if (ST->hasSVE())
5287 return LT.first * Mul64CostFactor;
5288
5289 // When SVE is not available, there is no MUL.2d instruction,
5290 // which means mul <2 x i64> is expensive as elements are extracted
5291 // from the vectors and the muls scalarized.
5292 // As getScalarizationOverhead is a bit too pessimistic, we
5293 // estimate the cost for a i64 vector directly here, which is:
5294 // - four 2-cost i64 extracts,
5295 // - two 2-cost i64 inserts, and
5296 // - two 1-cost muls.
5297 // So, for a v2i64 with LT.First = 1 the cost is 14, and for a v4i64 with
5298 // LT.first = 2 the cost is 28.
5299 return cast<VectorType>(Ty)->getElementCount().getKnownMinValue() *
5300 (getArithmeticInstrCost(Opcode, Ty->getScalarType(), CostKind) +
5301 getVectorInstrCost(Instruction::ExtractElement, Ty, CostKind, -1,
5302 nullptr, nullptr) *
5303 2 +
5304 getVectorInstrCost(Instruction::InsertElement, Ty, CostKind, -1,
5305 nullptr, nullptr));
5306 }
5307
5308 if (LT.second == MVT::nxv2i64)
5309 return LT.first * Mul64CostFactor;
5310
5311 return LT.first;
5312 }
5313 case ISD::SREM:
5314 case ISD::SDIV:
5315 /*
5316 Notes for sdiv/srem specific costs:
5317 1. This only considers the cases where the divisor is constant, uniform and
5318 (pow-of-2/non-pow-of-2). Other cases are not important since they either
5319 result in some form of (ldr + adrp), corresponding to constant vectors, or
5320 scalarization of the division operation.
5321 2. Constant divisors, either negative in whole or partially, don't result in
5322 significantly different codegen as compared to positive constant divisors.
5323 So, we don't consider negative divisors separately.
5324 3. If the codegen is significantly different with SVE, it has been indicated
5325 using comments at appropriate places.
5326
5327 sdiv specific cases:
5328 -----------------------------------------------------------------------
5329 codegen | pow-of-2 | Type
5330 -----------------------------------------------------------------------
5331 add + cmp + csel + asr | Y | i64
5332 add + cmp + csel + asr | Y | i32
5333 -----------------------------------------------------------------------
5334
5335 srem specific cases:
5336 -----------------------------------------------------------------------
5337 codegen | pow-of-2 | Type
5338 -----------------------------------------------------------------------
5339 negs + and + and + csneg | Y | i64
5340 negs + and + and + csneg | Y | i32
5341 -----------------------------------------------------------------------
5342
5343 other sdiv/srem cases:
5344 -------------------------------------------------------------------------
5345 common codegen | + srem | + sdiv | pow-of-2 | Type
5346 -------------------------------------------------------------------------
5347 smulh + asr + add + add | - | - | N | i64
5348 smull + lsr + add + add | - | - | N | i32
5349 usra | and + sub | sshr | Y | <2 x i64>
5350 2 * (scalar code) | - | - | N | <2 x i64>
5351 usra | bic + sub | sshr + neg | Y | <4 x i32>
5352 smull2 + smull + uzp2 | mls | - | N | <4 x i32>
5353 + sshr + usra | | | |
5354 -------------------------------------------------------------------------
5355 */
5356 if (Op2Info.isConstant() && Op2Info.isUniform()) {
5357 InstructionCost AddCost =
5358 getArithmeticInstrCost(Instruction::Add, Ty, CostKind,
5359 Op1Info.getNoProps(), Op2Info.getNoProps());
5360 InstructionCost AsrCost =
5361 getArithmeticInstrCost(Instruction::AShr, Ty, CostKind,
5362 Op1Info.getNoProps(), Op2Info.getNoProps());
5363 InstructionCost MulCost =
5364 getArithmeticInstrCost(Instruction::Mul, Ty, CostKind,
5365 Op1Info.getNoProps(), Op2Info.getNoProps());
5366 // add/cmp/csel/csneg should have similar cost while asr/negs/and should
5367 // have similar cost.
5368 auto VT = TLI->getValueType(DL, Ty);
5369 if (VT.isScalarInteger() && VT.getSizeInBits() <= 64) {
5370 if (Op2Info.isPowerOf2() || Op2Info.isNegatedPowerOf2()) {
5371 // Neg can be folded into the asr instruction.
5372 return ISD == ISD::SDIV ? (3 * AddCost + AsrCost)
5373 : (3 * AsrCost + AddCost);
5374 } else {
5375 return MulCost + AsrCost + 2 * AddCost;
5376 }
5377 } else if (VT.isVector()) {
5378 InstructionCost UsraCost = 2 * AsrCost;
5379 if (Op2Info.isPowerOf2() || Op2Info.isNegatedPowerOf2()) {
5380 // Division with scalable types corresponds to native 'asrd'
5381 // instruction when SVE is available.
5382 // e.g. %1 = sdiv <vscale x 4 x i32> %a, splat (i32 8)
5383
5384 // One more for the negation in SDIV
5386 (Op2Info.isNegatedPowerOf2() && ISD == ISD::SDIV) ? AsrCost : 0;
5387 if (Ty->isScalableTy() && ST->hasSVE())
5388 Cost += 2 * AsrCost;
5389 else {
5390 Cost +=
5391 UsraCost +
5392 (ISD == ISD::SDIV
5393 ? (LT.second.getScalarType() == MVT::i64 ? 1 : 2) * AsrCost
5394 : 2 * AddCost);
5395 }
5396 return Cost;
5397 } else if (LT.second == MVT::v2i64) {
5398 return VT.getVectorNumElements() *
5399 getArithmeticInstrCost(Opcode, Ty->getScalarType(), CostKind,
5400 Op1Info.getNoProps(),
5401 Op2Info.getNoProps());
5402 } else {
5403 // When SVE is available, we get:
5404 // smulh + lsr + add/sub + asr + add/sub.
5405 if (Ty->isScalableTy() && ST->hasSVE())
5406 return MulCost /*smulh cost*/ + 2 * AddCost + 2 * AsrCost;
5407 return 2 * MulCost + AddCost /*uzp2 cost*/ + AsrCost + UsraCost;
5408 }
5409 }
5410 }
5411 if (Op2Info.isConstant() && !Op2Info.isUniform() &&
5412 LT.second.isFixedLengthVector()) {
5413 // FIXME: When the constant vector is non-uniform, this may result in
5414 // loading the vector from constant pool or in some cases, may also result
5415 // in scalarization. For now, we are approximating this with the
5416 // scalarization cost.
5417 auto ExtractCost = 2 * getVectorInstrCost(Instruction::ExtractElement, Ty,
5418 CostKind, -1, nullptr, nullptr);
5419 auto InsertCost = getVectorInstrCost(Instruction::InsertElement, Ty,
5420 CostKind, -1, nullptr, nullptr);
5421 unsigned NElts = cast<FixedVectorType>(Ty)->getNumElements();
5422 return ExtractCost + InsertCost +
5423 NElts * getArithmeticInstrCost(Opcode, Ty->getScalarType(),
5424 CostKind, Op1Info.getNoProps(),
5425 Op2Info.getNoProps());
5426 }
5427 [[fallthrough]];
5428 case ISD::UDIV:
5429 case ISD::UREM: {
5430 auto VT = TLI->getValueType(DL, Ty);
5431 if (Op2Info.isConstant()) {
5432 // If the operand is a power of 2 we can use the shift or and cost.
5433 if (ISD == ISD::UDIV && Op2Info.isPowerOf2())
5434 return getArithmeticInstrCost(Instruction::LShr, Ty, CostKind,
5435 Op1Info.getNoProps(),
5436 Op2Info.getNoProps());
5437 if (ISD == ISD::UREM && Op2Info.isPowerOf2())
5438 return getArithmeticInstrCost(Instruction::And, Ty, CostKind,
5439 Op1Info.getNoProps(),
5440 Op2Info.getNoProps());
5441
5442 if (ISD == ISD::UDIV || ISD == ISD::UREM) {
5443 // Divides by a constant are expanded to MULHU + SUB + SRL + ADD + SRL.
5444 // The MULHU will be expanded to UMULL for the types not listed below,
5445 // and will become a pair of UMULL+MULL2 for 128bit vectors.
5446 bool HasMULH = VT == MVT::i64 || LT.second == MVT::nxv2i64 ||
5447 LT.second == MVT::nxv4i32 || LT.second == MVT::nxv8i16 ||
5448 LT.second == MVT::nxv16i8;
5449 bool Is128bit = LT.second.is128BitVector();
5450
5451 InstructionCost MulCost =
5452 getArithmeticInstrCost(Instruction::Mul, Ty, CostKind,
5453 Op1Info.getNoProps(), Op2Info.getNoProps());
5454 InstructionCost AddCost =
5455 getArithmeticInstrCost(Instruction::Add, Ty, CostKind,
5456 Op1Info.getNoProps(), Op2Info.getNoProps());
5457 InstructionCost ShrCost =
5458 getArithmeticInstrCost(Instruction::AShr, Ty, CostKind,
5459 Op1Info.getNoProps(), Op2Info.getNoProps());
5460 InstructionCost DivCost = MulCost * (Is128bit ? 2 : 1) + // UMULL/UMULH
5461 (HasMULH ? 0 : ShrCost) + // UMULL shift
5462 AddCost * 2 + ShrCost;
5463 return DivCost + (ISD == ISD::UREM ? MulCost + AddCost : 0);
5464 }
5465 }
5466
5467 // div i128's are lowered as libcalls. Pass nullptr as (u)divti3 calls are
5468 // emitted by the backend even when those functions are not declared in the
5469 // module.
5470 if (!VT.isVector() && VT.getSizeInBits() > 64)
5471 return getCallInstrCost(/*Function*/ nullptr, Ty, {Ty, Ty}, CostKind);
5472
5473 InstructionCost Cost = BaseT::getArithmeticInstrCost(
5474 Opcode, Ty, CostKind, Op1Info, Op2Info);
5475 if (Ty->isVectorTy() && (ISD == ISD::SDIV || ISD == ISD::UDIV)) {
5476 if (TLI->isOperationLegalOrCustom(ISD, LT.second) && ST->hasSVE()) {
5477 // SDIV/UDIV operations are lowered using SVE, then we can have less
5478 // costs.
5479 if (VT.isSimple() && isa<FixedVectorType>(Ty) &&
5480 Ty->getPrimitiveSizeInBits().getFixedValue() < 128) {
5481 static const CostTblEntry DivTbl[]{
5482 {ISD::SDIV, MVT::v2i8, 5}, {ISD::SDIV, MVT::v4i8, 8},
5483 {ISD::SDIV, MVT::v8i8, 8}, {ISD::SDIV, MVT::v2i16, 5},
5484 {ISD::SDIV, MVT::v4i16, 5}, {ISD::SDIV, MVT::v2i32, 1},
5485 {ISD::UDIV, MVT::v2i8, 5}, {ISD::UDIV, MVT::v4i8, 8},
5486 {ISD::UDIV, MVT::v8i8, 8}, {ISD::UDIV, MVT::v2i16, 5},
5487 {ISD::UDIV, MVT::v4i16, 5}, {ISD::UDIV, MVT::v2i32, 1}};
5488
5489 const auto *Entry = CostTableLookup(DivTbl, ISD, VT.getSimpleVT());
5490 if (nullptr != Entry)
5491 return Entry->Cost;
5492 }
5493 // A non-power-of-2 count can't divide as a single whole-register op
5494 // (an inactive lane's leftover value could be a zero divisor and
5495 // trap), so the legalizer emits one div per whole register plus one
5496 // per set bit of the remainder (e.g. <7 x i32> emits 3 divs, not 2).
5497 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty);
5498 FVTy && LT.second.isFixedLengthVector()) {
5499 unsigned NumElts = FVTy->getNumElements();
5500 unsigned RegElts = LT.second.getVectorNumElements();
5501 if (RegElts > 0)
5502 Cost = (NumElts / RegElts + popcount(NumElts % RegElts)) * 2;
5503 }
5504 // For 8/16-bit elements, the cost is higher because the type
5505 // requires promotion and possibly splitting:
5506 if (LT.second.getScalarType() == MVT::i8)
5507 Cost *= 8;
5508 else if (LT.second.getScalarType() == MVT::i16)
5509 Cost *= 4;
5510 return Cost;
5511 } else {
5512 // If one of the operands is a uniform constant then the cost for each
5513 // element is Cost for insertion, extraction and division.
5514 // Insertion cost = 2, Extraction Cost = 2, Division = cost for the
5515 // operation with scalar type
5516 if ((Op1Info.isConstant() && Op1Info.isUniform()) ||
5517 (Op2Info.isConstant() && Op2Info.isUniform())) {
5518 if (auto *VTy = dyn_cast<FixedVectorType>(Ty)) {
5519 InstructionCost DivCost = BaseT::getArithmeticInstrCost(
5520 Opcode, Ty->getScalarType(), CostKind, Op1Info, Op2Info);
5521 return (4 + DivCost) * VTy->getNumElements();
5522 }
5523 }
5524 // On AArch64, without SVE, vector divisions are expanded
5525 // into scalar divisions of each pair of elements.
5526 Cost += getVectorInstrCost(Instruction::ExtractElement, Ty, CostKind,
5527 -1, nullptr, nullptr);
5528 Cost += getVectorInstrCost(Instruction::InsertElement, Ty, CostKind, -1,
5529 nullptr, nullptr);
5530 }
5531
5532 // TODO: if one of the arguments is scalar, then it's not necessary to
5533 // double the cost of handling the vector elements.
5534 Cost += Cost;
5535 }
5536 return Cost;
5537 }
5538 case ISD::XOR:
5539 case ISD::OR:
5540 case ISD::AND:
5541 // TODO: revisit these costs as it's not accurate enough for non-uniform
5542 // constant.
5543 return LT.first;
5544 case ISD::SRL:
5545 case ISD::SRA:
5546 case ISD::SHL: {
5547 // Immediate vector shifts require uniform shift amounts. Non-uniform
5548 // constants therefore use variable shifts and require materializing the
5549 // shift vector. Account for a shift and materialization per legalized
5550 // vector, together with shared setup.
5551 // This cost is for (ldr, shl) + adrp
5552 // TODO: These costs are based on CodeSize only, consider other CostKinds.
5553 if (Op2Info.isConstant() && !Op2Info.isUniform() &&
5554 LT.second.isFixedLengthVector())
5555 return 2 * LT.first + 1;
5556
5557 // Marked 'custom' for combining purposes; a uniform shift amount still
5558 // lowers to a single legal instruction.
5559 return LT.first;
5560 }
5561
5562 case ISD::FNEG:
5563 // Scalar fmul(fneg) or fneg(fmul) can be converted to fnmul
5564 if ((Ty->isFloatTy() || Ty->isDoubleTy() ||
5565 (Ty->isHalfTy() && ST->hasFullFP16())) &&
5566 CtxI &&
5567 ((CtxI->hasOneUse() &&
5568 match(*CtxI->user_begin(), m_FMul(m_Value(), m_Value()))) ||
5569 match(CtxI->getOperand(0), m_FMul(m_Value(), m_Value()))))
5570 return 0;
5571 [[fallthrough]];
5572 case ISD::FADD:
5573 case ISD::FSUB:
5574 if (!Ty->getScalarType()->isFP128Ty())
5575 return LT.first;
5576 [[fallthrough]];
5577 case ISD::FMUL:
5578 case ISD::FDIV:
5579 // These nodes are marked as 'custom' just to lower them to SVE.
5580 // We know said lowering will incur no additional cost.
5581 if (!Ty->getScalarType()->isFP128Ty())
5582 return 2 * LT.first;
5583
5584 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
5585 Op2Info);
5586 case ISD::FREM:
5587 // Pass nullptr as fmod/fmodf calls are emitted by the backend even when
5588 // those functions are not declared in the module.
5589 if (!Ty->isVectorTy())
5590 return getCallInstrCost(/*Function*/ nullptr, Ty, {Ty, Ty}, CostKind);
5591 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
5592 Op2Info);
5593 }
5594}
5595
5598 const SCEV *Ptr,
5599 TTI::TargetCostKind CostKind) const {
5600 // Address computations in vectorized code with non-consecutive addresses will
5601 // likely result in more instructions compared to scalar code where the
5602 // computation can more often be merged into the index mode. The resulting
5603 // extra micro-ops can significantly decrease throughput.
5604 unsigned NumVectorInstToHideOverhead =
5605 ST->getCLOpts().neon_nonconst_stride_overhead;
5606 int MaxMergeDistance = 64;
5607
5608 if (PtrTy->isVectorTy() && SE &&
5609 !BaseT::isConstantStridedAccessLessThan(SE, Ptr, MaxMergeDistance + 1))
5610 return NumVectorInstToHideOverhead;
5611
5612 // In many cases the address computation is not merged into the instruction
5613 // addressing mode.
5614 return 1;
5615}
5616
5617/// Check whether Opcode1 has less throughput according to the scheduling
5618/// model than Opcode2. For variant scheduling classes, the lowest throughput
5619/// of Opcode1 and the highest throughput of Opcode2 are compared among all the
5620/// scheduling classes they may be resolved to.
5622 unsigned Opcode1, unsigned Opcode2) const {
5623 const MCSchedModel &Sched = ST->getSchedModel();
5624 const TargetInstrInfo *TII = ST->getInstrInfo();
5625 if (!Sched.hasInstrSchedModel())
5626 return false;
5627
5628 auto GetRThroughput = [&](unsigned Opcode, bool Highest) {
5629 std::optional<double> Res;
5630 SmallVector<unsigned> Worklist = {TII->get(Opcode).getSchedClass()};
5631 while (!Worklist.empty()) {
5632 unsigned SchedClass = Worklist.pop_back_val();
5633 const MCSchedClassDesc *SCD = Sched.getSchedClassDesc(SchedClass);
5634 if (SCD->isVariant()) {
5635 append_range(Worklist, AArch64_MC::getVariantSchedClassTargets(
5636 SchedClass, Sched.getProcessorID()));
5637 continue;
5638 }
5639 if (!SCD->isValid())
5640 continue;
5641 double RThroughput = MCSchedModel::getReciprocalThroughput(*ST, *SCD);
5642 if (!Res || (Highest ? RThroughput > *Res : RThroughput < *Res))
5643 Res = RThroughput;
5644 }
5645 return Res;
5646 };
5647 std::optional<double> RThroughput1 =
5648 GetRThroughput(Opcode1, /*Highest=*/true);
5649 std::optional<double> RThroughput2 =
5650 GetRThroughput(Opcode2, /*Highest=*/false);
5651 return RThroughput1 && RThroughput2 && *RThroughput1 > *RThroughput2;
5652}
5653
5655 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
5657 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
5658 // We don't lower some vector selects well that are wider than the register
5659 // width. TODO: Improve this with different cost kinds.
5660 if (isa<FixedVectorType>(ValTy) && Opcode == Instruction::Select) {
5661 // We would need this many instructions to hide the scalarization happening.
5662 const int AmortizationCost = 20;
5663
5664 // If VecPred is not set, check if we can get a predicate from the context
5665 // instruction, if its type matches the requested ValTy.
5666 if (VecPred == CmpInst::BAD_ICMP_PREDICATE && I && I->getType() == ValTy) {
5667 CmpPredicate CurrentPred;
5668 if (match(I, m_Select(m_Cmp(CurrentPred, m_Value(), m_Value()), m_Value(),
5669 m_Value())))
5670 VecPred = CurrentPred;
5671 }
5672 // Check if we have a compare/select chain that can be lowered using
5673 // a (F)CMxx & BFI pair.
5674 if (CmpInst::isIntPredicate(VecPred) || VecPred == CmpInst::FCMP_OLE ||
5675 VecPred == CmpInst::FCMP_OLT || VecPred == CmpInst::FCMP_OGT ||
5676 VecPred == CmpInst::FCMP_OGE || VecPred == CmpInst::FCMP_OEQ ||
5677 VecPred == CmpInst::FCMP_UNE) {
5678 static const auto ValidMinMaxTys = {
5679 MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
5680 MVT::v4i32, MVT::v2i64, MVT::v2f32, MVT::v4f32, MVT::v2f64};
5681 static const auto ValidFP16MinMaxTys = {MVT::v4f16, MVT::v8f16};
5682
5683 auto LT = getTypeLegalizationCost(ValTy);
5684 if (any_of(ValidMinMaxTys, equal_to(LT.second)) ||
5685 (ST->hasFullFP16() &&
5686 any_of(ValidFP16MinMaxTys, equal_to(LT.second))))
5687 return LT.first;
5688 }
5689
5690 static const TypeConversionCostTblEntry VectorSelectTbl[] = {
5691 {Instruction::Select, MVT::v2i1, MVT::v2f32, 2},
5692 {Instruction::Select, MVT::v2i1, MVT::v2f64, 2},
5693 {Instruction::Select, MVT::v4i1, MVT::v4f32, 2},
5694 {Instruction::Select, MVT::v4i1, MVT::v4f16, 2},
5695 {Instruction::Select, MVT::v8i1, MVT::v8f16, 2},
5696 {Instruction::Select, MVT::v16i1, MVT::v16i16, 16},
5697 {Instruction::Select, MVT::v8i1, MVT::v8i32, 8},
5698 {Instruction::Select, MVT::v16i1, MVT::v16i32, 16},
5699 {Instruction::Select, MVT::v4i1, MVT::v4i64, 4 * AmortizationCost},
5700 {Instruction::Select, MVT::v8i1, MVT::v8i64, 8 * AmortizationCost},
5701 {Instruction::Select, MVT::v16i1, MVT::v16i64, 16 * AmortizationCost}};
5702
5703 EVT SelCondTy = TLI->getValueType(DL, CondTy);
5704 EVT SelValTy = TLI->getValueType(DL, ValTy);
5705 if (SelCondTy.isSimple() && SelValTy.isSimple()) {
5706 if (const auto *Entry = ConvertCostTableLookup(VectorSelectTbl, Opcode,
5707 SelCondTy.getSimpleVT(),
5708 SelValTy.getSimpleVT()))
5709 return Entry->Cost;
5710 }
5711 }
5712
5713 if (Opcode == Instruction::FCmp) {
5714 if (auto PromotedCost = getFP16BF16PromoteCost(
5715 ValTy, CostKind, Op1Info, Op2Info, /*IncludeTrunc=*/false,
5716 // TODO: Consider costing SVE FCMPs.
5717 /*CanUseSVE=*/false, [&](Type *PromotedTy) {
5719 getCmpSelInstrCost(Opcode, PromotedTy, CondTy, VecPred,
5720 CostKind, Op1Info, Op2Info);
5721 if (isa<VectorType>(PromotedTy))
5723 Instruction::Trunc,
5726 TTI::CastContextHint::None, CostKind);
5727 return Cost;
5728 }))
5729 return *PromotedCost;
5730
5731 auto LT = getTypeLegalizationCost(ValTy);
5732 // Model unknown fp compares as a libcall.
5733 if (LT.second.getScalarType() != MVT::f64 &&
5734 LT.second.getScalarType() != MVT::f32 &&
5735 LT.second.getScalarType() != MVT::f16)
5736 return LT.first * getCallInstrCost(/*Function*/ nullptr, ValTy,
5737 {ValTy, ValTy}, CostKind);
5738
5739 // Some comparison operators require expanding to multiple compares + or.
5740 unsigned Factor = 1;
5741 if (!CondTy->isVectorTy() &&
5742 (VecPred == FCmpInst::FCMP_ONE || VecPred == FCmpInst::FCMP_UEQ))
5743 Factor = 2; // fcmp with 2 selects
5744 else if (isa<FixedVectorType>(ValTy) &&
5745 (VecPred == FCmpInst::FCMP_ONE || VecPred == FCmpInst::FCMP_UEQ ||
5746 VecPred == FCmpInst::FCMP_ORD || VecPred == FCmpInst::FCMP_UNO))
5747 Factor = 3; // fcmxx+fcmyy+or
5748 else if (isa<ScalableVectorType>(ValTy) &&
5749 (VecPred == FCmpInst::FCMP_ONE || VecPred == FCmpInst::FCMP_UEQ))
5750 Factor = 3; // fcmxx+fcmyy+or
5751
5752 if (isa<ScalableVectorType>(ValTy) &&
5753 CostKind == TTI::TCK_RecipThroughput &&
5754 hasKnownLowerThroughputFromSchedulingModel(AArch64::FCMEQ_PPzZZ_S,
5755 AArch64::FCMEQv4f32))
5756 Factor *= 2;
5757
5758 return Factor * (CostKind == TTI::TCK_Latency ? 2 : LT.first);
5759 }
5760
5761 // Treat the icmp in icmp(and, 0) or icmp(and, -1/1) when it can be folded to
5762 // icmp(and, 0) as free, as we can make use of ands, but only if the
5763 // comparison is not unsigned. FIXME: Enable for non-throughput cost kinds
5764 // providing it will not cause performance regressions.
5765 if (CostKind == TTI::TCK_RecipThroughput && ValTy->isIntegerTy() &&
5766 Opcode == Instruction::ICmp && I && !CmpInst::isUnsigned(VecPred) &&
5767 TLI->isTypeLegal(TLI->getValueType(DL, ValTy)) &&
5768 match(I->getOperand(0), m_And(m_Value(), m_Value()))) {
5769 if (match(I->getOperand(1), m_Zero()))
5770 return 0;
5771
5772 // x >= 1 / x < 1 -> x > 0 / x <= 0
5773 if (match(I->getOperand(1), m_One()) &&
5774 (VecPred == CmpInst::ICMP_SLT || VecPred == CmpInst::ICMP_SGE))
5775 return 0;
5776
5777 // x <= -1 / x > -1 -> x > 0 / x <= 0
5778 if (match(I->getOperand(1), m_AllOnes()) &&
5779 (VecPred == CmpInst::ICMP_SLE || VecPred == CmpInst::ICMP_SGT))
5780 return 0;
5781 }
5782
5783 // The base case handles scalable vectors fine for now, since it treats the
5784 // cost as 1 * legalization cost.
5786 Opcode, ValTy, CondTy, VecPred, CostKind, Op1Info, Op2Info, I);
5787 if (Opcode == Instruction::ICmp && isa<ScalableVectorType>(ValTy) &&
5788 CostKind == TTI::TCK_RecipThroughput &&
5789 hasKnownLowerThroughputFromSchedulingModel(AArch64::CMPEQ_PPzZZ_S,
5790 AArch64::CMEQv4i32))
5791 Cost *= 2;
5792 return Cost;
5793}
5794
5796AArch64TTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
5798 if (ST->requiresStrictAlign()) {
5799 // TODO: Add cost modeling for strict align. Misaligned loads expand to
5800 // a bunch of instructions when strict align is enabled.
5801 return Options;
5802 }
5803 Options.AllowOverlappingLoads = true;
5804 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
5805 Options.NumLoadsPerBlock = IsZeroCmp ? Options.MaxNumLoads : 1;
5806 // TODO: Though vector loads usually perform well on AArch64, in some targets
5807 // they may wake up the FP unit, which raises the power consumption. Perhaps
5808 // they could be used with no holds barred (-O3).
5809 Options.LoadSizes = {8, 4, 2, 1};
5810 Options.AllowedTailExpansions = {3, 5, 6};
5811 return Options;
5812}
5813
5815 return ST->hasSVE();
5816}
5817
5820 TTI::TargetCostKind CostKind) const {
5821 switch (MICA.getID()) {
5822 case Intrinsic::masked_scatter:
5823 case Intrinsic::masked_gather:
5824 return getGatherScatterOpCost(MICA, CostKind);
5825 case Intrinsic::masked_load:
5826 case Intrinsic::masked_store:
5827 case Intrinsic::masked_expandload:
5828 case Intrinsic::masked_compressstore:
5829 return getMaskedMemoryOpCost(MICA, CostKind);
5830 }
5831 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
5832}
5833
5836 TTI::TargetCostKind CostKind) const {
5837 Type *Src = MICA.getDataType();
5838
5839 if (useNeonVector(Src))
5840 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
5841 auto LT = getTypeLegalizationCost(Src);
5842 if (!LT.first.isValid())
5844
5845 // Return an invalid cost for element types that we are unable to lower.
5846 auto *VT = cast<VectorType>(Src);
5847 if (VT->getElementType()->isIntegerTy(1))
5849
5850 // <vscale x 1 x eltty> operations require mask adaptation.
5851 // Allow it for normal masked ld/st
5852 if (VT->getElementCount() == ElementCount::getScalable(1))
5853 return is_contained({Intrinsic::masked_load, Intrinsic::masked_store},
5854 MICA.getID())
5855 ? LT.first + 1
5857
5858 InstructionCost MemOpCost = LT.first;
5859 if (MICA.getID() == Intrinsic::masked_expandload) {
5860 // Fall back to the base cost (we can expand fixed-width expandloads).
5861 if (!isLegalMaskedExpandLoad(Src, MICA.getAlignment()))
5862 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
5863
5864 // Operation will be split into expand of masked.load
5865 // Something like:
5866 // cntp x8, p0, p0.s
5867 // whilelo p1.s, xzr, x8
5868 // ld1w { z1.s }, p1/z, [x0]
5869 // expand z1.s, p0, z1.s
5870 MemOpCost *= 4;
5871 }
5872
5873 if (MICA.getID() == Intrinsic::masked_compressstore) {
5874 // Fall back to the base cost (we can expand fixed-width compressstores).
5875 if (!isLegalMaskedCompressStore(Src, MICA.getAlignment()))
5876 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
5877
5878 // A compress store lowers to something like:
5879 // compact z0.s, p0, z0.s
5880 // cntp x8, p1, p0.s
5881 // whilelo p0.s, xzr, x8
5882 // st1w { z0.s }, p0, [x0]
5883 MemOpCost *= 4;
5884 }
5885
5886 // If we need to split the memory operation, we will also need to split the
5887 // mask. This will likely lead to overestimating the cost in some cases if
5888 // multiple memory operations use the same mask, but we often don't have
5889 // enough context to figure that out here.
5890 //
5891 // If the elements being loaded are bytes then the mask will already be split,
5892 // since the number of bits in a P register matches the number of bytes in a
5893 // Z register.
5894 if (LT.first > 1 && LT.second.getScalarSizeInBits() > 8)
5895 return MemOpCost * 2;
5896
5897 return MemOpCost;
5898}
5899
5900// This function returns gather/scatter overhead either from
5901// user-provided value or specialized values per-target from \p ST.
5902static unsigned getSVEGatherScatterOverhead(unsigned Opcode,
5903 const AArch64Subtarget *ST) {
5904 assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
5905 "Should be called on only load or stores.");
5906 switch (Opcode) {
5907 case Instruction::Load:
5908 return ST->getCLOpts().sve_gather_overhead.value_or(
5909 ST->getGatherOverhead());
5910 break;
5911 case Instruction::Store:
5912 return ST->getCLOpts().sve_scatter_overhead.value_or(
5913 ST->getScatterOverhead());
5914 break;
5915 default:
5916 llvm_unreachable("Shouldn't have reached here");
5917 }
5918}
5919
5922 TTI::TargetCostKind CostKind) const {
5923
5924 unsigned Opcode = (MICA.getID() == Intrinsic::masked_gather ||
5925 MICA.getID() == Intrinsic::vp_gather)
5926 ? Instruction::Load
5927 : Instruction::Store;
5928
5929 Type *DataTy = MICA.getDataType();
5930 Align Alignment = MICA.getAlignment();
5931 const Instruction *I = MICA.getInst();
5932
5933 // No need to call useNeonVector here since isLegalMaskedGatherScatter
5934 // determines if we're using SVE for fixed-length vectors.
5935 if (!isLegalMaskedGatherScatter(DataTy))
5936 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
5937 auto *VT = cast<VectorType>(DataTy);
5938 auto LT = getTypeLegalizationCost(DataTy);
5939 if (!LT.first.isValid())
5941
5942 // Return an invalid cost for element types that we are unable to lower.
5943 if (!LT.second.isVector() ||
5944 !isElementTypeLegalForScalableVector(VT->getElementType()) ||
5945 VT->getElementType()->isIntegerTy(1))
5947
5948 // The code-generator is currently not able to handle scalable vectors
5949 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
5950 // it. This change will be removed when code-generation for these types is
5951 // sufficiently reliable.
5952 if (VT->getElementCount() == ElementCount::getScalable(1))
5954
5955 ElementCount LegalVF = LT.second.getVectorElementCount();
5956 InstructionCost MemOpCost =
5957 getMemoryOpCost(Opcode, VT->getElementType(), Alignment, 0, CostKind,
5958 {TTI::OK_AnyValue, TTI::OP_None}, I);
5959 // Add on an overhead cost for using gathers/scatters.
5960 MemOpCost *= getSVEGatherScatterOverhead(Opcode, ST);
5961 return LT.first * MemOpCost * getMaxNumElements(LegalVF);
5962}
5963
5965 return isa<FixedVectorType>(Ty) && !ST->useSVEForFixedLengthVectors();
5966}
5967
5968// Get the latency of NumInst of Inst according to the scheduling model. The
5969// latency will be the latency of the last instruction plus the time it gets to
5970// get there, which will be the amount of other instructions before that (i.e.
5971// NumInst - 1) multiplied by how long it takes to get through them (the
5972// reciprocal of the throughput). If the model has no information for this
5973// instruction then we assume the reciprocal throughput is 1 and use
5974// DefaultLatency for the latency.
5975static InstructionCost
5977 unsigned NumInst, InstructionCost DefaultLatency) {
5978 assert(NumInst != 0 && "Can't calculate latency of zero instructions");
5979 const MCSchedModel &Sched = ST->getSchedModel();
5980 const TargetInstrInfo *TII = ST->getInstrInfo();
5981 unsigned SchedClass = TII->get(Inst).getSchedClass();
5982 const MCSchedClassDesc *SCD = Sched.hasInstrSchedModel()
5983 ? Sched.getSchedClassDesc(SchedClass)
5984 : nullptr;
5985 // If the cpu has no scheduling model, or it doesn't describe the
5986 // instruction, then fall back to the default latency. Variant
5987 // scheduling classes can't be resolved without a MachineInstr, so
5988 // treat them the same way.
5989 if (!SCD || !SCD->isValid() || SCD->isVariant())
5990 return (NumInst - 1) + DefaultLatency;
5991 // We need to convert the number of instructions before the last to a float
5992 // here, as the reciprocal throughput may be fractional.
5993 return (float)(NumInst - 1) * Sched.getReciprocalThroughput(*ST, *SCD) +
5994 Sched.computeInstrLatency(*ST, *SCD);
5995}
5996
5998 Align Alignment,
5999 unsigned AddressSpace,
6000 TTI::TargetCostKind CostKind,
6001 TTI::OperandValueInfo OpInfo,
6002 const Instruction *I) const {
6003 EVT VT = TLI->getValueType(DL, Ty, true);
6004 // Type legalization can't handle structs
6005 if (VT == MVT::Other)
6006 return BaseT::getMemoryOpCost(Opcode, Ty, Alignment, AddressSpace,
6007 CostKind);
6008
6009 auto LT = getTypeLegalizationCost(Ty);
6010 if (!LT.first.isValid())
6012
6013 if (auto *VTy = dyn_cast<ScalableVectorType>(Ty)) {
6014
6015 // We only support full register predicate loads and stores.
6016 if (VTy->getElementType()->isIntegerTy(1) &&
6017 !VTy->getElementCount().isKnownMultipleOf(
6020
6021 // <vscale x 1 x eltty> operations require crafting a new mask.
6022 if (VTy->getElementCount() == ElementCount::getScalable(1)) {
6023 Intrinsic::ID IID = Opcode == Instruction::Load ? Intrinsic::masked_load
6024 : Intrinsic::masked_store;
6025 return getMaskedMemoryOpCost(
6026 MemIntrinsicCostAttributes(IID, Ty, Alignment, AddressSpace),
6027 CostKind) +
6028 1;
6029 }
6030 }
6031
6032 // TODO: consider latency as well for TCK_SizeAndLatency.
6033 if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency)
6034 return LT.first;
6035
6036 if (CostKind == TTI::TCK_Latency) {
6037 // Latency doesn't make much sense for stores, so just return 1
6038 if (Opcode == Instruction::Store)
6039 return 1;
6040 // If the subtarget has overridden the load latency then use that instead of
6041 // querying the SchedModel.
6042 if (ST->getFixedLoadLatency())
6043 return (LT.first - 1) + ST->getFixedLoadLatency();
6044 // We expect the load to become LT.first loads of type LT.second. We
6045 // calculate the latency using the SchedModel, and assume that the loads
6046 // become the variant with unsigned integer offset.
6047 unsigned Inst = 0;
6048 if (LT.second.isScalableVector() ||
6049 ST->useSVEForFixedLengthVectors(LT.second)) {
6050 Inst = AArch64::LDR_ZXI;
6051 } else if (LT.second.isVector() || LT.second.isFloatingPoint()) {
6052 switch (LT.second.getSizeInBits()) {
6053 case 8:
6054 Inst = AArch64::LDRBui;
6055 break;
6056 case 16:
6057 Inst = AArch64::LDRHui;
6058 break;
6059 case 32:
6060 Inst = AArch64::LDRSui;
6061 break;
6062 case 64:
6063 Inst = AArch64::LDRDui;
6064 break;
6065 case 128:
6066 Inst = AArch64::LDRQui;
6067 break;
6068 default:
6069 llvm_unreachable("Unexpected float or vector type");
6070 }
6071 } else {
6072 switch (LT.second.getSizeInBits()) {
6073 case 8:
6074 Inst = AArch64::LDRBBui;
6075 break;
6076 case 16:
6077 Inst = AArch64::LDRHHui;
6078 break;
6079 case 32:
6080 Inst = AArch64::LDRWui;
6081 break;
6082 case 64:
6083 Inst = AArch64::LDRXui;
6084 break;
6085 default:
6086 llvm_unreachable("Unexpected integer type");
6087 }
6088 }
6089 return getLatencyFromSchedModel(ST, Inst, LT.first.getValue(),
6090 ST->getLoadLatency());
6091 }
6092
6093 if (ST->isMisaligned128StoreSlow() && Opcode == Instruction::Store &&
6094 LT.second.is128BitVector() && Alignment < Align(16)) {
6095 // Unaligned stores are extremely inefficient. We don't split all
6096 // unaligned 128-bit stores because the negative impact that has shown in
6097 // practice on inlined block copy code.
6098 // We make such stores expensive so that we will only vectorize if there
6099 // are 6 other instructions getting vectorized.
6100 const int AmortizationCost = 6;
6101
6102 return LT.first * 2 * AmortizationCost;
6103 }
6104
6105 // Opaque ptr or ptr vector types are i64s and can be lowered to STP/LDPs.
6106 if (Ty->isPtrOrPtrVectorTy())
6107 return LT.first;
6108
6109 if (useNeonVector(Ty)) {
6110 // Check truncating stores and extending loads.
6111 if (Ty->getScalarSizeInBits() != LT.second.getScalarSizeInBits()) {
6112 // v4i8 types are lowered to scalar a load/store and sshll/xtn.
6113 if (VT == MVT::v4i8)
6114 return 2;
6115 // Otherwise we need to scalarize.
6116 return cast<FixedVectorType>(Ty)->getNumElements() * 2;
6117 }
6118 EVT EltVT = VT.getVectorElementType();
6119 unsigned EltSize = EltVT.getScalarSizeInBits();
6120 if (!isPowerOf2_32(EltSize) || EltSize < 8 || EltSize > 64)
6121 return LT.first;
6122 // FIXME: v3i8 lowering currently is very inefficient, due to automatic
6123 // widening to v4i8, which produces suboptimal results.
6124 if (VT.getVectorNumElements() == 3 && EltVT == MVT::i8)
6125 return LT.first;
6126
6127 // Check non-power-of-2 loads/stores for legal vector element types with
6128 // NEON. Non-power-of-2 memory ops will get broken down to a set of
6129 // operations on smaller power-of-2 ops, including ld1/st1.
6130 InstructionCost Cost = VT.getVectorNumElements() / (128 / EltSize);
6131 unsigned Remainder = VT.getVectorNumElements() % (128 / EltSize);
6132 if (Remainder != 0) {
6134 TypeWorklist.push_back({Remainder, 0});
6135 while (!TypeWorklist.empty()) {
6136 auto [CurrNumElements, Offset] = TypeWorklist.pop_back_val();
6137 // Loads with sufficient alignment are widened to the next power of two,
6138 // so only a single load is needed.
6139 if (isPowerOf2_32(CurrNumElements) ||
6140 (Opcode == Instruction::Load &&
6141 Alignment >= PowerOf2Ceil(CurrNumElements) * EltSize / 8)) {
6142 // 1 per load/store + possible lane insert.
6143 Cost += 1 + (Offset == 0 ? 0 : 1);
6144 continue;
6145 }
6146
6147 unsigned PrevPow2 = NextPowerOf2(CurrNumElements) / 2;
6148 TypeWorklist.push_back({PrevPow2, Offset});
6149 TypeWorklist.push_back({CurrNumElements - PrevPow2, Offset + PrevPow2});
6150 }
6151 }
6152 return Cost;
6153 }
6154
6155 return LT.first;
6156}
6157
6159 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
6160 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
6161 bool UseMaskForCond, bool UseMaskForGaps) const {
6162 assert(Factor >= 2 && "Invalid interleave factor");
6163 auto *VecVTy = cast<VectorType>(VecTy);
6164
6165 if (VecTy->isScalableTy() && !ST->hasSVE())
6167
6168 // Scalable VFs emit vector.[de]interleave intrinsics, for which the target
6169 // supports factors up to the maximum supported interleave factor.
6170 if (VecTy->isScalableTy()) {
6171 if (Factor > TLI->getMaxSupportedInterleaveFactor())
6173
6174 if (Factor == 3 &&
6175 DL.getTypeSizeInBits(VecTy).getKnownMinValue() != (3 * 128))
6177 }
6178
6179 auto LT = getTypeLegalizationCost(VecTy);
6180 unsigned MaxNativeInterleaveFactor = TLI->getMaxSupportedInterleaveFactor();
6181 // Vectorization for masked interleaved accesses is only enabled for scalable
6182 // VF. For fixed-length SVE, avoid non-native interleave factor because
6183 // the generic fallback costs wide fixed-vector shuffles too optimistically.
6184 if (!VecTy->isScalableTy() &&
6185 (UseMaskForCond || UseMaskForGaps ||
6186 (Factor > MaxNativeInterleaveFactor &&
6187 TLI->useSVEForFixedLengthVectorVT(LT.second))))
6189
6190 if (!UseMaskForGaps && Factor <= MaxNativeInterleaveFactor) {
6191 ElementCount EC = VecVTy->getElementCount();
6192 auto *SubVecTy = VectorType::get(VecVTy->getElementType(),
6193 EC.divideCoefficientBy(Factor));
6194
6195 // ldN/stN only support legal vector types of size 64 or 128 in bits.
6196 // Accesses having vector types that are a multiple of 128 bits can be
6197 // matched to more than one ldN/stN instruction.
6198 bool UseScalable;
6199
6200 if (EC.isKnownMultipleOf(Factor) &&
6201 TLI->isLegalInterleavedAccessType(SubVecTy, DL, UseScalable)) {
6202 unsigned Num = TLI->getNumInterleavedAccesses(SubVecTy, DL, UseScalable);
6203 if (CostKind == TTI::TCK_Latency) {
6204 // Latency doesn't make much sense for stores, so just return 1
6205 if (Opcode == Instruction::Store)
6206 return 1;
6207 // If the subtarget has overridden the load latency then use that
6208 // instead of querying the SchedModel.
6209 if (ST->getFixedLoadLatency())
6210 return (LT.first - 1) + ST->getFixedLoadLatency();
6211 // Lookup the load opcode based on whether we're using SVE, interleaving
6212 // factor, and element size.
6213 const unsigned Opcodes[2][3][4] = {
6214 {
6215 {AArch64::LD2Twov16b, AArch64::LD2Twov8h, AArch64::LD2Twov4s,
6216 AArch64::LD2Twov2d},
6217 {AArch64::LD3Threev16b, AArch64::LD3Threev8h,
6218 AArch64::LD3Threev4s, AArch64::LD3Threev2d},
6219 {AArch64::LD4Fourv16b, AArch64::LD4Fourv8h, AArch64::LD4Fourv4s,
6220 AArch64::LD4Fourv2d},
6221 },
6222 {
6223 {AArch64::LD2B, AArch64::LD2H, AArch64::LD2W, AArch64::LD2D},
6224 {AArch64::LD3B, AArch64::LD3H, AArch64::LD3W, AArch64::LD3D},
6225 {AArch64::LD4B, AArch64::LD4H, AArch64::LD4W, AArch64::LD4D},
6226 }};
6227 unsigned LogElemSize =
6228 Log2_32(DL.getTypeSizeInBits(SubVecTy->getElementType()));
6229 assert(LogElemSize >= 3 && LogElemSize <= 6 &&
6230 "Unexpected element size");
6231 assert(Factor <= 4 && "Unexpected interleaving factor");
6232 unsigned Inst = Opcodes[UseScalable][Factor - 2][LogElemSize - 3];
6233 return getLatencyFromSchedModel(ST, Inst, Num, ST->getLoadLatency());
6234 }
6235 return Factor * Num;
6236 }
6237
6238 // Cost the alternative approach for scalable vectors where the interleave
6239 // factor is larger than the VF: use a contiguous load/store of the full
6240 // wide vector followed by deinterleave/interleave shuffles.
6241 if (VecTy->isScalableTy() && EC.isKnownMultipleOf(Factor)) {
6242 if (SubVecTy->getElementCount() == ElementCount::getScalable(1))
6244
6245 // Cost of the contiguous memory operation on the wide vector.
6246 InstructionCost MemCost;
6247 if (UseMaskForCond) {
6248 unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load
6249 : Intrinsic::masked_store;
6250 MemCost = getMemIntrinsicInstrCost(
6251 MemIntrinsicCostAttributes(IID, VecTy, Alignment, AddressSpace),
6252 CostKind);
6253 } else {
6254 MemCost =
6255 getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace, CostKind);
6256 }
6257
6258 // llvm.vector.deinterleaveN is lowered as a binary tree of deinterleave2
6259 // operations. The tree has Log2(Factor) levels, with Factor UZP/ZIP
6260 // operations at each level, giving a total shuffle cost of
6261 // Factor * Log2(Factor).
6262 auto SubVecCost = getTypeLegalizationCost(SubVecTy);
6263 auto ResultCost = getTypeLegalizationCost(VecTy);
6264 llvm::InstructionCost LegalizationCost = SubVecCost.first;
6265
6266 // FIXME: A temporary increase to the cost in cases where the input
6267 // element type is 4x the output type. Otherwise it produces an SVE tail
6268 // loop which is significantly larger than the NEON equivalent.
6269 if (Opcode == Instruction::Store && Factor == 4 &&
6270 SubVecCost.second.getScalarSizeInBits() ==
6271 (4 * ResultCost.second.getScalarSizeInBits()))
6272 LegalizationCost *= 4;
6273
6274 return MemCost + (Factor * LegalizationCost) + (Factor * Log2_64(Factor));
6275 }
6276 }
6277
6278 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
6279 Alignment, AddressSpace, CostKind,
6280 UseMaskForCond, UseMaskForGaps);
6281}
6282
6287 for (auto *I : Tys) {
6288 if (!I->isVectorTy())
6289 continue;
6290 if (I->getScalarSizeInBits() * cast<FixedVectorType>(I)->getNumElements() ==
6291 128)
6292 Cost += getMemoryOpCost(Instruction::Store, I, Align(128), 0, CostKind) +
6293 getMemoryOpCost(Instruction::Load, I, Align(128), 0, CostKind);
6294 }
6295 return Cost;
6296}
6297
6299 Align Alignment) const {
6300 // Neon types should be scalarised when we are not choosing to use SVE.
6301 if (useNeonVector(DataTy))
6302 return false;
6303
6305 return false;
6306
6307 // Return true only if we are able to lower using the SVE2p2/SME2p2
6308 // expand instruction.
6309 return (ST->isSVEAvailable() && ST->hasSVE2p2()) ||
6310 (ST->isSVEorStreamingSVEAvailable() && ST->hasSME2p2());
6311}
6312
6314 unsigned AddressSpace) const {
6315 // Matches AArch64TargetLowering::emitCanLoadSpeculatively: only address
6316 // space 0 and power-of-2 sizes up to the 16-byte MTE tag granule.
6317 // TODO: Support scalable vectors.
6318 if (AddressSpace != 0)
6319 return false;
6320 TypeSize Size = DL.getTypeStoreSize(DataType);
6321 return !Size.isScalable() && isPowerOf2_64(Size.getFixedValue()) &&
6322 Size.getFixedValue() <= 16;
6323}
6324
6326 unsigned NumVectors, TTI::MaskSource Mask, VectorType *VectorTy,
6327 bool IsStore, std::optional<Instruction::CastOps> CastHint) const {
6328 if (NumVectors <= 1 || !ST->enableSubRegLiveness() || !ST->hasSVE2p1())
6329 return false;
6330
6331 switch (Mask) {
6333 return false;
6336 break;
6337 }
6338
6339 // A null vector type queries whether the target supports multi-vector memory
6340 // operations in general.
6341 if (!VectorTy)
6342 return true;
6343
6344 if (!isa<ScalableVectorType>(VectorTy))
6345 return false;
6346
6347 // Conservatively, avoid using multi-vector loads when it's possible we could
6348 // use extending loads instead. Note: We can ignore stores as we only use
6349 // truncating stores when the store vector-width is < a full SVE vector.
6350 if (!IsStore &&
6351 (CastHint == Instruction::ZExt || CastHint == Instruction::SExt))
6352 return false;
6353
6354 TypeSize VectorWidth = DL.getTypeSizeInBits(VectorTy);
6356 return false;
6357
6358 // For masked loads/stores only allow operations that map to two or four
6359 // vectors (as these can use a single predicate-as-counter).
6360 if (Mask == TTI::MaskSource::ActiveLaneMask) {
6361 unsigned NumSVEVectors = (VectorWidth * NumVectors).getKnownMinValue() /
6363 return NumSVEVectors == 2 || NumSVEVectors == 4;
6364 }
6365
6366 // For unpredicated loads/stores allow any pow-of-two multiple of a vector >=
6367 // to a single z-register. We can split operations wider than a single
6368 // multi-vector load/store during ISEL.
6369 return isPowerOf2_32(NumVectors);
6370}
6371
6372unsigned
6374 bool HasUnorderedReductions) const {
6375 if (VF.isScalar() || (HasUnorderedReductions && VF.getKnownMinValue() <= 4))
6376 return 4;
6377 return ST->getMaxInterleaveFactor();
6378}
6379
6380// For Falkor, we want to avoid having too many strided loads in a loop since
6381// that can exhaust the HW prefetcher resources. We adjust the unroller
6382// MaxCount preference below to attempt to ensure unrolling doesn't create too
6383// many strided loads.
6384static void
6387 enum { MaxStridedLoads = 7 };
6388 auto countStridedLoads = [](Loop *L, ScalarEvolution &SE) {
6389 int StridedLoads = 0;
6390 // FIXME? We could make this more precise by looking at the CFG and
6391 // e.g. not counting loads in each side of an if-then-else diamond.
6392 for (const auto BB : L->blocks()) {
6393 for (auto &I : *BB) {
6394 LoadInst *LMemI = dyn_cast<LoadInst>(&I);
6395 if (!LMemI)
6396 continue;
6397
6398 Value *PtrValue = LMemI->getPointerOperand();
6399 if (L->isLoopInvariant(PtrValue))
6400 continue;
6401
6402 const SCEV *LSCEV = SE.getSCEV(PtrValue);
6403 const SCEVAddRecExpr *LSCEVAddRec = dyn_cast<SCEVAddRecExpr>(LSCEV);
6404 if (!LSCEVAddRec || !LSCEVAddRec->isAffine())
6405 continue;
6406
6407 // FIXME? We could take pairing of unrolled load copies into account
6408 // by looking at the AddRec, but we would probably have to limit this
6409 // to loops with no stores or other memory optimization barriers.
6410 ++StridedLoads;
6411 // We've seen enough strided loads that seeing more won't make a
6412 // difference.
6413 if (StridedLoads > MaxStridedLoads / 2)
6414 return StridedLoads;
6415 }
6416 }
6417 return StridedLoads;
6418 };
6419
6420 int StridedLoads = countStridedLoads(L, SE);
6421 LLVM_DEBUG(dbgs() << "falkor-hwpf: detected " << StridedLoads
6422 << " strided loads\n");
6423 // Pick the largest power of 2 unroll count that won't result in too many
6424 // strided loads.
6425 if (StridedLoads) {
6426 UP.MaxCount = 1 << Log2_32(MaxStridedLoads / StridedLoads);
6427 LLVM_DEBUG(dbgs() << "falkor-hwpf: setting unroll MaxCount to "
6428 << UP.MaxCount << '\n');
6429 }
6430}
6431
6432// This function returns true if the loop:
6433// 1. Has a valid cost, and
6434// 2. Has a cost within the supplied budget.
6435// Otherwise it returns false.
6437 InstructionCost Budget,
6438 unsigned *FinalSize) {
6439 // Estimate the size of the loop.
6440 InstructionCost LoopCost = 0;
6441
6442 for (auto *BB : L->getBlocks()) {
6443 for (auto &I : *BB) {
6444 SmallVector<const Value *, 4> Operands(I.operand_values());
6445 InstructionCost Cost =
6446 TTI.getInstructionCost(&I, Operands, TTI::TCK_CodeSize);
6447 // This can happen with intrinsics that don't currently have a cost model
6448 // or for some operations that require SVE.
6449 if (!Cost.isValid())
6450 return false;
6451
6452 LoopCost += Cost;
6453 if (LoopCost > Budget)
6454 return false;
6455 }
6456 }
6457
6458 if (FinalSize)
6459 *FinalSize = LoopCost.getValue();
6460 return true;
6461}
6462
6464 const AArch64TTIImpl &TTI) {
6465 // Only consider loops with unknown trip counts for which we can determine
6466 // a symbolic expression. Multi-exit loops with small known trip counts will
6467 // likely be unrolled anyway.
6468 const SCEV *BTC = SE.getSymbolicMaxBackedgeTakenCount(L);
6470 return false;
6471
6472 // It might not be worth unrolling loops with low max trip counts. Restrict
6473 // this to max trip counts > 32 for now.
6474 unsigned MaxTC = SE.getSmallConstantMaxTripCount(L);
6475 if (MaxTC > 0 && MaxTC <= 32)
6476 return false;
6477
6478 // Make sure the loop size is <= 5.
6479 if (!isLoopSizeWithinBudget(L, TTI, 5, nullptr))
6480 return false;
6481
6482 // Small search loops with multiple exits can be highly beneficial to unroll.
6483 // We only care about loops with exactly two exiting blocks, although each
6484 // block could jump to the same exit block.
6485 ArrayRef<BasicBlock *> Blocks = L->getBlocks();
6486 if (Blocks.size() != 2)
6487 return false;
6488
6489 if (any_of(Blocks, [](BasicBlock *BB) {
6491 }))
6492 return false;
6493
6494 return true;
6495}
6496
6497/// For Apple CPUs, we want to runtime-unroll loops to make better use if the
6498/// OOO engine's wide instruction window and various predictors.
6499static void
6502 const AArch64TTIImpl &TTI) {
6503 // Limit loops with structure that is highly likely to benefit from runtime
6504 // unrolling; that is we exclude outer loops and loops with many blocks (i.e.
6505 // likely with complex control flow). Note that the heuristics here may be
6506 // overly conservative and we err on the side of avoiding runtime unrolling
6507 // rather than unroll excessively. They are all subject to further refinement.
6508 if (!L->isInnermost() || L->getNumBlocks() > 8)
6509 return;
6510
6511 // Loops with multiple exits are handled by common code.
6512 if (!L->getExitBlock())
6513 return;
6514
6515 // Check if the loop contains any reductions that could be parallelized when
6516 // unrolling. If so, enable partial unrolling, if the trip count is know to be
6517 // a multiple of 2.
6518 bool HasParellelizableReductions =
6519 L->getNumBlocks() == 1 &&
6520 any_of(L->getHeader()->phis(),
6521 [&SE, L](PHINode &Phi) {
6522 return canParallelizeReductionWhenUnrolling(Phi, L, &SE);
6523 }) &&
6524 isLoopSizeWithinBudget(L, TTI, 12, nullptr);
6525 if (HasParellelizableReductions &&
6526 SE.getSmallConstantTripMultiple(L, L->getExitingBlock()) % 2 == 0) {
6527 UP.Partial = true;
6528 UP.MaxCount = 4;
6529 UP.AddAdditionalAccumulators = true;
6530 }
6531
6532 const SCEV *BTC = SE.getSymbolicMaxBackedgeTakenCount(L);
6534 (SE.getSmallConstantMaxTripCount(L) > 0 &&
6535 SE.getSmallConstantMaxTripCount(L) <= 32))
6536 return;
6537
6538 if (getBooleanLoopAttribute(L, "llvm.loop.unroll.runtime.disable"))
6539 return;
6540
6542 return;
6543
6544 // Limit to loops with trip counts that are cheap to expand.
6545 UP.SCEVExpansionBudget = 1;
6546
6547 if (HasParellelizableReductions) {
6548 UP.Runtime = true;
6550 UP.AddAdditionalAccumulators = true;
6551 }
6552
6553 // Try to unroll small, single-block loops with low budget, if they have
6554 // load/store dependencies, to expose more parallel memory access streams,
6555 // or if they do little work inside a block (i.e. load -> X -> store pattern).
6556 BasicBlock *Header = L->getHeader();
6557 BasicBlock *Latch = L->getLoopLatch();
6558 if (Header == Latch) {
6559 // Estimate the size of the loop.
6560 unsigned Size;
6561 unsigned Width = 10;
6562 if (!isLoopSizeWithinBudget(L, TTI, Width, &Size))
6563 return;
6564
6565 // Try to find an unroll count that maximizes the use of the instruction
6566 // window, i.e. trying to fetch as many instructions per cycle as possible.
6567 constexpr unsigned MaxInstsPerLine = 16;
6568 constexpr unsigned MaxLines = 3;
6569 constexpr unsigned MaxSizeWithUC = MaxInstsPerLine * MaxLines;
6570 constexpr unsigned MaxUC = 8;
6571
6572 unsigned BestUC = 1;
6573 unsigned BestLineRemainder = Size % MaxInstsPerLine;
6574 for (unsigned UC = 2; UC <= MaxUC; ++UC) {
6575 unsigned SizeWithUC = UC * Size;
6576 if (SizeWithUC > MaxSizeWithUC)
6577 break;
6578 unsigned LineRemainder = SizeWithUC % MaxInstsPerLine;
6579 if (LineRemainder == 0 || BestLineRemainder < LineRemainder) {
6580 BestUC = UC;
6581 BestLineRemainder = LineRemainder;
6582 }
6583 }
6584
6585 if (BestUC == 1)
6586 return;
6587
6588 SmallPtrSet<Value *, 8> LoadedValuesPlus;
6590 for (auto *BB : L->blocks()) {
6591 for (auto &I : *BB) {
6593 if (!Ptr)
6594 continue;
6595 const SCEV *PtrSCEV = SE.getSCEV(Ptr);
6596 if (SE.isLoopInvariant(PtrSCEV, L))
6597 continue;
6598 if (isa<LoadInst>(&I)) {
6599 LoadedValuesPlus.insert(&I);
6600 // Include in-loop 1st users of loaded values.
6601 for (auto *U : I.users())
6602 if (L->contains(cast<Instruction>(U)))
6603 LoadedValuesPlus.insert(U);
6604 } else
6605 Stores.push_back(cast<StoreInst>(&I));
6606 }
6607 }
6608
6609 if (none_of(Stores, [&LoadedValuesPlus](StoreInst *SI) {
6610 return LoadedValuesPlus.contains(SI->getOperand(0));
6611 }))
6612 return;
6613
6614 UP.Runtime = true;
6615 UP.DefaultUnrollRuntimeCount = BestUC;
6616 return;
6617 }
6618
6619 // Try to runtime-unroll loops with early-continues depending on loop-varying
6620 // loads; this helps with branch-prediction for the early-continues.
6621 auto *Term = dyn_cast<CondBrInst>(Header->getTerminator());
6623 if (!Term || Preds.size() == 1 || !llvm::is_contained(Preds, Header) ||
6624 none_of(Preds, [L](BasicBlock *Pred) { return L->contains(Pred); }))
6625 return;
6626
6627 std::function<bool(Instruction *, unsigned)> DependsOnLoopLoad =
6628 [&](Instruction *I, unsigned Depth) -> bool {
6629 if (isa<PHINode>(I) || L->isLoopInvariant(I) || Depth > 8)
6630 return false;
6631
6632 if (isa<LoadInst>(I))
6633 return true;
6634
6635 return any_of(I->operands(), [&](Value *V) {
6636 auto *I = dyn_cast<Instruction>(V);
6637 return I && DependsOnLoopLoad(I, Depth + 1);
6638 });
6639 };
6641 Instruction *I;
6642 if (match(Term, m_Br(m_ICmp(Pred, m_Instruction(I), m_Value()), m_Value(),
6643 m_Value())) &&
6644 DependsOnLoopLoad(I, 0)) {
6645 UP.Runtime = true;
6646 }
6647}
6648
6651 OptimizationRemarkEmitter *ORE) const {
6652 // Enable partial unrolling and runtime unrolling.
6653 BaseT::getUnrollingPreferences(L, SE, UP, ORE);
6654
6655 UP.UpperBound = true;
6656
6657 // A loop can have a small maximum trip count while SCEV still cannot
6658 // form an exact backedge count - typically a data-dependent exit, e.g.
6659 // shifting a value until it reaches zero. Unlike for counted loops, the
6660 // unrolled body keeps an exit test per iteration, and whether that pays
6661 // off depends on how many iterations the loop usually runs, which is
6662 // unknown at compile time; the code growth and extra branches are certain.
6663 // Be conservative and hold such loops to a lower upper bound; 5 still lets
6664 // smaller early-exit loops unroll. Also disable runtime unrolling, which
6665 // would clamp the unroll count to the known maximum trip count and produce
6666 // the same complete unroll.
6667 if (L->getExitingBlock() && !SE.isBackedgeTakenCountMaxOrZero(L) &&
6669 UP.MaxUpperBound = 5;
6670 UP.Runtime = false;
6671 }
6672
6673 // For inner loop, it is more likely to be a hot one, and the runtime check
6674 // can be promoted out from LICM pass, so the overhead is less, let's try
6675 // a larger threshold to unroll more loops.
6676 if (L->getLoopDepth() > 1)
6677 UP.PartialThreshold *= 2;
6678
6679 // Disable partial & runtime unrolling on -Os.
6681
6682 // Scan the loop: don't unroll loops with calls as this could prevent
6683 // inlining. Don't unroll auto-vectorized loops either, though do allow
6684 // unrolling of the scalar remainder.
6685 bool IsVectorized = getBooleanLoopAttribute(L, "llvm.loop.isvectorized");
6687 for (auto *BB : L->getBlocks()) {
6688 for (auto &I : *BB) {
6689 // Both auto-vectorized loops and the scalar remainder have the
6690 // isvectorized attribute, so differentiate between them by the presence
6691 // of vector instructions.
6692 if (IsVectorized && I.getType()->isVectorTy())
6693 return;
6694 if (isa<CallBase>(I)) {
6697 if (!isLoweredToCall(F))
6698 continue;
6699 return;
6700 }
6701
6702 // The cost is only compared against Aarch64ForceUnrollThreshold below.
6703 if (Cost >= ST->getCLOpts().force_unroll_threshold)
6704 continue;
6705 SmallVector<const Value *, 4> Operands(I.operand_values());
6708 }
6709 }
6710
6711 // Apply subtarget-specific unrolling preferences.
6712 if (ST->isAppleMLike())
6713 getAppleRuntimeUnrollPreferences(L, SE, UP, *this);
6714 else if (ST->getProcFamily() == AArch64Subtarget::Falkor &&
6715 ST->getCLOpts().enable_falkor_hwpf_unroll_fix)
6717
6718 // If this is a small, multi-exit loop similar to something like std::find,
6719 // then there is typically a performance improvement achieved by unrolling.
6720 if (!L->getExitBlock() && shouldUnrollMultiExitLoop(L, SE, *this)) {
6721 UP.RuntimeUnrollMultiExit = true;
6722 UP.Runtime = true;
6723 // Limit unroll count.
6725 // Allow slightly more costly trip-count expansion to catch search loops
6726 // with pointer inductions.
6727 UP.SCEVExpansionBudget = 5;
6728 return;
6729 }
6730
6731 // Enable runtime unrolling for in-order models
6732 // If mcpu is omitted, getProcFamily() returns AArch64Subtarget::Others, so by
6733 // checking for that case, we can ensure that the default behaviour is
6734 // unchanged
6735 if (ST->getProcFamily() != AArch64Subtarget::Generic &&
6736 !ST->getSchedModel().isOutOfOrder()) {
6737 UP.Runtime = true;
6738 UP.Partial = true;
6739 UP.UnrollRemainder = true;
6741
6742 UP.UnrollAndJam = true;
6744 }
6745
6746 // Force unrolling small loops can be very useful because of the branch
6747 // taken cost of the backedge.
6748 if (Cost < ST->getCLOpts().force_unroll_threshold)
6749 UP.Force = true;
6750}
6751
6756
6758 Type *ExpectedType,
6759 bool CanCreate) const {
6760 switch (Inst->getIntrinsicID()) {
6761 default:
6762 return nullptr;
6763 case Intrinsic::aarch64_neon_st1x2:
6764 case Intrinsic::aarch64_neon_st1x3:
6765 case Intrinsic::aarch64_neon_st1x4:
6766 case Intrinsic::aarch64_neon_st2:
6767 case Intrinsic::aarch64_neon_st3:
6768 case Intrinsic::aarch64_neon_st4: {
6769 // Create a struct type
6770 StructType *ST = dyn_cast<StructType>(ExpectedType);
6771 if (!CanCreate || !ST)
6772 return nullptr;
6773 unsigned NumElts = Inst->arg_size() - 1;
6774 if (ST->getNumElements() != NumElts)
6775 return nullptr;
6776 for (unsigned i = 0, e = NumElts; i != e; ++i) {
6777 if (Inst->getArgOperand(i)->getType() != ST->getElementType(i))
6778 return nullptr;
6779 }
6780 Value *Res = PoisonValue::get(ExpectedType);
6781 IRBuilder<> Builder(Inst);
6782 for (unsigned i = 0, e = NumElts; i != e; ++i) {
6783 Value *L = Inst->getArgOperand(i);
6784 Res = Builder.CreateInsertValue(Res, L, i);
6785 }
6786 return Res;
6787 }
6788 case Intrinsic::aarch64_neon_ld1x2:
6789 case Intrinsic::aarch64_neon_ld1x3:
6790 case Intrinsic::aarch64_neon_ld1x4:
6791 case Intrinsic::aarch64_neon_ld2:
6792 case Intrinsic::aarch64_neon_ld3:
6793 case Intrinsic::aarch64_neon_ld4:
6794 if (Inst->getType() == ExpectedType)
6795 return Inst;
6796 return nullptr;
6797 }
6798}
6799
6801 MemIntrinsicInfo &Info) const {
6802 switch (Inst->getIntrinsicID()) {
6803 default:
6804 break;
6805 case Intrinsic::aarch64_neon_ld1x2:
6806 case Intrinsic::aarch64_neon_ld1x3:
6807 case Intrinsic::aarch64_neon_ld1x4:
6808 case Intrinsic::aarch64_neon_ld2:
6809 case Intrinsic::aarch64_neon_ld3:
6810 case Intrinsic::aarch64_neon_ld4:
6811 Info.ReadMem = true;
6812 Info.WriteMem = false;
6813 Info.PtrVal = Inst->getArgOperand(0);
6814 break;
6815 case Intrinsic::aarch64_neon_st1x2:
6816 case Intrinsic::aarch64_neon_st1x3:
6817 case Intrinsic::aarch64_neon_st1x4:
6818 case Intrinsic::aarch64_neon_st2:
6819 case Intrinsic::aarch64_neon_st3:
6820 case Intrinsic::aarch64_neon_st4:
6821 Info.ReadMem = false;
6822 Info.WriteMem = true;
6823 Info.PtrVal = Inst->getArgOperand(Inst->arg_size() - 1);
6824 break;
6825 }
6826
6827 // Use the ID of neon load as the "matching id".
6828 switch (Inst->getIntrinsicID()) {
6829 default:
6830 return false;
6831 case Intrinsic::aarch64_neon_ld1x2:
6832 case Intrinsic::aarch64_neon_st1x2:
6833 Info.MatchingId = Intrinsic::aarch64_neon_ld1x2;
6834 break;
6835 case Intrinsic::aarch64_neon_ld1x3:
6836 case Intrinsic::aarch64_neon_st1x3:
6837 Info.MatchingId = Intrinsic::aarch64_neon_ld1x3;
6838 break;
6839 case Intrinsic::aarch64_neon_ld1x4:
6840 case Intrinsic::aarch64_neon_st1x4:
6841 Info.MatchingId = Intrinsic::aarch64_neon_ld1x4;
6842 break;
6843 case Intrinsic::aarch64_neon_ld2:
6844 case Intrinsic::aarch64_neon_st2:
6845 Info.MatchingId = Intrinsic::aarch64_neon_ld2;
6846 break;
6847 case Intrinsic::aarch64_neon_ld3:
6848 case Intrinsic::aarch64_neon_st3:
6849 Info.MatchingId = Intrinsic::aarch64_neon_ld3;
6850 break;
6851 case Intrinsic::aarch64_neon_ld4:
6852 case Intrinsic::aarch64_neon_st4:
6853 Info.MatchingId = Intrinsic::aarch64_neon_ld4;
6854 break;
6855 }
6856 return true;
6857}
6858
6859/// See if \p I should be considered for address type promotion. We check if \p
6860/// I is a sext with right type and used in memory accesses. If it used in a
6861/// "complex" getelementptr, we allow it to be promoted without finding other
6862/// sext instructions that sign extended the same initial value. A getelementptr
6863/// is considered as "complex" if it has more than 2 operands.
6865 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
6866 bool Considerable = false;
6867 AllowPromotionWithoutCommonHeader = false;
6868 if (!isa<SExtInst>(&I))
6869 return false;
6870 Type *ConsideredSExtType =
6871 Type::getInt64Ty(I.getParent()->getParent()->getContext());
6872 if (I.getType() != ConsideredSExtType)
6873 return false;
6874 // See if the sext is the one with the right type and used in at least one
6875 // GetElementPtrInst.
6876 for (const User *U : I.users()) {
6877 if (const GetElementPtrInst *GEPInst = dyn_cast<GetElementPtrInst>(U)) {
6878 Considerable = true;
6879 // A getelementptr is considered as "complex" if it has more than 2
6880 // operands. We will promote a SExt used in such complex GEP as we
6881 // expect some computation to be merged if they are done on 64 bits.
6882 if (GEPInst->getNumOperands() > 2) {
6883 AllowPromotionWithoutCommonHeader = true;
6884 break;
6885 }
6886 }
6887 }
6888 return Considerable;
6889}
6890
6892 const RecurrenceDescriptor &RdxDesc, ElementCount VF) const {
6893 if (!VF.isScalable())
6894 return true;
6895
6896 Type *Ty = RdxDesc.getRecurrenceType();
6897 if (Ty->isBFloatTy() || !isElementTypeLegalForScalableVector(Ty))
6898 return false;
6899
6900 switch (RdxDesc.getRecurrenceKind()) {
6901 case RecurKind::Sub:
6902 case RecurKind::FSub:
6905 case RecurKind::Add:
6906 case RecurKind::FAdd:
6907 case RecurKind::And:
6908 case RecurKind::Or:
6909 case RecurKind::Xor:
6910 case RecurKind::SMin:
6911 case RecurKind::SMax:
6912 case RecurKind::UMin:
6913 case RecurKind::UMax:
6914 case RecurKind::FMin:
6915 case RecurKind::FMax:
6916 case RecurKind::FMulAdd:
6917 case RecurKind::AnyOf:
6919 return true;
6920 default:
6921 return false;
6922 }
6923}
6924
6927 FastMathFlags FMF,
6928 TTI::TargetCostKind CostKind) const {
6929 // The code-generator is currently not able to handle scalable vectors
6930 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
6931 // it. This change will be removed when code-generation for these types is
6932 // sufficiently reliable.
6933 if (auto *VTy = dyn_cast<ScalableVectorType>(Ty))
6934 if (VTy->getElementCount() == ElementCount::getScalable(1))
6936
6937 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
6938
6939 if (LT.second.getScalarType() == MVT::f16 && !ST->hasFullFP16())
6940 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
6941
6942 InstructionCost LegalizationCost = 0;
6943 if (LT.first > 1) {
6944 Type *LegalVTy = EVT(LT.second).getTypeForEVT(Ty->getContext());
6945 IntrinsicCostAttributes Attrs(IID, LegalVTy, {LegalVTy, LegalVTy}, FMF);
6946 LegalizationCost = getIntrinsicInstrCost(Attrs, CostKind) * (LT.first - 1);
6947 }
6948
6949 return LegalizationCost + /*Cost of horizontal reduction*/ 2;
6950}
6951
6953 unsigned Opcode, VectorType *ValTy, TTI::TargetCostKind CostKind) const {
6954 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
6955 InstructionCost LegalizationCost = 0;
6956 if (LT.first > 1) {
6957 Type *LegalVTy = EVT(LT.second).getTypeForEVT(ValTy->getContext());
6958 LegalizationCost = getArithmeticInstrCost(Opcode, LegalVTy, CostKind);
6959 LegalizationCost *= LT.first - 1;
6960 }
6961
6962 int ISD = TLI->InstructionOpcodeToISD(Opcode);
6963 assert(ISD && "Invalid opcode");
6964 // Add the final reduction cost for the legal horizontal reduction
6965 switch (ISD) {
6966 case ISD::ADD:
6967 case ISD::AND:
6968 case ISD::OR:
6969 case ISD::XOR:
6970 case ISD::FADD:
6971 return LegalizationCost + 2;
6972 default:
6974 }
6975}
6976
6979 std::optional<FastMathFlags> FMF,
6980 TTI::TargetCostKind CostKind) const {
6981 // The code-generator is currently not able to handle scalable vectors
6982 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
6983 // it. This change will be removed when code-generation for these types is
6984 // sufficiently reliable.
6985 if (auto *VTy = dyn_cast<ScalableVectorType>(ValTy))
6986 if (VTy->getElementCount() == ElementCount::getScalable(1))
6988
6990 if (auto *FixedVTy = dyn_cast<FixedVectorType>(ValTy)) {
6991 InstructionCost BaseCost =
6992 BaseT::getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind);
6993 // Add on extra cost to reflect the extra overhead on some CPUs. We still
6994 // end up vectorizing for more computationally intensive loops.
6995 return BaseCost + FixedVTy->getNumElements();
6996 }
6997
6998 if (Opcode != Instruction::FAdd || ValTy->getElementType()->isBFloatTy())
7000
7001 auto *VTy = cast<ScalableVectorType>(ValTy);
7003 getArithmeticInstrCost(Opcode, VTy->getScalarType(), CostKind);
7004 Cost *= getMaxNumElements(VTy->getElementCount());
7005 return Cost;
7006 }
7007
7008 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
7009 MVT MTy = LT.second;
7010
7011 if (isa<ScalableVectorType>(ValTy) || TLI->useSVEForFixedLengthVectorVT(MTy))
7012 return getArithmeticReductionCostSVE(Opcode, ValTy, CostKind);
7013
7014 int ISD = TLI->InstructionOpcodeToISD(Opcode);
7015 assert(ISD && "Invalid opcode");
7016
7017 // Horizontal adds can use the 'addv' instruction. We model the cost of these
7018 // instructions as twice a normal vector add, plus 1 for each legalization
7019 // step (LT.first). This is the only arithmetic vector reduction operation for
7020 // which we have an instruction.
7021 // OR, XOR and AND costs should match the codegen from:
7022 // OR: llvm/test/CodeGen/AArch64/reduce-or.ll
7023 // XOR: llvm/test/CodeGen/AArch64/reduce-xor.ll
7024 // AND: llvm/test/CodeGen/AArch64/reduce-and.ll
7025 static const CostTblEntry CostTblNoPairwise[]{
7026 {ISD::ADD, MVT::v8i8, 2},
7027 {ISD::ADD, MVT::v16i8, 2},
7028 {ISD::ADD, MVT::v4i16, 2},
7029 {ISD::ADD, MVT::v8i16, 2},
7030 {ISD::ADD, MVT::v2i32, 2},
7031 {ISD::ADD, MVT::v4i32, 2},
7032 {ISD::ADD, MVT::v2i64, 2},
7033 {ISD::OR, MVT::v8i8, 5}, // fmov + orr_lsr + orr_lsr + lsr + orr
7034 {ISD::OR, MVT::v16i8, 7}, // ext + orr + same as v8i8
7035 {ISD::OR, MVT::v4i16, 4}, // fmov + orr_lsr + lsr + orr
7036 {ISD::OR, MVT::v8i16, 6}, // ext + orr + same as v4i16
7037 {ISD::OR, MVT::v2i32, 3}, // fmov + lsr + orr
7038 {ISD::OR, MVT::v4i32, 5}, // ext + orr + same as v2i32
7039 {ISD::OR, MVT::v2i64, 3}, // ext + orr + fmov
7040 {ISD::XOR, MVT::v8i8, 5}, // Same as above for or...
7041 {ISD::XOR, MVT::v16i8, 7},
7042 {ISD::XOR, MVT::v4i16, 4},
7043 {ISD::XOR, MVT::v8i16, 6},
7044 {ISD::XOR, MVT::v2i32, 3},
7045 {ISD::XOR, MVT::v4i32, 5},
7046 {ISD::XOR, MVT::v2i64, 3},
7047 {ISD::AND, MVT::v8i8, 5}, // Same as above for or...
7048 {ISD::AND, MVT::v16i8, 7},
7049 {ISD::AND, MVT::v4i16, 4},
7050 {ISD::AND, MVT::v8i16, 6},
7051 {ISD::AND, MVT::v2i32, 3},
7052 {ISD::AND, MVT::v4i32, 5},
7053 {ISD::AND, MVT::v2i64, 3},
7054 };
7055 switch (ISD) {
7056 default:
7057 break;
7058 case ISD::FADD:
7059 if (Type *EltTy = ValTy->getScalarType();
7060 // FIXME: For half types without fullfp16 support, this could extend and
7061 // use a fp32 faddp reduction but current codegen unrolls.
7062 MTy.isVector() && (EltTy->isFloatTy() || EltTy->isDoubleTy() ||
7063 (EltTy->isHalfTy() && ST->hasFullFP16()))) {
7064 const unsigned NElts = MTy.getVectorNumElements();
7065 if (ValTy->getElementCount().getFixedValue() >= 2 && NElts >= 2 &&
7066 isPowerOf2_32(NElts))
7067 // Reduction corresponding to series of fadd instructions is lowered to
7068 // series of faddp instructions. faddp has latency/throughput that
7069 // matches fadd instruction and hence, every faddp instruction can be
7070 // considered to have a relative cost = 1 with
7071 // CostKind = TCK_RecipThroughput.
7072 // An faddp will pairwise add vector elements, so the size of input
7073 // vector reduces by half every time, requiring
7074 // #(faddp instructions) = log2_32(NElts).
7075 return (LT.first - 1) + /*No of faddp instructions*/ Log2_32(NElts);
7076 }
7077 break;
7078 case ISD::ADD:
7079 if (const auto *Entry = CostTableLookup(CostTblNoPairwise, ISD, MTy))
7080 return (LT.first - 1) + Entry->Cost;
7081 break;
7082 case ISD::XOR:
7083 case ISD::AND:
7084 case ISD::OR:
7085 const auto *Entry = CostTableLookup(CostTblNoPairwise, ISD, MTy);
7086 if (!Entry)
7087 break;
7088 auto *ValVTy = cast<FixedVectorType>(ValTy);
7089 // A non-power-of-2 i1 vector is widened with the identity and reduced as
7090 // the promoted integer vector, the i1 min/max lowering does not apply. The
7091 // zero identity of or/xor is set by a single masking and.
7092 if (ValTy->isIntOrIntVectorTy(1) &&
7093 !isPowerOf2_32(ValVTy->getNumElements()) && LT.first == 1) {
7094 unsigned PaddedElts = MTy.getVectorNumElements();
7095 auto *PaddedTy =
7096 FixedVectorType::get(ValVTy->getElementType(), PaddedElts);
7097 if (ISD != ISD::AND)
7098 return Entry->Cost +
7099 getArithmeticInstrCost(Instruction::And, PaddedTy, CostKind);
7100 return Entry->Cost +
7102 PaddedTy,
7103 APInt::getBitsSetFrom(PaddedElts, ValVTy->getNumElements()),
7104 /*Insert=*/true, /*Extract=*/false, CostKind);
7105 }
7106 if (MTy.getVectorNumElements() <= ValVTy->getNumElements() &&
7107 isPowerOf2_32(ValVTy->getNumElements())) {
7108 InstructionCost ExtraCost = 0;
7109 if (LT.first != 1) {
7110 // Type needs to be split, so there is an extra cost of LT.first - 1
7111 // arithmetic ops.
7112 auto *Ty = FixedVectorType::get(ValTy->getElementType(),
7113 MTy.getVectorNumElements());
7114 ExtraCost = getArithmeticInstrCost(Opcode, Ty, CostKind);
7115 ExtraCost *= LT.first - 1;
7116 }
7117 // All and/or/xor of i1 will be lowered with maxv/minv/addv + fmov
7118 auto Cost = ValVTy->getElementType()->isIntegerTy(1) ? 2 : Entry->Cost;
7119 return Cost + ExtraCost;
7120 }
7121 break;
7122 }
7123 return BaseT::getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind);
7124}
7125
7127 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *VecTy,
7128 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
7129 EVT VecVT = TLI->getValueType(DL, VecTy);
7130 EVT ResVT = TLI->getValueType(DL, ResTy);
7131
7132 if (IsUnsigned && Opcode == Instruction::Add &&
7133 VecVT.isScalableVectorOf(MVT::i1) && ResVT.isSimple() &&
7134 getTLI()->isTypeLegal(VecVT)) {
7135 // This is vecreduce.add(ZExt(mask)) which can lower as cntp(mask).
7136 return 1;
7137 }
7138
7139 if (Opcode == Instruction::Add && VecVT.isSimple() && ResVT.isSimple() &&
7140 !VecVT.isScalableVT() && VecVT.getFixedSizeInBits() >= 64) {
7141 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(VecTy);
7142
7143 // The legal cases are:
7144 // UADDLV 8/16/32->32
7145 // UADDLP 32->64
7146 unsigned RevVTSize = ResVT.getSizeInBits();
7147 if (((LT.second == MVT::v8i8 || LT.second == MVT::v16i8) &&
7148 RevVTSize <= 32) ||
7149 ((LT.second == MVT::v4i16 || LT.second == MVT::v8i16) &&
7150 RevVTSize <= 32) ||
7151 ((LT.second == MVT::v2i32 || LT.second == MVT::v4i32) &&
7152 RevVTSize <= 64))
7153 return (LT.first - 1) * 2 + 2;
7154 }
7155
7156 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, VecTy, FMF,
7157 CostKind);
7158}
7159
7161AArch64TTIImpl::getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode,
7162 Type *ResTy, VectorType *VecTy,
7163 TTI::TargetCostKind CostKind) const {
7164 EVT VecVT = TLI->getValueType(DL, VecTy);
7165 EVT ResVT = TLI->getValueType(DL, ResTy);
7166
7167 if (ST->hasDotProd() && VecVT.isSimple() && ResVT.isSimple() &&
7168 RedOpcode == Instruction::Add) {
7169 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(VecTy);
7170
7171 // The legal cases with dotprod are
7172 // UDOT 8->32
7173 // Which requires an additional uaddv to sum the i32 values.
7174 if ((LT.second == MVT::v8i8 || LT.second == MVT::v16i8) &&
7175 ResVT == MVT::i32)
7176 return LT.first + 2;
7177 }
7178
7179 return BaseT::getMulAccReductionCost(IsUnsigned, RedOpcode, ResTy, VecTy,
7180 CostKind);
7181}
7182
7185 TTI::TargetCostKind CostKind) const {
7186 static const CostTblEntry ShuffleTbl[] = {
7187 { TTI::SK_Splice, MVT::nxv16i8, 1 },
7188 { TTI::SK_Splice, MVT::nxv8i16, 1 },
7189 { TTI::SK_Splice, MVT::nxv4i32, 1 },
7190 { TTI::SK_Splice, MVT::nxv2i64, 1 },
7191 { TTI::SK_Splice, MVT::nxv2f16, 1 },
7192 { TTI::SK_Splice, MVT::nxv4f16, 1 },
7193 { TTI::SK_Splice, MVT::nxv8f16, 1 },
7194 { TTI::SK_Splice, MVT::nxv2bf16, 1 },
7195 { TTI::SK_Splice, MVT::nxv4bf16, 1 },
7196 { TTI::SK_Splice, MVT::nxv8bf16, 1 },
7197 { TTI::SK_Splice, MVT::nxv2f32, 1 },
7198 { TTI::SK_Splice, MVT::nxv4f32, 1 },
7199 { TTI::SK_Splice, MVT::nxv2f64, 1 },
7200 };
7201
7202 // The code-generator is currently not able to handle scalable vectors
7203 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
7204 // it. This change will be removed when code-generation for these types is
7205 // sufficiently reliable.
7208
7209 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp);
7210 Type *LegalVTy = EVT(LT.second).getTypeForEVT(Tp->getContext());
7211 EVT PromotedVT = LT.second.getScalarType() == MVT::i1
7212 ? TLI->getPromotedVTForPredicate(EVT(LT.second))
7213 : LT.second;
7214 Type *PromotedVTy = EVT(PromotedVT).getTypeForEVT(Tp->getContext());
7215 InstructionCost LegalizationCost = 0;
7216 if (Index < 0) {
7217 LegalizationCost =
7218 getCmpSelInstrCost(Instruction::ICmp, PromotedVTy, PromotedVTy,
7219 CmpInst::BAD_ICMP_PREDICATE, CostKind) +
7220 getCmpSelInstrCost(Instruction::Select, PromotedVTy, LegalVTy,
7221 CmpInst::BAD_ICMP_PREDICATE, CostKind);
7222 }
7223
7224 // Predicated splice are promoted when lowering. See AArch64ISelLowering.cpp
7225 // Cost performed on a promoted type.
7226 if (LT.second.getScalarType() == MVT::i1) {
7227 LegalizationCost +=
7228 getCastInstrCost(Instruction::ZExt, PromotedVTy, LegalVTy,
7229 TTI::CastContextHint::None, CostKind) +
7230 getCastInstrCost(Instruction::Trunc, LegalVTy, PromotedVTy,
7231 TTI::CastContextHint::None, CostKind);
7232 }
7233
7234 const auto *Entry =
7235 CostTableLookup(ShuffleTbl, TTI::SK_Splice, PromotedVT.getSimpleVT());
7236 if (!Entry)
7238
7239 LegalizationCost += Entry->Cost;
7240 return LegalizationCost * LT.first;
7241}
7242
7244 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
7246 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
7247 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
7249
7250 if (CostKind != TTI::TCK_RecipThroughput)
7251 return Invalid;
7252
7253 if ((Opcode != Instruction::Add && Opcode != Instruction::Sub &&
7254 Opcode != Instruction::FAdd && Opcode != Instruction::FSub))
7255 return Invalid;
7256
7257 // Floating-point partial reductions are invalid if `reassoc` and `contract`
7258 // are not allowed.
7259 if (AccumType->isFloatingPointTy()) {
7260 assert(FMF && "Missing FastMathFlags for floating-point partial reduction");
7261 if (!FMF->allowReassoc() || !FMF->allowContract())
7262 return Invalid;
7263 } else {
7264 assert(!FMF &&
7265 "FastMathFlags only apply to floating-point partial reductions");
7266 }
7267
7268 assert((BinOp || (OpBExtend == TTI::PR_None && !InputTypeB)) &&
7269 (!BinOp || (OpBExtend != TTI::PR_None && InputTypeB)) &&
7270 "Unexpected values for OpBExtend or InputTypeB");
7271
7272 // We only support multiply binary operations for now, and for muls we
7273 // require the types being extended to be the same.
7274 if (BinOp && ((*BinOp != Instruction::Mul && *BinOp != Instruction::FMul) ||
7275 InputTypeA != InputTypeB))
7276 return Invalid;
7277
7278 // We only support the following element sizes.
7279 if (!is_contained({8u, 16u, 32u, 64u}, AccumType->getScalarSizeInBits()))
7280 return Invalid;
7281
7282 // If none of the operands are extended and there's no extra BinOp, just
7283 // cost this as the equivalent arithmetic instruction.
7284 // TODO: Depending on VF and element type, we may be able to improve on this.
7285 if (!OpAExtend) {
7286 assert(!OpBExtend && "Extended second operand without extended first.");
7287 assert(InputTypeA == AccumType && "Type mismatch with no extensions.");
7288
7289 VectorType *VTy = VectorType::get(AccumType, VF);
7290 return getArithmeticInstrCost(Opcode, VTy, CostKind);
7291 }
7292
7293 bool IsUSDot = OpBExtend != TTI::PR_None && OpAExtend != OpBExtend;
7294 // USDot is natively supported with +i8mm. With plain +dotprod, SUMLA is
7295 // lowered to two udots plus an eor and a sub.
7296 if (IsUSDot && !ST->hasMatMulInt8() && !ST->hasDotProd())
7297 // FIXME: Remove this early bailout in favour of expand cost.
7298 return Invalid;
7299
7300 unsigned Ratio =
7301 AccumType->getScalarSizeInBits() / InputTypeA->getScalarSizeInBits();
7302 if (VF.getKnownMinValue() <= Ratio)
7303 return Invalid;
7304
7305 VectorType *InputVectorType = VectorType::get(InputTypeA, VF);
7306 VectorType *AccumVectorType =
7307 VectorType::get(AccumType, VF.divideCoefficientBy(Ratio));
7308 // We don't yet support all kinds of legalization.
7309 auto TC = TLI->getTypeConversion(AccumVectorType->getContext(),
7310 EVT::getEVT(AccumVectorType));
7311 switch (TC.first) {
7312 default:
7313 return Invalid;
7317 // The legalised type (e.g. after splitting) must be legal too.
7318 if (TLI->getTypeAction(AccumVectorType->getContext(), TC.second) !=
7320 return Invalid;
7321 break;
7322 }
7323
7324 std::pair<InstructionCost, MVT> AccumLT =
7325 getTypeLegalizationCost(AccumVectorType);
7326 std::pair<InstructionCost, MVT> InputLT =
7327 getTypeLegalizationCost(InputVectorType);
7328
7329 // Returns true if the subtarget supports the operation for a given type.
7330 auto IsSupported = [&](bool SVEPred, bool NEONPred) -> bool {
7331 return (ST->isSVEorStreamingSVEAvailable() && SVEPred) ||
7332 (AccumLT.second.isFixedLengthVector() &&
7333 AccumLT.second.getSizeInBits() <= 128 && ST->isNeonAvailable() &&
7334 NEONPred);
7335 };
7336
7337 bool IsSub = Opcode == Instruction::Sub || Opcode == Instruction::FSub;
7338 InstructionCost Cost = InputLT.first * TTI::TCC_Basic;
7339 // Integer partial sub-reductions that don't map to a specific instruction,
7340 // carry an extra cost for implementing a double negation:
7341 // partial_reduce_umls acc, lhs, rhs
7342 // <=> -partial_reduce_umla -acc, lhs, rhs
7343 InstructionCost INegCost = IsSub ? 2 * InputLT.first * TTI::TCC_Basic : 0;
7344
7345 if (AccumLT.second.getScalarType() == MVT::i32 &&
7346 InputLT.second.getScalarType() == MVT::i8) {
7347 // i8 -> i32 is natively supported with udot/sdot for both NEON and SVE.
7348 if (!IsUSDot && IsSupported(true, ST->hasDotProd()))
7349 return Cost + INegCost;
7350 // i8 -> i32 usdot requires +i8mm
7351 if (IsUSDot && IsSupported(ST->hasMatMulInt8(), ST->hasMatMulInt8()))
7352 return Cost + INegCost;
7353 // Without +i8mm, lower SUMLA via two udots plus an eor and a sub on plain
7354 // +dotprod targets. Note that this is only implemented for NEON, as all
7355 // modern CPUs with SVE also have +i8mm. Charge an extra factor for the
7356 // expansion.
7357 if (IsUSDot && IsSupported(false, ST->hasDotProd()))
7358 return Cost * 3 + INegCost;
7359 }
7360
7361 if (ST->isSVEorStreamingSVEAvailable() && !IsUSDot) {
7362 // i16 -> i64 is natively supported for udot/sdot
7363 if (AccumLT.second.getScalarType() == MVT::i64 &&
7364 InputLT.second.getScalarType() == MVT::i16)
7365 return Cost + INegCost;
7366 // i16 -> i32 is natively supported with SVE2p1 udot/sdot.
7367 // For sub-reductions, we prefer using the *mlslb/t instructions.
7368 if (AccumLT.second.getScalarType() == MVT::i32 &&
7369 InputLT.second.getScalarType() == MVT::i16 &&
7370 (ST->hasSVE2p1() || ST->hasSME2()) && !IsSub)
7371 return Cost;
7372 // i8 -> i64 is supported with an extra level of extends
7373 if (AccumLT.second.getScalarType() == MVT::i64 &&
7374 InputLT.second.getScalarType() == MVT::i8)
7375 // FIXME: Also account for folding the i32 dot product results into the
7376 // i64 accumulator, as the NEON case below does.
7377 return Cost + INegCost;
7378 // i8 -> i16 is natively supported with SVE2p3 udot/sdot
7379 // For sub-reductions, we prefer using the *mlslb/t instructions.
7380 if (AccumLT.second.getScalarType() == MVT::i16 &&
7381 InputLT.second.getScalarType() == MVT::i8 &&
7382 (ST->hasSVE2p3() || ST->hasSME2p3()) && !IsSub)
7383 return Cost;
7384 }
7385
7386 // With +dotprod, NEON lowers an extended i8 -> i64 partial reduction to a
7387 // udot/sdot into a zeroed i32 vector followed by an [SU]ADALP into the i64
7388 // accumulator. Charge one basic cost for each of the two steps per legal
7389 // input vector. This must stay after the SVE cases, as IsSupported's NEON
7390 // predicate also holds for fixed-length vectors on SVE targets.
7391 // TODO: Handle zero-extended multiplies and sub-reductions. In loops,
7392 // CodeGenPrepare turns zero extends that do not directly feed the partial
7393 // reduction (such as the operands of a multiply, or of the negation in a
7394 // sub-reduction) into TBLs, which hides the udot from instruction selection.
7395 // Sign-extended sub-reductions have not been measured yet.
7396 // TODO: Without +dotprod, pure adds with a ratio of 4 or 8 lower to a chain
7397 // of [SU]ADDLPs and an [SU]ADALP, which is not costed yet either.
7398 bool IsSExtMul = BinOp && OpAExtend == TTI::PR_SignExtend &&
7399 OpBExtend == TTI::PR_SignExtend;
7400 if ((!BinOp || IsSExtMul) && !IsSub &&
7401 AccumLT.second.getScalarType() == MVT::i64 &&
7402 InputLT.second.getScalarType() == MVT::i8 &&
7403 IsSupported(false, ST->hasDotProd()))
7404 return Cost * 2;
7405
7406 // f16 -> f32 is natively supported for fdot using either
7407 // SVE or NEON instruction.
7408 if (Opcode == Instruction::FAdd && !IsSub &&
7409 IsSupported(ST->hasSME2() || ST->hasSVE2p1(), ST->hasF16F32DOT()) &&
7410 AccumLT.second.getScalarType() == MVT::f32 &&
7411 InputLT.second.getScalarType() == MVT::f16)
7412 return Cost;
7413
7414 // For a ratio of 2, we can use *mlal and *mlsl top/bottom instructions.
7415 if (Ratio == 2 && !IsUSDot) {
7416 MVT InVT = InputLT.second.getScalarType();
7417
7418 // SVE2 [us]ml[as]lb/t and NEON [us]ml[as]l(2). A pure widening add with a
7419 // ratio of 2 can use [SU]ADALP instead.
7420 if (IsSupported(ST->hasSVE2() || ST->hasSME(), true) &&
7421 llvm::is_contained({MVT::i8, MVT::i16, MVT::i32}, InVT.SimpleTy))
7422 return (BinOp || IsSub) ? Cost * 2 : Cost;
7423
7424 // SVE2 fml[as]lb/t and NEON fml[as]l(2)
7425 if (IsSupported(ST->hasSVE2(), ST->hasFP16FML()) && InVT == MVT::f16)
7426 return Cost * 2;
7427
7428 // SME2/SVE2p1 bfmlslb/t
7429 if (IsSupported(ST->hasSVE2p1() || ST->hasSME2(), false) &&
7430 InVT == MVT::bf16 && IsSub)
7431 return Cost * 2;
7432
7433 // FP partial sub-reductions that don't map to a specific instruction,
7434 // carry an extra cost for implementing an extra negation:
7435 // partial_reduce_fmls acc, lhs, rhs
7436 // <=> partial_reduce_fmla acc, lhs, -rhs
7437 InstructionCost FNegCost = IsSub ? InputLT.first * TTI::TCC_Basic : 0;
7438
7439 // SVE and NEON bfmlalb/t
7440 if (IsSupported(ST->hasBF16(), ST->hasBF16()) && InVT == MVT::bf16)
7441 return Cost * 2 + FNegCost;
7442 }
7443
7444 return BaseT::getPartialReductionCost(Opcode, InputTypeA, InputTypeB,
7445 AccumType, VF, OpAExtend, OpBExtend,
7446 BinOp, CostKind, FMF);
7447}
7448
7450 TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
7451 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
7452 VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
7453 TTI::VectorInstrContext VIC) const {
7454 assert((Mask.empty() || DstTy->isScalableTy() ||
7455 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
7456 "Expected the Mask to match the return size if given");
7457 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
7458 "Expected the same scalar types");
7459 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(SrcTy);
7460
7461 // If we have a Mask, and the LT is being legalized somehow, split the Mask
7462 // into smaller vectors and sum the cost of each shuffle.
7463 if (!Mask.empty() && isa<FixedVectorType>(SrcTy) && LT.second.isVector() &&
7464 LT.second.getScalarSizeInBits() * Mask.size() > 128 &&
7465 SrcTy->getScalarSizeInBits() == LT.second.getScalarSizeInBits() &&
7466 Mask.size() > LT.second.getVectorNumElements() && !Index && !SubTp) {
7467 // Check for LD3/LD4 instructions, which are represented in llvm IR as
7468 // deinterleaving-shuffle(load). The shuffle cost could potentially be free,
7469 // but we model it with a cost of LT.first so that LD3/LD4 have a higher
7470 // cost than just the load.
7471 if (Args.size() >= 1 && isa<LoadInst>(Args[0]) &&
7474 return std::max<InstructionCost>(1, LT.first / 4);
7475
7476 // Check for ST3/ST4 instructions, which are represented in llvm IR as
7477 // store(interleaving-shuffle). The shuffle cost could potentially be free,
7478 // but we model it with a cost of LT.first so that ST3/ST4 have a higher
7479 // cost than just the store.
7480 if (CtxI && CtxI->hasOneUse() && isa<StoreInst>(*CtxI->user_begin()) &&
7482 Mask, 4, SrcTy->getElementCount().getKnownMinValue() * 2) ||
7484 Mask, 3, SrcTy->getElementCount().getKnownMinValue() * 2)))
7485 return LT.first;
7486
7487 unsigned TpNumElts = Mask.size();
7488 unsigned LTNumElts = LT.second.getVectorNumElements();
7489 unsigned NumVecs = (TpNumElts + LTNumElts - 1) / LTNumElts;
7490 VectorType *NTp = VectorType::get(SrcTy->getScalarType(),
7491 LT.second.getVectorElementCount());
7493 std::map<std::tuple<unsigned, unsigned, SmallVector<int>>, InstructionCost>
7494 PreviousCosts;
7495 for (unsigned N = 0; N < NumVecs; N++) {
7496 SmallVector<int> NMask;
7497 // Split the existing mask into chunks of size LTNumElts. Track the source
7498 // sub-vectors to ensure the result has at most 2 inputs.
7499 unsigned Source1 = -1U, Source2 = -1U;
7500 unsigned NumSources = 0;
7501 for (unsigned E = 0; E < LTNumElts; E++) {
7502 int MaskElt = (N * LTNumElts + E < TpNumElts) ? Mask[N * LTNumElts + E]
7504 if (MaskElt < 0) {
7506 continue;
7507 }
7508
7509 // Calculate which source from the input this comes from and whether it
7510 // is new to us.
7511 unsigned Source = MaskElt / LTNumElts;
7512 if (NumSources == 0) {
7513 Source1 = Source;
7514 NumSources = 1;
7515 } else if (NumSources == 1 && Source != Source1) {
7516 Source2 = Source;
7517 NumSources = 2;
7518 } else if (NumSources >= 2 && Source != Source1 && Source != Source2) {
7519 NumSources++;
7520 }
7521
7522 // Add to the new mask. For the NumSources>2 case these are not correct,
7523 // but are only used for the modular lane number.
7524 if (Source == Source1)
7525 NMask.push_back(MaskElt % LTNumElts);
7526 else if (Source == Source2)
7527 NMask.push_back(MaskElt % LTNumElts + LTNumElts);
7528 else
7529 NMask.push_back(MaskElt % LTNumElts);
7530 }
7531 // Check if we have already generated this sub-shuffle, which means we
7532 // will have already generated the output. For example a <16 x i32> splat
7533 // will be the same sub-splat 4 times, which only needs to be generated
7534 // once and reused.
7535 auto Result =
7536 PreviousCosts.insert({std::make_tuple(Source1, Source2, NMask), 0});
7537 // Check if it was already in the map (already costed).
7538 if (!Result.second)
7539 continue;
7540 // If the sub-mask has at most 2 input sub-vectors then re-cost it using
7541 // getShuffleCost. If not then cost it using the worst case as the number
7542 // of element moves into a new vector.
7543 InstructionCost NCost =
7544 NumSources <= 2
7545 ? getShuffleCost(NumSources <= 1 ? TTI::SK_PermuteSingleSrc
7547 NTp, NTp, CostKind, NMask, 0, nullptr, Args,
7548 CtxI)
7549 : LTNumElts;
7550 Result.first->second = NCost;
7551 Cost += NCost;
7552 }
7553 return Cost;
7554 }
7555
7556 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp);
7557 bool IsExtractSubvector = Kind == TTI::SK_ExtractSubvector;
7558 // A subvector extract can be implemented with a NEON/SVE ext (or trivial
7559 // extract, if from lane 0) for 128-bit NEON vectors or legal SVE vectors.
7560 // This currently only handles low or high extracts to prevent SLP vectorizer
7561 // regressions.
7562 // Note that SVE's ext instruction is destructive, but it can be fused with
7563 // a movprfx to act like a constructive instruction.
7564 if (IsExtractSubvector && LT.second.isFixedLengthVector()) {
7565 if (LT.second.getFixedSizeInBits() >= 128 &&
7566 cast<FixedVectorType>(SubTp)->getNumElements() ==
7567 LT.second.getVectorNumElements() / 2) {
7568 if (Index == 0)
7569 return 0;
7570 if (Index == (int)LT.second.getVectorNumElements() / 2)
7571 return 1;
7572 }
7574 }
7575 // FIXME: This was added to keep the costs equal when adding DstTys. Update
7576 // the code to handle length-changing shuffles.
7577 if (Kind == TTI::SK_InsertSubvector) {
7578 LT = getTypeLegalizationCost(DstTy);
7579 SrcTy = DstTy;
7580 }
7581
7582 // Check for identity masks, which we can treat as free for both fixed and
7583 // scalable vector paths.
7584 if (!Mask.empty() && LT.second.isFixedLengthVector() &&
7585 (Kind == TTI::SK_PermuteTwoSrc || Kind == TTI::SK_PermuteSingleSrc) &&
7586 all_of(enumerate(Mask), [](const auto &M) {
7587 return M.value() < 0 || M.value() == (int)M.index();
7588 }))
7589 return 0;
7590
7591 // Segmented shuffle matching.
7592 if (Kind == TTI::SK_PermuteSingleSrc && isa<FixedVectorType>(SrcTy) &&
7593 !Mask.empty() && SrcTy->getPrimitiveSizeInBits().isNonZero() &&
7594 SrcTy->getPrimitiveSizeInBits().isKnownMultipleOf(
7596
7598 unsigned Segments =
7600 unsigned SegmentElts = VTy->getNumElements() / Segments;
7601
7602 // dupq zd.t, zn.t[idx]
7603 if ((ST->hasSVE2p1() || ST->hasSME2p1()) &&
7604 ST->isSVEorStreamingSVEAvailable() &&
7605 isDUPQMask(Mask, Segments, SegmentElts))
7606 return LT.first;
7607
7608 // mov zd.q, vn
7609 if (ST->isSVEorStreamingSVEAvailable() &&
7610 isDUPFirstSegmentMask(Mask, Segments, SegmentElts))
7611 return LT.first;
7612 }
7613
7614 // Check for broadcast loads, which are supported by the LD1R instruction.
7615 // In terms of code-size, the shuffle vector is free when a load + dup get
7616 // folded into a LD1R. That's what we check and return here. For performance
7617 // and reciprocal throughput, a LD1R is not completely free. In this case, we
7618 // return the cost for the broadcast below (i.e. 1 for most/all types), so
7619 // that we model the load + dup sequence slightly higher because LD1R is a
7620 // high latency instruction.
7621 if (CostKind == TTI::TCK_CodeSize && Kind == TTI::SK_Broadcast) {
7622 bool IsLoad = !Args.empty() && isa<LoadInst>(Args[0]);
7623 if (IsLoad && LT.second.isVector() &&
7624 isLegalBroadcastLoad(SrcTy->getElementType(),
7625 LT.second.getVectorElementCount()))
7626 return 0;
7627 }
7628
7629 // If we have 4 elements for the shuffle and a Mask, get the cost straight
7630 // from the perfect shuffle tables.
7631 if (Mask.size() == 4 &&
7632 SrcTy->getElementCount() == ElementCount::getFixed(4) &&
7633 (SrcTy->getScalarSizeInBits() == 16 ||
7634 SrcTy->getScalarSizeInBits() == 32) &&
7635 all_of(Mask, [](int E) { return E < 8; }))
7636 return getPerfectShuffleCost(Mask);
7637
7638 // Check for other shuffles that are not SK_ kinds but we have native
7639 // instructions for, for example ZIP and UZP.
7640 unsigned Unused;
7641 if (LT.second.isFixedLengthVector() &&
7642 LT.second.getVectorNumElements() == Mask.size() &&
7643 (Kind == TTI::SK_PermuteTwoSrc || Kind == TTI::SK_PermuteSingleSrc ||
7644 // Discrepancies between isTRNMask and ShuffleVectorInst::isTransposeMask
7645 // mean that we can end up with shuffles that satisfy isTRNMask, but end
7646 // up labelled as TTI::SK_InsertSubvector. (e.g. {2, 0}).
7647 Kind == TTI::SK_InsertSubvector) &&
7648 (isZIPMask(Mask, LT.second.getVectorNumElements(), Unused, Unused) ||
7649 isTRNMask(Mask, LT.second.getVectorNumElements(), Unused, Unused) ||
7650 isUZPMask(Mask, LT.second.getVectorNumElements(), Unused) ||
7651 isREVMask(Mask, LT.second.getScalarSizeInBits(),
7652 LT.second.getVectorNumElements(), 16) ||
7653 isREVMask(Mask, LT.second.getScalarSizeInBits(),
7654 LT.second.getVectorNumElements(), 32) ||
7655 isREVMask(Mask, LT.second.getScalarSizeInBits(),
7656 LT.second.getVectorNumElements(), 64) ||
7657 // Check for non-zero lane splats
7658 all_of(drop_begin(Mask),
7659 [&Mask](int M) { return M < 0 || M == Mask[0]; })))
7660 return 1;
7661
7662 if (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Transpose ||
7663 Kind == TTI::SK_Select || Kind == TTI::SK_PermuteSingleSrc ||
7664 Kind == TTI::SK_Reverse || Kind == TTI::SK_Splice) {
7665 static const CostTblEntry ShuffleTbl[] = {
7666 // Broadcast shuffle kinds can be performed with 'dup'.
7667 {TTI::SK_Broadcast, MVT::v8i8, 1},
7668 {TTI::SK_Broadcast, MVT::v16i8, 1},
7669 {TTI::SK_Broadcast, MVT::v4i16, 1},
7670 {TTI::SK_Broadcast, MVT::v8i16, 1},
7671 {TTI::SK_Broadcast, MVT::v2i32, 1},
7672 {TTI::SK_Broadcast, MVT::v4i32, 1},
7673 {TTI::SK_Broadcast, MVT::v2i64, 1},
7674 {TTI::SK_Broadcast, MVT::v4f16, 1},
7675 {TTI::SK_Broadcast, MVT::v8f16, 1},
7676 {TTI::SK_Broadcast, MVT::v4bf16, 1},
7677 {TTI::SK_Broadcast, MVT::v8bf16, 1},
7678 {TTI::SK_Broadcast, MVT::v2f32, 1},
7679 {TTI::SK_Broadcast, MVT::v4f32, 1},
7680 {TTI::SK_Broadcast, MVT::v2f64, 1},
7681 // Transpose shuffle kinds can be performed with 'trn1/trn2' and
7682 // 'zip1/zip2' instructions.
7683 {TTI::SK_Transpose, MVT::v8i8, 1},
7684 {TTI::SK_Transpose, MVT::v16i8, 1},
7685 {TTI::SK_Transpose, MVT::v4i16, 1},
7686 {TTI::SK_Transpose, MVT::v8i16, 1},
7687 {TTI::SK_Transpose, MVT::v2i32, 1},
7688 {TTI::SK_Transpose, MVT::v4i32, 1},
7689 {TTI::SK_Transpose, MVT::v2i64, 1},
7690 {TTI::SK_Transpose, MVT::v4f16, 1},
7691 {TTI::SK_Transpose, MVT::v8f16, 1},
7692 {TTI::SK_Transpose, MVT::v4bf16, 1},
7693 {TTI::SK_Transpose, MVT::v8bf16, 1},
7694 {TTI::SK_Transpose, MVT::v2f32, 1},
7695 {TTI::SK_Transpose, MVT::v4f32, 1},
7696 {TTI::SK_Transpose, MVT::v2f64, 1},
7697 // Select shuffle kinds.
7698 // TODO: handle vXi8/vXi16.
7699 {TTI::SK_Select, MVT::v2i32, 1}, // mov.
7700 {TTI::SK_Select, MVT::v4i32, 2}, // rev+trn (or similar).
7701 {TTI::SK_Select, MVT::v2i64, 1}, // mov.
7702 {TTI::SK_Select, MVT::v2f32, 1}, // mov.
7703 {TTI::SK_Select, MVT::v4f32, 2}, // rev+trn (or similar).
7704 {TTI::SK_Select, MVT::v2f64, 1}, // mov.
7705 // PermuteSingleSrc shuffle kinds.
7706 {TTI::SK_PermuteSingleSrc, MVT::v2i32, 1}, // mov.
7707 {TTI::SK_PermuteSingleSrc, MVT::v4i32, 3}, // perfectshuffle worst case.
7708 {TTI::SK_PermuteSingleSrc, MVT::v2i64, 1}, // mov.
7709 {TTI::SK_PermuteSingleSrc, MVT::v2f32, 1}, // mov.
7710 {TTI::SK_PermuteSingleSrc, MVT::v4f32, 3}, // perfectshuffle worst case.
7711 {TTI::SK_PermuteSingleSrc, MVT::v2f64, 1}, // mov.
7712 {TTI::SK_PermuteSingleSrc, MVT::v4i16, 3}, // perfectshuffle worst case.
7713 {TTI::SK_PermuteSingleSrc, MVT::v4f16, 3}, // perfectshuffle worst case.
7714 {TTI::SK_PermuteSingleSrc, MVT::v4bf16, 3}, // same
7715 {TTI::SK_PermuteSingleSrc, MVT::v8i16, 8}, // constpool + load + tbl
7716 {TTI::SK_PermuteSingleSrc, MVT::v8f16, 8}, // constpool + load + tbl
7717 {TTI::SK_PermuteSingleSrc, MVT::v8bf16, 8}, // constpool + load + tbl
7718 {TTI::SK_PermuteSingleSrc, MVT::v8i8, 8}, // constpool + load + tbl
7719 {TTI::SK_PermuteSingleSrc, MVT::v16i8, 8}, // constpool + load + tbl
7720 // Reverse can be lowered with `rev`.
7721 {TTI::SK_Reverse, MVT::v2i32, 1}, // REV64
7722 {TTI::SK_Reverse, MVT::v4i32, 2}, // REV64; EXT
7723 {TTI::SK_Reverse, MVT::v2i64, 1}, // EXT
7724 {TTI::SK_Reverse, MVT::v2f32, 1}, // REV64
7725 {TTI::SK_Reverse, MVT::v4f32, 2}, // REV64; EXT
7726 {TTI::SK_Reverse, MVT::v2f64, 1}, // EXT
7727 {TTI::SK_Reverse, MVT::v8f16, 2}, // REV64; EXT
7728 {TTI::SK_Reverse, MVT::v8bf16, 2}, // REV64; EXT
7729 {TTI::SK_Reverse, MVT::v8i16, 2}, // REV64; EXT
7730 {TTI::SK_Reverse, MVT::v16i8, 2}, // REV64; EXT
7731 {TTI::SK_Reverse, MVT::v4f16, 1}, // REV64
7732 {TTI::SK_Reverse, MVT::v4bf16, 1}, // REV64
7733 {TTI::SK_Reverse, MVT::v4i16, 1}, // REV64
7734 {TTI::SK_Reverse, MVT::v8i8, 1}, // REV64
7735 // Splice can all be lowered as `ext`.
7736 {TTI::SK_Splice, MVT::v2i32, 1},
7737 {TTI::SK_Splice, MVT::v4i32, 1},
7738 {TTI::SK_Splice, MVT::v2i64, 1},
7739 {TTI::SK_Splice, MVT::v2f32, 1},
7740 {TTI::SK_Splice, MVT::v4f32, 1},
7741 {TTI::SK_Splice, MVT::v2f64, 1},
7742 {TTI::SK_Splice, MVT::v8f16, 1},
7743 {TTI::SK_Splice, MVT::v8bf16, 1},
7744 {TTI::SK_Splice, MVT::v8i16, 1},
7745 {TTI::SK_Splice, MVT::v16i8, 1},
7746 {TTI::SK_Splice, MVT::v4f16, 1},
7747 {TTI::SK_Splice, MVT::v4bf16, 1},
7748 {TTI::SK_Splice, MVT::v4i16, 1},
7749 {TTI::SK_Splice, MVT::v8i8, 1},
7750 // Broadcast shuffle kinds for scalable vectors
7751 {TTI::SK_Broadcast, MVT::nxv16i8, 1},
7752 {TTI::SK_Broadcast, MVT::nxv8i16, 1},
7753 {TTI::SK_Broadcast, MVT::nxv4i32, 1},
7754 {TTI::SK_Broadcast, MVT::nxv2i64, 1},
7755 {TTI::SK_Broadcast, MVT::nxv2f16, 1},
7756 {TTI::SK_Broadcast, MVT::nxv4f16, 1},
7757 {TTI::SK_Broadcast, MVT::nxv8f16, 1},
7758 {TTI::SK_Broadcast, MVT::nxv2bf16, 1},
7759 {TTI::SK_Broadcast, MVT::nxv4bf16, 1},
7760 {TTI::SK_Broadcast, MVT::nxv8bf16, 1},
7761 {TTI::SK_Broadcast, MVT::nxv2f32, 1},
7762 {TTI::SK_Broadcast, MVT::nxv4f32, 1},
7763 {TTI::SK_Broadcast, MVT::nxv2f64, 1},
7764 {TTI::SK_Broadcast, MVT::nxv16i1, 1},
7765 {TTI::SK_Broadcast, MVT::nxv8i1, 1},
7766 {TTI::SK_Broadcast, MVT::nxv4i1, 1},
7767 {TTI::SK_Broadcast, MVT::nxv2i1, 1},
7768 // Handle the cases for vector.reverse with scalable vectors
7769 {TTI::SK_Reverse, MVT::nxv16i8, 1},
7770 {TTI::SK_Reverse, MVT::nxv8i16, 1},
7771 {TTI::SK_Reverse, MVT::nxv4i32, 1},
7772 {TTI::SK_Reverse, MVT::nxv2i64, 1},
7773 {TTI::SK_Reverse, MVT::nxv2f16, 1},
7774 {TTI::SK_Reverse, MVT::nxv4f16, 1},
7775 {TTI::SK_Reverse, MVT::nxv8f16, 1},
7776 {TTI::SK_Reverse, MVT::nxv2bf16, 1},
7777 {TTI::SK_Reverse, MVT::nxv4bf16, 1},
7778 {TTI::SK_Reverse, MVT::nxv8bf16, 1},
7779 {TTI::SK_Reverse, MVT::nxv2f32, 1},
7780 {TTI::SK_Reverse, MVT::nxv4f32, 1},
7781 {TTI::SK_Reverse, MVT::nxv2f64, 1},
7782 {TTI::SK_Reverse, MVT::nxv16i1, 1},
7783 {TTI::SK_Reverse, MVT::nxv8i1, 1},
7784 {TTI::SK_Reverse, MVT::nxv4i1, 1},
7785 {TTI::SK_Reverse, MVT::nxv2i1, 1},
7786 };
7787 if (const auto *Entry = CostTableLookup(ShuffleTbl, Kind, LT.second))
7788 return LT.first * Entry->Cost;
7789 }
7790
7791 if (Kind == TTI::SK_Splice && isa<ScalableVectorType>(SrcTy))
7792 return getSpliceCost(SrcTy, Index, CostKind);
7793
7794 // Inserting a subvector can often be done with either a D, S or H register
7795 // move, so long as the inserted vector is "aligned".
7796 if (Kind == TTI::SK_InsertSubvector && LT.second.isFixedLengthVector() &&
7797 LT.second.getSizeInBits() <= 128 && SubTp) {
7798 std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(SubTp);
7799 if (SubLT.second.isVector()) {
7800 int NumElts = LT.second.getVectorNumElements();
7801 int NumSubElts = SubLT.second.getVectorNumElements();
7802 if ((Index % NumSubElts) == 0 && (NumElts % NumSubElts) == 0)
7803 return SubLT.first;
7804 }
7805 }
7806
7807 // Restore optimal kind.
7808 if (IsExtractSubvector)
7810 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index, SubTp,
7811 Args, CtxI);
7812}
7813
7816 const DominatorTree &DT) {
7817 const auto &Strides = SymbolicStrideMap();
7818 for (BasicBlock *BB : TheLoop->blocks()) {
7819 // Scan the instructions in the block and look for addresses that are
7820 // consecutive and decreasing.
7821 for (Instruction &I : *BB) {
7822 if (isa<LoadInst>(&I) || isa<StoreInst>(&I)) {
7824 Type *AccessTy = getLoadStoreType(&I);
7825 // Analyze assuming predicates will be added, but discard them; this
7826 // query only guides tail-folding and must not add runtime checks to the
7827 // loop.
7829 if (getPtrStride(*PSE, AccessTy, Ptr, TheLoop, DT, Strides,
7830 /*ShouldCheckWrap=*/false, &Predicates)
7831 .value_or(0) < 0)
7832 return true;
7833 }
7834 }
7835 }
7836 return false;
7837}
7838
7840 return valueOr(ST->getCLOpts().sve_prefer_fixed_over_scalable_if_equal,
7841 ST->useFixedOverScalableIfEqualCost());
7842}
7843
7845 return ST->getEpilogueVectorizationMinVF();
7846}
7847
7849 TailFoldingOpts DefaultOpts = ST->getSVETailFoldingDefaultOpts();
7850 if (!ST->hasSVE() || TailFoldingOptionLoc.isDisabled(DefaultOpts))
7851 return false;
7852
7853 // We don't currently support vectorisation with interleaving for SVE - with
7854 // such loops we're better off not using tail-folding. This gives us a chance
7855 // to fall back on fixed-width vectorisation using NEON's ld2/st2/etc.
7856 if (TFI->IAI->hasGroups())
7857 return false;
7858
7860 if (TFI->LVL->getReductionVars().size())
7861 Required |= TailFoldingOpts::Reductions;
7862 if (TFI->LVL->getFixedOrderRecurrences().size())
7863 Required |= TailFoldingOpts::Recurrences;
7864
7865 // We call this to discover whether any load/store pointers in the loop have
7866 // negative strides. This will require extra work to reverse the loop
7867 // predicate, which may be expensive.
7870 *TFI->LVL->getDominatorTree()))
7871 Required |= TailFoldingOpts::Reverse;
7872 if (Required == TailFoldingOpts::Disabled)
7873 Required |= TailFoldingOpts::Simple;
7874
7875 if (!TailFoldingOptionLoc.satisfies(DefaultOpts, Required))
7876 return false;
7877
7878 // Don't tail-fold for tight loops where we would be better off interleaving
7879 // with an unpredicated loop.
7880 unsigned NumInsns = 0;
7881 for (BasicBlock *BB : TFI->LVL->getLoop()->blocks()) {
7882 NumInsns += BB->size();
7883 }
7884
7885 // We expect 4 of these to be a IV PHI, IV add, IV compare and branch.
7886 return NumInsns >= ST->getCLOpts().sve_tail_folding_insn_threshold;
7887}
7888
7891 StackOffset BaseOffset, bool HasBaseReg,
7892 int64_t Scale, unsigned AddrSpace) const {
7893 // Scaling factors are not free at all.
7894 // Operands | Rt Latency
7895 // -------------------------------------------
7896 // Rt, [Xn, Xm] | 4
7897 // -------------------------------------------
7898 // Rt, [Xn, Xm, lsl #imm] | Rn: 4 Rm: 5
7899 // Rt, [Xn, Wm, <extend> #imm] |
7901 AM.BaseGV = BaseGV;
7902 AM.BaseOffs = BaseOffset.getFixed();
7903 AM.HasBaseReg = HasBaseReg;
7904 AM.Scale = Scale;
7905 AM.ScalableOffset = BaseOffset.getScalable();
7906 if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace))
7907 // Scale represents reg2 * scale, thus account for 1 if
7908 // it is not equal to 0 or 1.
7909 return AM.Scale != 0 && AM.Scale != 1;
7911}
7912
7914 const Instruction *I) const {
7915 if (ST->getCLOpts().enable_aarch64_or_like_select) {
7916 // For the binary operators (e.g. or) we need to be more careful than
7917 // selects, here we only transform them if they are already at a natural
7918 // break point in the code - the end of a block with an unconditional
7919 // terminator.
7920 if (I->getOpcode() == Instruction::Or &&
7921 isa<UncondBrInst>(I->getNextNode()))
7922 return true;
7923
7924 if (I->getOpcode() == Instruction::Add ||
7925 I->getOpcode() == Instruction::Sub)
7926 return true;
7927 }
7929}
7930
7933 const TargetTransformInfo::LSRCost &C2) const {
7934 // AArch64 specific here is adding the number of instructions to the
7935 // comparison (though not as the first consideration, as some targets do),
7936 // preferring lower number of pre/post-index if numregs and insns are equal,
7937 // as typically pre/post-index instructions use extra processor resources,
7938 // along with changing the priority of the base additions.
7939 // TODO: Maybe a more nuanced tradeoff between instruction count
7940 // and number of registers? To be investigated at a later date.
7941 if (ST->getCLOpts().enable_aarch64_lsr_cost_opt)
7942 return std::tie(C1.NumRegs, C1.Insns, C1.NumPrePostIndex, C1.NumBaseAdds,
7943 C1.AddRecCost, C1.NumIVMuls, C1.ScaleCost, C1.ImmCost,
7944 C1.SetupCost) <
7945 std::tie(C2.NumRegs, C2.Insns, C2.NumPrePostIndex, C2.NumBaseAdds,
7946 C2.AddRecCost, C2.NumIVMuls, C2.ScaleCost, C2.ImmCost,
7947 C2.SetupCost);
7948
7950}
7951
7952static bool isSplatShuffle(Value *V) {
7953 if (auto *Shuf = dyn_cast<ShuffleVectorInst>(V))
7954 return all_equal(Shuf->getShuffleMask());
7955 return false;
7956}
7957
7958/// Check if both Op1 and Op2 are shufflevector extracts of either the lower
7959/// or upper half of the vector elements.
7960static bool areExtractShuffleVectors(Value *Op1, Value *Op2,
7961 bool AllowSplat = false) {
7962 // Scalable types can't be extract shuffle vectors.
7963 if (Op1->getType()->isScalableTy() || Op2->getType()->isScalableTy())
7964 return false;
7965
7966 auto areTypesHalfed = [](Value *FullV, Value *HalfV) {
7967 auto *FullTy = FullV->getType();
7968 auto *HalfTy = HalfV->getType();
7969 return FullTy->getPrimitiveSizeInBits().getFixedValue() ==
7970 2 * HalfTy->getPrimitiveSizeInBits().getFixedValue();
7971 };
7972
7973 auto extractHalf = [](Value *FullV, Value *HalfV) {
7974 auto *FullVT = cast<FixedVectorType>(FullV->getType());
7975 auto *HalfVT = cast<FixedVectorType>(HalfV->getType());
7976 return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
7977 };
7978
7979 ArrayRef<int> M1, M2;
7980 Value *S1Op1 = nullptr, *S2Op1 = nullptr;
7981 if (!match(Op1, m_Shuffle(m_Value(S1Op1), m_Undef(), m_Mask(M1))) ||
7982 !match(Op2, m_Shuffle(m_Value(S2Op1), m_Undef(), m_Mask(M2))))
7983 return false;
7984
7985 // If we allow splats, set S1Op1/S2Op1 to nullptr for the relevant arg so that
7986 // it is not checked as an extract below.
7987 if (AllowSplat && isSplatShuffle(Op1))
7988 S1Op1 = nullptr;
7989 if (AllowSplat && isSplatShuffle(Op2))
7990 S2Op1 = nullptr;
7991
7992 // Check that the operands are half as wide as the result and we extract
7993 // half of the elements of the input vectors.
7994 if ((S1Op1 && (!areTypesHalfed(S1Op1, Op1) || !extractHalf(S1Op1, Op1))) ||
7995 (S2Op1 && (!areTypesHalfed(S2Op1, Op2) || !extractHalf(S2Op1, Op2))))
7996 return false;
7997
7998 // Check the mask extracts either the lower or upper half of vector
7999 // elements.
8000 int M1Start = 0;
8001 int M2Start = 0;
8002 int NumElements = cast<FixedVectorType>(Op1->getType())->getNumElements() * 2;
8003 if ((S1Op1 &&
8004 !ShuffleVectorInst::isExtractSubvectorMask(M1, NumElements, M1Start)) ||
8005 (S2Op1 &&
8006 !ShuffleVectorInst::isExtractSubvectorMask(M2, NumElements, M2Start)))
8007 return false;
8008
8009 if ((M1Start != 0 && M1Start != (NumElements / 2)) ||
8010 (M2Start != 0 && M2Start != (NumElements / 2)))
8011 return false;
8012 if (S1Op1 && S2Op1 && M1Start != M2Start)
8013 return false;
8014
8015 return true;
8016}
8017
8018/// Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth
8019/// of the vector elements.
8020static bool areExtractExts(Value *Ext1, Value *Ext2) {
8021 auto areExtDoubled = [](Instruction *Ext) {
8022 return Ext->getType()->getScalarSizeInBits() ==
8023 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
8024 };
8025
8026 if (!match(Ext1, m_ZExtOrSExt(m_Value())) ||
8027 !match(Ext2, m_ZExtOrSExt(m_Value())) ||
8028 !areExtDoubled(cast<Instruction>(Ext1)) ||
8029 !areExtDoubled(cast<Instruction>(Ext2)))
8030 return false;
8031
8032 return true;
8033}
8034
8035/// Check if Op could be used with vmull_high_p64 intrinsic.
8037 Value *VectorOperand = nullptr;
8038 ConstantInt *ElementIndex = nullptr;
8039 return match(Op, m_ExtractElt(m_Value(VectorOperand),
8040 m_ConstantInt(ElementIndex))) &&
8041 ElementIndex->getValue() == 1 &&
8042 isa<FixedVectorType>(VectorOperand->getType()) &&
8043 cast<FixedVectorType>(VectorOperand->getType())->getNumElements() == 2;
8044}
8045
8046/// Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
8047static bool areOperandsOfVmullHighP64(Value *Op1, Value *Op2) {
8049}
8050
8052 // Restrict ourselves to the form CodeGenPrepare typically constructs.
8053 auto *GEP = dyn_cast<GetElementPtrInst>(Ptrs);
8054 if (!GEP || GEP->getNumOperands() != 2)
8055 return false;
8056
8057 Value *Base = GEP->getOperand(0);
8058 Value *Offsets = GEP->getOperand(1);
8059
8060 // We only care about scalar_base+vector_offsets.
8061 if (Base->getType()->isVectorTy() || !Offsets->getType()->isVectorTy())
8062 return false;
8063
8064 // Sink extends that would allow us to use 32-bit offset vectors.
8065 if (isa<SExtInst>(Offsets) || isa<ZExtInst>(Offsets)) {
8066 auto *OffsetsInst = cast<Instruction>(Offsets);
8067 if (OffsetsInst->getType()->getScalarSizeInBits() > 32 &&
8068 OffsetsInst->getOperand(0)->getType()->getScalarSizeInBits() <= 32)
8069 Ops.push_back(&GEP->getOperandUse(1));
8070 }
8071
8072 // Sink the GEP.
8073 return true;
8074}
8075
8076/// We want to sink following cases:
8077/// (add|sub|gep) A, ((mul|shl) vscale, imm); (add|sub|gep) A, vscale;
8078/// (add|sub|gep) A, ((mul|shl) zext(vscale), imm);
8080 if (match(Op, m_VScale()))
8081 return true;
8082 if (match(Op, m_Shl(m_VScale(), m_ConstantInt())) ||
8084 Ops.push_back(&cast<Instruction>(Op)->getOperandUse(0));
8085 return true;
8086 }
8087 if (match(Op, m_Shl(m_ZExt(m_VScale()), m_ConstantInt())) ||
8089 Value *ZExtOp = cast<Instruction>(Op)->getOperand(0);
8090 Ops.push_back(&cast<Instruction>(ZExtOp)->getOperandUse(0));
8091 Ops.push_back(&cast<Instruction>(Op)->getOperandUse(0));
8092 return true;
8093 }
8094 return false;
8095}
8096
8097static bool isFNeg(Value *Op) {
8098 // TODO: Confirm whether m_FNegOrFSubNegZero is correct.
8099 return match(Op, m_FNegOrFSubNegZero(m_Value()));
8100}
8101
8102/// Check if sinking \p I's operands to I's basic block is profitable, because
8103/// the operands can be folded into a target instruction, e.g.
8104/// shufflevectors extracts and/or sext/zext can be folded into (u,s)subl(2).
8107 const LoopInfo *LI) const {
8109 switch (II->getIntrinsicID()) {
8110 case Intrinsic::aarch64_neon_smull:
8111 case Intrinsic::aarch64_neon_umull:
8112 if (areExtractShuffleVectors(II->getOperand(0), II->getOperand(1),
8113 /*AllowSplat=*/true)) {
8114 Ops.push_back(&II->getOperandUse(0));
8115 Ops.push_back(&II->getOperandUse(1));
8116 return true;
8117 }
8118 [[fallthrough]];
8119
8120 case Intrinsic::fma:
8121 case Intrinsic::fmuladd:
8122 if (isa<VectorType>(I->getType()) &&
8123 cast<VectorType>(I->getType())->getElementType()->isHalfTy() &&
8124 !ST->hasFullFP16())
8125 return false;
8126
8127 if (isFNeg(II->getOperand(0)))
8128 Ops.push_back(&II->getOperandUse(0));
8129 if (isFNeg(II->getOperand(1)))
8130 Ops.push_back(&II->getOperandUse(1));
8131
8132 [[fallthrough]];
8133 case Intrinsic::aarch64_neon_sqdmull:
8134 case Intrinsic::aarch64_neon_sqdmulh:
8135 case Intrinsic::aarch64_neon_sqrdmulh:
8136 // Sink splats for index lane variants
8137 if (isSplatShuffle(II->getOperand(0)))
8138 Ops.push_back(&II->getOperandUse(0));
8139 if (isSplatShuffle(II->getOperand(1)))
8140 Ops.push_back(&II->getOperandUse(1));
8141 return !Ops.empty();
8142 case Intrinsic::aarch64_neon_fmlal:
8143 case Intrinsic::aarch64_neon_fmlal2:
8144 case Intrinsic::aarch64_neon_fmlsl:
8145 case Intrinsic::aarch64_neon_fmlsl2:
8146 // Sink splats for index lane variants
8147 if (isSplatShuffle(II->getOperand(1)))
8148 Ops.push_back(&II->getOperandUse(1));
8149 if (isSplatShuffle(II->getOperand(2)))
8150 Ops.push_back(&II->getOperandUse(2));
8151 return !Ops.empty();
8152 case Intrinsic::aarch64_sve_ptest_first:
8153 case Intrinsic::aarch64_sve_ptest_last:
8154 if (auto *IIOp = dyn_cast<IntrinsicInst>(II->getOperand(0)))
8155 if (IIOp->getIntrinsicID() == Intrinsic::aarch64_sve_ptrue)
8156 Ops.push_back(&II->getOperandUse(0));
8157 return !Ops.empty();
8158 case Intrinsic::aarch64_sme_write_horiz:
8159 case Intrinsic::aarch64_sme_write_vert:
8160 case Intrinsic::aarch64_sme_writeq_horiz:
8161 case Intrinsic::aarch64_sme_writeq_vert: {
8162 auto *Idx = dyn_cast<Instruction>(II->getOperand(1));
8163 if (!Idx || Idx->getOpcode() != Instruction::Add)
8164 return false;
8165 Ops.push_back(&II->getOperandUse(1));
8166 return true;
8167 }
8168 case Intrinsic::aarch64_sme_read_horiz:
8169 case Intrinsic::aarch64_sme_read_vert:
8170 case Intrinsic::aarch64_sme_readq_horiz:
8171 case Intrinsic::aarch64_sme_readq_vert:
8172 case Intrinsic::aarch64_sme_ld1b_vert:
8173 case Intrinsic::aarch64_sme_ld1h_vert:
8174 case Intrinsic::aarch64_sme_ld1w_vert:
8175 case Intrinsic::aarch64_sme_ld1d_vert:
8176 case Intrinsic::aarch64_sme_ld1q_vert:
8177 case Intrinsic::aarch64_sme_st1b_vert:
8178 case Intrinsic::aarch64_sme_st1h_vert:
8179 case Intrinsic::aarch64_sme_st1w_vert:
8180 case Intrinsic::aarch64_sme_st1d_vert:
8181 case Intrinsic::aarch64_sme_st1q_vert:
8182 case Intrinsic::aarch64_sme_ld1b_horiz:
8183 case Intrinsic::aarch64_sme_ld1h_horiz:
8184 case Intrinsic::aarch64_sme_ld1w_horiz:
8185 case Intrinsic::aarch64_sme_ld1d_horiz:
8186 case Intrinsic::aarch64_sme_ld1q_horiz:
8187 case Intrinsic::aarch64_sme_st1b_horiz:
8188 case Intrinsic::aarch64_sme_st1h_horiz:
8189 case Intrinsic::aarch64_sme_st1w_horiz:
8190 case Intrinsic::aarch64_sme_st1d_horiz:
8191 case Intrinsic::aarch64_sme_st1q_horiz: {
8192 auto *Idx = dyn_cast<Instruction>(II->getOperand(3));
8193 if (!Idx || Idx->getOpcode() != Instruction::Add)
8194 return false;
8195 Ops.push_back(&II->getOperandUse(3));
8196 return true;
8197 }
8198 case Intrinsic::aarch64_neon_pmull:
8199 if (!areExtractShuffleVectors(II->getOperand(0), II->getOperand(1)))
8200 return false;
8201 Ops.push_back(&II->getOperandUse(0));
8202 Ops.push_back(&II->getOperandUse(1));
8203 return true;
8204 case Intrinsic::aarch64_neon_pmull64:
8205 if (!areOperandsOfVmullHighP64(II->getArgOperand(0),
8206 II->getArgOperand(1)))
8207 return false;
8208 Ops.push_back(&II->getArgOperandUse(0));
8209 Ops.push_back(&II->getArgOperandUse(1));
8210 return true;
8211 case Intrinsic::masked_gather:
8212 if (!shouldSinkVectorOfPtrs(II->getArgOperand(0), Ops))
8213 return false;
8214 Ops.push_back(&II->getArgOperandUse(0));
8215 return true;
8216 case Intrinsic::masked_scatter:
8217 if (!shouldSinkVectorOfPtrs(II->getArgOperand(1), Ops))
8218 return false;
8219 Ops.push_back(&II->getArgOperandUse(1));
8220 return true;
8221 default:
8222 return false;
8223 }
8224 }
8225
8226 auto ShouldSinkCondition = [](Value *Cond,
8227 SmallVectorImpl<Use *> &Ops) -> bool {
8229 return false;
8231 if (II->getIntrinsicID() != Intrinsic::vector_reduce_or ||
8232 !isa<ScalableVectorType>(II->getOperand(0)->getType()))
8233 return false;
8234 if (isa<CmpInst>(II->getOperand(0)))
8235 Ops.push_back(&II->getOperandUse(0));
8236 return true;
8237 };
8238
8239 switch (I->getOpcode()) {
8240 case Instruction::Store: {
8241 auto *SI = cast<StoreInst>(I);
8242 auto *II = dyn_cast<IntrinsicInst>(SI->getValueOperand());
8243 if (!SI->isSimple() || !II)
8244 break;
8245
8246 if (!LI ||
8247 LI->getLoopFor(SI->getParent()) != LI->getLoopFor(II->getParent()))
8248 break;
8249
8250 unsigned Factor = getInterleaveIntrinsicFactor(II->getIntrinsicID());
8251 if (Factor) {
8252 Ops.push_back(&SI->getOperandUse(0));
8253 return true;
8254 }
8255 break;
8256 }
8257 case Instruction::GetElementPtr:
8258 case Instruction::Add:
8259 case Instruction::Sub:
8260 // Sink vscales closer to uses for better isel
8261 for (unsigned Op = 0; Op < I->getNumOperands(); ++Op) {
8262 if (shouldSinkVScale(I->getOperand(Op), Ops)) {
8263 Ops.push_back(&I->getOperandUse(Op));
8264 return true;
8265 }
8266 }
8267 break;
8268 case Instruction::Select: {
8269 if (!ShouldSinkCondition(I->getOperand(0), Ops))
8270 return false;
8271
8272 Ops.push_back(&I->getOperandUse(0));
8273 return true;
8274 }
8275 case Instruction::UncondBr:
8276 return false;
8277 case Instruction::CondBr: {
8278 if (!ShouldSinkCondition(cast<CondBrInst>(I)->getCondition(), Ops))
8279 return false;
8280
8281 Ops.push_back(&I->getOperandUse(0));
8282 return true;
8283 }
8284 case Instruction::FMul:
8285 // fmul with contract flag can be combined with fadd into fma.
8286 // Sinking fneg into this block enables fmls pattern.
8287 if (cast<FPMathOperator>(I)->hasAllowContract()) {
8288 if (isFNeg(I->getOperand(0)))
8289 Ops.push_back(&I->getOperandUse(0));
8290 if (isFNeg(I->getOperand(1)))
8291 Ops.push_back(&I->getOperandUse(1));
8292 }
8293 break;
8294
8295 // Type | BIC | ORN | EON
8296 // ----------------+-----------+-----------+-----------
8297 // scalar | Base | Base | Base
8298 // scalar w/shift | - | - | -
8299 // fixed vector | NEON/Base | NEON/Base | BSL2N/Base
8300 // scalable vector | SVE | - | BSL2N
8301 case Instruction::Xor:
8302 // EON only for scalars (possibly expanded fixed vectors)
8303 // and vectors using the SVE2/SME BSL2N instruction.
8304 if (I->getType()->isVectorTy() && ST->isNeonAvailable()) {
8305 bool HasBSL2N =
8306 ST->isSVEorStreamingSVEAvailable() && (ST->hasSVE2() || ST->hasSME());
8307 if (!HasBSL2N)
8308 break;
8309 }
8310 [[fallthrough]];
8311 case Instruction::And:
8312 case Instruction::Or:
8313 // Even though we could use the SVE2/SME BSL2N instruction,
8314 // it might pessimize with an extra MOV depending on register allocation.
8315 if (I->getOpcode() == Instruction::Or &&
8316 isa<ScalableVectorType>(I->getType()))
8317 break;
8318 // Shift can be fold into scalar AND/ORR/EOR,
8319 // but not the non-negated operand of BIC/ORN/EON.
8320 if (!(I->getType()->isVectorTy() && ST->hasNEON()) &&
8322 break;
8323 for (auto &Op : I->operands()) {
8324 // (and/or/xor X, (not Y)) -> (bic/orn/eon X, Y)
8325 if (match(Op.get(), m_Not(m_Value()))) {
8326 Ops.push_back(&Op);
8327 return true;
8328 }
8329 // (and/or/xor X, (splat (not Y))) -> (bic/orn/eon X, (splat Y))
8330 if (match(Op.get(),
8332 m_Value(), m_ZeroMask()))) {
8333 Use &InsertElt = cast<Instruction>(Op)->getOperandUse(0);
8334 Use &Not = cast<Instruction>(InsertElt)->getOperandUse(1);
8335 Ops.push_back(&Not);
8336 Ops.push_back(&InsertElt);
8337 Ops.push_back(&Op);
8338 return true;
8339 }
8340 }
8341 break;
8342 default:
8343 break;
8344 }
8345
8346 if (!I->getType()->isVectorTy())
8347 return !Ops.empty();
8348
8349 switch (I->getOpcode()) {
8350 case Instruction::Sub:
8351 case Instruction::Add: {
8352 if (!areExtractExts(I->getOperand(0), I->getOperand(1)))
8353 return false;
8354
8355 // If the exts' operands extract either the lower or upper elements, we
8356 // can sink them too.
8357 auto Ext1 = cast<Instruction>(I->getOperand(0));
8358 auto Ext2 = cast<Instruction>(I->getOperand(1));
8359 if (areExtractShuffleVectors(Ext1->getOperand(0), Ext2->getOperand(0))) {
8360 Ops.push_back(&Ext1->getOperandUse(0));
8361 Ops.push_back(&Ext2->getOperandUse(0));
8362 }
8363
8364 Ops.push_back(&I->getOperandUse(0));
8365 Ops.push_back(&I->getOperandUse(1));
8366
8367 return true;
8368 }
8369 case Instruction::Or: {
8370 // Pattern: Or(And(MaskValue, A), And(Not(MaskValue), B)) ->
8371 // bitselect(MaskValue, A, B) where Not(MaskValue) = Xor(MaskValue, -1)
8372 if (ST->hasNEON()) {
8373 // MainAnd refers to And instruction that has 'Not' as one of its operands
8374 Instruction *MainAnd, *OtherAnd;
8375 Value *A, *B;
8376 if (matchBSL(I, MainAnd, OtherAnd, A, B)) {
8377 Ops.push_back(
8378 &MainAnd->getOperandUse(MainAnd->getOperand(0) == A ? 1 : 0));
8379 Ops.push_back(&I->getOperandUse(0));
8380 Ops.push_back(&I->getOperandUse(1));
8381
8382 return true;
8383 }
8384 }
8385
8386 return false;
8387 }
8388 case Instruction::Mul: {
8389 auto ShouldSinkSplatForIndexedVariant = [](Value *V) {
8390 auto *Ty = cast<VectorType>(V->getType());
8391 // For SVE the lane-indexing is within 128-bits, so we can't fold splats.
8392 if (Ty->isScalableTy())
8393 return false;
8394
8395 // Indexed variants of Mul exist for i16 and i32 element types only.
8396 return Ty->getScalarSizeInBits() == 16 || Ty->getScalarSizeInBits() == 32;
8397 };
8398
8399 int NumZExts = 0, NumSExts = 0;
8400 for (auto &Op : I->operands()) {
8401 // Make sure we are not already sinking this operand
8402 if (any_of(Ops, [&](Use *U) { return U->get() == Op; }))
8403 continue;
8404
8405 if (match(&Op, m_ZExtOrSExt(m_Value()))) {
8406 auto *Ext = cast<Instruction>(Op);
8407 auto *ExtOp = Ext->getOperand(0);
8408 if (isSplatShuffle(ExtOp) && ShouldSinkSplatForIndexedVariant(ExtOp))
8409 Ops.push_back(&Ext->getOperandUse(0));
8410 Ops.push_back(&Op);
8411
8412 if (isa<SExtInst>(Ext)) {
8413 NumSExts++;
8414 } else {
8415 NumZExts++;
8416 // A zext(a) is also a sext(zext(a)), if we take more than 2 steps.
8417 if (Ext->getOperand(0)->getType()->getScalarSizeInBits() * 2 <
8418 I->getType()->getScalarSizeInBits())
8419 NumSExts++;
8420 }
8421
8422 continue;
8423 }
8424
8426 if (!Shuffle)
8427 continue;
8428
8429 // If the Shuffle is a splat and the operand is a zext/sext, sinking the
8430 // operand and the s/zext can help create indexed s/umull. This is
8431 // especially useful to prevent i64 mul being scalarized.
8432 if (isSplatShuffle(Shuffle) &&
8433 match(Shuffle->getOperand(0), m_ZExtOrSExt(m_Value()))) {
8434 Ops.push_back(&Shuffle->getOperandUse(0));
8435 Ops.push_back(&Op);
8436 if (match(Shuffle->getOperand(0), m_SExt(m_Value())))
8437 NumSExts++;
8438 else
8439 NumZExts++;
8440 continue;
8441 }
8442
8443 Value *ShuffleOperand = Shuffle->getOperand(0);
8444 InsertElementInst *Insert = dyn_cast<InsertElementInst>(ShuffleOperand);
8445 if (!Insert)
8446 continue;
8447
8448 Instruction *OperandInstr = dyn_cast<Instruction>(Insert->getOperand(1));
8449 if (!OperandInstr)
8450 continue;
8451
8452 ConstantInt *ElementConstant =
8453 dyn_cast<ConstantInt>(Insert->getOperand(2));
8454 // Check that the insertelement is inserting into element 0
8455 if (!ElementConstant || !ElementConstant->isZero())
8456 continue;
8457
8458 unsigned Opcode = OperandInstr->getOpcode();
8459 if (Opcode == Instruction::SExt)
8460 NumSExts++;
8461 else if (Opcode == Instruction::ZExt)
8462 NumZExts++;
8463 else {
8464 // If we find that the top bits are known 0, then we can sink and allow
8465 // the backend to generate a umull.
8466 unsigned Bitwidth = I->getType()->getScalarSizeInBits();
8467 APInt UpperMask = APInt::getHighBitsSet(Bitwidth, Bitwidth / 2);
8468 if (!MaskedValueIsZero(OperandInstr, UpperMask, DL))
8469 continue;
8470 NumZExts++;
8471 }
8472
8473 // And(Load) is excluded to prevent CGP getting stuck in a loop of sinking
8474 // the And, just to hoist it again back to the load.
8475 if (!match(OperandInstr, m_And(m_Load(m_Value()), m_Value())))
8476 Ops.push_back(&Insert->getOperandUse(1));
8477 Ops.push_back(&Shuffle->getOperandUse(0));
8478 Ops.push_back(&Op);
8479 }
8480
8481 // It is profitable to sink if we found two of the same type of extends.
8482 if (!Ops.empty() && (NumSExts == 2 || NumZExts == 2))
8483 return true;
8484
8485 // Otherwise, see if we should sink splats for indexed variants.
8486 if (!ShouldSinkSplatForIndexedVariant(I))
8487 return false;
8488
8489 Ops.clear();
8490 if (isSplatShuffle(I->getOperand(0)))
8491 Ops.push_back(&I->getOperandUse(0));
8492 if (isSplatShuffle(I->getOperand(1)))
8493 Ops.push_back(&I->getOperandUse(1));
8494
8495 return !Ops.empty();
8496 }
8497 case Instruction::FMul: {
8498 // For SVE the lane-indexing is within 128-bits, so we can't fold splats.
8499 if (I->getType()->isScalableTy())
8500 return !Ops.empty();
8501
8502 if (cast<VectorType>(I->getType())->getElementType()->isHalfTy() &&
8503 !ST->hasFullFP16())
8504 return !Ops.empty();
8505
8506 // Sink splats for index lane variants
8507 if (isSplatShuffle(I->getOperand(0)))
8508 Ops.push_back(&I->getOperandUse(0));
8509 if (isSplatShuffle(I->getOperand(1)))
8510 Ops.push_back(&I->getOperandUse(1));
8511 return !Ops.empty();
8512 }
8513 default:
8514 return false;
8515 }
8516 return false;
8517}
8518
8520 Align Alignment) const {
8521 if (!(ST->isSVEAvailable() ||
8522 (ST->isSVEorStreamingSVEAvailable() && ST->hasSME2p2())))
8523 return false;
8524
8525 if (isa<FixedVectorType>(DataType) &&
8526 DataType->getPrimitiveSizeInBits().getFixedValue() < 128)
8527 return false;
8528
8529 if (!isa<VectorType>(DataType))
8530 return isElementTypeLegalForScalableVector(DataType);
8531
8532 auto LT = getTypeLegalizationCost(DataType);
8533 if (!LT.first.isValid() || !LT.second.isVector())
8534 return false;
8535
8536 // Use the i32 or i64 compact instructions for f16/bf16 unpacked types.
8537 LLVMContext &Ctx = DataType->getContext();
8538 switch (LT.second.SimpleTy) {
8539 case MVT::nxv2f16:
8540 case MVT::nxv2bf16:
8542 case MVT::nxv4f16:
8543 case MVT::nxv4bf16:
8545 default:
8546 break;
8547 }
8548
8550 EVT(LT.second.getScalarType()).getTypeForEVT(Ctx));
8551}
const TargetLowering & TLI
SDValue Pred
static bool isAllActivePredicate(const SelectionDAG &DAG, SDValue N)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static std::optional< Instruction * > instCombinePTrue(InstCombiner &IC, IntrinsicInst &II)
TailFoldingOption TailFoldingOptionLoc
static std::optional< Instruction * > instCombineSVEVectorFAdd(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEVectorFuseMulAddSub(InstCombiner &IC, IntrinsicInst &II, bool MergeIntoAddendOp)
static std::optional< Instruction * > instCombineZExtSVECmpNE(InstCombiner &IC, IntrinsicInst &II)
static void getFalkorUnrollingPreferences(Loop *L, ScalarEvolution &SE, TargetTransformInfo::UnrollingPreferences &UP)
bool SimplifyValuePattern(SmallVector< Value * > &Vec, bool AllowPoison)
static std::optional< Instruction * > instCombineSVESel(InstCombiner &IC, IntrinsicInst &II)
static bool shouldSinkVScale(Value *Op, SmallVectorImpl< Use * > &Ops)
We want to sink following cases: (add|sub|gep) A, ((mul|shl) vscale, imm); (add|sub|gep) A,...
static InstructionCost getHistogramCost(const AArch64Subtarget *ST, const IntrinsicCostAttributes &ICA)
static std::optional< Instruction * > tryCombineFromSVBoolBinOp(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEUnpack(InstCombiner &IC, IntrinsicInst &II)
static void getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE, TargetTransformInfo::UnrollingPreferences &UP, const AArch64TTIImpl &TTI)
For Apple CPUs, we want to runtime-unroll loops to make better use if the OOO engine's wide instructi...
static std::optional< Instruction * > instCombineWhilelo(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEVectorFAddU(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEPairwiseAddLong(InstCombiner &IC, IntrinsicInst &II)
static bool areExtractExts(Value *Ext1, Value *Ext2)
Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
static std::optional< Instruction * > instCombineSVEUMin(InstCombiner &IC, IntrinsicInst &II)
static bool shouldSinkVectorOfPtrs(Value *Ptrs, SmallVectorImpl< Use * > &Ops)
static bool shouldUnrollMultiExitLoop(Loop *L, ScalarEvolution &SE, const AArch64TTIImpl &TTI)
static std::optional< Instruction * > simplifySVEIntrinsicBinOp(InstCombiner &IC, IntrinsicInst &II, const SVEIntrinsicInfo &IInfo)
static std::optional< Instruction * > instCombineSVEVectorSub(InstCombiner &IC, IntrinsicInst &II)
static bool isLoopSizeWithinBudget(Loop *L, const AArch64TTIImpl &TTI, InstructionCost Budget, unsigned *FinalSize)
static std::optional< Instruction * > instCombineLD1GatherIndex(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEVectorFSub(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > processPhiNode(InstCombiner &IC, IntrinsicInst &II)
The function will remove redundant reinterprets casting in the presence of the control flow.
static std::optional< Instruction * > instCombineSVEInsr(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSMECntsd(InstCombiner &IC, IntrinsicInst &II, const AArch64Subtarget *ST)
static void extractAttrFeatures(const Function &F, const AArch64TTIImpl *TTI, SmallVectorImpl< StringRef > &Features)
static std::optional< Instruction * > instCombineST1ScatterIndex(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVESDIV(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEST1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL)
static Value * stripInactiveLanes(Value *V, const Value *Pg)
static bool isUnpackedVectorVT(EVT VecVT)
static std::optional< Instruction * > instCombineSVEDupX(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVECmpNE(InstCombiner &IC, IntrinsicInst &II)
static SVEIntrinsicInfo constructSVEIntrinsicInfo(IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEVectorFSubU(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineRDFFR(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineMaxMinNM(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > simplifySVEIntrinsicCompare(InstCombiner &IC, IntrinsicInst &II, const SVEIntrinsicInfo &IInfo)
static InstructionCost getLatencyFromSchedModel(const AArch64Subtarget *ST, unsigned Inst, unsigned NumInst, InstructionCost DefaultLatency)
static std::optional< Instruction * > instCombineSVECondLast(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEPTest(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEZip(InstCombiner &IC, IntrinsicInst &II)
static bool matchBSL(const Instruction *I, Instruction *&MainAnd, Instruction *&OtherAnd, Value *&A, Value *&B)
Match Or(And(MASK, A), And(Not(MASK), B)), which ISel selects as BSL/BIT/BIF.
static std::optional< Instruction * > instCombineSVEDup(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineConvertFromSVBool(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEUzp1(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEVectorBinOp(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVETBL(InstCombiner &IC, IntrinsicInst &II)
static bool areOperandsOfVmullHighP64(Value *Op1, Value *Op2)
Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
static bool isFNeg(Value *Op)
static Instruction::BinaryOps intrinsicIDToBinOpCode(unsigned Intrinsic)
static bool containsDecreasingPointers(Loop *TheLoop, PredicatedScalarEvolution *PSE, const DominatorTree &DT)
static std::optional< Instruction * > instCombineSVEAnd(InstCombiner &IC, IntrinsicInst &II)
static bool isSplatShuffle(Value *V)
static std::optional< Instruction * > instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL)
static std::optional< Instruction * > instCombineSVESrshl(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineXorSVECmpCC(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > simplifySVEIntrinsic(InstCombiner &IC, IntrinsicInst &II, const SVEIntrinsicInfo &IInfo)
static unsigned getSVEGatherScatterOverhead(unsigned Opcode, const AArch64Subtarget *ST)
static std::optional< Instruction * > instCombineSVEVectorMlaU(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineDMB(InstCombiner &IC, IntrinsicInst &II, unsigned LookaheadThreshold)
static bool isOperandOfVmullHighP64(Value *Op)
Check if Op could be used with vmull_high_p64 intrinsic.
static std::optional< Instruction * > instCombineInStreamingMode(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVELast(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEOrr(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVECntElts(InstCombiner &IC, IntrinsicInst &II, unsigned NumElts)
static std::optional< Instruction * > instCombineSVEUxt(InstCombiner &IC, IntrinsicInst &II, unsigned NumBits)
static cl::opt< TailFoldingOption, true, cl::parser< std::string > > SVETailFolding("sve-tail-folding", cl::desc("Control the use of vectorisation using tail-folding for SVE where the" " option is specified in the form (Initial)[+(Flag1|Flag2|...)]:" "\ndisabled (Initial) No loop types will vectorize using " "tail-folding" "\ndefault (Initial) Uses the default tail-folding settings for " "the target CPU" "\nall (Initial) All legal loop types will vectorize using " "tail-folding" "\nsimple (Initial) Use tail-folding for simple loops (not " "reductions or recurrences)" "\nreductions Use tail-folding for loops containing reductions" "\nnoreductions Inverse of above" "\nrecurrences Use tail-folding for loops containing fixed order " "recurrences" "\nnorecurrences Inverse of above" "\nreverse Use tail-folding for loops requiring reversed " "predicates" "\nnoreverse Inverse of above"), cl::location(TailFoldingOptionLoc))
static bool areExtractShuffleVectors(Value *Op1, Value *Op2, bool AllowSplat=false)
Check if both Op1 and Op2 are shufflevector extracts of either the lower or upper half of the vector ...
static std::optional< Instruction * > instCombineSVEVectorAdd(InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEDupqLane(InstCombiner &IC, IntrinsicInst &II)
This file a TargetTransformInfoImplBase conforming object specific to the AArch64 target machine.
const AArch64InstrInfo & TII
uint64_t uint64_t
uint64_t Imm
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file provides a helper that implements much of the TTI interface in terms of the target-independ...
static Error reportError(StringRef Message)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Cost tables and simple lookup functions.
This file defines the DenseMap class.
@ Default
static Value * getCondition(Instruction *I)
Hexagon Common GEP
#define _
This file provides the interface for the instcombine pass implementation.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static LVOptions Options
Definition LVOptions.cpp:25
This file defines the LoopVectorizationLegality class.
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
static const Function * getCalledFunction(const Value *V)
#define T
AttributeSet CallAttrs
uint64_t IntrinsicInst * II
#define P(N)
SmallVector< MachineOperand, 4 > Cond
int64_t C1
static uint64_t getBits(uint64_t Val, int Start, int End)
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
SI Fold Operands
static unsigned getFastMathFlags(const MachineInstr &I, const SPIRVSubtarget &ST)
#define LLVM_DEBUG(...)
Definition Debug.h:119
static unsigned getScalarSizeInBits(Type *Ty)
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
This file describes how to lower LLVM code to machine code.
This pass exposes codegen information to IR-level passes.
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type.
Value * RHS
Value * LHS
BinaryOperator * Mul
This file implements the C++20 <bit> header.
unsigned getVectorInsertExtractBaseCost() const
bool useSVEForFixedLengthVectors() const
InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty, std::optional< FastMathFlags > FMF, TTI::TargetCostKind CostKind) const override
InstructionCost getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract, TTI::TargetCostKind CostKind, bool ForPoisonSrc=true, ArrayRef< Value * > VL={}, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
InstructionCost getCostOfKeepingLiveOverCall(ArrayRef< Type * > Tys) const override
InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const
InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const
bool hasPossibleIncompatibleOps(const Function *F, bool ConsiderZA, bool ConsiderZT, bool ConsiderSM) const
Returns true if the function has explicit operations that can only be lowered using incompatible inst...
bool isLegalBroadcastLoad(Type *ElementTy, ElementCount NumElements) const override
InstructionCost getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr, TTI::TargetCostKind CostKind) const override
bool isExtPartOfAvgExpr(const Instruction *ExtUser, Type *Dst, Type *Src) const
InstructionCost getIntImmCost(int64_t Val) const
Calculate the cost of materializing a 64-bit value.
InstructionCost getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, unsigned Index) const override
std::optional< InstructionCost > getFP16BF16PromoteCost(Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info, bool IncludeTrunc, bool CanUseSVE, std::function< InstructionCost(Type *)> InstCost) const
FP16 and BF16 operations are lowered to fptrunc(op(fpext, fpext) if the architecture features are not...
bool prefersVectorizedAddressing() const override
bool preferFixedOverScalableIfEqualCost() const override
InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) const override
bool isProfitableToSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops, const LoopInfo *LI) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
InstructionCost getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode, Type *ResTy, VectorType *Ty, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput) const override
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, unsigned Index, const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
bool shouldMaximizeVectorBandwidth(TargetTransformInfo::VectorKind K, Type *ElementTy) const override
InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx, const APInt &Imm, Type *Ty, TTI::TargetCostKind CostKind, Instruction *Inst=nullptr) const override
bool isElementTypeLegalForScalableVector(Type *Ty) const override
bool hasMultiVectorLoadStore(unsigned NumVectors, TTI::MaskSource Mask, VectorType *VectorTy, bool IsStore, std::optional< Instruction::CastOps > CastHint) const override
void getPeelingPreferences(Loop *L, ScalarEvolution &SE, TTI::PeelingPreferences &PP) const override
InstructionCost getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType, ElementCount VF, TTI::PartialReductionExtendKind OpAExtend, TTI::PartialReductionExtendKind OpBExtend, std::optional< unsigned > BinOp, TTI::TargetCostKind CostKind, std::optional< FastMathFlags > FMF) const override
InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, const Instruction *I=nullptr) const override
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP, OptimizationRemarkEmitter *ORE) const override
bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const override
bool preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const override
InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF, TTI::TargetCostKind CostKind) const override
InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo={TTI::OK_AnyValue, TTI::OP_None}, const Instruction *I=nullptr) const override
APInt getPriorityMask(const Function &F) const override
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef< int > Mask, int Index, VectorType *SubTp, ArrayRef< const Value * > Args={}, const Instruction *CtxI=nullptr, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1, const TargetTransformInfo::LSRCost &C2) const override
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I=nullptr) const override
std::optional< Value * > simplifyDemandedVectorEltsIntrinsic(InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3, std::function< void(Instruction *, unsigned, APInt, APInt &)> SimplifyAndSetOp) const override
bool useNeonVector(const Type *Ty) const
std::optional< Instruction * > instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const override
InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Op2Info={TTI::OK_AnyValue, TTI::OP_None}, const Instruction *I=nullptr) const override
InstructionCost getExtendedReductionCost(unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy, std::optional< FastMathFlags > FMF, TTI::TargetCostKind CostKind) const override
bool isLegalSpeculativeLoad(Type *DataType, unsigned AddressSpace) const override
bool isLegalMaskedExpandLoad(Type *DataTy, Align Alignment) const override
TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const override
bool isElementTypeLegalForCompressStore(Type *Ty) const
InstructionCost getExtractWithExtendCost(unsigned Opcode, Type *Dst, VectorType *VecTy, unsigned Index, TTI::TargetCostKind CostKind) const override
unsigned getInlineCallPenalty(const Function *F, const CallBase &Call, unsigned DefaultCallPenalty) const override
bool areInlineCompatible(const Function *Caller, const Function *Callee) const override
The compiler must not inline when that may alter the behavior of the program.
unsigned getMaxNumElements(ElementCount VF) const
Try to return an estimate cost factor that can be used as a multiplier when scalarizing an operation ...
bool shouldTreatInstructionLikeSelect(const Instruction *I) const override
bool isMultiversionedFunction(const Function &F) const override
TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const override
bool isLegalToVectorizeReduction(const RecurrenceDescriptor &RdxDesc, ElementCount VF) const override
TTI::MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const override
bool isLegalMaskedCompressStore(Type *DataType, Align Alignment) const override
InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, Type *Ty, TTI::TargetCostKind CostKind) const override
bool isLegalMaskedGatherScatter(Type *DataType) const
InstructionCost getBranchMispredictPenalty() const override
bool shouldConsiderAddressTypePromotion(const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const override
See if I should be considered for address type promotion.
APInt getFeatureMask(const Function &F) const override
InstructionCost getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef< unsigned > Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, bool UseMaskForCond=false, bool UseMaskForGaps=false) const override
bool areTypesABICompatible(const Function *Caller, const Function *Callee, ArrayRef< Type * > Types) const override
bool enableScalableVectorization() const override
InstructionCost getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Op2Info={TTI::OK_AnyValue, TTI::OP_None}, ArrayRef< const Value * > Args={}, const Instruction *CtxI=nullptr) const override
InstructionCost getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const override
Value * getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, Type *ExpectedType, bool CanCreate=true) const override
bool hasKnownLowerThroughputFromSchedulingModel(unsigned Opcode1, unsigned Opcode2) const
Check whether Opcode1 has less throughput according to the scheduling model than Opcode2.
unsigned getEpilogueVectorizationMinVF() const override
InstructionCost getSpliceCost(VectorType *Tp, int Index, TTI::TargetCostKind CostKind) const
InstructionCost getArithmeticReductionCostSVE(unsigned Opcode, VectorType *ValTy, TTI::TargetCostKind CostKind) const
InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) const override
Return the cost of the scaling factor used in the addressing mode represented by AM for this target,...
unsigned getMaxInterleaveFactor(ElementCount VF, bool HasUnorderedReductions) const override
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
bool isNegatedPowerOf2() const
Check if this APInt's negated value is a power of two greater than zero.
Definition APInt.h:445
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
unsigned popcount() const
Count the number of bits set.
Definition APInt.h:1690
void negate()
Negate this APInt in place.
Definition APInt.h:1488
LLVM_ABI APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition APInt.cpp:1086
unsigned logBase2() const
Definition APInt.h:1781
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition APInt.h:829
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition APInt.h:1154
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:436
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition APInt.h:292
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition APInt.h:282
int64_t getSExtValue() const
Get sign extended value.
Definition APInt.h:1582
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
LLVM Basic Block Representation.
Definition BasicBlock.h:62
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
InstructionCost getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef< unsigned > Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, bool UseMaskForCond=false, bool UseMaskForGaps=false) const override
InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF, TTI::TargetCostKind CostKind) const override
TTI::ShuffleKind improveShuffleKindFromMask(TTI::ShuffleKind Kind, ArrayRef< int > Mask, VectorType *SrcTy, int &Index, VectorType *&SubTy) const
bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace, Instruction *I=nullptr, int64_t ScalableOffset=0) const override
bool areInlineCompatible(const Function *Caller, const Function *Callee) const override
InstructionCost getScalarizationOverhead(VectorType *InTy, const APInt &DemandedElts, bool Insert, bool Extract, TTI::TargetCostKind CostKind, bool ForPoisonSrc=true, ArrayRef< Value * > VL={}, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty, std::optional< FastMathFlags > FMF, TTI::TargetCostKind CostKind) const override
InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Op2Info={TTI::OK_AnyValue, TTI::OP_None}, const Instruction *I=nullptr) const override
InstructionCost getCallInstrCost(Function *F, Type *RetTy, ArrayRef< Type * > Tys, TTI::TargetCostKind CostKind) const override
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP, OptimizationRemarkEmitter *ORE) const override
void getPeelingPreferences(Loop *L, ScalarEvolution &SE, TTI::PeelingPreferences &PP) const override
InstructionCost getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode, Type *ResTy, VectorType *Ty, TTI::TargetCostKind CostKind) const override
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef< int > Mask, int Index, VectorType *SubTp, ArrayRef< const Value * > Args={}, const Instruction *CtxI=nullptr, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
InstructionCost getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index) const override
InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, const Instruction *I=nullptr) const override
std::pair< InstructionCost, MVT > getTypeLegalizationCost(Type *Ty) const
InstructionCost getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType, ElementCount VF, TTI::PartialReductionExtendKind OpAExtend, TTI::PartialReductionExtendKind OpBExtend, std::optional< unsigned > BinOp, TTI::TargetCostKind CostKind, std::optional< FastMathFlags > FMF) const override
InstructionCost getExtendedReductionCost(unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *Ty, std::optional< FastMathFlags > FMF, TTI::TargetCostKind CostKind) const override
InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) const override
InstructionCost getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const override
InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo={TTI::OK_AnyValue, TTI::OP_None}, const Instruction *I=nullptr) const override
bool isTypeLegal(Type *Ty) const override
static BinaryOperator * CreateWithCopiedFlags(BinaryOps Opc, Value *V1, Value *V2, Value *CopyO, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Definition InstrTypes.h:254
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Value * getArgOperand(unsigned i) const
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
@ ICMP_NE
not equal
Definition InstrTypes.h:762
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
static bool isFPPredicate(Predicate P)
Definition InstrTypes.h:833
static bool isIntPredicate(Predicate P)
Definition InstrTypes.h:839
bool isUnsigned() const
Definition InstrTypes.h:999
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
static LLVM_ABI ConstantInt * getBool(LLVMContext &Context, bool V)
static LLVM_ABI Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
This is an important base class in LLVM.
Definition Constant.h:44
LLVM_ABI Constant * getSplatValue(bool AllowPoison=false) const
If all elements of the vector constant have the same value, return that value.
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition DataLayout.h:794
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition DenseMap.h:757
bool empty() const
Definition DenseMap.h:716
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
static constexpr ElementCount getScalable(ScalarTy MinVal)
Definition TypeSize.h:309
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:306
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:317
static ExtractElementInst * Create(Value *Vec, Value *Idx, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
static bool isCommutative(Predicate Pred)
This provides a helper for copying FMF from an instruction or setting specified flags.
Definition IRBuilder.h:90
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
bool noSignedZeros() const
Definition FMF.h:67
bool noInfs() const
Definition FMF.h:66
bool approxFunc() const
Definition FMF.h:70
bool allowContract() const
Definition FMF.h:69
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static bool isCommutative(Predicate P)
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Definition IRBuilder.h:2748
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition IRBuilder.h:2736
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
Definition IRBuilder.h:541
Type * getDoubleTy()
Fetch the type representing a 64-bit floating point value.
Definition IRBuilder.h:561
LLVM_ABI Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
LLVM_ABI CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
LLVM_ABI Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition IRBuilder.h:528
Type * getHalfTy()
Fetch the type representing a 16-bit floating point value.
Definition IRBuilder.h:546
LLVM_ABI Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition IRBuilder.h:476
Value * CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition IRBuilder.h:1772
Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")
Definition IRBuilder.h:2408
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition IRBuilder.h:2626
Value * CreateBinOpFMF(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, FMFSource FMFSource, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition IRBuilder.h:1739
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition IRBuilder.h:1441
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition IRBuilder.h:2319
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition IRBuilder.h:1913
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Definition IRBuilder.h:2770
LLVM_ABI Value * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > OverloadTypes, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="", ArrayRef< OperandBundleDef > OpBundles={}, function_ref< void(CallInst *)> SetFn=[](CallInst *) {})
Variant to create a possibly constant-folded intrinsic.
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition IRBuilder.h:1931
LLVM_ABI CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition IRBuilder.h:1424
Type * getFloatTy()
Fetch the type representing a 32-bit floating point value.
Definition IRBuilder.h:556
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition IRBuilder.h:2399
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition IRBuilder.h:197
Value * CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec, Value *Idx, const Twine &Name="")
Create a call to the vector.insert intrinsic.
Definition IRBuilder.h:1128
LLVM_ABI Value * CreateElementCount(Type *Ty, ElementCount EC)
Create an expression which evaluates to the number of elements in EC at runtime.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
This instruction inserts a single (scalar) element into a VectorType value.
The core instruction combiner logic.
virtual Instruction * eraseInstFromFunction(Instruction &I)=0
Combiner aware instruction erasure.
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
static InstructionCost getInvalid(CostType Val=0)
CostType getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
LLVM_ABI bool isCommutative() const LLVM_READONLY
Return true if the instruction is commutative:
bool isBinaryOp() const
Instruction * user_back()
iterator_range< user_iterator > users()
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
user_iterator user_begin()
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Class to represent integer types.
bool hasGroups() const
Returns true if we have any interleave groups.
const SmallVectorImpl< Type * > & getArgTypes() const
const SmallVectorImpl< const Value * > & getArgs() const
const IntrinsicInst * getInst() const
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
An instruction for reading from memory.
Value * getPointerOperand()
iterator_range< block_iterator > blocks() const
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
RecurrenceSet & getFixedOrderRecurrences()
Return the fixed-order recurrences found in the loop.
PredicatedScalarEvolution * getPredicatedScalarEvolution() const
const ReductionList & getReductionVars() const
Returns the reduction variables found in the loop.
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
Machine Value Type.
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getScalableVectorVT(MVT VT, unsigned NumElements)
bool isFixedLengthVector() const
MVT getVectorElementType() const
size_type size() const
Definition MapVector.h:59
Information for memory intrinsic cost model.
const Instruction * getInst() const
The optimization diagnostic interface.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
Type * getRecurrenceType() const
Returns the type of the recurrence.
RecurKind getRecurrenceKind() const
This node represents a polynomial recurrence on the trip count of the specified loop.
bool isAffine() const
Return true if this represents an expression A + B*x where A and B are loop invariant values.
This class represents an analyzed expression in the program.
SMEAttrs is a utility class to parse the SME ACLE attributes on functions.
bool hasStreamingCompatibleInterface() const
bool hasStreamingInterfaceOrBody() const
SMECallAttrs is a utility class to hold the SMEAttrs for a callsite.
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:860
static ScalableVectorType * getDoubleElementsVectorType(ScalableVectorType *VTy)
The main scalar evolution driver.
LLVM_ABI const SCEV * getBackedgeTakenCount(const Loop *L, ExitCountKind Kind=Exact)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
LLVM_ABI unsigned getSmallConstantTripMultiple(const Loop *L, const SCEV *ExitCount)
Returns the largest constant divisor of the trip count as a normal unsigned value,...
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
LLVM_ABI unsigned getSmallConstantMaxTripCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > *Predicates=nullptr)
Returns the upper bound of the loop trip count as a normal unsigned value.
LLVM_ABI bool isBackedgeTakenCountMaxOrZero(const Loop *L)
Return true if the backedge taken count is either the value returned by getConstantMaxBackedgeTakenCo...
LLVM_ABI bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
const SCEV * getSymbolicMaxBackedgeTakenCount(const Loop *L)
When successful, this returns a SCEV that is greater than or equal to (i.e.
This instruction constructs a fixed permutation of two input vectors.
static LLVM_ABI bool isDeInterleaveMaskOfFactor(ArrayRef< int > Mask, unsigned Factor, unsigned &Index)
Check if the mask is a DE-interleave mask of the given factor Factor like: <Index,...
static LLVM_ABI bool isExtractSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &Index)
Return true if this shuffle mask is an extract subvector mask.
static LLVM_ABI bool isInterleaveMask(ArrayRef< int > Mask, unsigned Factor, unsigned NumInputElts, SmallVectorImpl< unsigned > &StartIndexes)
Return true if the mask interleaves one or more input vectors together.
size_type size() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
iterator insert(iterator I, T &&Elt)
void resize(size_type N)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
Definition TypeSize.h:31
static StackOffset getScalable(int64_t Scalable)
Definition TypeSize.h:41
static StackOffset getFixed(int64_t Fixed)
Definition TypeSize.h:40
An instruction for storing to memory.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
Class to represent struct types.
TargetInstrInfo - Interface to description of machine instruction set.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
std::pair< LegalizeTypeAction, EVT > LegalizeKind
LegalizeKind holds the legalization kind that needs to happen to EVT in order to type-legalize it.
virtual const DataLayout & getDataLayout() const
virtual bool shouldTreatInstructionLikeSelect(const Instruction *I) const
virtual bool isLoweredToCall(const Function *F) const
virtual bool isLSRCostLess(const TTI::LSRCost &C1, const TTI::LSRCost &C2) const
bool isConstantStridedAccessLessThan(ScalarEvolution *SE, const SCEV *Ptr, int64_t MergeDistance) const
virtual bool areTypesABICompatible(const Function *Caller, const Function *Callee, ArrayRef< Type * > Types) const
InstructionCost getInstructionCost(const User *U, ArrayRef< const Value * > Operands, TTI::TargetCostKind CostKind) const override
static LLVM_ABI OperandValueInfo getOperandInfo(const Value *V)
Collect properties of V used in cost analysis, e.g. OP_PowerOf2.
TargetCostKind
The kind of cost model.
@ TCK_RecipThroughput
Reciprocal throughput.
@ TCK_CodeSize
Instruction code size.
@ TCK_SizeAndLatency
The weighted sum of size and latency.
@ TCK_Latency
The latency of instruction.
static bool requiresOrderedReduction(std::optional< FastMathFlags > FMF)
A helper function to determine the type of reduction algorithm used for a given Opcode and set of Fas...
MaskSource
Enum describing the source/producer of a mask.
@ ArbitraryPredicate
The operation is masked with an arbitrary predicate.
@ ActiveLaneMask
The operation is masked with a contiguous active lane mask.
PopcntSupportKind
Flags indicating the kind of support for population count.
llvm::VectorInstrContext VectorInstrContext
@ TCC_Free
Expected to fold away in lowering.
@ TCC_Basic
The cost of a typical 'add' instruction.
ShuffleKind
The various kinds of shuffle patterns for vector queries.
@ SK_InsertSubvector
InsertSubvector. Index indicates start offset.
@ SK_Select
Selects elements from the corresponding lane of either source operand.
@ SK_PermuteSingleSrc
Shuffle elements of single source vector with any shuffle mask.
@ SK_Transpose
Transpose two vectors.
@ SK_Splice
Concatenates elements from the first input vector with elements of the second input vector.
@ SK_Broadcast
Broadcast element 0 to all other elements.
@ SK_PermuteTwoSrc
Merge elements from two source vectors into one with any shuffle mask.
@ SK_Reverse
Reverse the order of the vector.
@ SK_ExtractSubvector
ExtractSubvector Index indicates start offset.
CastContextHint
Represents a hint about the context in which a cast is used.
@ Masked
The cast is used with a masked load/store.
@ Normal
The cast is used with a normal load/store.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
Definition TypeSize.h:355
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:295
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Definition Type.h:258
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
bool isFP128Ty() const
Return true if this is 'fp128'.
Definition Type.h:164
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.h:354
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:292
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
LLVM_ABI Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
Definition Type.h:144
LLVM_ABI Type * getWithNewType(Type *EltTy) const
Given vector type, change the element type, whilst keeping the old number of elements.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:130
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
Definition Type.h:158
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:291
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
LLVM_ABI bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
Definition Type.cpp:61
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:298
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:271
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
const Use & getOperandUse(unsigned i) const
Definition User.h:220
Value * getOperand(unsigned i) const
Definition User.h:207
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
LLVM_ABI Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
Definition Value.cpp:985
bool use_empty() const
Definition Value.h:348
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:395
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Type * getElementType() const
constexpr bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
Definition TypeSize.h:181
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition TypeSize.h:253
const ParentTy * getParent() const
Definition ilist_node.h:34
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
LLVM_ABI APInt getCpuSupportsMask(ArrayRef< StringRef > Features)
static constexpr unsigned SVEBitsPerBlock
LLVM_ABI APInt getFMVPriority(ArrayRef< StringRef > Features)
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
@ Entry
Definition COFF.h:862
ISD namespace - This namespace contains an enum which represents all of the SelectionDAG node types a...
Definition ISDOpcodes.h:26
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ CTTZ_ELTS
Returns the number of number of trailing (least significant) zero elements in a vector.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
AllOnesConstantMatch m_AllOnes()
CheckType m_SpecificType(LLT Ty)
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
auto m_Cmp()
Matches any compare instruction and ignore it.
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
LogicalOp_match< LHS, RHS, Instruction::And > m_LogicalAnd(const LHS &L, const RHS &R)
Matches L && R either in the form of L & R or L ?
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
BinaryOp_match< LHS, RHS, Instruction::FMul > m_FMul(const LHS &L, const RHS &R)
bool match(Val *V, const Pattern &P)
match_bind< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
cst_pred_ty< is_nonnegative > m_NonNegative()
Match an integer or vector of non-negative values.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
auto m_BinOp()
Match an arbitrary binary operation and ignore it.
auto m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Xor, true > m_c_Xor(const LHS &L, const RHS &R)
Matches an Xor with LHS and RHS in either order.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
auto m_VScale()
Matches a call to llvm.vscale().
OneOps_match< OpTy, Instruction::Load > m_Load(const OpTy &Op)
Matches LoadInst.
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
is_zero< true > m_Zero()
Match any null constant or a vector with all elements equal to 0.
BinaryOp_match< LHS, RHS, Instruction::Add, true > m_c_Add(const LHS &L, const RHS &R)
Matches a Add with LHS and RHS in either order.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
FNeg_match< OpTy, true > m_FNegOrFSubNegZero(const OpTy &X)
Match fneg X or fsub -0.0, X.
AnyBinaryOp_match< LHS, RHS, true > m_c_BinOp(const LHS &L, const RHS &R)
Matches a BinaryOperator with LHS and RHS in either order.
CmpClass_match< LHS, RHS, ICmpInst > m_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
BinOpPred_match< LHS, RHS, is_shift_op > m_Shift(const LHS &L, const RHS &R)
Matches shift operations.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
brc_match< Cond_t, match_bind< BasicBlock >, match_bind< BasicBlock > > m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F)
auto m_Undef()
Match an arbitrary undef constant.
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
BinaryOp_match< LHS, RHS, Instruction::Or, true > m_c_Or(const LHS &L, const RHS &R)
Matches an Or with LHS and RHS in either order.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
auto m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
LocationClass< Ty > location(Ty &L)
InstructionCost getVectorInstrCost(const TargetTransformInfo &TTI, bool ReVec, Type *ScalarTy, unsigned Opcode, Type *Val, const TTI::TargetCostKind CostKind, unsigned Index, Value *Scalar, ArrayRef< std::tuple< Value *, User *, int > > ScalarUserAndIdx, TTI::VectorInstrContext VIC)
This is similar to TargetTransformInfo::getVectorInstrCost, but if ScalarTy is a FixedVectorType,...
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:577
std::optional< unsigned > isDUPQMask(ArrayRef< int > Mask, unsigned Segments, unsigned SegmentSize)
isDUPQMask - matches a splat of equivalent lanes within segments of a given number of elements.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
const CostTblEntryT< CostType > * CostTableLookup(ArrayRef< CostTblEntryT< CostType > > Tbl, int ISD, MVT Ty)
Find in cost table.
Definition CostTable.h:36
LLVM_ABI bool getBooleanLoopAttribute(const Loop *TheLoop, StringRef Name)
Returns true if Name is applied to TheLoop and enabled.
bool isZIPMask(ArrayRef< int > M, unsigned NumElts, unsigned &WhichResultOut, unsigned &OperandOrderOut)
Return true for zip1 or zip2 masks of the form: <0, 8, 1, 9, 2, 10, 3, 11> (WhichResultOut = 0,...
TailFoldingOpts
An enum to describe what types of loops we should attempt to tail-fold: Disabled: None Reductions: Lo...
InstructionCost Cost
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
@ Known
Known to have no common set bits.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
bool isDUPFirstSegmentMask(ArrayRef< int > Mask, unsigned Segments, unsigned SegmentSize)
isDUPFirstSegmentMask - matches a splat of the first 128b segment.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
@ Uninitialized
Definition Threading.h:60
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2224
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
LLVM_ABI Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
LLVM_ABI std::optional< int64_t > getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, const Loop *Lp, const DominatorTree &DT, const SymbolicStrideMap &StridesMap=SymbolicStrideMap(), bool ShouldCheckWrap=true, SmallVectorImpl< const SCEVPredicate * > *Predicates=nullptr)
If the pointer has a constant stride return it in units of the access type size.
constexpr auto equal_to(T &&Arg)
Functor variant of std::equal_to that can be used as a UnaryPredicate in functional algorithms like a...
Definition STLExtras.h:2189
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:332
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
Definition MathExtras.h:380
LLVM_ABI bool MaskedValueIsZero(const Value *V, const APInt &Mask, const SimplifyQuery &SQ, unsigned Depth=0)
Return true if 'V & Mask' is known to be zero.
unsigned M1(unsigned Val)
Definition VE.h:394
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
LLVM_ABI bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
unsigned getPerfectShuffleCost(llvm::ArrayRef< int > M)
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
DenseMap< Value *, const SCEVUnknown * > SymbolicStrideMap
Maps a pointer to its symbolic (non-constant) stride.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
bool isUZPMask(ArrayRef< int > M, unsigned NumElts, unsigned &WhichResultOut)
Return true for uzp1 or uzp2 masks of the form: <0, 2, 4, 6, 8, 10, 12, 14> or <1,...
LLVM_ABI unsigned getInterleaveIntrinsicFactor(Intrinsic::ID ID)
Returns the corresponding factor of llvm.vector.interleaveN intrinsics.
bool isREVMask(ArrayRef< int > M, unsigned EltSize, unsigned NumElts, unsigned BlockSize)
isREVMask - Check if a vector shuffle corresponds to a REV instruction with the specified blocksize.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
constexpr int PoisonMaskElem
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:389
TargetTransformInfo TTI
LLVM_ABI Value * simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a BinaryOperator, fold the result or return null.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ FSub
Subtraction of floats.
@ FAddChainWithSubs
A chain of fadds and fsubs.
@ AnyOf
AnyOf reduction with select(cmp(),x,y) where one of (x,y) is loop invariant, and both x and y are int...
@ Xor
Bitwise or logical XOR of integers.
@ FindLast
FindLast reduction with select(cmp(),x,y) where x and y.
@ FMax
FP max implemented in terms of select(cmp()).
@ FMulAdd
Sum of float products with llvm.fmuladd(a * b + sum).
@ FMul
Product of floats.
@ SMax
Signed integer max implemented in terms of select(cmp()).
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
@ FMin
FP min implemented in terms of select(cmp()).
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
@ AddChainWithSubs
A chain of adds and subs.
@ FAdd
Sum of floats.
@ UMax
Unsigned integer max implemented in terms of select(cmp()).
DWARFExpression::Operation Op
TypeConversionCostTblEntryT< uint16_t > TypeConversionCostTblEntry
Definition CostTable.h:62
CostTblEntryT< uint16_t > CostTblEntry
Definition CostTable.h:31
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
unsigned getNumElementsFromSVEPredPattern(unsigned Pattern)
Return the number of active elements for VL1 to VL256 predicate pattern, zero for all other patterns.
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
constexpr bool valueOr(BoolOrDefault X, bool Default)
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition STLExtras.h:2182
LLVM_ABI Value * simplifyCmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a CmpInst, fold the result or return null.
Type * toVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
const TypeConversionCostTblEntryT< CostType > * ConvertCostTableLookup(ArrayRef< TypeConversionCostTblEntryT< CostType > > Tbl, int ISD, MVT Dst, MVT Src)
Find in type conversion cost table.
Definition CostTable.h:67
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition MathExtras.h:368
bool isTRNMask(ArrayRef< int > M, unsigned NumElts, unsigned &WhichResultOut, unsigned &OperandOrderOut)
Return true for trn1 or trn2 masks of the form: <0, 8, 2, 10, 4, 12, 6, 14> (WhichResultOut = 0,...
#define N
static SVEIntrinsicInfo defaultMergingUnaryNarrowingTopOp()
static SVEIntrinsicInfo defaultZeroingOp()
SVEIntrinsicInfo & setOperandIdxInactiveLanesTakenFrom(unsigned Index)
static SVEIntrinsicInfo defaultMergingOp(Intrinsic::ID IID=Intrinsic::not_intrinsic)
SVEIntrinsicInfo & setOperandIdxWithNoActiveLanes(unsigned Index)
unsigned getOperandIdxWithNoActiveLanes() const
CmpInst::Predicate getCmpPredicate() const
SVEIntrinsicInfo & setInactiveLanesAreUnused()
SVEIntrinsicInfo & setInactiveLanesAreNotDefined()
SVEIntrinsicInfo & setGoverningPredicateOperandIdx(unsigned Index)
static SVEIntrinsicInfo defaultUndefOp()
Intrinsic::ID getMatchingUndefIntrinsic() const
SVEIntrinsicInfo & setResultIsZeroInitialized()
static SVEIntrinsicInfo defaultMergingUnaryOp()
SVEIntrinsicInfo & setMatchingUndefIntrinsic(Intrinsic::ID IID)
unsigned getGoverningPredicateOperandIdx() const
SVEIntrinsicInfo & setCmpPredicate(CmpInst::Predicate Pred)
SVEIntrinsicInfo & setMatchingIROpcode(unsigned Opcode)
unsigned getOperandIdxInactiveLanesTakenFrom() const
static SVEIntrinsicInfo defaultVoidOp(unsigned GPIndex)
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Extended Value Type.
Definition ValueTypes.h:35
bool isScalableVectorOf(EVT EltVT) const
Return true if this is a scalable vector with matching element type.
Definition ValueTypes.h:207
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition ValueTypes.h:322
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition ValueTypes.h:397
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition ValueTypes.h:419
bool isScalableVT() const
Return true if the type is a scalable type.
Definition ValueTypes.h:225
bool isFixedLengthVector() const
Definition ValueTypes.h:214
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition ValueTypes.h:202
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
Summarize the scheduling resources required for an instruction of a particular scheduling class.
Definition MCSchedule.h:124
bool isVariant() const
Definition MCSchedule.h:145
Machine model for scheduling, bundling, and heuristics.
Definition MCSchedule.h:268
static LLVM_ABI double getReciprocalThroughput(const MCSubtargetInfo &STI, const MCSchedClassDesc &SCDesc)
Matching combinators.
Information about a load/store intrinsic defined by the target.
InterleavedAccessInfo * IAI
LoopVectorizationLegality * LVL
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
unsigned Insns
TODO: Some of these could be merged.
Returns options for expansion of memcmp. IsZeroCmp is.
Parameters that control the generic loop unrolling transformation.
bool UpperBound
Allow using trip count upper bound to unroll loops.
bool Force
Apply loop unroll on any kind of loop (mainly to loops that fail runtime unrolling).
unsigned PartialOptSizeThreshold
The cost threshold for the unrolled loop when optimizing for size, like OptSizeThreshold,...
unsigned DefaultUnrollRuntimeCount
Default unroll count for loops with run-time trip count.
bool RuntimeUnrollMultiExit
Allow runtime unrolling multi-exit loops.
unsigned SCEVExpansionBudget
Don't allow runtime unrolling if expanding the trip count takes more than SCEVExpansionBudget.
bool AddAdditionalAccumulators
Allow unrolling to add parallel reduction phis.
unsigned UnrollAndJamInnerLoopThreshold
Threshold for unroll and jam, for inner loop size.
bool UnrollAndJam
Allow unroll and jam. Used to enable unroll and jam for the target.
bool UnrollRemainder
Allow unrolling of all the iterations of the runtime loop remainder.
unsigned PartialThreshold
The cost threshold for the unrolled loop, like Threshold, but used for partial/runtime unrolling (set...
bool Runtime
Allow runtime unrolling (unrolling of loops to expand the size of the loop body even when the number ...
bool Partial
Allow partial unrolling (unrolling of loops to expand the size of the loop body, not only to eliminat...
unsigned MaxUpperBound
Set the maximum upper bound of trip count.