LLVM 22.0.0git
ExpandVectorPredication.cpp
Go to the documentation of this file.
1//===----- CodeGen/ExpandVectorPredication.cpp - Expand VP intrinsics -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements IR expansion for vector predication intrinsics, allowing
10// targets to enable vector predication until just before codegen.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/Statistic.h"
19#include "llvm/IR/Constants.h"
20#include "llvm/IR/Function.h"
21#include "llvm/IR/IRBuilder.h"
24#include "llvm/IR/Intrinsics.h"
27#include "llvm/Support/Debug.h"
29#include <optional>
30
31using namespace llvm;
32
35
36// Keep this in sync with TargetTransformInfo::VPLegalization.
37#define VPINTERNAL_VPLEGAL_CASES \
38 VPINTERNAL_CASE(Legal) \
39 VPINTERNAL_CASE(Discard) \
40 VPINTERNAL_CASE(Convert)
41
42#define VPINTERNAL_CASE(X) "|" #X
43
44// Override options.
46 "expandvp-override-evl-transform", cl::init(""), cl::Hidden,
47 cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
48 ". If non-empty, ignore "
49 "TargetTransformInfo and "
50 "always use this transformation for the %evl parameter (Used in "
51 "testing)."));
52
54 "expandvp-override-mask-transform", cl::init(""), cl::Hidden,
55 cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
56 ". If non-empty, Ignore "
57 "TargetTransformInfo and "
58 "always use this transformation for the %mask parameter (Used in "
59 "testing)."));
60
61#undef VPINTERNAL_CASE
62#define VPINTERNAL_CASE(X) .Case(#X, VPLegalization::X)
63
64static VPTransform parseOverrideOption(const std::string &TextOpt) {
66}
67
68#undef VPINTERNAL_VPLEGAL_CASES
69
70// Whether any override options are set.
72 return !EVLTransformOverride.empty() || !MaskTransformOverride.empty();
73}
74
75#define DEBUG_TYPE "expandvp"
76
77STATISTIC(NumFoldedVL, "Number of folded vector length params");
78STATISTIC(NumLoweredVPOps, "Number of folded vector predication operations");
79
80///// Helpers {
81
82/// \returns Whether the vector mask \p MaskVal has all lane bits set.
83static bool isAllTrueMask(Value *MaskVal) {
84 if (Value *SplattedVal = getSplatValue(MaskVal))
85 if (auto *ConstValue = dyn_cast<Constant>(SplattedVal))
86 return ConstValue->isAllOnesValue();
87
88 return false;
89}
90
91/// \returns A non-excepting divisor constant for this type.
92static Constant *getSafeDivisor(Type *DivTy) {
93 assert(DivTy->isIntOrIntVectorTy() && "Unsupported divisor type");
94 return ConstantInt::get(DivTy, 1u, false);
95}
96
97/// Transfer operation properties from \p OldVPI to \p NewVal.
98static void transferDecorations(Value &NewVal, VPIntrinsic &VPI) {
99 auto *NewInst = dyn_cast<Instruction>(&NewVal);
100 if (!NewInst || !isa<FPMathOperator>(NewVal))
101 return;
102
103 auto *OldFMOp = dyn_cast<FPMathOperator>(&VPI);
104 if (!OldFMOp)
105 return;
106
107 NewInst->setFastMathFlags(OldFMOp->getFastMathFlags());
108}
109
110/// Transfer all properties from \p OldOp to \p NewOp and replace all uses.
111/// OldVP gets erased.
112static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp) {
113 transferDecorations(NewOp, OldOp);
114 OldOp.replaceAllUsesWith(&NewOp);
115 OldOp.eraseFromParent();
116}
117
119 // The result of VP reductions depends on the mask and evl.
121 return false;
122 // Fallback to whether the intrinsic is speculatable.
123 if (auto IntrID = VPI.getFunctionalIntrinsicID())
124 return Intrinsic::getFnAttributes(VPI.getContext(), *IntrID)
125 .hasAttribute(Attribute::AttrKind::Speculatable);
126 if (auto Opc = VPI.getFunctionalOpcode())
128 return false;
129}
130
131//// } Helpers
132
133namespace {
134
135// Expansion pass state at function scope.
136struct CachingVPExpander {
137 const TargetTransformInfo &TTI;
138
139 /// \returns A bitmask that is true where the lane position is less-than \p
140 /// EVLParam
141 ///
142 /// \p Builder
143 /// Used for instruction creation.
144 /// \p VLParam
145 /// The explicit vector length parameter to test against the lane
146 /// positions.
147 /// \p ElemCount
148 /// Static (potentially scalable) number of vector elements.
149 Value *convertEVLToMask(IRBuilder<> &Builder, Value *EVLParam,
150 ElementCount ElemCount);
151
152 /// If needed, folds the EVL in the mask operand and discards the EVL
153 /// parameter. Returns true if the mask was actually folded.
154 bool foldEVLIntoMask(VPIntrinsic &VPI);
155
156 /// "Remove" the %evl parameter of \p PI by setting it to the static vector
157 /// length of the operation. Returns true if the %evl (if any) was effectively
158 /// changed.
159 bool discardEVLParameter(VPIntrinsic &PI);
160
161 /// Lower this VP binary operator to a unpredicated binary operator.
162 bool expandPredicationInBinaryOperator(IRBuilder<> &Builder, VPIntrinsic &PI);
163
164 /// Lower this VP int call to a unpredicated int call.
165 bool expandPredicationToIntCall(IRBuilder<> &Builder, VPIntrinsic &PI);
166
167 /// Lower this VP fp call to a unpredicated fp call.
168 bool expandPredicationToFPCall(IRBuilder<> &Builder, VPIntrinsic &PI,
169 unsigned UnpredicatedIntrinsicID);
170
171 /// Lower this VP reduction to a call to an unpredicated reduction intrinsic.
172 bool expandPredicationInReduction(IRBuilder<> &Builder,
173 VPReductionIntrinsic &PI);
174
175 /// Lower this VP cast operation to a non-VP intrinsic.
176 bool expandPredicationToCastIntrinsic(IRBuilder<> &Builder, VPIntrinsic &VPI);
177
178 /// Lower this VP memory operation to a non-VP intrinsic.
179 bool expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder,
180 VPIntrinsic &VPI);
181
182 /// Lower this VP comparison to a call to an unpredicated comparison.
183 bool expandPredicationInComparison(IRBuilder<> &Builder, VPCmpIntrinsic &PI);
184
185 /// Query TTI and expand the vector predication in \p P accordingly.
186 bool expandPredication(VPIntrinsic &PI);
187
188 /// Determine how and whether the VPIntrinsic \p VPI shall be expanded. This
189 /// overrides TTI with the cl::opts listed at the top of this file.
190 VPLegalization getVPLegalizationStrategy(const VPIntrinsic &VPI) const;
191 bool UsingTTIOverrides;
192
193public:
194 CachingVPExpander(const TargetTransformInfo &TTI)
195 : TTI(TTI), UsingTTIOverrides(anyExpandVPOverridesSet()) {}
196
197 /// Expand llvm.vp.* intrinsics as requested by \p TTI.
198 /// Returns the details of the expansion.
199 VPExpansionDetails expandVectorPredication(VPIntrinsic &VPI);
200};
201
202//// CachingVPExpander {
203
204Value *CachingVPExpander::convertEVLToMask(IRBuilder<> &Builder,
205 Value *EVLParam,
206 ElementCount ElemCount) {
207 // TODO add caching
208 // Scalable vector %evl conversion.
209 if (ElemCount.isScalable()) {
210 Type *BoolVecTy = VectorType::get(Builder.getInt1Ty(), ElemCount);
211 // `get_active_lane_mask` performs an implicit less-than comparison.
212 Value *ConstZero = Builder.getInt32(0);
213 return Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask,
214 {BoolVecTy, EVLParam->getType()},
215 {ConstZero, EVLParam});
216 }
217
218 // Fixed vector %evl conversion.
219 Type *LaneTy = EVLParam->getType();
220 unsigned NumElems = ElemCount.getFixedValue();
221 Value *VLSplat = Builder.CreateVectorSplat(NumElems, EVLParam);
222 Value *IdxVec = Builder.CreateStepVector(VectorType::get(LaneTy, ElemCount));
223 return Builder.CreateICmp(CmpInst::ICMP_ULT, IdxVec, VLSplat);
224}
225
226bool CachingVPExpander::expandPredicationInBinaryOperator(IRBuilder<> &Builder,
227 VPIntrinsic &VPI) {
229 "Implicitly dropping %evl in non-speculatable operator!");
230
231 auto OC = static_cast<Instruction::BinaryOps>(*VPI.getFunctionalOpcode());
233
234 Value *Op0 = VPI.getOperand(0);
235 Value *Op1 = VPI.getOperand(1);
236 Value *Mask = VPI.getMaskParam();
237
238 // Blend in safe operands.
239 if (Mask && !isAllTrueMask(Mask)) {
240 switch (OC) {
241 default:
242 // Can safely ignore the predicate.
243 break;
244
245 // Division operators need a safe divisor on masked-off lanes (1).
246 case Instruction::UDiv:
247 case Instruction::SDiv:
248 case Instruction::URem:
249 case Instruction::SRem:
250 // 2nd operand must not be zero.
251 Value *SafeDivisor = getSafeDivisor(VPI.getType());
252 Op1 = Builder.CreateSelect(Mask, Op1, SafeDivisor);
253 }
254 }
255
256 Value *NewBinOp = Builder.CreateBinOp(OC, Op0, Op1, VPI.getName());
257
258 replaceOperation(*NewBinOp, VPI);
259 return true;
260}
261
262bool CachingVPExpander::expandPredicationToIntCall(IRBuilder<> &Builder,
263 VPIntrinsic &VPI) {
264 std::optional<unsigned> FID = VPI.getFunctionalIntrinsicID();
265 if (!FID)
266 return false;
267 SmallVector<Value *, 2> Argument;
268 for (unsigned i = 0; i < VPI.getNumOperands() - 3; i++) {
269 Argument.push_back(VPI.getOperand(i));
270 }
271 Value *NewOp = Builder.CreateIntrinsic(FID.value(), {VPI.getType()}, Argument,
272 /*FMFSource=*/nullptr, VPI.getName());
273 replaceOperation(*NewOp, VPI);
274 return true;
275}
276
277bool CachingVPExpander::expandPredicationToFPCall(
278 IRBuilder<> &Builder, VPIntrinsic &VPI, unsigned UnpredicatedIntrinsicID) {
280 "Implicitly dropping %evl in non-speculatable operator!");
281
282 switch (UnpredicatedIntrinsicID) {
283 case Intrinsic::fabs:
284 case Intrinsic::sqrt:
285 case Intrinsic::maxnum:
286 case Intrinsic::minnum: {
287 SmallVector<Value *, 2> Argument;
288 for (unsigned i = 0; i < VPI.getNumOperands() - 3; i++) {
289 Argument.push_back(VPI.getOperand(i));
290 }
291 Value *NewOp = Builder.CreateIntrinsic(
292 UnpredicatedIntrinsicID, {VPI.getType()}, Argument,
293 /*FMFSource=*/nullptr, VPI.getName());
294 replaceOperation(*NewOp, VPI);
295 return true;
296 }
297 case Intrinsic::fma:
298 case Intrinsic::fmuladd:
299 case Intrinsic::experimental_constrained_fma:
300 case Intrinsic::experimental_constrained_fmuladd: {
301 Value *Op0 = VPI.getOperand(0);
302 Value *Op1 = VPI.getOperand(1);
303 Value *Op2 = VPI.getOperand(2);
305 VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
306 Value *NewOp;
307 if (Intrinsic::isConstrainedFPIntrinsic(UnpredicatedIntrinsicID))
308 NewOp =
309 Builder.CreateConstrainedFPCall(Fn, {Op0, Op1, Op2}, VPI.getName());
310 else
311 NewOp = Builder.CreateCall(Fn, {Op0, Op1, Op2}, VPI.getName());
312 replaceOperation(*NewOp, VPI);
313 return true;
314 }
315 }
316
317 return false;
318}
319
320static Value *getNeutralReductionElement(const VPReductionIntrinsic &VPI,
321 Type *EltTy) {
323 FastMathFlags FMF;
324 if (isa<FPMathOperator>(VPI))
325 FMF = VPI.getFastMathFlags();
326 return getReductionIdentity(RdxID, EltTy, FMF);
327}
328
329bool CachingVPExpander::expandPredicationInReduction(
330 IRBuilder<> &Builder, VPReductionIntrinsic &VPI) {
332 "Implicitly dropping %evl in non-speculatable operator!");
333
334 Value *Mask = VPI.getMaskParam();
335 Value *RedOp = VPI.getOperand(VPI.getVectorParamPos());
336
337 // Insert neutral element in masked-out positions
338 if (Mask && !isAllTrueMask(Mask)) {
339 auto *NeutralElt = getNeutralReductionElement(VPI, VPI.getType());
340 auto *NeutralVector = Builder.CreateVectorSplat(
341 cast<VectorType>(RedOp->getType())->getElementCount(), NeutralElt);
342 RedOp = Builder.CreateSelect(Mask, RedOp, NeutralVector);
343 }
344
347
348 switch (VPI.getIntrinsicID()) {
349 default:
350 llvm_unreachable("Impossible reduction kind");
351 case Intrinsic::vp_reduce_add:
352 case Intrinsic::vp_reduce_mul:
353 case Intrinsic::vp_reduce_and:
354 case Intrinsic::vp_reduce_or:
355 case Intrinsic::vp_reduce_xor: {
357 unsigned Opc = getArithmeticReductionInstruction(RedID);
359 Reduction = Builder.CreateUnaryIntrinsic(RedID, RedOp);
360 Reduction =
362 break;
363 }
364 case Intrinsic::vp_reduce_smax:
365 case Intrinsic::vp_reduce_smin:
366 case Intrinsic::vp_reduce_umax:
367 case Intrinsic::vp_reduce_umin:
368 case Intrinsic::vp_reduce_fmax:
369 case Intrinsic::vp_reduce_fmin:
370 case Intrinsic::vp_reduce_fmaximum:
371 case Intrinsic::vp_reduce_fminimum: {
374 Reduction = Builder.CreateUnaryIntrinsic(RedID, RedOp);
376 Reduction = Builder.CreateBinaryIntrinsic(ScalarID, Reduction, Start);
377 break;
378 }
379 case Intrinsic::vp_reduce_fadd:
380 Reduction = Builder.CreateFAddReduce(Start, RedOp);
381 break;
382 case Intrinsic::vp_reduce_fmul:
383 Reduction = Builder.CreateFMulReduce(Start, RedOp);
384 break;
385 }
386
388 return true;
389}
390
391bool CachingVPExpander::expandPredicationToCastIntrinsic(IRBuilder<> &Builder,
392 VPIntrinsic &VPI) {
393 Intrinsic::ID VPID = VPI.getIntrinsicID();
394 unsigned CastOpcode = VPIntrinsic::getFunctionalOpcodeForVP(VPID).value();
395 assert(Instruction::isCast(CastOpcode));
396 Value *CastOp =
397 Builder.CreateCast(Instruction::CastOps(CastOpcode), VPI.getOperand(0),
398 VPI.getType(), VPI.getName());
399
400 replaceOperation(*CastOp, VPI);
401 return true;
402}
403
404bool CachingVPExpander::expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder,
405 VPIntrinsic &VPI) {
407
408 const auto &DL = VPI.getDataLayout();
409
410 Value *MaskParam = VPI.getMaskParam();
411 Value *PtrParam = VPI.getMemoryPointerParam();
412 Value *DataParam = VPI.getMemoryDataParam();
413 bool IsUnmasked = isAllTrueMask(MaskParam);
414
415 MaybeAlign AlignOpt = VPI.getPointerAlignment();
416
417 Value *NewMemoryInst = nullptr;
418 switch (VPI.getIntrinsicID()) {
419 default:
420 llvm_unreachable("Not a VP memory intrinsic");
421 case Intrinsic::vp_store:
422 if (IsUnmasked) {
423 StoreInst *NewStore =
424 Builder.CreateStore(DataParam, PtrParam, /*IsVolatile*/ false);
425 if (AlignOpt.has_value())
426 NewStore->setAlignment(*AlignOpt);
427 NewMemoryInst = NewStore;
428 } else
429 NewMemoryInst = Builder.CreateMaskedStore(
430 DataParam, PtrParam, AlignOpt.valueOrOne(), MaskParam);
431
432 break;
433 case Intrinsic::vp_load:
434 if (IsUnmasked) {
435 LoadInst *NewLoad =
436 Builder.CreateLoad(VPI.getType(), PtrParam, /*IsVolatile*/ false);
437 if (AlignOpt.has_value())
438 NewLoad->setAlignment(*AlignOpt);
439 NewMemoryInst = NewLoad;
440 } else
441 NewMemoryInst = Builder.CreateMaskedLoad(
442 VPI.getType(), PtrParam, AlignOpt.valueOrOne(), MaskParam);
443
444 break;
445 case Intrinsic::vp_scatter: {
446 auto *ElementType =
447 cast<VectorType>(DataParam->getType())->getElementType();
448 NewMemoryInst = Builder.CreateMaskedScatter(
449 DataParam, PtrParam,
450 AlignOpt.value_or(DL.getPrefTypeAlign(ElementType)), MaskParam);
451 break;
452 }
453 case Intrinsic::vp_gather: {
454 auto *ElementType = cast<VectorType>(VPI.getType())->getElementType();
455 NewMemoryInst = Builder.CreateMaskedGather(
456 VPI.getType(), PtrParam,
457 AlignOpt.value_or(DL.getPrefTypeAlign(ElementType)), MaskParam, nullptr,
458 VPI.getName());
459 break;
460 }
461 }
462
463 assert(NewMemoryInst);
464 replaceOperation(*NewMemoryInst, VPI);
465 return true;
466}
467
468bool CachingVPExpander::expandPredicationInComparison(IRBuilder<> &Builder,
469 VPCmpIntrinsic &VPI) {
471 "Implicitly dropping %evl in non-speculatable operator!");
472
473 assert(*VPI.getFunctionalOpcode() == Instruction::ICmp ||
474 *VPI.getFunctionalOpcode() == Instruction::FCmp);
475
476 Value *Op0 = VPI.getOperand(0);
477 Value *Op1 = VPI.getOperand(1);
478 auto Pred = VPI.getPredicate();
479
480 auto *NewCmp = Builder.CreateCmp(Pred, Op0, Op1);
481
482 replaceOperation(*NewCmp, VPI);
483 return true;
484}
485
486bool CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) {
487 LLVM_DEBUG(dbgs() << "Discard EVL parameter in " << VPI << "\n");
488
490 return false;
491
492 Value *EVLParam = VPI.getVectorLengthParam();
493 if (!EVLParam)
494 return false;
495
496 ElementCount StaticElemCount = VPI.getStaticVectorLength();
497 Value *MaxEVL = nullptr;
498 Type *Int32Ty = Type::getInt32Ty(VPI.getContext());
499 if (StaticElemCount.isScalable()) {
500 // TODO add caching
501 IRBuilder<> Builder(VPI.getParent(), VPI.getIterator());
502 Value *FactorConst = Builder.getInt32(StaticElemCount.getKnownMinValue());
503 Value *VScale = Builder.CreateVScale(Int32Ty, "vscale");
504 MaxEVL = Builder.CreateNUWMul(VScale, FactorConst, "scalable_size");
505 } else {
506 MaxEVL = ConstantInt::get(Int32Ty, StaticElemCount.getFixedValue(), false);
507 }
508 VPI.setVectorLengthParam(MaxEVL);
509 return true;
510}
511
512bool CachingVPExpander::foldEVLIntoMask(VPIntrinsic &VPI) {
513 LLVM_DEBUG(dbgs() << "Folding vlen for " << VPI << '\n');
514
515 IRBuilder<> Builder(&VPI);
516
517 // Ineffective %evl parameter and so nothing to do here.
519 return false;
520
521 // Only VP intrinsics can have an %evl parameter.
522 Value *OldMaskParam = VPI.getMaskParam();
523 if (!OldMaskParam) {
524 assert((VPI.getIntrinsicID() == Intrinsic::vp_merge ||
525 VPI.getIntrinsicID() == Intrinsic::vp_select) &&
526 "Unexpected VP intrinsic without mask operand");
527 OldMaskParam = VPI.getArgOperand(0);
528 }
529
530 Value *OldEVLParam = VPI.getVectorLengthParam();
531 assert(OldMaskParam && "no mask param to fold the vl param into");
532 assert(OldEVLParam && "no EVL param to fold away");
533
534 LLVM_DEBUG(dbgs() << "OLD evl: " << *OldEVLParam << '\n');
535 LLVM_DEBUG(dbgs() << "OLD mask: " << *OldMaskParam << '\n');
536
537 // Convert the %evl predication into vector mask predication.
538 ElementCount ElemCount = VPI.getStaticVectorLength();
539 Value *VLMask = convertEVLToMask(Builder, OldEVLParam, ElemCount);
540 Value *NewMaskParam = Builder.CreateAnd(VLMask, OldMaskParam);
541 if (VPI.getIntrinsicID() == Intrinsic::vp_merge ||
542 VPI.getIntrinsicID() == Intrinsic::vp_select)
543 VPI.setArgOperand(0, NewMaskParam);
544 else
545 VPI.setMaskParam(NewMaskParam);
546
547 // Drop the %evl parameter.
548 discardEVLParameter(VPI);
550 "transformation did not render the evl param ineffective!");
551
552 // Reassess the modified instruction.
553 return true;
554}
555
556bool CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
557 LLVM_DEBUG(dbgs() << "Lowering to unpredicated op: " << VPI << '\n');
558
559 IRBuilder<> Builder(&VPI);
560
561 // Try lowering to a LLVM instruction first.
562 auto OC = VPI.getFunctionalOpcode();
563
564 if (OC && Instruction::isBinaryOp(*OC))
565 return expandPredicationInBinaryOperator(Builder, VPI);
566
567 if (auto *VPRI = dyn_cast<VPReductionIntrinsic>(&VPI))
568 return expandPredicationInReduction(Builder, *VPRI);
569
570 if (auto *VPCmp = dyn_cast<VPCmpIntrinsic>(&VPI))
571 return expandPredicationInComparison(Builder, *VPCmp);
572
574 return expandPredicationToCastIntrinsic(Builder, VPI);
575
576 switch (VPI.getIntrinsicID()) {
577 default:
578 break;
579 case Intrinsic::vp_fneg: {
580 Value *NewNegOp = Builder.CreateFNeg(VPI.getOperand(0), VPI.getName());
581 replaceOperation(*NewNegOp, VPI);
582 return NewNegOp;
583 }
584 case Intrinsic::vp_select:
585 case Intrinsic::vp_merge: {
587 Value *NewSelectOp = Builder.CreateSelect(
588 VPI.getOperand(0), VPI.getOperand(1), VPI.getOperand(2), VPI.getName());
589 replaceOperation(*NewSelectOp, VPI);
590 return NewSelectOp;
591 }
592 case Intrinsic::vp_abs:
593 case Intrinsic::vp_smax:
594 case Intrinsic::vp_smin:
595 case Intrinsic::vp_umax:
596 case Intrinsic::vp_umin:
597 case Intrinsic::vp_bswap:
598 case Intrinsic::vp_bitreverse:
599 case Intrinsic::vp_ctpop:
600 case Intrinsic::vp_ctlz:
601 case Intrinsic::vp_cttz:
602 case Intrinsic::vp_sadd_sat:
603 case Intrinsic::vp_uadd_sat:
604 case Intrinsic::vp_ssub_sat:
605 case Intrinsic::vp_usub_sat:
606 case Intrinsic::vp_fshl:
607 case Intrinsic::vp_fshr:
608 return expandPredicationToIntCall(Builder, VPI);
609 case Intrinsic::vp_fabs:
610 case Intrinsic::vp_sqrt:
611 case Intrinsic::vp_maxnum:
612 case Intrinsic::vp_minnum:
613 case Intrinsic::vp_maximum:
614 case Intrinsic::vp_minimum:
615 case Intrinsic::vp_fma:
616 case Intrinsic::vp_fmuladd:
617 return expandPredicationToFPCall(Builder, VPI,
618 VPI.getFunctionalIntrinsicID().value());
619 case Intrinsic::vp_load:
620 case Intrinsic::vp_store:
621 case Intrinsic::vp_gather:
622 case Intrinsic::vp_scatter:
623 return expandPredicationInMemoryIntrinsic(Builder, VPI);
624 }
625
626 if (auto CID = VPI.getConstrainedIntrinsicID())
627 if (expandPredicationToFPCall(Builder, VPI, *CID))
628 return true;
629
630 return false;
631}
632
633//// } CachingVPExpander
634
635void sanitizeStrategy(VPIntrinsic &VPI, VPLegalization &LegalizeStrat) {
636 // Operations with speculatable lanes do not strictly need predication.
637 if (maySpeculateLanes(VPI)) {
638 // Converting a speculatable VP intrinsic means dropping %mask and %evl.
639 // No need to expand %evl into the %mask only to ignore that code.
640 if (LegalizeStrat.OpStrategy == VPLegalization::Convert)
642 return;
643 }
644
645 // We have to preserve the predicating effect of %evl for this
646 // non-speculatable VP intrinsic.
647 // 1) Never discard %evl.
648 // 2) If this VP intrinsic will be expanded to non-VP code, make sure that
649 // %evl gets folded into %mask.
650 if ((LegalizeStrat.EVLParamStrategy == VPLegalization::Discard) ||
651 (LegalizeStrat.OpStrategy == VPLegalization::Convert)) {
653 }
654}
655
657CachingVPExpander::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
658 auto VPStrat = TTI.getVPLegalizationStrategy(VPI);
659 if (LLVM_LIKELY(!UsingTTIOverrides)) {
660 // No overrides - we are in production.
661 return VPStrat;
662 }
663
664 // Overrides set - we are in testing, the following does not need to be
665 // efficient.
667 VPStrat.OpStrategy = parseOverrideOption(MaskTransformOverride);
668 return VPStrat;
669}
670
672CachingVPExpander::expandVectorPredication(VPIntrinsic &VPI) {
673 auto Strategy = getVPLegalizationStrategy(VPI);
674 sanitizeStrategy(VPI, Strategy);
675
676 VPExpansionDetails Changed = VPExpansionDetails::IntrinsicUnchanged;
677
678 // Transform the EVL parameter.
679 switch (Strategy.EVLParamStrategy) {
681 break;
683 if (discardEVLParameter(VPI))
684 Changed = VPExpansionDetails::IntrinsicUpdated;
685 break;
687 if (foldEVLIntoMask(VPI)) {
688 Changed = VPExpansionDetails::IntrinsicUpdated;
689 ++NumFoldedVL;
690 }
691 break;
692 }
693
694 // Replace with a non-predicated operation.
695 switch (Strategy.OpStrategy) {
697 break;
699 llvm_unreachable("Invalid strategy for operators.");
701 if (expandPredication(VPI)) {
702 ++NumLoweredVPOps;
703 Changed = VPExpansionDetails::IntrinsicReplaced;
704 }
705 break;
706 }
707
708 return Changed;
709}
710} // namespace
711
714 const TargetTransformInfo &TTI) {
715 return CachingVPExpander(TTI).expandVectorPredication(VPI);
716}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
#define LLVM_LIKELY(EXPR)
Definition Compiler.h:335
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static VPTransform parseOverrideOption(const std::string &TextOpt)
static cl::opt< std::string > MaskTransformOverride("expandvp-override-mask-transform", cl::init(""), cl::Hidden, cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES ". If non-empty, Ignore " "TargetTransformInfo and " "always use this transformation for the %mask parameter (Used in " "testing)."))
static cl::opt< std::string > EVLTransformOverride("expandvp-override-evl-transform", cl::init(""), cl::Hidden, cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES ". If non-empty, ignore " "TargetTransformInfo and " "always use this transformation for the %evl parameter (Used in " "testing)."))
static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp)
Transfer all properties from OldOp to NewOp and replace all uses.
static bool isAllTrueMask(Value *MaskVal)
static void transferDecorations(Value &NewVal, VPIntrinsic &VPI)
Transfer operation properties from OldVPI to NewVal.
TargetTransformInfo::VPLegalization VPLegalization
TargetTransformInfo::VPLegalization::VPTransform VPTransform
static bool anyExpandVPOverridesSet()
static bool maySpeculateLanes(VPIntrinsic &VPI)
static Constant * getSafeDivisor(Type *DivTy)
#define VPINTERNAL_VPLEGAL_CASES
loop Loop Strength Reduction
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:167
#define LLVM_DEBUG(...)
Definition Debug.h:119
This pass exposes codegen information to IR-level passes.
LLVM_ABI bool hasAttribute(Attribute::AttrKind Kind) const
Return true if the attribute exists in this set.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:703
This is an important base class in LLVM.
Definition Constant.h:43
Value * CreateNUWMul(Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:1450
LLVM_ABI CallInst * CreateFAddReduce(Value *Acc, Value *Src)
Create a sequential vector fadd reduction intrinsic of the source vector.
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Definition IRBuilder.h:547
LLVM_ABI Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
LLVM_ABI CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
LLVM_ABI CallInst * CreateConstrainedFPCall(Function *Callee, ArrayRef< Value * > Args, const Twine &Name="", std::optional< RoundingMode > Rounding=std::nullopt, std::optional< fp::ExceptionBehavior > Except=std::nullopt)
LLVM_ABI Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr, FMFSource FMFSource={})
Definition IRBuilder.h:2238
Value * CreateVScale(Type *Ty, const Twine &Name="")
Create a call to llvm.vscale.<Ty>().
Definition IRBuilder.h:958
LLVM_ABI Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
LLVM_ABI CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition IRBuilder.h:522
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition IRBuilder.h:2463
LLVM_ABI CallInst * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition IRBuilder.h:1847
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:1551
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition IRBuilder.h:1860
LLVM_ABI CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition IRBuilder.h:2508
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition IRBuilder.h:1708
LLVM_ABI CallInst * CreateFMulReduce(Value *Acc, Value *Src)
Create a sequential vector fmul reduction intrinsic of the source vector.
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:2439
Value * CreateFNeg(Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition IRBuilder.h:1790
LLVM_ABI Value * CreateStepVector(Type *DstType, const Twine &Name="")
Creates a vector of type DstType with the linear sequence <0, 1, ...>
LLVM_ABI CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
LLVM_ABI CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2780
bool isCast() const
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
bool isBinaryOp() const
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
void setAlignment(Align Align)
void setAlignment(Align Align)
A switch()-like statement whose cases are string literals.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
LLVM_ABI VPLegalization getVPLegalizationStrategy(const VPIntrinsic &PI) const
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Definition Type.h:246
Value * getOperand(unsigned i) const
Definition User.h:232
unsigned getNumOperands() const
Definition User.h:254
static LLVM_ABI bool isVPCast(Intrinsic::ID ID)
LLVM_ABI CmpInst::Predicate getPredicate() const
This is the common base class for vector predication intrinsics.
std::optional< unsigned > getFunctionalIntrinsicID() const
LLVM_ABI bool canIgnoreVectorLengthParam() const
LLVM_ABI void setMaskParam(Value *)
static LLVM_ABI std::optional< unsigned > getFunctionalOpcodeForVP(Intrinsic::ID ID)
LLVM_ABI Value * getVectorLengthParam() const
LLVM_ABI void setVectorLengthParam(Value *)
LLVM_ABI Value * getMemoryDataParam() const
LLVM_ABI Value * getMemoryPointerParam() const
std::optional< unsigned > getConstrainedIntrinsicID() const
LLVM_ABI MaybeAlign getPointerAlignment() const
LLVM_ABI Value * getMaskParam() const
LLVM_ABI ElementCount getStaticVectorLength() const
std::optional< unsigned > getFunctionalOpcode() const
LLVM_ABI unsigned getStartParamPos() const
LLVM_ABI unsigned getVectorParamPos() const
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:256
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:546
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.cpp:1101
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:322
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:134
Changed
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
LLVM_ABI AttributeSet getFnAttributes(LLVMContext &C, ID id)
Return the function attributes for an intrinsic.
LLVM_ABI bool isConstrainedFPIntrinsic(ID QID)
Returns true if the intrinsic ID is for one of the "ConstrainedFloating-Point Intrinsics".
initializer< Ty > init(const Ty &Val)
ElementType
The element type of an SRV or UAV resource.
Definition DXILABI.h:60
This is an optimization pass for GlobalISel generic memory operations.
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)
Returns the min/max intrinsic used when expanding a min/max reduction.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:649
FunctionAddr VTableAddr uintptr_t uintptr_t Int32Ty
Definition InstrProf.h:296
LLVM_ABI Value * getReductionIdentity(Intrinsic::ID RdxID, Type *Ty, FastMathFlags FMF)
Given information about an @llvm.vector.reduce.
LLVM_ABI unsigned getArithmeticReductionInstruction(Intrinsic::ID RdxID)
Returns the arithmetic instruction opcode used when expanding a reduction.
LLVM_ABI Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:548
VPExpansionDetails expandVectorPredicationIntrinsic(VPIntrinsic &VPI, const TargetTransformInfo &TTI)
Expand a vector predication intrinsic.
LLVM_ABI bool isSafeToSpeculativelyExecuteWithOpcode(unsigned Opcode, const Instruction *Inst, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
This returns the same result as isSafeToSpeculativelyExecute if Opcode is the actual opcode of Inst.
TargetTransformInfo TTI
IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:565
VPExpansionDetails
Represents the details the expansion of a VP intrinsic.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition Alignment.h:141