LLVM 19.0.0git
ExpandVectorPredication.cpp
Go to the documentation of this file.
1//===----- CodeGen/ExpandVectorPredication.cpp - Expand VP intrinsics -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements IR expansion for vector predication intrinsics, allowing
10// targets to enable vector predication until just before codegen.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/Statistic.h"
19#include "llvm/CodeGen/Passes.h"
20#include "llvm/IR/Constants.h"
21#include "llvm/IR/Function.h"
22#include "llvm/IR/IRBuilder.h"
26#include "llvm/IR/Intrinsics.h"
28#include "llvm/Pass.h"
31#include "llvm/Support/Debug.h"
32#include <optional>
33
34using namespace llvm;
35
38
39// Keep this in sync with TargetTransformInfo::VPLegalization.
40#define VPINTERNAL_VPLEGAL_CASES \
41 VPINTERNAL_CASE(Legal) \
42 VPINTERNAL_CASE(Discard) \
43 VPINTERNAL_CASE(Convert)
44
45#define VPINTERNAL_CASE(X) "|" #X
46
47// Override options.
49 "expandvp-override-evl-transform", cl::init(""), cl::Hidden,
50 cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
51 ". If non-empty, ignore "
52 "TargetTransformInfo and "
53 "always use this transformation for the %evl parameter (Used in "
54 "testing)."));
55
57 "expandvp-override-mask-transform", cl::init(""), cl::Hidden,
58 cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
59 ". If non-empty, Ignore "
60 "TargetTransformInfo and "
61 "always use this transformation for the %mask parameter (Used in "
62 "testing)."));
63
64#undef VPINTERNAL_CASE
65#define VPINTERNAL_CASE(X) .Case(#X, VPLegalization::X)
66
67static VPTransform parseOverrideOption(const std::string &TextOpt) {
69}
70
71#undef VPINTERNAL_VPLEGAL_CASES
72
73// Whether any override options are set.
75 return !EVLTransformOverride.empty() || !MaskTransformOverride.empty();
76}
77
78#define DEBUG_TYPE "expandvp"
79
80STATISTIC(NumFoldedVL, "Number of folded vector length params");
81STATISTIC(NumLoweredVPOps, "Number of folded vector predication operations");
82
83///// Helpers {
84
85/// \returns Whether the vector mask \p MaskVal has all lane bits set.
86static bool isAllTrueMask(Value *MaskVal) {
87 if (Value *SplattedVal = getSplatValue(MaskVal))
88 if (auto *ConstValue = dyn_cast<Constant>(SplattedVal))
89 return ConstValue->isAllOnesValue();
90
91 return false;
92}
93
94/// \returns A non-excepting divisor constant for this type.
95static Constant *getSafeDivisor(Type *DivTy) {
96 assert(DivTy->isIntOrIntVectorTy() && "Unsupported divisor type");
97 return ConstantInt::get(DivTy, 1u, false);
98}
99
100/// Transfer operation properties from \p OldVPI to \p NewVal.
101static void transferDecorations(Value &NewVal, VPIntrinsic &VPI) {
102 auto *NewInst = dyn_cast<Instruction>(&NewVal);
103 if (!NewInst || !isa<FPMathOperator>(NewVal))
104 return;
105
106 auto *OldFMOp = dyn_cast<FPMathOperator>(&VPI);
107 if (!OldFMOp)
108 return;
109
110 NewInst->setFastMathFlags(OldFMOp->getFastMathFlags());
111}
112
113/// Transfer all properties from \p OldOp to \p NewOp and replace all uses.
114/// OldVP gets erased.
115static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp) {
116 transferDecorations(NewOp, OldOp);
117 OldOp.replaceAllUsesWith(&NewOp);
118 OldOp.eraseFromParent();
119}
120
122 // The result of VP reductions depends on the mask and evl.
123 if (isa<VPReductionIntrinsic>(VPI))
124 return false;
125 // Fallback to whether the intrinsic is speculatable.
126 if (auto IntrID = VPI.getFunctionalIntrinsicID())
127 return Intrinsic::getAttributes(VPI.getContext(), *IntrID)
128 .hasFnAttr(Attribute::AttrKind::Speculatable);
129 if (auto Opc = VPI.getFunctionalOpcode())
131 return false;
132}
133
134//// } Helpers
135
136namespace {
137
138// Expansion pass state at function scope.
139struct CachingVPExpander {
140 Function &F;
142
143 /// \returns A (fixed length) vector with ascending integer indices
144 /// (<0, 1, ..., NumElems-1>).
145 /// \p Builder
146 /// Used for instruction creation.
147 /// \p LaneTy
148 /// Integer element type of the result vector.
149 /// \p NumElems
150 /// Number of vector elements.
151 Value *createStepVector(IRBuilder<> &Builder, Type *LaneTy,
152 unsigned NumElems);
153
154 /// \returns A bitmask that is true where the lane position is less-than \p
155 /// EVLParam
156 ///
157 /// \p Builder
158 /// Used for instruction creation.
159 /// \p VLParam
160 /// The explicit vector length parameter to test against the lane
161 /// positions.
162 /// \p ElemCount
163 /// Static (potentially scalable) number of vector elements.
164 Value *convertEVLToMask(IRBuilder<> &Builder, Value *EVLParam,
165 ElementCount ElemCount);
166
167 Value *foldEVLIntoMask(VPIntrinsic &VPI);
168
169 /// "Remove" the %evl parameter of \p PI by setting it to the static vector
170 /// length of the operation.
171 void discardEVLParameter(VPIntrinsic &PI);
172
173 /// Lower this VP binary operator to a unpredicated binary operator.
174 Value *expandPredicationInBinaryOperator(IRBuilder<> &Builder,
175 VPIntrinsic &PI);
176
177 /// Lower this VP int call to a unpredicated int call.
178 Value *expandPredicationToIntCall(IRBuilder<> &Builder, VPIntrinsic &PI,
179 unsigned UnpredicatedIntrinsicID);
180
181 /// Lower this VP fp call to a unpredicated fp call.
182 Value *expandPredicationToFPCall(IRBuilder<> &Builder, VPIntrinsic &PI,
183 unsigned UnpredicatedIntrinsicID);
184
185 /// Lower this VP reduction to a call to an unpredicated reduction intrinsic.
186 Value *expandPredicationInReduction(IRBuilder<> &Builder,
188
189 /// Lower this VP cast operation to a non-VP intrinsic.
190 Value *expandPredicationToCastIntrinsic(IRBuilder<> &Builder,
191 VPIntrinsic &VPI);
192
193 /// Lower this VP memory operation to a non-VP intrinsic.
194 Value *expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder,
195 VPIntrinsic &VPI);
196
197 /// Lower this VP comparison to a call to an unpredicated comparison.
198 Value *expandPredicationInComparison(IRBuilder<> &Builder,
199 VPCmpIntrinsic &PI);
200
201 /// Query TTI and expand the vector predication in \p P accordingly.
202 Value *expandPredication(VPIntrinsic &PI);
203
204 /// Determine how and whether the VPIntrinsic \p VPI shall be expanded. This
205 /// overrides TTI with the cl::opts listed at the top of this file.
206 VPLegalization getVPLegalizationStrategy(const VPIntrinsic &VPI) const;
207 bool UsingTTIOverrides;
208
209public:
210 CachingVPExpander(Function &F, const TargetTransformInfo &TTI)
211 : F(F), TTI(TTI), UsingTTIOverrides(anyExpandVPOverridesSet()) {}
212
213 bool expandVectorPredication();
214};
215
216//// CachingVPExpander {
217
218Value *CachingVPExpander::createStepVector(IRBuilder<> &Builder, Type *LaneTy,
219 unsigned NumElems) {
220 // TODO add caching
222
223 for (unsigned Idx = 0; Idx < NumElems; ++Idx)
224 ConstElems.push_back(ConstantInt::get(LaneTy, Idx, false));
225
226 return ConstantVector::get(ConstElems);
227}
228
229Value *CachingVPExpander::convertEVLToMask(IRBuilder<> &Builder,
230 Value *EVLParam,
231 ElementCount ElemCount) {
232 // TODO add caching
233 // Scalable vector %evl conversion.
234 if (ElemCount.isScalable()) {
235 auto *M = Builder.GetInsertBlock()->getModule();
236 Type *BoolVecTy = VectorType::get(Builder.getInt1Ty(), ElemCount);
237 Function *ActiveMaskFunc = Intrinsic::getDeclaration(
238 M, Intrinsic::get_active_lane_mask, {BoolVecTy, EVLParam->getType()});
239 // `get_active_lane_mask` performs an implicit less-than comparison.
240 Value *ConstZero = Builder.getInt32(0);
241 return Builder.CreateCall(ActiveMaskFunc, {ConstZero, EVLParam});
242 }
243
244 // Fixed vector %evl conversion.
245 Type *LaneTy = EVLParam->getType();
246 unsigned NumElems = ElemCount.getFixedValue();
247 Value *VLSplat = Builder.CreateVectorSplat(NumElems, EVLParam);
248 Value *IdxVec = createStepVector(Builder, LaneTy, NumElems);
249 return Builder.CreateICmp(CmpInst::ICMP_ULT, IdxVec, VLSplat);
250}
251
252Value *
253CachingVPExpander::expandPredicationInBinaryOperator(IRBuilder<> &Builder,
254 VPIntrinsic &VPI) {
256 "Implicitly dropping %evl in non-speculatable operator!");
257
258 auto OC = static_cast<Instruction::BinaryOps>(*VPI.getFunctionalOpcode());
260
261 Value *Op0 = VPI.getOperand(0);
262 Value *Op1 = VPI.getOperand(1);
263 Value *Mask = VPI.getMaskParam();
264
265 // Blend in safe operands.
266 if (Mask && !isAllTrueMask(Mask)) {
267 switch (OC) {
268 default:
269 // Can safely ignore the predicate.
270 break;
271
272 // Division operators need a safe divisor on masked-off lanes (1).
273 case Instruction::UDiv:
274 case Instruction::SDiv:
275 case Instruction::URem:
276 case Instruction::SRem:
277 // 2nd operand must not be zero.
278 Value *SafeDivisor = getSafeDivisor(VPI.getType());
279 Op1 = Builder.CreateSelect(Mask, Op1, SafeDivisor);
280 }
281 }
282
283 Value *NewBinOp = Builder.CreateBinOp(OC, Op0, Op1, VPI.getName());
284
285 replaceOperation(*NewBinOp, VPI);
286 return NewBinOp;
287}
288
289Value *CachingVPExpander::expandPredicationToIntCall(
290 IRBuilder<> &Builder, VPIntrinsic &VPI, unsigned UnpredicatedIntrinsicID) {
291 switch (UnpredicatedIntrinsicID) {
292 case Intrinsic::abs:
293 case Intrinsic::smax:
294 case Intrinsic::smin:
295 case Intrinsic::umax:
296 case Intrinsic::umin: {
297 Value *Op0 = VPI.getOperand(0);
298 Value *Op1 = VPI.getOperand(1);
300 VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
301 Value *NewOp = Builder.CreateCall(Fn, {Op0, Op1}, VPI.getName());
302 replaceOperation(*NewOp, VPI);
303 return NewOp;
304 }
305 case Intrinsic::bswap:
306 case Intrinsic::bitreverse: {
307 Value *Op = VPI.getOperand(0);
309 VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
310 Value *NewOp = Builder.CreateCall(Fn, {Op}, VPI.getName());
311 replaceOperation(*NewOp, VPI);
312 return NewOp;
313 }
314 }
315 return nullptr;
316}
317
318Value *CachingVPExpander::expandPredicationToFPCall(
319 IRBuilder<> &Builder, VPIntrinsic &VPI, unsigned UnpredicatedIntrinsicID) {
321 "Implicitly dropping %evl in non-speculatable operator!");
322
323 switch (UnpredicatedIntrinsicID) {
324 case Intrinsic::fabs:
325 case Intrinsic::sqrt: {
326 Value *Op0 = VPI.getOperand(0);
328 VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
329 Value *NewOp = Builder.CreateCall(Fn, {Op0}, VPI.getName());
330 replaceOperation(*NewOp, VPI);
331 return NewOp;
332 }
333 case Intrinsic::maxnum:
334 case Intrinsic::minnum: {
335 Value *Op0 = VPI.getOperand(0);
336 Value *Op1 = VPI.getOperand(1);
338 VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
339 Value *NewOp = Builder.CreateCall(Fn, {Op0, Op1}, VPI.getName());
340 replaceOperation(*NewOp, VPI);
341 return NewOp;
342 }
343 case Intrinsic::experimental_constrained_fma:
344 case Intrinsic::experimental_constrained_fmuladd: {
345 Value *Op0 = VPI.getOperand(0);
346 Value *Op1 = VPI.getOperand(1);
347 Value *Op2 = VPI.getOperand(2);
349 VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
350 Value *NewOp =
351 Builder.CreateConstrainedFPCall(Fn, {Op0, Op1, Op2}, VPI.getName());
352 replaceOperation(*NewOp, VPI);
353 return NewOp;
354 }
355 }
356
357 return nullptr;
358}
359
360static Value *getNeutralReductionElement(const VPReductionIntrinsic &VPI,
361 Type *EltTy) {
362 bool Negative = false;
363 unsigned EltBits = EltTy->getScalarSizeInBits();
364 switch (VPI.getIntrinsicID()) {
365 default:
366 llvm_unreachable("Expecting a VP reduction intrinsic");
367 case Intrinsic::vp_reduce_add:
368 case Intrinsic::vp_reduce_or:
369 case Intrinsic::vp_reduce_xor:
370 case Intrinsic::vp_reduce_umax:
371 return Constant::getNullValue(EltTy);
372 case Intrinsic::vp_reduce_mul:
373 return ConstantInt::get(EltTy, 1, /*IsSigned*/ false);
374 case Intrinsic::vp_reduce_and:
375 case Intrinsic::vp_reduce_umin:
376 return ConstantInt::getAllOnesValue(EltTy);
377 case Intrinsic::vp_reduce_smin:
378 return ConstantInt::get(EltTy->getContext(),
379 APInt::getSignedMaxValue(EltBits));
380 case Intrinsic::vp_reduce_smax:
381 return ConstantInt::get(EltTy->getContext(),
382 APInt::getSignedMinValue(EltBits));
383 case Intrinsic::vp_reduce_fmax:
384 Negative = true;
385 [[fallthrough]];
386 case Intrinsic::vp_reduce_fmin: {
388 const fltSemantics &Semantics = EltTy->getFltSemantics();
389 return !Flags.noNaNs() ? ConstantFP::getQNaN(EltTy, Negative)
390 : !Flags.noInfs()
391 ? ConstantFP::getInfinity(EltTy, Negative)
392 : ConstantFP::get(EltTy,
393 APFloat::getLargest(Semantics, Negative));
394 }
395 case Intrinsic::vp_reduce_fadd:
396 return ConstantFP::getNegativeZero(EltTy);
397 case Intrinsic::vp_reduce_fmul:
398 return ConstantFP::get(EltTy, 1.0);
399 }
400}
401
402Value *
403CachingVPExpander::expandPredicationInReduction(IRBuilder<> &Builder,
406 "Implicitly dropping %evl in non-speculatable operator!");
407
408 Value *Mask = VPI.getMaskParam();
409 Value *RedOp = VPI.getOperand(VPI.getVectorParamPos());
410
411 // Insert neutral element in masked-out positions
412 if (Mask && !isAllTrueMask(Mask)) {
413 auto *NeutralElt = getNeutralReductionElement(VPI, VPI.getType());
414 auto *NeutralVector = Builder.CreateVectorSplat(
415 cast<VectorType>(RedOp->getType())->getElementCount(), NeutralElt);
416 RedOp = Builder.CreateSelect(Mask, RedOp, NeutralVector);
417 }
418
420 Value *Start = VPI.getOperand(VPI.getStartParamPos());
421
422 switch (VPI.getIntrinsicID()) {
423 default:
424 llvm_unreachable("Impossible reduction kind");
425 case Intrinsic::vp_reduce_add:
426 Reduction = Builder.CreateAddReduce(RedOp);
427 Reduction = Builder.CreateAdd(Reduction, Start);
428 break;
429 case Intrinsic::vp_reduce_mul:
430 Reduction = Builder.CreateMulReduce(RedOp);
431 Reduction = Builder.CreateMul(Reduction, Start);
432 break;
433 case Intrinsic::vp_reduce_and:
434 Reduction = Builder.CreateAndReduce(RedOp);
435 Reduction = Builder.CreateAnd(Reduction, Start);
436 break;
437 case Intrinsic::vp_reduce_or:
438 Reduction = Builder.CreateOrReduce(RedOp);
439 Reduction = Builder.CreateOr(Reduction, Start);
440 break;
441 case Intrinsic::vp_reduce_xor:
442 Reduction = Builder.CreateXorReduce(RedOp);
443 Reduction = Builder.CreateXor(Reduction, Start);
444 break;
445 case Intrinsic::vp_reduce_smax:
446 Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ true);
447 Reduction =
448 Builder.CreateBinaryIntrinsic(Intrinsic::smax, Reduction, Start);
449 break;
450 case Intrinsic::vp_reduce_smin:
451 Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ true);
452 Reduction =
453 Builder.CreateBinaryIntrinsic(Intrinsic::smin, Reduction, Start);
454 break;
455 case Intrinsic::vp_reduce_umax:
456 Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ false);
457 Reduction =
458 Builder.CreateBinaryIntrinsic(Intrinsic::umax, Reduction, Start);
459 break;
460 case Intrinsic::vp_reduce_umin:
461 Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ false);
462 Reduction =
463 Builder.CreateBinaryIntrinsic(Intrinsic::umin, Reduction, Start);
464 break;
465 case Intrinsic::vp_reduce_fmax:
466 Reduction = Builder.CreateFPMaxReduce(RedOp);
467 transferDecorations(*Reduction, VPI);
468 Reduction =
469 Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, Reduction, Start);
470 break;
471 case Intrinsic::vp_reduce_fmin:
472 Reduction = Builder.CreateFPMinReduce(RedOp);
473 transferDecorations(*Reduction, VPI);
474 Reduction =
475 Builder.CreateBinaryIntrinsic(Intrinsic::minnum, Reduction, Start);
476 break;
477 case Intrinsic::vp_reduce_fadd:
478 Reduction = Builder.CreateFAddReduce(Start, RedOp);
479 break;
480 case Intrinsic::vp_reduce_fmul:
481 Reduction = Builder.CreateFMulReduce(Start, RedOp);
482 break;
483 }
484
485 replaceOperation(*Reduction, VPI);
486 return Reduction;
487}
488
489Value *CachingVPExpander::expandPredicationToCastIntrinsic(IRBuilder<> &Builder,
490 VPIntrinsic &VPI) {
491 Value *CastOp = nullptr;
492 switch (VPI.getIntrinsicID()) {
493 default:
494 llvm_unreachable("Not a VP cast intrinsic");
495 case Intrinsic::vp_sext:
496 CastOp =
497 Builder.CreateSExt(VPI.getOperand(0), VPI.getType(), VPI.getName());
498 break;
499 case Intrinsic::vp_zext:
500 CastOp =
501 Builder.CreateZExt(VPI.getOperand(0), VPI.getType(), VPI.getName());
502 break;
503 case Intrinsic::vp_trunc:
504 CastOp =
505 Builder.CreateTrunc(VPI.getOperand(0), VPI.getType(), VPI.getName());
506 break;
507 case Intrinsic::vp_inttoptr:
508 CastOp =
509 Builder.CreateIntToPtr(VPI.getOperand(0), VPI.getType(), VPI.getName());
510 break;
511 case Intrinsic::vp_ptrtoint:
512 CastOp =
513 Builder.CreatePtrToInt(VPI.getOperand(0), VPI.getType(), VPI.getName());
514 break;
515 case Intrinsic::vp_fptosi:
516 CastOp =
517 Builder.CreateFPToSI(VPI.getOperand(0), VPI.getType(), VPI.getName());
518 break;
519
520 case Intrinsic::vp_fptoui:
521 CastOp =
522 Builder.CreateFPToUI(VPI.getOperand(0), VPI.getType(), VPI.getName());
523 break;
524 case Intrinsic::vp_sitofp:
525 CastOp =
526 Builder.CreateSIToFP(VPI.getOperand(0), VPI.getType(), VPI.getName());
527 break;
528 case Intrinsic::vp_uitofp:
529 CastOp =
530 Builder.CreateUIToFP(VPI.getOperand(0), VPI.getType(), VPI.getName());
531 break;
532 case Intrinsic::vp_fptrunc:
533 CastOp =
534 Builder.CreateFPTrunc(VPI.getOperand(0), VPI.getType(), VPI.getName());
535 break;
536 case Intrinsic::vp_fpext:
537 CastOp =
538 Builder.CreateFPExt(VPI.getOperand(0), VPI.getType(), VPI.getName());
539 break;
540 }
541 replaceOperation(*CastOp, VPI);
542 return CastOp;
543}
544
545Value *
546CachingVPExpander::expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder,
547 VPIntrinsic &VPI) {
549
550 const auto &DL = F.getParent()->getDataLayout();
551
552 Value *MaskParam = VPI.getMaskParam();
553 Value *PtrParam = VPI.getMemoryPointerParam();
554 Value *DataParam = VPI.getMemoryDataParam();
555 bool IsUnmasked = isAllTrueMask(MaskParam);
556
557 MaybeAlign AlignOpt = VPI.getPointerAlignment();
558
559 Value *NewMemoryInst = nullptr;
560 switch (VPI.getIntrinsicID()) {
561 default:
562 llvm_unreachable("Not a VP memory intrinsic");
563 case Intrinsic::vp_store:
564 if (IsUnmasked) {
565 StoreInst *NewStore =
566 Builder.CreateStore(DataParam, PtrParam, /*IsVolatile*/ false);
567 if (AlignOpt.has_value())
568 NewStore->setAlignment(*AlignOpt);
569 NewMemoryInst = NewStore;
570 } else
571 NewMemoryInst = Builder.CreateMaskedStore(
572 DataParam, PtrParam, AlignOpt.valueOrOne(), MaskParam);
573
574 break;
575 case Intrinsic::vp_load:
576 if (IsUnmasked) {
577 LoadInst *NewLoad =
578 Builder.CreateLoad(VPI.getType(), PtrParam, /*IsVolatile*/ false);
579 if (AlignOpt.has_value())
580 NewLoad->setAlignment(*AlignOpt);
581 NewMemoryInst = NewLoad;
582 } else
583 NewMemoryInst = Builder.CreateMaskedLoad(
584 VPI.getType(), PtrParam, AlignOpt.valueOrOne(), MaskParam);
585
586 break;
587 case Intrinsic::vp_scatter: {
588 auto *ElementType =
589 cast<VectorType>(DataParam->getType())->getElementType();
590 NewMemoryInst = Builder.CreateMaskedScatter(
591 DataParam, PtrParam,
592 AlignOpt.value_or(DL.getPrefTypeAlign(ElementType)), MaskParam);
593 break;
594 }
595 case Intrinsic::vp_gather: {
596 auto *ElementType = cast<VectorType>(VPI.getType())->getElementType();
597 NewMemoryInst = Builder.CreateMaskedGather(
598 VPI.getType(), PtrParam,
599 AlignOpt.value_or(DL.getPrefTypeAlign(ElementType)), MaskParam, nullptr,
600 VPI.getName());
601 break;
602 }
603 }
604
605 assert(NewMemoryInst);
606 replaceOperation(*NewMemoryInst, VPI);
607 return NewMemoryInst;
608}
609
610Value *CachingVPExpander::expandPredicationInComparison(IRBuilder<> &Builder,
611 VPCmpIntrinsic &VPI) {
613 "Implicitly dropping %evl in non-speculatable operator!");
614
615 assert(*VPI.getFunctionalOpcode() == Instruction::ICmp ||
616 *VPI.getFunctionalOpcode() == Instruction::FCmp);
617
618 Value *Op0 = VPI.getOperand(0);
619 Value *Op1 = VPI.getOperand(1);
620 auto Pred = VPI.getPredicate();
621
622 auto *NewCmp = Builder.CreateCmp(Pred, Op0, Op1);
623
624 replaceOperation(*NewCmp, VPI);
625 return NewCmp;
626}
627
628void CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) {
629 LLVM_DEBUG(dbgs() << "Discard EVL parameter in " << VPI << "\n");
630
632 return;
633
634 Value *EVLParam = VPI.getVectorLengthParam();
635 if (!EVLParam)
636 return;
637
638 ElementCount StaticElemCount = VPI.getStaticVectorLength();
639 Value *MaxEVL = nullptr;
641 if (StaticElemCount.isScalable()) {
642 // TODO add caching
643 auto *M = VPI.getModule();
644 Function *VScaleFunc =
645 Intrinsic::getDeclaration(M, Intrinsic::vscale, Int32Ty);
646 IRBuilder<> Builder(VPI.getParent(), VPI.getIterator());
647 Value *FactorConst = Builder.getInt32(StaticElemCount.getKnownMinValue());
648 Value *VScale = Builder.CreateCall(VScaleFunc, {}, "vscale");
649 MaxEVL = Builder.CreateMul(VScale, FactorConst, "scalable_size",
650 /*NUW*/ true, /*NSW*/ false);
651 } else {
652 MaxEVL = ConstantInt::get(Int32Ty, StaticElemCount.getFixedValue(), false);
653 }
654 VPI.setVectorLengthParam(MaxEVL);
655}
656
657Value *CachingVPExpander::foldEVLIntoMask(VPIntrinsic &VPI) {
658 LLVM_DEBUG(dbgs() << "Folding vlen for " << VPI << '\n');
659
660 IRBuilder<> Builder(&VPI);
661
662 // Ineffective %evl parameter and so nothing to do here.
664 return &VPI;
665
666 // Only VP intrinsics can have an %evl parameter.
667 Value *OldMaskParam = VPI.getMaskParam();
668 Value *OldEVLParam = VPI.getVectorLengthParam();
669 assert(OldMaskParam && "no mask param to fold the vl param into");
670 assert(OldEVLParam && "no EVL param to fold away");
671
672 LLVM_DEBUG(dbgs() << "OLD evl: " << *OldEVLParam << '\n');
673 LLVM_DEBUG(dbgs() << "OLD mask: " << *OldMaskParam << '\n');
674
675 // Convert the %evl predication into vector mask predication.
676 ElementCount ElemCount = VPI.getStaticVectorLength();
677 Value *VLMask = convertEVLToMask(Builder, OldEVLParam, ElemCount);
678 Value *NewMaskParam = Builder.CreateAnd(VLMask, OldMaskParam);
679 VPI.setMaskParam(NewMaskParam);
680
681 // Drop the %evl parameter.
682 discardEVLParameter(VPI);
684 "transformation did not render the evl param ineffective!");
685
686 // Reassess the modified instruction.
687 return &VPI;
688}
689
690Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
691 LLVM_DEBUG(dbgs() << "Lowering to unpredicated op: " << VPI << '\n');
692
693 IRBuilder<> Builder(&VPI);
694
695 // Try lowering to a LLVM instruction first.
696 auto OC = VPI.getFunctionalOpcode();
697
698 if (OC && Instruction::isBinaryOp(*OC))
699 return expandPredicationInBinaryOperator(Builder, VPI);
700
701 if (auto *VPRI = dyn_cast<VPReductionIntrinsic>(&VPI))
702 return expandPredicationInReduction(Builder, *VPRI);
703
704 if (auto *VPCmp = dyn_cast<VPCmpIntrinsic>(&VPI))
705 return expandPredicationInComparison(Builder, *VPCmp);
706
708 return expandPredicationToCastIntrinsic(Builder, VPI);
709 }
710
711 switch (VPI.getIntrinsicID()) {
712 default:
713 break;
714 case Intrinsic::vp_fneg: {
715 Value *NewNegOp = Builder.CreateFNeg(VPI.getOperand(0), VPI.getName());
716 replaceOperation(*NewNegOp, VPI);
717 return NewNegOp;
718 }
719 case Intrinsic::vp_abs:
720 case Intrinsic::vp_smax:
721 case Intrinsic::vp_smin:
722 case Intrinsic::vp_umax:
723 case Intrinsic::vp_umin:
724 case Intrinsic::vp_bswap:
725 case Intrinsic::vp_bitreverse:
726 return expandPredicationToIntCall(Builder, VPI,
727 VPI.getFunctionalIntrinsicID().value());
728 case Intrinsic::vp_fabs:
729 case Intrinsic::vp_sqrt:
730 case Intrinsic::vp_maxnum:
731 case Intrinsic::vp_minnum:
732 case Intrinsic::vp_maximum:
733 case Intrinsic::vp_minimum:
734 return expandPredicationToFPCall(Builder, VPI,
735 VPI.getFunctionalIntrinsicID().value());
736 case Intrinsic::vp_load:
737 case Intrinsic::vp_store:
738 case Intrinsic::vp_gather:
739 case Intrinsic::vp_scatter:
740 return expandPredicationInMemoryIntrinsic(Builder, VPI);
741 }
742
743 if (auto CID = VPI.getConstrainedIntrinsicID())
744 if (Value *Call = expandPredicationToFPCall(Builder, VPI, *CID))
745 return Call;
746
747 return &VPI;
748}
749
750//// } CachingVPExpander
751
752struct TransformJob {
753 VPIntrinsic *PI;
755 TransformJob(VPIntrinsic *PI, TargetTransformInfo::VPLegalization InitStrat)
756 : PI(PI), Strategy(InitStrat) {}
757
758 bool isDone() const { return Strategy.shouldDoNothing(); }
759};
760
761void sanitizeStrategy(VPIntrinsic &VPI, VPLegalization &LegalizeStrat) {
762 // Operations with speculatable lanes do not strictly need predication.
763 if (maySpeculateLanes(VPI)) {
764 // Converting a speculatable VP intrinsic means dropping %mask and %evl.
765 // No need to expand %evl into the %mask only to ignore that code.
766 if (LegalizeStrat.OpStrategy == VPLegalization::Convert)
768 return;
769 }
770
771 // We have to preserve the predicating effect of %evl for this
772 // non-speculatable VP intrinsic.
773 // 1) Never discard %evl.
774 // 2) If this VP intrinsic will be expanded to non-VP code, make sure that
775 // %evl gets folded into %mask.
776 if ((LegalizeStrat.EVLParamStrategy == VPLegalization::Discard) ||
777 (LegalizeStrat.OpStrategy == VPLegalization::Convert)) {
779 }
780}
781
783CachingVPExpander::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
784 auto VPStrat = TTI.getVPLegalizationStrategy(VPI);
785 if (LLVM_LIKELY(!UsingTTIOverrides)) {
786 // No overrides - we are in production.
787 return VPStrat;
788 }
789
790 // Overrides set - we are in testing, the following does not need to be
791 // efficient.
793 VPStrat.OpStrategy = parseOverrideOption(MaskTransformOverride);
794 return VPStrat;
795}
796
797/// Expand llvm.vp.* intrinsics as requested by \p TTI.
798bool CachingVPExpander::expandVectorPredication() {
800
801 // Collect all VPIntrinsics that need expansion and determine their expansion
802 // strategy.
803 for (auto &I : instructions(F)) {
804 auto *VPI = dyn_cast<VPIntrinsic>(&I);
805 if (!VPI)
806 continue;
807 auto VPStrat = getVPLegalizationStrategy(*VPI);
808 sanitizeStrategy(*VPI, VPStrat);
809 if (!VPStrat.shouldDoNothing())
810 Worklist.emplace_back(VPI, VPStrat);
811 }
812 if (Worklist.empty())
813 return false;
814
815 // Transform all VPIntrinsics on the worklist.
816 LLVM_DEBUG(dbgs() << "\n:::: Transforming " << Worklist.size()
817 << " instructions ::::\n");
818 for (TransformJob Job : Worklist) {
819 // Transform the EVL parameter.
820 switch (Job.Strategy.EVLParamStrategy) {
822 break;
824 discardEVLParameter(*Job.PI);
825 break;
827 if (foldEVLIntoMask(*Job.PI))
828 ++NumFoldedVL;
829 break;
830 }
831 Job.Strategy.EVLParamStrategy = VPLegalization::Legal;
832
833 // Replace with a non-predicated operation.
834 switch (Job.Strategy.OpStrategy) {
836 break;
838 llvm_unreachable("Invalid strategy for operators.");
840 expandPredication(*Job.PI);
841 ++NumLoweredVPOps;
842 break;
843 }
844 Job.Strategy.OpStrategy = VPLegalization::Legal;
845
846 assert(Job.isDone() && "incomplete transformation");
847 }
848
849 return true;
850}
851class ExpandVectorPredication : public FunctionPass {
852public:
853 static char ID;
854 ExpandVectorPredication() : FunctionPass(ID) {
856 }
857
858 bool runOnFunction(Function &F) override {
859 const auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
860 CachingVPExpander VPExpander(F, *TTI);
861 return VPExpander.expandVectorPredication();
862 }
863
864 void getAnalysisUsage(AnalysisUsage &AU) const override {
866 AU.setPreservesCFG();
867 }
868};
869} // namespace
870
871char ExpandVectorPredication::ID;
872INITIALIZE_PASS_BEGIN(ExpandVectorPredication, "expandvp",
873 "Expand vector predication intrinsics", false, false)
876INITIALIZE_PASS_END(ExpandVectorPredication, "expandvp",
877 "Expand vector predication intrinsics", false, false)
878
880 return new ExpandVectorPredication();
881}
882
885 const auto &TTI = AM.getResult<TargetIRAnalysis>(F);
886 CachingVPExpander VPExpander(F, TTI);
887 if (!VPExpander.expandVectorPredication())
888 return PreservedAnalyses::all();
891 return PA;
892}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
#define LLVM_LIKELY(EXPR)
Definition: Compiler.h:240
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
static VPTransform parseOverrideOption(const std::string &TextOpt)
static cl::opt< std::string > MaskTransformOverride("expandvp-override-mask-transform", cl::init(""), cl::Hidden, cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES ". If non-empty, Ignore " "TargetTransformInfo and " "always use this transformation for the %mask parameter (Used in " "testing)."))
static cl::opt< std::string > EVLTransformOverride("expandvp-override-evl-transform", cl::init(""), cl::Hidden, cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES ". If non-empty, ignore " "TargetTransformInfo and " "always use this transformation for the %evl parameter (Used in " "testing)."))
static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp)
Transfer all properties from OldOp to NewOp and replace all uses.
static bool isAllTrueMask(Value *MaskVal)
static void transferDecorations(Value &NewVal, VPIntrinsic &VPI)
Transfer operation properties from OldVPI to NewVal.
static bool anyExpandVPOverridesSet()
static bool maySpeculateLanes(VPIntrinsic &VPI)
Expand vector predication intrinsics
static Constant * getSafeDivisor(Type *DivTy)
#define VPINTERNAL_VPLEGAL_CASES
loop Loop Strength Reduction
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
IntegerType * Int32Ty
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
This pass exposes codegen information to IR-level passes.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:187
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:197
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:348
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:500
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:269
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Definition: BasicBlock.cpp:336
Represents analyses that only rely on functions' control flow.
Definition: Analysis.h:70
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:992
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:267
static Constant * getNegativeZero(Type *Ty)
Definition: Constants.h:305
static Constant * getQNaN(Type *Ty, bool Negative=false, APInt *Payload=nullptr)
Definition: Constants.cpp:1015
static Constant * get(ArrayRef< Constant * > V)
Definition: Constants.cpp:1398
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
This class represents an Operation in the Expression.
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:313
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:20
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
virtual bool runOnFunction(Function &F)=0
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.
CallInst * CreateMulReduce(Value *Src)
Create a vector int mul reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:437
CallInst * CreateFAddReduce(Value *Acc, Value *Src)
Create a sequential vector fadd reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:417
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2006
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Definition: IRBuilder.h:505
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2077
Value * CreateFPTrunc(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2084
CallInst * CreateAndReduce(Value *Src)
Create a vector int AND reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:441
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Definition: IRBuilder.cpp:1212
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
Definition: IRBuilder.cpp:578
CallInst * CreateConstrainedFPCall(Function *Callee, ArrayRef< Value * > Args, const Twine &Name="", std::optional< RoundingMode > Rounding=std::nullopt, std::optional< fp::ExceptionBehavior > Except=std::nullopt)
Definition: IRBuilder.cpp:1082
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.cpp:1108
Value * CreateFPToUI(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2056
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2022
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2105
CallInst * CreateAddReduce(Value *Src)
Create a vector int add reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:433
BasicBlock * GetInsertBlock() const
Definition: IRBuilder.h:174
Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2070
CallInst * CreateXorReduce(Value *Src)
Create a vector int XOR reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:449
CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:445
CallInst * CreateFPMinReduce(Value *Src)
Create a vector float min reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:469
CallInst * CreateFPMaxReduce(Value *Src)
Create a vector float max reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:465
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:480
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2349
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition: IRBuilder.h:1789
CallInst * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Definition: IRBuilder.cpp:921
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Definition: IRBuilder.h:2010
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1469
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition: IRBuilder.h:1802
CallInst * CreateIntMaxReduce(Value *Src, bool IsSigned=false)
Create a vector integer max reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:453
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Definition: IRBuilder.cpp:598
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1321
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2100
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1491
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1660
CallInst * CreateIntMinReduce(Value *Src, bool IsSigned=false)
Create a vector integer min reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:459
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2395
Value * CreateFPExt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2093
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1513
CallInst * CreateFMulReduce(Value *Acc, Value *Src)
Create a sequential vector fmul reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:425
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2334
Value * CreateFNeg(Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1729
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1355
CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
Definition: IRBuilder.cpp:661
CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
Definition: IRBuilder.cpp:630
Value * CreateFPToSI(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2063
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2649
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:81
bool isBinaryOp() const
Definition: Instruction.h:255
const BasicBlock * getParent() const
Definition: Instruction.h:150
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
An instruction for reading from memory.
Definition: Instructions.h:184
void setAlignment(Align Align)
Definition: Instructions.h:240
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Definition: Pass.cpp:98
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:109
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:115
void preserveSet()
Mark an analysis set as preserved.
Definition: Analysis.h:144
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
An instruction for storing to memory.
Definition: Instructions.h:317
void setAlignment(Align Align)
Definition: Instructions.h:373
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:44
Analysis pass providing the TargetTransformInfo.
Wrapper pass for TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
VPLegalization getVPLegalizationStrategy(const VPIntrinsic &PI) const
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
const fltSemantics & getFltSemantics() const
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Definition: Type.h:234
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
static IntegerType * getInt32Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
Definition: User.h:169
static bool isVPCast(Intrinsic::ID ID)
CmpInst::Predicate getPredicate() const
This is the common base class for vector predication intrinsics.
std::optional< unsigned > getFunctionalIntrinsicID() const
bool canIgnoreVectorLengthParam() const
void setMaskParam(Value *)
Value * getVectorLengthParam() const
void setVectorLengthParam(Value *)
Value * getMemoryDataParam() const
Value * getMemoryPointerParam() const
std::optional< unsigned > getConstrainedIntrinsicID() const
MaybeAlign getPointerAlignment() const
Value * getMaskParam() const
ElementCount getStaticVectorLength() const
std::optional< unsigned > getFunctionalOpcode() const
This represents vector predication reduction intrinsics.
unsigned getStartParamPos() const
unsigned getVectorParamPos() const
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:187
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
self_iterator getIterator()
Definition: ilist_node.h:109
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
AttributeList getAttributes(LLVMContext &C, ID id)
Return the attributes for an intrinsic.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1447
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
ElementType
The element type of an SRV or UAV resource.
Definition: DXILABI.h:68
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
FunctionPass * createExpandVectorPredicationPass()
This pass expands the vector predication intrinsics into unpredicated instructions with selects or ju...
void initializeExpandVectorPredicationPass(PassRegistry &)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isSafeToSpeculativelyExecuteWithOpcode(unsigned Opcode, const Instruction *Inst, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
This returns the same result as isSafeToSpeculativelyExecute if Opcode is the actual opcode of Inst.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition: Alignment.h:141