LLVM 24.0.0git
AMDGPUInstCombineIntrinsic.cpp
Go to the documentation of this file.
1//===- AMDGPInstCombineIntrinsic.cpp - AMDGPU specific InstCombine pass ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// This file implements a TargetTransformInfo analysis pass specific to the
11// AMDGPU target machine. It uses the target's detailed information to provide
12// more precise answers to certain TTI queries, while letting the target
13// independent and default TTI implementations handle the rest.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUInstrInfo.h"
19#include "GCNSubtarget.h"
20#include "SIDefines.h"
22#include "llvm/ADT/STLExtras.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/Dominators.h"
27#include "llvm/IR/IntrinsicsAMDGPU.h"
30#include <optional>
31
32using namespace llvm;
33using namespace llvm::PatternMatch;
34
35#define DEBUG_TYPE "AMDGPUtti"
36
37namespace {
38
39struct AMDGPUImageDMaskIntrinsic {
40 unsigned Intr;
41};
42
43#define GET_AMDGPUImageDMaskIntrinsicTable_IMPL
44#include "AMDGPUGenSearchableTables.inc"
45
46} // end anonymous namespace
47
48// Constant fold llvm.amdgcn.fmed3 intrinsics for standard inputs.
49//
50// A single NaN input is folded to minnum, so we rely on that folding for
51// handling NaNs.
52static APFloat fmed3AMDGCN(const APFloat &Src0, const APFloat &Src1,
53 const APFloat &Src2) {
54 assert(!Src0.isNaN() && !Src1.isNaN() && !Src2.isNaN() &&
55 "nans handled separately");
56 APFloat Max3 = maxnum(maxnum(Src0, Src1), Src2);
57
58 if (Max3.bitwiseIsEqual(Src0))
59 return maxnum(Src1, Src2);
60
61 if (Max3.bitwiseIsEqual(Src1))
62 return maxnum(Src0, Src2);
63
64 return maxnum(Src0, Src1);
65}
66
67// Check if a value can be converted to a 16-bit value without losing precision.
68// The value is expected to be either a float (IsFloat = true) or an unsigned
69// integer (IsFloat = false). When AllowI16SExt is set, a sext from i16 is also
70// accepted: for unsigned addresses sext and zext only differ for a negative
71// i16, which is out of bounds anyway (see caller).
72static bool canSafelyConvertTo16Bit(Value &V, bool IsFloat,
73 bool AllowI16SExt = false) {
74 Type *VTy = V.getType();
75 if (VTy->isHalfTy() || VTy->isIntegerTy(16)) {
76 // The value is already 16-bit, so we don't want to convert to 16-bit again!
77 return false;
78 }
79 if (IsFloat) {
80 if (ConstantFP *ConstFloat = dyn_cast<ConstantFP>(&V)) {
81 // We need to check that if we cast the index down to a half, we do not
82 // lose precision.
83 APFloat FloatValue(ConstFloat->getValueAPF());
84 bool LosesInfo = true;
86 &LosesInfo);
87 return !LosesInfo;
88 }
89 } else {
90 if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(&V)) {
91 // We need to check that if we cast the index down to an i16, we do not
92 // lose precision.
93 APInt IntValue(ConstInt->getValue());
94 return IntValue.getActiveBits() <= 16;
95 }
96 }
97
98 // Coordinates may arrive as extractelement((s|z|fp)ext Vec), Idx. The
99 // widening cast has one use per lane, so it is never sunk into the extract;
100 // strip the extract here so the cast check below is common to scalar and
101 // vector coords.
102 Value *CastCandidate;
103 if (!match(&V, m_ExtractElt(m_Value(CastCandidate), m_Value())))
104 CastCandidate = &V;
105
106 Value *CastSrc;
107 bool IsExt = IsFloat ? match(CastCandidate, m_FPExt(m_Value(CastSrc)))
108 : match(CastCandidate, m_ZExt(m_Value(CastSrc)));
109 if (!IsExt && !IsFloat && AllowI16SExt)
110 IsExt = match(CastCandidate, m_SExt(m_Value(CastSrc)));
111 if (IsExt) {
112 Type *CastSrcTy = CastSrc->getType()->getScalarType();
113 if (CastSrcTy->isHalfTy() || CastSrcTy->isIntegerTy(16))
114 return true;
115 }
116
117 return false;
118}
119
120// Convert a value to 16-bit.
122 Type *VTy = V.getType();
124 return cast<Instruction>(&V)->getOperand(0);
125 // Vector form: extractelement((s|z|fp)ext Vec), Idx -> extractelement(Vec,
126 // Idx), taking the narrow lane directly so the widening cast can be removed.
127 Instruction *VecCast;
128 Value *Idx;
129 if (match(&V, m_ExtractElt(m_Instruction(VecCast), m_Value(Idx))) &&
131 return Builder.CreateExtractElement(VecCast->getOperand(0), Idx);
132 if (VTy->isIntegerTy())
133 return Builder.CreateIntCast(&V, Type::getInt16Ty(V.getContext()), false);
134 if (VTy->isFloatingPointTy())
135 return Builder.CreateFPCast(&V, Type::getHalfTy(V.getContext()));
136
137 llvm_unreachable("Should never be called!");
138}
139
140/// Applies Func(OldIntr.Args, OldIntr.ArgTys), creates intrinsic call with
141/// modified arguments (based on OldIntr) and replaces InstToReplace with
142/// this newly created intrinsic call.
143static std::optional<Instruction *> modifyIntrinsicCall(
144 IntrinsicInst &OldIntr, Instruction &InstToReplace, unsigned NewIntr,
145 InstCombiner &IC,
146 std::function<void(SmallVectorImpl<Value *> &, SmallVectorImpl<Type *> &)>
147 Func) {
148 SmallVector<Type *, 4> OverloadTys;
149 if (!Intrinsic::isSignatureValid(OldIntr.getCalledFunction(), OverloadTys))
150 return std::nullopt;
151
152 SmallVector<Value *, 8> Args(OldIntr.args());
153
154 // Modify arguments and types
155 Func(Args, OverloadTys);
156
157 CallInst *NewCall =
158 IC.Builder.CreateIntrinsicWithoutFolding(NewIntr, OverloadTys, Args);
159 NewCall->takeName(&OldIntr);
160 NewCall->copyMetadata(OldIntr);
161 if (isa<FPMathOperator>(NewCall))
162 NewCall->copyFastMathFlags(&OldIntr);
163 // Copy attributes
164 AttributeList OldAttrList = OldIntr.getAttributes();
165 NewCall->setAttributes(OldAttrList);
166
167 // Erase and replace uses
168 if (!InstToReplace.getType()->isVoidTy())
169 IC.replaceInstUsesWith(InstToReplace, NewCall);
170
171 bool RemoveOldIntr = &OldIntr != &InstToReplace;
172
173 auto *RetValue = IC.eraseInstFromFunction(InstToReplace);
174 if (RemoveOldIntr)
175 IC.eraseInstFromFunction(OldIntr);
176
177 return RetValue;
178}
179
180static std::optional<Instruction *>
182 const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr,
184 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
186
187 // Optimize _L to _LZ when _L is zero
188 if (const auto *LZMappingInfo =
190 if (auto *ConstantLod =
191 dyn_cast<ConstantFP>(II.getOperand(ImageDimIntr->LodIndex))) {
192 if (ConstantLod->isZero() || ConstantLod->isNegative()) {
193 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
195 ImageDimIntr->Dim);
196 return modifyIntrinsicCall(
197 II, II, NewImageDimIntr->Intr, IC, [&](auto &Args, auto &ArgTys) {
198 Args.erase(Args.begin() + ImageDimIntr->LodIndex);
199 });
200 }
201 }
202 }
203
204 // Optimize _mip away, when 'lod' is zero
205 if (const auto *MIPMappingInfo =
207 if (auto *ConstantMip =
208 dyn_cast<ConstantInt>(II.getOperand(ImageDimIntr->MipIndex))) {
209 if (ConstantMip->isZero()) {
210 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
211 AMDGPU::getImageDimIntrinsicByBaseOpcode(MIPMappingInfo->NONMIP,
212 ImageDimIntr->Dim);
213 return modifyIntrinsicCall(
214 II, II, NewImageDimIntr->Intr, IC, [&](auto &Args, auto &ArgTys) {
215 Args.erase(Args.begin() + ImageDimIntr->MipIndex);
216 });
217 }
218 }
219 }
220
221 // Optimize _bias away when 'bias' is zero
222 if (const auto *BiasMappingInfo =
224 if (auto *ConstantBias =
225 dyn_cast<ConstantFP>(II.getOperand(ImageDimIntr->BiasIndex))) {
226 if (ConstantBias->isZero()) {
227 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
228 AMDGPU::getImageDimIntrinsicByBaseOpcode(BiasMappingInfo->NoBias,
229 ImageDimIntr->Dim);
230 return modifyIntrinsicCall(
231 II, II, NewImageDimIntr->Intr, IC, [&](auto &Args, auto &ArgTys) {
232 Args.erase(Args.begin() + ImageDimIntr->BiasIndex);
233 ArgTys.erase(ArgTys.begin() + ImageDimIntr->BiasTyArg);
234 });
235 }
236 }
237 }
238
239 // Optimize _offset away when 'offset' is zero
240 if (const auto *OffsetMappingInfo =
242 if (auto *ConstantOffset =
243 dyn_cast<ConstantInt>(II.getOperand(ImageDimIntr->OffsetIndex))) {
244 if (ConstantOffset->isZero()) {
245 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
247 OffsetMappingInfo->NoOffset, ImageDimIntr->Dim);
248 return modifyIntrinsicCall(
249 II, II, NewImageDimIntr->Intr, IC, [&](auto &Args, auto &ArgTys) {
250 Args.erase(Args.begin() + ImageDimIntr->OffsetIndex);
251 });
252 }
253 }
254 }
255
256 // Optimize the arrayed dim away when the array slice is zero, since slice 0
257 // is the base layer. Restricted to non-atomic, non-sampled image loads and
258 // stores for now.
259 const AMDGPU::MIMGDimInfo *DimInfo =
260 AMDGPU::getMIMGDimInfo(ImageDimIntr->Dim);
261 if (!BaseOpcode->Atomic && !BaseOpcode->Sampler && BaseOpcode->Coordinates &&
262 DimInfo->NonArrayDim != ImageDimIntr->Dim) {
263 // Address is [coords..., slice, (fragid)] plus an optional mip operand.
264 // The slice is the last coordinate, so index it from CoordStart.
265 unsigned SliceIndex = ImageDimIntr->CoordStart + DimInfo->NumCoords - 1 -
266 (DimInfo->MSAA ? 1 : 0);
267 auto *ConstantSlice = dyn_cast<ConstantInt>(II.getOperand(SliceIndex));
268 if (ConstantSlice && ConstantSlice->isZero()) {
269 if (const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
271 DimInfo->NonArrayDim)) {
272 return modifyIntrinsicCall(II, II, NewImageDimIntr->Intr, IC,
273 [&](auto &Args, auto &ArgTys) {
274 Args.erase(Args.begin() + SliceIndex);
275 });
276 }
277 }
278 }
279
280 // Try to use D16
281 if (ST->hasD16Images()) {
282 if (BaseOpcode->HasD16) {
283
284 // If the only use of image intrinsic is a fptrunc (with conversion to
285 // half) then both fptrunc and image intrinsic will be replaced with image
286 // intrinsic with D16 flag.
287 if (II.hasOneUse()) {
288 Instruction *User = II.user_back();
289
290 if (User->getOpcode() == Instruction::FPTrunc &&
292
293 return modifyIntrinsicCall(II, *User, ImageDimIntr->Intr, IC,
294 [&](auto &Args, auto &ArgTys) {
295 // Change return type of image intrinsic.
296 // Set it to return type of fptrunc.
297 ArgTys[0] = User->getType();
298 });
299 }
300 }
301
302 // Only perform D16 folding if every user of the image sample is
303 // an ExtractElementInst immediately followed by an FPTrunc to half.
305 ExtractTruncPairs;
306 bool AllHalfExtracts = true;
307
308 for (User *U : II.users()) {
309 auto *Ext = dyn_cast<ExtractElementInst>(U);
310 if (!Ext || !Ext->hasOneUse()) {
311 AllHalfExtracts = false;
312 break;
313 }
314
315 auto *Tr = dyn_cast<FPTruncInst>(*Ext->user_begin());
316 if (!Tr || !Tr->getType()->isHalfTy()) {
317 AllHalfExtracts = false;
318 break;
319 }
320
321 ExtractTruncPairs.emplace_back(Ext, Tr);
322 }
323
324 if (!ExtractTruncPairs.empty() && AllHalfExtracts) {
325 auto *VecTy = cast<VectorType>(II.getType());
326 Type *HalfVecTy =
327 VecTy->getWithNewType(Type::getHalfTy(II.getContext()));
328
329 // Obtain the original image sample intrinsic's signature
330 // and replace its return type with the half-vector for D16 folding
331 SmallVector<Type *, 8> OverloadTys;
332 if (!Intrinsic::isSignatureValid(II.getCalledFunction(), OverloadTys))
333 return std::nullopt;
334
335 OverloadTys[0] = HalfVecTy;
336 Module *M = II.getModule();
338 M, ImageDimIntr->Intr, OverloadTys);
339
340 II.mutateType(HalfVecTy);
341 II.setCalledFunction(HalfDecl);
342
343 IRBuilder<> Builder(*M);
344 for (auto &[Ext, Tr] : ExtractTruncPairs) {
345 Value *Idx = Ext->getIndexOperand();
346
347 Builder.SetInsertPoint(Tr);
348
349 Value *HalfExtract = Builder.CreateExtractElement(&II, Idx);
350 HalfExtract->takeName(Tr);
351
352 Tr->replaceAllUsesWith(HalfExtract);
353 }
354
355 for (auto &[Ext, Tr] : ExtractTruncPairs) {
356 IC.eraseInstFromFunction(*Tr);
357 IC.eraseInstFromFunction(*Ext);
358 }
359
360 return &II;
361 }
362 }
363 }
364
365 // Try to use A16 or G16
366 if (!ST->hasA16() && !ST->hasG16())
367 return std::nullopt;
368
369 // Address is interpreted as float if the instruction has a sampler or as
370 // unsigned int if there is no sampler.
371 bool HasSampler = BaseOpcode->Sampler;
372 bool FloatCoord = false;
373 // true means derivatives can be converted to 16 bit, coordinates not
374 bool OnlyDerivatives = false;
375
376 // Sampler-less addresses are unsigned, so a sext from i16 folds to a16 like a
377 // zext: they only disagree for a negative i16 (>= 0x8000), which is out of
378 // bounds while the max image dimension is <= 0x8000.
379 bool AllowI16SExt = !HasSampler;
380
381 for (unsigned OperandIndex = ImageDimIntr->GradientStart;
382 OperandIndex < ImageDimIntr->VAddrEnd; OperandIndex++) {
383 Value *Coord = II.getOperand(OperandIndex);
384 // If the values are not derived from 16-bit values, we cannot optimize.
385 if (!canSafelyConvertTo16Bit(*Coord, HasSampler, AllowI16SExt)) {
386 if (OperandIndex < ImageDimIntr->CoordStart ||
387 ImageDimIntr->GradientStart == ImageDimIntr->CoordStart) {
388 return std::nullopt;
389 }
390 // All gradients can be converted, so convert only them
391 OnlyDerivatives = true;
392 break;
393 }
394
395 assert(OperandIndex == ImageDimIntr->GradientStart ||
396 FloatCoord == Coord->getType()->isFloatingPointTy());
397 FloatCoord = Coord->getType()->isFloatingPointTy();
398 }
399
400 if (!OnlyDerivatives && !ST->hasA16())
401 OnlyDerivatives = true; // Only supports G16
402
403 // Check if there is a bias parameter and if it can be converted to f16
404 if (!OnlyDerivatives && ImageDimIntr->NumBiasArgs != 0) {
405 Value *Bias = II.getOperand(ImageDimIntr->BiasIndex);
406 assert(HasSampler &&
407 "Only image instructions with a sampler can have a bias");
408 if (!canSafelyConvertTo16Bit(*Bias, HasSampler))
409 OnlyDerivatives = true;
410 }
411
412 if (OnlyDerivatives && (!ST->hasG16() || ImageDimIntr->GradientStart ==
413 ImageDimIntr->CoordStart))
414 return std::nullopt;
415
416 Type *CoordType = FloatCoord ? Type::getHalfTy(II.getContext())
417 : Type::getInt16Ty(II.getContext());
418
419 return modifyIntrinsicCall(
420 II, II, II.getIntrinsicID(), IC, [&](auto &Args, auto &ArgTys) {
421 ArgTys[ImageDimIntr->GradientTyArg] = CoordType;
422 if (!OnlyDerivatives) {
423 ArgTys[ImageDimIntr->CoordTyArg] = CoordType;
424
425 // Change the bias type
426 if (ImageDimIntr->NumBiasArgs != 0)
427 ArgTys[ImageDimIntr->BiasTyArg] = Type::getHalfTy(II.getContext());
428 }
429
430 unsigned EndIndex =
431 OnlyDerivatives ? ImageDimIntr->CoordStart : ImageDimIntr->VAddrEnd;
432 for (unsigned OperandIndex = ImageDimIntr->GradientStart;
433 OperandIndex < EndIndex; OperandIndex++) {
434 Args[OperandIndex] =
435 convertTo16Bit(*II.getOperand(OperandIndex), IC.Builder);
436 }
437
438 // Convert the bias
439 if (!OnlyDerivatives && ImageDimIntr->NumBiasArgs != 0) {
440 Value *Bias = II.getOperand(ImageDimIntr->BiasIndex);
441 Args[ImageDimIntr->BiasIndex] = convertTo16Bit(*Bias, IC.Builder);
442 }
443 });
444}
445
447 const Value *Op0, const Value *Op1,
448 InstCombiner &IC) const {
449 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
450 // infinity, gives +0.0. If we can prove we don't have one of the special
451 // cases then we can use a normal multiply instead.
453 KnownFPClass Known0 =
455 DenormalMode Mode = I.getFunction()->getDenormalMode(APFloat::IEEEsingle());
456
457 // Bail early if Op0 may be zero and nsz is not set -- Op1 cannot help.
458 if (!Known0.isKnownNeverLogicalZero(Mode) && !I.hasNoSignedZeros())
459 return false;
460
461 KnownFPClass Known1 =
463
464 // Simplify if both operands are known non-zero.
465 if (Known0.isKnownNeverLogicalZero(Mode) &&
466 Known1.isKnownNeverLogicalZero(Mode))
467 return true;
468
469 // With nsz, two additional cases allow simplification:
470 // 1. One operand is not zero or infinity or NaN:
471 // Op0 NeverLogicalZero && NeverInfOrNaN, or symmetric for Op1.
472 // 2. Neither operand is infinity or NaN:
473 // Op0 NeverInfOrNaN && Op1 NeverInfOrNaN.
474 // The following condition captures both cases.
475 if (I.hasNoSignedZeros() &&
476 (Known0.isKnownNeverLogicalZero(Mode) || Known1.isKnownNeverInfOrNaN()) &&
477 (Known1.isKnownNeverLogicalZero(Mode) || Known0.isKnownNeverInfOrNaN()))
478 return true;
479
480 return false;
481}
482
483/// Match an fpext from half to float, or a constant we can convert.
485 Value *Src = nullptr;
486 ConstantFP *CFP = nullptr;
487 if (match(Arg, m_OneUse(m_FPExt(m_Value(Src))))) {
488 if (Src->getType()->isHalfTy())
489 return Src;
490 } else if (match(Arg, m_ConstantFP(CFP))) {
491 bool LosesInfo;
492 APFloat Val(CFP->getValueAPF());
494 if (!LosesInfo)
495 return ConstantFP::get(Type::getHalfTy(Arg->getContext()), Val);
496 }
497 return nullptr;
498}
499
500// Trim all zero components from the end of the vector \p UseV and return
501// an appropriate bitset with known elements.
503 Instruction *I) {
504 auto *VTy = cast<FixedVectorType>(UseV->getType());
505 unsigned VWidth = VTy->getNumElements();
506 APInt DemandedElts = APInt::getAllOnes(VWidth);
507
508 for (int i = VWidth - 1; i > 0; --i) {
509 auto *Elt = findScalarElement(UseV, i);
510 if (!Elt)
511 break;
512
513 if (auto *ConstElt = dyn_cast<Constant>(Elt)) {
514 if (!ConstElt->isNullValue() && !isa<UndefValue>(Elt))
515 break;
516 } else {
517 break;
518 }
519
520 DemandedElts.clearBit(i);
521 }
522
523 return DemandedElts;
524}
525
526// Trim elements of the end of the vector \p V, if they are
527// equal to the first element of the vector.
529 auto *VTy = cast<FixedVectorType>(V->getType());
530 unsigned VWidth = VTy->getNumElements();
531 APInt DemandedElts = APInt::getAllOnes(VWidth);
532 Value *FirstComponent = findScalarElement(V, 0);
533
534 SmallVector<int> ShuffleMask;
535 if (auto *SVI = dyn_cast<ShuffleVectorInst>(V))
536 SVI->getShuffleMask(ShuffleMask);
537
538 for (int I = VWidth - 1; I > 0; --I) {
539 if (ShuffleMask.empty()) {
540 auto *Elt = findScalarElement(V, I);
541 if (!Elt || (Elt != FirstComponent && !isa<UndefValue>(Elt)))
542 break;
543 } else {
544 // Detect identical elements in the shufflevector result, even though
545 // findScalarElement cannot tell us what that element is.
546 if (ShuffleMask[I] != ShuffleMask[0] && ShuffleMask[I] != PoisonMaskElem)
547 break;
548 }
549 DemandedElts.clearBit(I);
550 }
551
552 return DemandedElts;
553}
554
555/// Return true if it's legal to contract llvm.amdgcn.rcp(llvm.sqrt)
556static bool canContractSqrtToRsq(const FPMathOperator *SqrtOp) {
557 return (SqrtOp->getType()->isFloatTy() &&
558 (SqrtOp->hasApproxFunc() || SqrtOp->getFPAccuracy() >= 1.0f)) ||
559 SqrtOp->getType()->isHalfTy();
560}
561
562/// Simplify a lane index operand (e.g. llvm.amdgcn.readlane src1).
563///
564/// The instruction only reads the low 5 bits for wave32, and 6 bits for wave64.
567 unsigned LaneArgIdx) const {
568 unsigned MaskBits = ST->getWavefrontSizeLog2();
569 APInt DemandedMask(32, maskTrailingOnes<unsigned>(MaskBits));
570
571 KnownBits Known(32);
572 if (IC.SimplifyDemandedBits(&II, LaneArgIdx, DemandedMask, Known))
573 return true;
574
575 if (!Known.isConstant())
576 return false;
577
578 // Out of bounds indexes may appear in wave64 code compiled for wave32.
579 // Unlike the DAG version, SimplifyDemandedBits does not change constants, so
580 // manually fix it up.
581
582 Value *LaneArg = II.getArgOperand(LaneArgIdx);
583 Constant *MaskedConst =
584 ConstantInt::get(LaneArg->getType(), Known.getConstant() & DemandedMask);
585 if (MaskedConst != LaneArg) {
586 II.getOperandUse(LaneArgIdx).set(MaskedConst);
587 return true;
588 }
589
590 return false;
591}
592
593/// Simplify the f32 scale operand of a V_CVT_SCALEF32_* conversion, of which
594/// only the exponent field is read.
596 unsigned ScaleArgIdx) {
597 Value *Scale = II.getArgOperand(ScaleArgIdx);
598 Value *StrippedSign = InstCombiner::stripSignOnlyFPOps(Scale);
599 if (StrippedSign != Scale) {
600 IC.replaceOperand(II, ScaleArgIdx, StrippedSign);
601 return true;
602 }
603
604 auto *BC = dyn_cast<BitCastInst>(Scale);
605 if (!BC || !BC->hasOneUse() || !BC->getSrcTy()->isIntegerTy(32))
606 return false;
607
608 KnownBits Known(32);
609 return IC.SimplifyDemandedBits(
611}
612
614 Function &NewCallee, ArrayRef<Value *> Ops) {
616 Old.getOperandBundlesAsDefs(OpBundles);
617
618 CallInst *NewCall = B.CreateCall(&NewCallee, Ops, OpBundles);
619 NewCall->takeName(&Old);
620 return NewCall;
621}
622
623// Return true for sequences of instructions that effectively assign
624// each lane to its thread ID
625static bool isThreadID(const GCNSubtarget &ST, Value *V) {
626 // Case 1:
627 // wave32: mbcnt_lo(-1, 0)
628 // wave64: mbcnt_hi(-1, mbcnt_lo(-1, 0))
634 if (ST.isWave32() && match(V, W32Pred))
635 return true;
636 if (ST.isWave64() && match(V, W64Pred))
637 return true;
638
639 // Case 2:
640 // workitem.id.x & (wavefront size - 1), or bare workitem.id.x when the
641 // workgroup is one wave long or less.
642 auto *I = dyn_cast<Instruction>(V);
643 if (!I || !ST.isWaveSizeKnown())
644 return false;
645
646 const Function &F = *I->getFunction();
647 if (!ST.hasWavefrontsEvenlySplittingXDim(F, /*RequiresUniformYZ=*/true))
648 return false;
649
650 unsigned WaveSize = ST.getWavefrontSize();
652 if (match(V, m_c_And(TidX, m_SpecificInt(WaveSize - 1))))
653 return true;
654
655 return match(V, TidX) && ST.getMaxWorkitemID(F, 0) < WaveSize;
656}
657
660 IntrinsicInst &II) const {
661 const auto IID = II.getIntrinsicID();
662 assert(IID == Intrinsic::amdgcn_readlane ||
663 IID == Intrinsic::amdgcn_readfirstlane ||
664 IID == Intrinsic::amdgcn_permlane64);
665
666 Instruction *OpInst = dyn_cast<Instruction>(II.getOperand(0));
667
668 // Only do this if both instructions are in the same block
669 // (so the exec mask won't change) and the readlane is the only user of its
670 // operand.
671 if (!OpInst || !OpInst->hasOneUser() || OpInst->getParent() != II.getParent())
672 return nullptr;
673
674 const bool IsReadLane = (IID == Intrinsic::amdgcn_readlane);
675
676 // If this is a readlane, check that the second operand is a constant, or is
677 // defined before OpInst so we know it's safe to move this intrinsic higher.
678 Value *LaneID = nullptr;
679 if (IsReadLane) {
680 LaneID = II.getOperand(1);
681
682 // readlane take an extra operand for the lane ID, so we must check if that
683 // LaneID value can be used at the point where we want to move the
684 // intrinsic.
685 if (auto *LaneIDInst = dyn_cast<Instruction>(LaneID)) {
686 if (!IC.getDominatorTree().dominates(LaneIDInst, OpInst))
687 return nullptr;
688 }
689 }
690
691 // Hoist the intrinsic (II) through OpInst.
692 //
693 // (II (OpInst x)) -> (OpInst (II x))
694 const auto DoIt = [&](unsigned OpIdx,
695 Function *NewIntrinsic) -> Instruction * {
696 SmallVector<Value *, 2> Ops{OpInst->getOperand(OpIdx)};
697 if (IsReadLane)
698 Ops.push_back(LaneID);
699
700 // Rewrite the intrinsic call.
701 CallInst *NewII = rewriteCall(IC.Builder, II, *NewIntrinsic, Ops);
702
703 // Rewrite OpInst so it takes the result of the intrinsic now.
704 Instruction &NewOp = *OpInst->clone();
705 NewOp.setOperand(OpIdx, NewII);
706 return &NewOp;
707 };
708
709 // TODO(?): Should we do more with permlane64?
710 if (IID == Intrinsic::amdgcn_permlane64 && !isa<BitCastInst>(OpInst))
711 return nullptr;
712
713 if (isa<UnaryOperator>(OpInst))
714 return DoIt(0, II.getCalledFunction());
715
716 if (isa<CastInst>(OpInst)) {
717 Value *Src = OpInst->getOperand(0);
718 Type *SrcTy = Src->getType();
719 if (!isTypeLegal(SrcTy))
720 return nullptr;
721
722 Function *Remangled =
723 Intrinsic::getOrInsertDeclaration(II.getModule(), IID, {SrcTy});
724 return DoIt(0, Remangled);
725 }
726
727 // We can also hoist through binary operators if the other operand is uniform.
728 if (isa<BinaryOperator>(OpInst)) {
729 // FIXME: If we had access to UniformityInfo here we could just check
730 // if the operand is uniform.
732 return DoIt(1, II.getCalledFunction());
734 return DoIt(0, II.getCalledFunction());
735 }
736
737 return nullptr;
738}
739
740/// Evaluate V as a function of the lane ID and return its value on Lane, or
741/// std::nullopt if V is not a closed-form expression of the lane ID.
742static std::optional<uint64_t> evalLaneExpr(Value *V, unsigned Lane,
743 const GCNSubtarget &ST,
744 const DataLayout &DL,
745 unsigned Depth = 0) {
747 return std::nullopt;
748
749 // Poison/undef in the index expression: bail and let InstCombine fold the
750 // intrinsic the usual way.
751 if (isa<UndefValue>(V))
752 return std::nullopt;
753
754 if (const ConstantInt *CI = dyn_cast<ConstantInt>(V))
755 return CI->getZExtValue();
756
757 if (isThreadID(ST, V))
758 return Lane;
759
761 if (!BO)
762 return std::nullopt;
763
764 std::optional<uint64_t> LHS =
765 evalLaneExpr(BO->getOperand(0), Lane, ST, DL, Depth + 1);
766 if (!LHS)
767 return std::nullopt;
768 std::optional<uint64_t> RHS =
769 evalLaneExpr(BO->getOperand(1), Lane, ST, DL, Depth + 1);
770 if (!RHS)
771 return std::nullopt;
772
773 Type *Ty = BO->getType();
774 Constant *Ops[] = {ConstantInt::get(Ty, *LHS), ConstantInt::get(Ty, *RHS)};
775 auto *CI =
777 return CI ? std::optional<uint64_t>(CI->getZExtValue()) : std::nullopt;
778}
779
780/// Build the per-lane shuffle map by evaluating Index for every lane in the
781/// wave. Returns false if any lane index is non-constant or out of range.
782static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST,
784 const DataLayout &DL) {
785 unsigned WaveSize = ST.getWavefrontSize();
786 Ids.resize(WaveSize);
787 for (unsigned Lane : seq(WaveSize)) {
788 std::optional<uint64_t> Val = evalLaneExpr(Index, Lane, ST, DL);
789 if (!Val || *Val >= WaveSize)
790 return false;
791 Ids[Lane] = static_cast<uint8_t>(*Val);
792 }
793 return true;
794}
795
796/// Lanes are partitioned into groups of Period; each group is a translated
797/// copy of the first: Ids[I] = Ids[I % Period] + (I & ~(Period - 1)).
798template <unsigned Period>
800 static_assert(isPowerOf2_32(Period), "Period must be a power of two");
801 for (size_t I = Period, E = Ids.size(); I < E; ++I)
802 if (Ids[I] != Ids[I % Period] + (I & ~(Period - 1)))
803 return false;
804 return true;
805}
806
807/// Match an N-lane row pattern: each lane in [0, N) reads from a source lane
808/// in the same N-lane row, and the pattern repeats periodically across rows.
809template <unsigned N> static bool isRowPattern(ArrayRef<uint8_t> Ids) {
810 for (unsigned I = 0; I < N; ++I)
811 if (Ids[I] >= N)
812 return false;
813 return hasPeriodicLayout<N>(Ids);
814}
815
816static constexpr auto isQuadPattern = isRowPattern<4>;
817static constexpr auto isHalfRowPattern = isRowPattern<8>;
818static constexpr auto isFullRowPattern = isRowPattern<16>;
819
820/// Match a 4-lane (quad) permutation, encoded as the v_mov_b32_dpp
821/// QUAD_PERM control word: bits[1:0]=Ids[0], [3:2]=Ids[1], [5:4]=Ids[2],
822/// [7:6]=Ids[3].
823static std::optional<unsigned> matchQuadPermPattern(ArrayRef<uint8_t> Ids) {
824 if (!isQuadPattern(Ids))
825 return std::nullopt;
826 return Ids[3] << 6 | Ids[2] << 4 | Ids[1] << 2 | Ids[0];
827}
828
829/// Match an N-lane reversal (mirror) pattern.
830template <unsigned N> static bool matchMirrorPattern(ArrayRef<uint8_t> Ids) {
831 if (!isRowPattern<N>(Ids))
832 return false;
833 for (unsigned J = 0; J < N; ++J)
834 if (Ids[J] != (N - 1) - J)
835 return false;
836 return true;
837}
838
841
842/// Match a 16-lane cyclic rotation; returns the rotation amount in [1, 15].
843static std::optional<unsigned> matchRowRotatePattern(ArrayRef<uint8_t> Ids) {
844 if (Ids[0] == 0 || !isFullRowPattern(Ids))
845 return std::nullopt;
846 for (unsigned J = 1; J < 16; ++J)
847 if (Ids[J] != (Ids[0] + J) % 16)
848 return std::nullopt;
849 return 16u - Ids[0];
850}
851
852/// Match a row-share pattern: all 16 lanes of each row read the same source
853/// lane. Returns the shared source lane index in [0, 16).
854static std::optional<unsigned> matchRowSharePattern(ArrayRef<uint8_t> Ids) {
855 if (!isFullRowPattern(Ids))
856 return std::nullopt;
857 if (!all_equal(Ids.take_front(16)))
858 return std::nullopt;
859 return Ids[0];
860}
861
862/// Match an XOR mask pattern within each 16-lane row: Ids[J] == Mask ^ J,
863/// with Mask in [1, 15].
864static std::optional<unsigned> matchRowXMaskPattern(ArrayRef<uint8_t> Ids) {
865 unsigned Mask = Ids[0];
866 if (Mask == 0 || !isFullRowPattern(Ids))
867 return std::nullopt;
868 for (unsigned J = 0; J < 16; ++J)
869 if (Ids[J] != (Mask ^ J))
870 return std::nullopt;
871 return Mask;
872}
873
874/// Match an 8-lane arbitrary permutation, encoded as the v_mov_b32_dpp8
875/// 24-bit selector (three bits per output lane).
876static std::optional<unsigned> matchHalfRowPermPattern(ArrayRef<uint8_t> Ids) {
877 if (!isHalfRowPattern(Ids))
878 return std::nullopt;
879 unsigned Selector = 0;
880 for (unsigned J = 0; J < 8; ++J)
881 Selector |= Ids[J] << (J * 3);
882 return Selector;
883}
884
885/// Pack a 16-lane permutation into a single 64-bit value: four bits per output
886/// lane, lane J in bits [J*4 + 3 : J*4]. The caller splits it into the low and
887/// high 32-bit selector operands of v_permlane16 / v_permlanex16.
889 uint64_t Sel = 0;
890 for (unsigned J = 0; J < 16; ++J)
891 Sel |= static_cast<uint64_t>(Ids[J] & 0xF) << (J * 4);
892 return Sel;
893}
894
895/// Match a half-wave swap: lane J reads from lane J ^ 32. Only meaningful on
896/// wave64 targets.
898 if (Ids.size() != 64)
899 return false;
900 for (unsigned J = 0; J < 64; ++J)
901 if (Ids[J] != (J ^ 32))
902 return false;
903 return true;
904}
905
906/// Match a cross-row permutation suitable for v_permlanex16: every lane in
907/// the low 16-lane half reads from the high half of its own row, and vice
908/// versa.
910 if (!hasPeriodicLayout<32>(Ids))
911 return false;
912 for (unsigned J = 0; J < 16; ++J) {
913 if (Ids[J] < 16 || Ids[J] >= 32)
914 return false;
915 if (Ids[J + 16] != Ids[J] - 16)
916 return false;
917 }
918 return true;
919}
920
921/// Match a DS_SWIZZLE bitmask-mode permutation:
922/// dst_lane = ((src_lane & AND) | OR) ^ XOR
923/// with each mask being five bits. Returns the encoded swizzle immediate.
924/// The hardware applies the formula independently within each 32-lane group,
925/// so on wave64 the high group must replicate the low one (translated by 32).
926static std::optional<unsigned>
928 if (!hasPeriodicLayout<32>(Ids))
929 return std::nullopt;
930
931 // The formula is per-bit: output bit B depends only on input bit B. Probe
932 // each bit with src=0 and src=(1<<B); if the output bit flipped, AND[B]=1
933 // and XOR[B] carries the constant offset; otherwise it is a constant bit
934 // encoded in OR (with AND[B]=0, XOR[B]=0).
935 unsigned AndMask = 0, OrMask = 0, XorMask = 0;
936 for (unsigned B = 0; B < 5; ++B) {
937 unsigned Bit0 = (Ids[0] >> B) & 1;
938 unsigned Bit1 = (Ids[1ull << B] >> B) & 1;
939 if (Bit0 != Bit1) {
940 AndMask |= 1u << B;
941 XorMask |= Bit0 << B;
942 } else {
943 OrMask |= Bit0 << B;
944 }
945 }
946
947 // The per-bit derivation assumes bit independence; verify the masks
948 // actually reproduce every lane in the 32-lane group.
949 for (unsigned I : seq(32u)) {
950 unsigned Expected = ((I & AndMask) | OrMask) ^ XorMask;
951 if (Ids[I] != Expected)
952 return std::nullopt;
953 }
954
959}
960
961/// Match a GFX9+ DS_SWIZZLE rotate-mode permutation: a cyclic left-rotation
962/// of all 32 lanes within each 32-lane group by a constant N in [0, 31],
963/// i.e. dst_lane = (src_lane + N) % 32. On wave64, hasPeriodicLayout<32>
964/// ensures both 32-lane groups rotate by the same amount.
965static std::optional<unsigned>
967 if (!hasPeriodicLayout<32>(Ids))
968 return std::nullopt;
969
970 // Determine the rotation amount from lane 0: every lane must read from
971 // lane (I + N) % 32 where N = Ids[0] and 0 <= N <= 31.
972 unsigned N = Ids[0];
973 if (N >= 32)
974 return std::nullopt;
975
976 for (unsigned I = 0; I < 32; ++I)
977 if (Ids[I] != (I + N) % 32)
978 return std::nullopt;
979
982}
983
984/// Emit v_mov_b32_dpp with the given control word, row/bank masks 0xF, and
985/// bound_ctrl=1 so out-of-bounds lanes are well-defined and the DPP mov can
986/// be folded into a consuming VALU op by GCNDPPCombine.
987static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl) {
988 Type *Ty = Val->getType();
989 return B.CreateIntrinsic(Intrinsic::amdgcn_update_dpp, {Ty},
990 {PoisonValue::get(Ty), Val, B.getInt32(Ctrl),
991 B.getInt32(0xF), B.getInt32(0xF), B.getTrue()});
992}
993
994/// Emit v_mov_b32_dpp8 with the given 24-bit lane selector.
995static Value *createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector) {
996 return B.CreateIntrinsic(Intrinsic::amdgcn_mov_dpp8, {Val->getType()},
997 {Val, B.getInt32(Selector)});
998}
999
1000/// Emit v_permlane16 with the precomputed lane-select halves.
1002 uint32_t Hi) {
1003 Type *Ty = Val->getType();
1004 return B.CreateIntrinsic(Intrinsic::amdgcn_permlane16, {Ty},
1005 {PoisonValue::get(Ty), Val, B.getInt32(Lo),
1006 B.getInt32(Hi), B.getFalse(), B.getFalse()});
1007}
1008
1009/// Emit v_permlanex16 with the precomputed lane-select halves. Each output
1010/// lane reads from the other 16-lane half of the same row.
1012 uint32_t Hi) {
1013 Type *Ty = Val->getType();
1014 return B.CreateIntrinsic(Intrinsic::amdgcn_permlanex16, {Ty},
1015 {PoisonValue::get(Ty), Val, B.getInt32(Lo),
1016 B.getInt32(Hi), B.getFalse(), B.getFalse()});
1017}
1018
1019/// Emit ds_swizzle with the given immediate, bitcasting/converting between
1020/// pointer/float types and i32 as required by the intrinsic signature.
1022 const DataLayout &DL) {
1023 Type *OrigTy = Val->getType();
1024 assert(DL.getTypeSizeInBits(OrigTy) == 32 &&
1025 "ds_swizzle only supports 32-bit operands");
1026 IntegerType *I32Ty = B.getInt32Ty();
1027 Value *Src = Val;
1028 if (OrigTy->isPointerTy())
1029 Src = B.CreatePtrToInt(Src, I32Ty);
1030 else if (OrigTy != I32Ty)
1031 Src = B.CreateBitCast(Src, I32Ty);
1032 Value *Result = B.CreateIntrinsic(Intrinsic::amdgcn_ds_swizzle, {},
1033 {Src, B.getInt32(Offset)});
1034 if (OrigTy->isPointerTy())
1035 return B.CreateIntToPtr(Result, OrigTy);
1036 if (OrigTy != I32Ty)
1037 return B.CreateBitCast(Result, OrigTy);
1038 return Result;
1039}
1040
1041/// Emit v_permlane64 (swap of the two 32-lane halves of a wave64).
1043 return B.CreateIntrinsic(Intrinsic::amdgcn_permlane64, {Val->getType()},
1044 {Val});
1045}
1046
1047/// Given a shuffle map, try to emit the best hardware intrinsic.
1050 const GCNSubtarget &ST,
1051 const DataLayout &DL) {
1052 // Identity shuffle (every lane reads itself) folds to the source value.
1053 if (all_of(enumerate(Ids),
1054 [](const auto &E) { return E.value() == E.index(); }))
1055 return Src;
1056
1057 // Uniform shuffle (all lanes read the same value) is handled by cheaper
1058 // broadcast/readlane intrinsics.
1059 if (all_equal(Ids))
1060 return nullptr;
1061
1062 if (std::optional<unsigned> QP = matchQuadPermPattern(Ids)) {
1063 if (ST.hasDPP())
1064 return createUpdateDpp(B, Src, *QP);
1066 }
1067
1068 if (ST.hasDPP()) {
1073 if (std::optional<unsigned> Amt = matchRowRotatePattern(Ids))
1074 return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1);
1075 }
1076
1077 // row_share is supported on GFX90A and GFX10+; row_xmask is GFX10+ only.
1078 if (ST.hasDPPRowShare()) {
1079 if (std::optional<unsigned> Lane = matchRowSharePattern(Ids))
1080 return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_SHARE_FIRST + *Lane);
1081 }
1082
1083 if (ST.hasDPP() && ST.hasGFX10Insts()) {
1084 if (std::optional<unsigned> Mask = matchRowXMaskPattern(Ids))
1085 return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_XMASK_FIRST + *Mask);
1086 }
1087
1088 if (ST.hasDPP8()) {
1089 if (std::optional<unsigned> Sel = matchHalfRowPermPattern(Ids))
1090 return createMovDpp8(B, Src, *Sel);
1091 }
1092
1093 if (ST.hasPermlane16Insts()) {
1094 if (isFullRowPattern(Ids)) {
1096 return createPermlane16(B, Src, Lo_32(Sel), Hi_32(Sel));
1097 }
1098 // Cross-row shuffles (e.g. XOR 16..31) — covered by permlanex16.
1099 if (isCrossRowPattern(Ids)) {
1101 return createPermlaneX16(B, Src, Lo_32(Sel), Hi_32(Sel));
1102 }
1103 }
1104
1105 // Generic DS_SWIZZLE bitmask-mode fallback: handles any 32-lane shuffle that
1106 // can be expressed as dst = ((src & AND) | OR) ^ XOR with 5-bit masks. This
1107 // is available on every target that has ds_swizzle.
1108 if (std::optional<unsigned> Imm = matchDsSwizzleBitmaskPattern(Ids))
1109 return createDsSwizzle(B, Src, *Imm, DL);
1110
1111 // DS_SWIZZLE rotate mode (GFX9+): handles cyclic 32-lane rotations that
1112 // bitmask mode cannot express (e.g. +1 mod 32 requires inter-bit carry).
1113 if (ST.hasDsSwizzleRotateMode()) {
1114 if (std::optional<unsigned> Imm = matchDsSwizzleRotatePattern(Ids))
1115 return createDsSwizzle(B, Src, *Imm, DL);
1116 }
1117
1118 if (ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
1119 return createPermlane64(B, Src);
1120
1121 return nullptr;
1122}
1123
1124/// Try to fold a wave_shuffle/ds_bpermute whose lane index is a constant
1125/// function of the lane ID into a hardware-specific lane permutation intrinsic.
1126static std::optional<Instruction *>
1128 const GCNSubtarget &ST) {
1129 const DataLayout &DL = IC.getDataLayout();
1130 if (DL.getTypeSizeInBits(II.getType()) != 32)
1131 return std::nullopt;
1132
1133 if (!ST.isWaveSizeKnown())
1134 return std::nullopt;
1135
1136 unsigned WaveSize = ST.getWavefrontSize();
1137 bool IsBpermute = II.getIntrinsicID() == Intrinsic::amdgcn_ds_bpermute;
1138 Value *Src = II.getArgOperand(IsBpermute ? 1 : 0);
1139 Value *Index = II.getArgOperand(IsBpermute ? 0 : 1);
1140
1142 if (IsBpermute) {
1143 Ids.resize(WaveSize);
1144 for (unsigned Lane : seq(WaveSize)) {
1145 std::optional<uint64_t> Val = evalLaneExpr(Index, Lane, ST, DL);
1146 if (!Val || (*Val & 3) || (*Val >> 2) >= WaveSize)
1147 return std::nullopt;
1148 Ids[Lane] = static_cast<uint8_t>(*Val >> 2);
1149 }
1150 } else {
1151 if (!tryBuildShuffleMap(Index, ST, Ids, DL))
1152 return std::nullopt;
1153 }
1154
1155 Value *Result = matchShuffleToHWIntrinsic(IC.Builder, Src, Ids, ST, DL);
1156 if (!Result)
1157 return std::nullopt;
1158
1159 return IC.replaceInstUsesWith(II, Result);
1160}
1161
1162/// Try to fold an addition into the accumulator when saturation is disabled.
1164 unsigned AccIdx,
1165 unsigned ClampIdx,
1166 InstCombiner &IC) {
1167 // Reassociating across a saturating accumulation is not valid.
1168 if (!match(II.getArgOperand(ClampIdx), m_Zero()) || !II.hasOneUse())
1169 return nullptr;
1170
1171 const APInt *Acc = nullptr;
1172 if (!match(II.getArgOperand(AccIdx), m_APInt(Acc)))
1173 return nullptr;
1174
1175 auto *AccumUser = dyn_cast<BinaryOperator>(II.user_back());
1176 if (!AccumUser)
1177 return nullptr;
1178
1179 Value *AccumDelta = nullptr;
1180 if (!match(AccumUser, m_c_Add(m_Specific(&II), m_Value(AccumDelta))))
1181 return nullptr;
1182
1183 Value *NewAcc = nullptr;
1184 const APInt *AccumDeltaC = nullptr;
1185 if (Acc->isZero())
1186 NewAcc = AccumDelta;
1187 else if (match(AccumDelta, m_APInt(AccumDeltaC)))
1188 NewAcc = ConstantInt::get(II.getType(), *Acc + *AccumDeltaC);
1189 else
1190 return nullptr;
1191
1192 Instruction *NewII =
1193 IC.InsertNewInstWith(II.clone(), AccumUser->getIterator());
1194 NewII->setOperand(AccIdx, NewAcc);
1197 IC.replaceInstUsesWith(*AccumUser, NewII);
1198 IC.eraseInstFromFunction(*AccumUser);
1199 return &II;
1200}
1201
1202std::optional<Instruction *>
1204 Intrinsic::ID IID = II.getIntrinsicID();
1205 switch (IID) {
1206 case Intrinsic::amdgcn_implicitarg_ptr: {
1207 if (II.getFunction()->hasFnAttribute("amdgpu-no-implicitarg-ptr"))
1208 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
1209 uint64_t ImplicitArgBytes = ST->getImplicitArgNumBytes(*II.getFunction());
1210
1211 uint64_t CurrentOrNullBytes =
1212 II.getAttributes().getRetDereferenceableOrNullBytes();
1213 if (CurrentOrNullBytes != 0) {
1214 // Refine "dereferenceable (A) meets dereferenceable_or_null(B)"
1215 // into dereferenceable(max(A, B))
1216 uint64_t NewBytes = std::max(CurrentOrNullBytes, ImplicitArgBytes);
1217 II.addRetAttr(
1218 Attribute::getWithDereferenceableBytes(II.getContext(), NewBytes));
1219 II.removeRetAttr(Attribute::DereferenceableOrNull);
1220 return &II;
1221 }
1222
1223 uint64_t CurrentBytes = II.getAttributes().getRetDereferenceableBytes();
1224 uint64_t NewBytes = std::max(CurrentBytes, ImplicitArgBytes);
1225 if (NewBytes != CurrentBytes) {
1226 II.addRetAttr(
1227 Attribute::getWithDereferenceableBytes(II.getContext(), NewBytes));
1228 return &II;
1229 }
1230
1231 return std::nullopt;
1232 }
1233 case Intrinsic::amdgcn_rcp: {
1234 Value *Src = II.getArgOperand(0);
1235 if (isa<PoisonValue>(Src))
1236 return IC.replaceInstUsesWith(II, Src);
1237
1238 // TODO: Move to ConstantFolding/InstSimplify?
1239 if (isa<UndefValue>(Src)) {
1240 Type *Ty = II.getType();
1241 auto *QNaN = ConstantFP::get(Ty, APFloat::getQNaN(Ty->getFltSemantics()));
1242 return IC.replaceInstUsesWith(II, QNaN);
1243 }
1244
1245 if (II.isStrictFP())
1246 break;
1247
1248 if (const ConstantFP *C = dyn_cast<ConstantFP>(Src)) {
1249 std::optional<APFloat> Val = AMDGPU::evaluateRcp(C->getValueAPF());
1250 if (!Val)
1251 break;
1252
1253 return IC.replaceInstUsesWith(II, ConstantFP::get(II.getContext(), *Val));
1254 }
1255
1256 FastMathFlags FMF = cast<FPMathOperator>(II).getFastMathFlags();
1257 if (!FMF.allowContract())
1258 break;
1259 auto *SrcCI = dyn_cast<IntrinsicInst>(Src);
1260 if (!SrcCI)
1261 break;
1262
1263 auto IID = SrcCI->getIntrinsicID();
1264 // llvm.amdgcn.rcp(llvm.amdgcn.sqrt(x)) -> llvm.amdgcn.rsq(x) if contractable
1265 //
1266 // llvm.amdgcn.rcp(llvm.sqrt(x)) -> llvm.amdgcn.rsq(x) if contractable and
1267 // relaxed.
1268 if (IID == Intrinsic::amdgcn_sqrt || IID == Intrinsic::sqrt) {
1269 const FPMathOperator *SqrtOp = cast<FPMathOperator>(SrcCI);
1270 FastMathFlags InnerFMF = SqrtOp->getFastMathFlags();
1271 if (!InnerFMF.allowContract() || !SrcCI->hasOneUse())
1272 break;
1273
1274 if (IID == Intrinsic::sqrt && !canContractSqrtToRsq(SqrtOp))
1275 break;
1276
1278 SrcCI->getModule(), Intrinsic::amdgcn_rsq, {SrcCI->getType()});
1279
1280 InnerFMF |= FMF;
1281 II.setFastMathFlags(InnerFMF);
1282
1283 II.setCalledFunction(NewDecl);
1284 return IC.replaceOperand(II, 0, SrcCI->getArgOperand(0));
1285 }
1286
1287 break;
1288 }
1289 case Intrinsic::amdgcn_sqrt:
1290 case Intrinsic::amdgcn_rsq:
1291 case Intrinsic::amdgcn_tanh: {
1292 Value *Src = II.getArgOperand(0);
1293 if (isa<PoisonValue>(Src))
1294 return IC.replaceInstUsesWith(II, Src);
1295
1296 // TODO: Move to ConstantFolding/InstSimplify?
1297 if (isa<UndefValue>(Src)) {
1298 Type *Ty = II.getType();
1299 auto *QNaN = ConstantFP::get(Ty, APFloat::getQNaN(Ty->getFltSemantics()));
1300 return IC.replaceInstUsesWith(II, QNaN);
1301 }
1302
1303 // f16 amdgcn.sqrt is identical to regular sqrt.
1304 if (IID == Intrinsic::amdgcn_sqrt && Src->getType()->isHalfTy()) {
1306 II.getModule(), Intrinsic::sqrt, {II.getType()});
1307 II.setCalledFunction(NewDecl);
1308 return &II;
1309 }
1310
1311 break;
1312 }
1313 case Intrinsic::amdgcn_log:
1314 case Intrinsic::amdgcn_exp2: {
1315 const bool IsLog = IID == Intrinsic::amdgcn_log;
1316 const bool IsExp = IID == Intrinsic::amdgcn_exp2;
1317 Value *Src = II.getArgOperand(0);
1318 Type *Ty = II.getType();
1319
1320 if (isa<PoisonValue>(Src))
1321 return IC.replaceInstUsesWith(II, Src);
1322
1323 if (IC.getSimplifyQuery().isUndefValue(Src))
1325
1326 if (ConstantFP *C = dyn_cast<ConstantFP>(Src)) {
1327 if (C->isInfinity()) {
1328 // exp2(+inf) -> +inf
1329 // log2(+inf) -> +inf
1330 if (!C->isNegative())
1331 return IC.replaceInstUsesWith(II, C);
1332
1333 // exp2(-inf) -> 0
1334 if (IsExp && C->isNegative())
1336 }
1337
1338 if (II.isStrictFP())
1339 break;
1340
1341 if (C->isNaN()) {
1342 Constant *Quieted = ConstantFP::get(Ty, C->getValue().makeQuiet());
1343 return IC.replaceInstUsesWith(II, Quieted);
1344 }
1345
1346 // f32 instruction doesn't handle denormals, f16 does.
1347 if (C->isZero() || (C->getValue().isDenormal() && Ty->isFloatTy())) {
1348 Constant *FoldedValue = IsLog ? ConstantFP::getInfinity(Ty, true)
1349 : ConstantFP::get(Ty, 1.0);
1350 return IC.replaceInstUsesWith(II, FoldedValue);
1351 }
1352
1353 if (IsLog && C->isNegative())
1355
1356 // TODO: Full constant folding matching hardware behavior.
1357 }
1358
1359 break;
1360 }
1361 case Intrinsic::amdgcn_frexp_mant:
1362 case Intrinsic::amdgcn_frexp_exp: {
1363 Value *Src = II.getArgOperand(0);
1364 if (const ConstantFP *C = dyn_cast<ConstantFP>(Src)) {
1365 int Exp;
1366 APFloat Significand =
1367 frexp(C->getValueAPF(), Exp, APFloat::rmNearestTiesToEven);
1368
1369 if (IID == Intrinsic::amdgcn_frexp_mant) {
1370 return IC.replaceInstUsesWith(
1371 II, ConstantFP::get(II.getContext(), Significand));
1372 }
1373
1374 // Match instruction special case behavior.
1375 if (Exp == APFloat::IEK_NaN || Exp == APFloat::IEK_Inf)
1376 Exp = 0;
1377
1378 return IC.replaceInstUsesWith(II,
1379 ConstantInt::getSigned(II.getType(), Exp));
1380 }
1381
1382 if (isa<PoisonValue>(Src))
1383 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
1384
1385 if (isa<UndefValue>(Src)) {
1386 return IC.replaceInstUsesWith(II, UndefValue::get(II.getType()));
1387 }
1388
1389 break;
1390 }
1391 case Intrinsic::amdgcn_class: {
1392 Value *Src0 = II.getArgOperand(0);
1393 Value *Src1 = II.getArgOperand(1);
1394 const ConstantInt *CMask = dyn_cast<ConstantInt>(Src1);
1395 if (CMask) {
1396 II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
1397 II.getModule(), Intrinsic::is_fpclass, Src0->getType()));
1398
1399 // Clamp any excess bits, as they're illegal for the generic intrinsic.
1400 II.setArgOperand(1, ConstantInt::get(Src1->getType(),
1401 CMask->getZExtValue() & fcAllFlags));
1402 return &II;
1403 }
1404
1405 // Propagate poison.
1406 if (isa<PoisonValue>(Src0) || isa<PoisonValue>(Src1))
1407 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
1408
1409 // llvm.amdgcn.class(_, undef) -> false
1410 if (IC.getSimplifyQuery().isUndefValue(Src1))
1411 return IC.replaceInstUsesWith(II, ConstantInt::get(II.getType(), false));
1412
1413 // llvm.amdgcn.class(undef, mask) -> mask != 0
1414 if (IC.getSimplifyQuery().isUndefValue(Src0)) {
1415 Value *CmpMask = IC.Builder.CreateICmpNE(
1416 Src1, ConstantInt::getNullValue(Src1->getType()));
1417 return IC.replaceInstUsesWith(II, CmpMask);
1418 }
1419 break;
1420 }
1421 case Intrinsic::amdgcn_cvt_pkrtz: {
1422 auto foldFPTruncToF16RTZ = [](Value *Arg) -> Value * {
1423 Type *HalfTy = Type::getHalfTy(Arg->getContext());
1424
1425 if (isa<PoisonValue>(Arg))
1426 return PoisonValue::get(HalfTy);
1427 if (isa<UndefValue>(Arg))
1428 return UndefValue::get(HalfTy);
1429
1430 ConstantFP *CFP = nullptr;
1431 if (match(Arg, m_ConstantFP(CFP))) {
1432 bool LosesInfo;
1433 APFloat Val(CFP->getValueAPF());
1435 return ConstantFP::get(HalfTy, Val);
1436 }
1437
1438 Value *Src = nullptr;
1439 if (match(Arg, m_FPExt(m_Value(Src)))) {
1440 if (Src->getType()->isHalfTy())
1441 return Src;
1442 }
1443
1444 return nullptr;
1445 };
1446
1447 if (Value *Src0 = foldFPTruncToF16RTZ(II.getArgOperand(0))) {
1448 if (Value *Src1 = foldFPTruncToF16RTZ(II.getArgOperand(1))) {
1449 Value *V = PoisonValue::get(II.getType());
1450 V = IC.Builder.CreateInsertElement(V, Src0, (uint64_t)0);
1451 V = IC.Builder.CreateInsertElement(V, Src1, (uint64_t)1);
1452 return IC.replaceInstUsesWith(II, V);
1453 }
1454 }
1455
1456 break;
1457 }
1458 case Intrinsic::amdgcn_cvt_pknorm_i16:
1459 case Intrinsic::amdgcn_cvt_pknorm_u16:
1460 case Intrinsic::amdgcn_cvt_pk_i16:
1461 case Intrinsic::amdgcn_cvt_pk_u16: {
1462 Value *Src0 = II.getArgOperand(0);
1463 Value *Src1 = II.getArgOperand(1);
1464
1465 // TODO: Replace call with scalar operation if only one element is poison.
1466 if (isa<PoisonValue>(Src0) && isa<PoisonValue>(Src1))
1467 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
1468
1469 if (isa<UndefValue>(Src0) && isa<UndefValue>(Src1)) {
1470 return IC.replaceInstUsesWith(II, UndefValue::get(II.getType()));
1471 }
1472
1473 break;
1474 }
1475 case Intrinsic::amdgcn_cvt_off_f32_i4: {
1476 Value* Arg = II.getArgOperand(0);
1477 Type *Ty = II.getType();
1478
1479 if (isa<PoisonValue>(Arg))
1480 return IC.replaceInstUsesWith(II, PoisonValue::get(Ty));
1481
1482 if(IC.getSimplifyQuery().isUndefValue(Arg))
1484
1485 ConstantInt *CArg = dyn_cast<ConstantInt>(II.getArgOperand(0));
1486 if (!CArg)
1487 break;
1488
1489 // Tabulated 0.0625 * (sext (CArg & 0xf)).
1490 constexpr size_t ResValsSize = 16;
1491 static constexpr float ResVals[ResValsSize] = {
1492 0.0, 0.0625, 0.125, 0.1875, 0.25, 0.3125, 0.375, 0.4375,
1493 -0.5, -0.4375, -0.375, -0.3125, -0.25, -0.1875, -0.125, -0.0625};
1494 Constant *Res =
1495 ConstantFP::get(Ty, ResVals[CArg->getZExtValue() & (ResValsSize - 1)]);
1496 return IC.replaceInstUsesWith(II, Res);
1497 }
1498 case Intrinsic::amdgcn_ubfe:
1499 case Intrinsic::amdgcn_sbfe: {
1500 // Decompose simple cases into standard shifts.
1501 Value *Src = II.getArgOperand(0);
1502 if (isa<UndefValue>(Src)) {
1503 return IC.replaceInstUsesWith(II, Src);
1504 }
1505
1506 uint64_t Width;
1507 Type *Ty = II.getType();
1508 unsigned IntSize = Ty->getIntegerBitWidth();
1509
1510 ConstantInt *CWidth = dyn_cast<ConstantInt>(II.getArgOperand(2));
1511 if (CWidth) {
1512 Width = CWidth->getZExtValue();
1513 if ((Width & (IntSize - 1)) == 0) {
1515 }
1516
1517 // Hardware ignores high bits, so remove those.
1518 if (Width >= IntSize) {
1519 return IC.replaceOperand(
1520 II, 2, ConstantInt::get(CWidth->getType(), Width & (IntSize - 1)));
1521 }
1522 }
1523
1524 uint64_t Offset;
1525 ConstantInt *COffset = dyn_cast<ConstantInt>(II.getArgOperand(1));
1526 if (COffset) {
1527 Offset = COffset->getZExtValue();
1528 if (Offset >= IntSize) {
1529 return IC.replaceOperand(
1530 II, 1,
1531 ConstantInt::get(COffset->getType(), Offset & (IntSize - 1)));
1532 }
1533 }
1534
1535 bool Signed = IID == Intrinsic::amdgcn_sbfe;
1536
1537 if (!CWidth || !COffset)
1538 break;
1539
1540 // The case of Width == 0 is handled above, which makes this transformation
1541 // safe. If Width == 0, then the ashr and lshr instructions become poison
1542 // value since the shift amount would be equal to the bit size.
1543 assert(Width != 0);
1544
1545 // TODO: This allows folding to undef when the hardware has specific
1546 // behavior?
1547 if (Offset + Width < IntSize) {
1548 Value *Shl = IC.Builder.CreateShl(Src, IntSize - Offset - Width);
1549 Value *RightShift = Signed ? IC.Builder.CreateAShr(Shl, IntSize - Width)
1550 : IC.Builder.CreateLShr(Shl, IntSize - Width);
1551 RightShift->takeName(&II);
1552 return IC.replaceInstUsesWith(II, RightShift);
1553 }
1554
1555 Value *RightShift = Signed ? IC.Builder.CreateAShr(Src, Offset)
1556 : IC.Builder.CreateLShr(Src, Offset);
1557
1558 RightShift->takeName(&II);
1559 return IC.replaceInstUsesWith(II, RightShift);
1560 }
1561 case Intrinsic::amdgcn_exp:
1562 case Intrinsic::amdgcn_exp_row:
1563 case Intrinsic::amdgcn_exp_compr: {
1564 ConstantInt *En = cast<ConstantInt>(II.getArgOperand(1));
1565 uint64_t EnBits = En->getZExtValue();
1566 if (EnBits == 0xf)
1567 break; // All inputs enabled.
1568
1569 bool IsCompr = IID == Intrinsic::amdgcn_exp_compr;
1570 bool Changed = false;
1571 for (int I = 0; I < (IsCompr ? 2 : 4); ++I) {
1572 if ((!IsCompr && (EnBits & (1 << I)) == 0) ||
1573 (IsCompr && ((EnBits & (0x3 << (2 * I))) == 0))) {
1574 Value *Src = II.getArgOperand(I + 2);
1575 if (!isa<PoisonValue>(Src)) {
1576 IC.replaceOperand(II, I + 2, PoisonValue::get(Src->getType()));
1577 Changed = true;
1578 }
1579 }
1580 }
1581
1582 if (Changed) {
1583 return &II;
1584 }
1585
1586 break;
1587 }
1588 case Intrinsic::amdgcn_fmed3: {
1589 Value *Src0 = II.getArgOperand(0);
1590 Value *Src1 = II.getArgOperand(1);
1591 Value *Src2 = II.getArgOperand(2);
1592
1593 for (Value *Src : {Src0, Src1, Src2}) {
1594 if (isa<PoisonValue>(Src))
1595 return IC.replaceInstUsesWith(II, Src);
1596 }
1597
1598 if (II.isStrictFP())
1599 break;
1600
1601 // med3 with a nan input acts like
1602 // v_min_f32(v_min_f32(s0, s1), s2)
1603 //
1604 // Signalingness is ignored with ieee=0, so we fold to
1605 // minimumnum/maximumnum. With ieee=1, the v_min_f32 acts like llvm.minnum
1606 // with signaling nan handling. With ieee=0, like llvm.minimumnum except a
1607 // returned signaling nan will not be quieted.
1608
1609 // ieee=1
1610 // s0 snan: s2
1611 // s1 snan: s2
1612 // s2 snan: qnan
1613
1614 // s0 qnan: min(s1, s2)
1615 // s1 qnan: min(s0, s2)
1616 // s2 qnan: min(s0, s1)
1617
1618 // ieee=0
1619 // s0 _nan: min(s1, s2)
1620 // s1 _nan: min(s0, s2)
1621 // s2 _nan: min(s0, s1)
1622
1623 // med3 behavior with infinity
1624 // s0 +inf: max(s1, s2)
1625 // s1 +inf: max(s0, s2)
1626 // s2 +inf: max(s0, s1)
1627 // s0 -inf: min(s1, s2)
1628 // s1 -inf: min(s0, s2)
1629 // s2 -inf: min(s0, s1)
1630
1631 // Checking for NaN before canonicalization provides better fidelity when
1632 // mapping other operations onto fmed3 since the order of operands is
1633 // unchanged.
1634 Value *V = nullptr;
1635 const APFloat *ConstSrc0 = nullptr;
1636 const APFloat *ConstSrc1 = nullptr;
1637 const APFloat *ConstSrc2 = nullptr;
1638
1639 if ((match(Src0, m_APFloat(ConstSrc0)) &&
1640 (ConstSrc0->isNaN() || ConstSrc0->isInfinity())) ||
1641 isa<UndefValue>(Src0)) {
1642 const bool IsPosInfinity = ConstSrc0 && ConstSrc0->isPosInfinity();
1643 switch (fpenvIEEEMode(II)) {
1644 case KnownIEEEMode::On:
1645 // TODO: If Src2 is snan, does it need quieting?
1646 if (ConstSrc0 && ConstSrc0->isNaN() && ConstSrc0->isSignaling())
1647 return IC.replaceInstUsesWith(II, Src2);
1648
1649 V = IsPosInfinity ? IC.Builder.CreateMaxNum(Src1, Src2)
1650 : IC.Builder.CreateMinNum(Src1, Src2);
1651 break;
1652 case KnownIEEEMode::Off:
1653 V = IsPosInfinity ? IC.Builder.CreateMaximumNum(Src1, Src2)
1654 : IC.Builder.CreateMinimumNum(Src1, Src2);
1655 break;
1657 break;
1658 }
1659 } else if ((match(Src1, m_APFloat(ConstSrc1)) &&
1660 (ConstSrc1->isNaN() || ConstSrc1->isInfinity())) ||
1661 isa<UndefValue>(Src1)) {
1662 const bool IsPosInfinity = ConstSrc1 && ConstSrc1->isPosInfinity();
1663 switch (fpenvIEEEMode(II)) {
1664 case KnownIEEEMode::On:
1665 // TODO: If Src2 is snan, does it need quieting?
1666 if (ConstSrc1 && ConstSrc1->isNaN() && ConstSrc1->isSignaling())
1667 return IC.replaceInstUsesWith(II, Src2);
1668
1669 V = IsPosInfinity ? IC.Builder.CreateMaxNum(Src0, Src2)
1670 : IC.Builder.CreateMinNum(Src0, Src2);
1671 break;
1672 case KnownIEEEMode::Off:
1673 V = IsPosInfinity ? IC.Builder.CreateMaximumNum(Src0, Src2)
1674 : IC.Builder.CreateMinimumNum(Src0, Src2);
1675 break;
1677 break;
1678 }
1679 } else if ((match(Src2, m_APFloat(ConstSrc2)) &&
1680 (ConstSrc2->isNaN() || ConstSrc2->isInfinity())) ||
1681 isa<UndefValue>(Src2)) {
1682 switch (fpenvIEEEMode(II)) {
1683 case KnownIEEEMode::On:
1684 if (ConstSrc2 && ConstSrc2->isNaN() && ConstSrc2->isSignaling()) {
1685 auto *Quieted = ConstantFP::get(II.getType(), ConstSrc2->makeQuiet());
1686 return IC.replaceInstUsesWith(II, Quieted);
1687 }
1688
1689 V = (ConstSrc2 && ConstSrc2->isPosInfinity())
1690 ? IC.Builder.CreateMaxNum(Src0, Src1)
1691 : IC.Builder.CreateMinNum(Src0, Src1);
1692 break;
1693 case KnownIEEEMode::Off:
1694 V = (ConstSrc2 && ConstSrc2->isPosInfinity())
1695 ? IC.Builder.CreateMaximumNum(Src0, Src1)
1696 : IC.Builder.CreateMinimumNum(Src0, Src1);
1697 break;
1699 break;
1700 }
1701 }
1702
1703 if (V) {
1704 if (auto *CI = dyn_cast<CallInst>(V)) {
1705 CI->copyFastMathFlags(&II);
1706 CI->takeName(&II);
1707 }
1708 return IC.replaceInstUsesWith(II, V);
1709 }
1710
1711 bool Swap = false;
1712 // Canonicalize constants to RHS operands.
1713 //
1714 // fmed3(c0, x, c1) -> fmed3(x, c0, c1)
1715 if (isa<Constant>(Src0) && !isa<Constant>(Src1)) {
1716 std::swap(Src0, Src1);
1717 Swap = true;
1718 }
1719
1720 if (isa<Constant>(Src1) && !isa<Constant>(Src2)) {
1721 std::swap(Src1, Src2);
1722 Swap = true;
1723 }
1724
1725 if (isa<Constant>(Src0) && !isa<Constant>(Src1)) {
1726 std::swap(Src0, Src1);
1727 Swap = true;
1728 }
1729
1730 if (Swap) {
1731 II.setArgOperand(0, Src0);
1732 II.setArgOperand(1, Src1);
1733 II.setArgOperand(2, Src2);
1734 return &II;
1735 }
1736
1737 if (const ConstantFP *C0 = dyn_cast<ConstantFP>(Src0)) {
1738 if (const ConstantFP *C1 = dyn_cast<ConstantFP>(Src1)) {
1739 if (const ConstantFP *C2 = dyn_cast<ConstantFP>(Src2)) {
1740 APFloat Result = fmed3AMDGCN(C0->getValueAPF(), C1->getValueAPF(),
1741 C2->getValueAPF());
1742 return IC.replaceInstUsesWith(II,
1743 ConstantFP::get(II.getType(), Result));
1744 }
1745 }
1746 }
1747
1748 if (!ST->hasMed3_16())
1749 break;
1750
1751 // Repeat floating-point width reduction done for minnum/maxnum.
1752 // fmed3((fpext X), (fpext Y), (fpext Z)) -> fpext (fmed3(X, Y, Z))
1753 if (Value *X = matchFPExtFromF16(Src0)) {
1754 if (Value *Y = matchFPExtFromF16(Src1)) {
1755 if (Value *Z = matchFPExtFromF16(Src2)) {
1756 Value *NewCall = IC.Builder.CreateIntrinsic(
1757 IID, {X->getType()}, {X, Y, Z}, &II, II.getName());
1758 return new FPExtInst(NewCall, II.getType());
1759 }
1760 }
1761 }
1762
1763 break;
1764 }
1765 case Intrinsic::amdgcn_mbcnt_hi:
1766 // exec_hi is all 0, so this is just a copy.
1767 if (ST->isWave32())
1768 return IC.replaceInstUsesWith(II, II.getArgOperand(1));
1769 [[fallthrough]];
1770 case Intrinsic::amdgcn_mbcnt_lo: {
1771 ConstantRange AccRange =
1772 computeConstantRange(II.getArgOperand(1),
1773 /*ForSigned=*/false, IC.getSimplifyQuery());
1774 if (AccRange.isFullSet())
1775 return nullptr;
1776
1777 // TODO: Can raise lower bound by inspecting first argument.
1778 // Wave64 mbcnt_lo adds at most 32. Otherwise these add at most 31.
1779 unsigned MaxCount =
1780 IID == Intrinsic::amdgcn_mbcnt_lo && ST->isWave64() ? 32 : 31;
1781 ConstantRange MbcntRange(APInt(32, 0), APInt(32, MaxCount + 1));
1782 ConstantRange ComputedRange = AccRange.add(MbcntRange);
1783 if (ComputedRange.isFullSet())
1784 return nullptr;
1785
1786 if (std::optional<ConstantRange> ExistingRange = II.getRange()) {
1787 ComputedRange = ComputedRange.intersectWith(*ExistingRange);
1788 if (ComputedRange == *ExistingRange)
1789 return nullptr;
1790 }
1791
1792 II.addRangeRetAttr(ComputedRange);
1793 return nullptr;
1794 }
1795 case Intrinsic::amdgcn_ballot: {
1796 Value *Arg = II.getArgOperand(0);
1797 if (isa<PoisonValue>(Arg))
1798 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
1799
1800 if (auto *Src = dyn_cast<ConstantInt>(Arg)) {
1801 if (Src->isZero()) {
1802 // amdgcn.ballot(i1 0) is zero.
1803 return IC.replaceInstUsesWith(II, Constant::getNullValue(II.getType()));
1804 }
1805 }
1806 if (ST->isWave32() && II.getType()->getIntegerBitWidth() == 64) {
1807 // %b64 = call i64 ballot.i64(...)
1808 // =>
1809 // %b32 = call i32 ballot.i32(...)
1810 // %b64 = zext i32 %b32 to i64
1812 IC.Builder.CreateIntrinsic(Intrinsic::amdgcn_ballot,
1813 {IC.Builder.getInt32Ty()},
1814 {II.getArgOperand(0)}),
1815 II.getType());
1816 Call->takeName(&II);
1817 return IC.replaceInstUsesWith(II, Call);
1818 }
1819 break;
1820 }
1821 case Intrinsic::amdgcn_wavefrontsize: {
1822 if (ST->isWaveSizeKnown())
1823 return IC.replaceInstUsesWith(
1824 II, ConstantInt::get(II.getType(), ST->getWavefrontSize()));
1825 break;
1826 }
1827 case Intrinsic::amdgcn_wqm_vote: {
1828 // wqm_vote is identity when the argument is constant.
1829 if (!isa<Constant>(II.getArgOperand(0)))
1830 break;
1831
1832 return IC.replaceInstUsesWith(II, II.getArgOperand(0));
1833 }
1834 case Intrinsic::amdgcn_kill: {
1835 const ConstantInt *C = dyn_cast<ConstantInt>(II.getArgOperand(0));
1836 if (!C || !C->getZExtValue())
1837 break;
1838
1839 // amdgcn.kill(i1 1) is a no-op
1840 return IC.eraseInstFromFunction(II);
1841 }
1842 case Intrinsic::amdgcn_s_sendmsg:
1843 case Intrinsic::amdgcn_s_sendmsghalt: {
1844 // The second operand is copied to m0, but is only actually used for
1845 // certain message types. For message types that are known to not use m0,
1846 // fold it to poison.
1847 using namespace AMDGPU::SendMsg;
1848
1849 Value *M0Val = II.getArgOperand(1);
1850 if (isa<PoisonValue>(M0Val))
1851 break;
1852
1853 auto *MsgImm = cast<ConstantInt>(II.getArgOperand(0));
1854 uint16_t MsgId, OpId, StreamId;
1855 decodeMsg(static_cast<unsigned>(MsgImm->getZExtValue()), MsgId, OpId,
1856 StreamId, *ST);
1857
1858 if (!msgDoesNotUseM0(MsgId, *ST))
1859 break;
1860
1861 // Drop UB-implying attributes since we're replacing with poison.
1862 II.dropUBImplyingAttrsAndMetadata();
1863 IC.replaceOperand(II, 1, PoisonValue::get(M0Val->getType()));
1864 return nullptr;
1865 }
1866 case Intrinsic::amdgcn_update_dpp: {
1867 Value *Old = II.getArgOperand(0);
1868
1869 auto *BC = cast<ConstantInt>(II.getArgOperand(5));
1870 auto *RM = cast<ConstantInt>(II.getArgOperand(3));
1871 auto *BM = cast<ConstantInt>(II.getArgOperand(4));
1872 if (BC->isNullValue() || RM->getZExtValue() != 0xF ||
1873 BM->getZExtValue() != 0xF || isa<PoisonValue>(Old))
1874 break;
1875
1876 // If bound_ctrl = 1, row mask = bank mask = 0xf we can omit old value.
1877 return IC.replaceOperand(II, 0, PoisonValue::get(Old->getType()));
1878 }
1879 case Intrinsic::amdgcn_permlane16:
1880 case Intrinsic::amdgcn_permlane16_var:
1881 case Intrinsic::amdgcn_permlanex16:
1882 case Intrinsic::amdgcn_permlanex16_var: {
1883 // Discard vdst_in if it's not going to be read.
1884 Value *VDstIn = II.getArgOperand(0);
1885 if (isa<PoisonValue>(VDstIn))
1886 break;
1887
1888 // FetchInvalid operand idx.
1889 unsigned int FiIdx = (IID == Intrinsic::amdgcn_permlane16 ||
1890 IID == Intrinsic::amdgcn_permlanex16)
1891 ? 4 /* for permlane16 and permlanex16 */
1892 : 3; /* for permlane16_var and permlanex16_var */
1893
1894 // BoundCtrl operand idx.
1895 // For permlane16 and permlanex16 it should be 5
1896 // For Permlane16_var and permlanex16_var it should be 4
1897 unsigned int BcIdx = FiIdx + 1;
1898
1899 ConstantInt *FetchInvalid = cast<ConstantInt>(II.getArgOperand(FiIdx));
1900 ConstantInt *BoundCtrl = cast<ConstantInt>(II.getArgOperand(BcIdx));
1901 if (!FetchInvalid->getZExtValue() && !BoundCtrl->getZExtValue())
1902 break;
1903
1904 return IC.replaceOperand(II, 0, PoisonValue::get(VDstIn->getType()));
1905 }
1906 case Intrinsic::amdgcn_wave_shuffle:
1907 return tryOptimizeShufflePattern(IC, II, *ST);
1908 case Intrinsic::amdgcn_permlane64:
1909 case Intrinsic::amdgcn_readfirstlane:
1910 case Intrinsic::amdgcn_readlane:
1911 case Intrinsic::amdgcn_ds_bpermute: {
1912 // If the data argument is uniform these intrinsics return it unchanged.
1913 unsigned SrcIdx = IID == Intrinsic::amdgcn_ds_bpermute ? 1 : 0;
1914 const Use &Src = II.getArgOperandUse(SrcIdx);
1916 return IC.replaceInstUsesWith(II, Src.get());
1917
1918 if (IID == Intrinsic::amdgcn_readlane &&
1920 return &II;
1921
1922 // If the lane argument of bpermute is uniform, change it to readlane. This
1923 // generates better code and can enable further optimizations because
1924 // readlane is AlwaysUniform.
1925 if (IID == Intrinsic::amdgcn_ds_bpermute) {
1926 const Use &Lane = II.getArgOperandUse(0);
1927 if (AMDGPU::isTriviallyUniform(Lane)) {
1928 Value *NewLane = IC.Builder.CreateLShr(Lane, 2);
1930 II.getModule(), Intrinsic::amdgcn_readlane, II.getType());
1931 II.setCalledFunction(NewDecl);
1932 II.setOperand(0, Src);
1933 II.setOperand(1, NewLane);
1934 return &II;
1935 }
1936 }
1937
1938 if (IID == Intrinsic::amdgcn_ds_bpermute)
1939 return tryOptimizeShufflePattern(IC, II, *ST);
1940
1942 return Res;
1943
1944 return std::nullopt;
1945 }
1946 case Intrinsic::amdgcn_wave_match_b32: {
1947 const Use &Src0 = II.getArgOperandUse(0);
1948 const Use &Src1 = II.getArgOperandUse(1);
1949 if (Src0.get() == Src1.get() && AMDGPU::isTriviallyUniform(Src0)) {
1951 II.getModule(), Intrinsic::amdgcn_ballot, II.getType());
1952 CallInst *NewCall =
1953 IC.Builder.CreateCall(NewF, {IC.Builder.getInt1(true)});
1954 return IC.replaceInstUsesWith(II, NewCall);
1955 }
1956 break;
1957 }
1958 case Intrinsic::amdgcn_writelane: {
1959 // TODO: Fold bitcast like readlane.
1960 if (simplifyDemandedLaneMaskArg(IC, II, 1))
1961 return &II;
1962 return std::nullopt;
1963 }
1964 case Intrinsic::amdgcn_trig_preop: {
1965 // The intrinsic is declared with name mangling, but currently the
1966 // instruction only exists for f64
1967 if (!II.getType()->isDoubleTy())
1968 break;
1969
1970 Value *Src = II.getArgOperand(0);
1971 Value *Segment = II.getArgOperand(1);
1972 if (isa<PoisonValue>(Src) || isa<PoisonValue>(Segment))
1973 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
1974
1975 if (isa<UndefValue>(Segment))
1976 return IC.replaceInstUsesWith(II, ConstantFP::getZero(II.getType()));
1977
1978 // Sign bit is not used.
1979 Value *StrippedSign = InstCombiner::stripSignOnlyFPOps(Src);
1980 if (StrippedSign != Src)
1981 return IC.replaceOperand(II, 0, StrippedSign);
1982
1983 if (II.isStrictFP())
1984 break;
1985
1986 const ConstantFP *CSrc = dyn_cast<ConstantFP>(Src);
1987 if (!CSrc && !isa<UndefValue>(Src))
1988 break;
1989
1990 // The instruction ignores special cases, and literally just extracts the
1991 // exponents. Fold undef to nan, and index the table as normal.
1992 APInt FSrcInt = CSrc ? CSrc->getValueAPF().bitcastToAPInt()
1993 : APFloat::getQNaN(II.getType()->getFltSemantics())
1994 .bitcastToAPInt();
1995
1996 const ConstantInt *Cseg = dyn_cast<ConstantInt>(Segment);
1997 if (!Cseg) {
1998 if (isa<UndefValue>(Src))
1999 return IC.replaceInstUsesWith(II, ConstantFP::getZero(II.getType()));
2000 break;
2001 }
2002
2003 unsigned Exponent =
2004 static_cast<unsigned>(FSrcInt.extractBitsAsZExtValue(11, 52));
2005 unsigned SegmentVal =
2006 static_cast<unsigned>(Cseg->getValue().trunc(5).getZExtValue());
2007 unsigned Shift = SegmentVal * 53;
2008 if (Exponent > 1077)
2009 Shift += Exponent - 1077;
2010
2011 // 2.0/PI table.
2012 static const uint32_t TwoByPi[] = {
2013 0xa2f9836e, 0x4e441529, 0xfc2757d1, 0xf534ddc0, 0xdb629599, 0x3c439041,
2014 0xfe5163ab, 0xdebbc561, 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c,
2015 0xfe1deb1c, 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41,
2016 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, 0xde05980f,
2017 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, 0x4f463f66, 0x9e5fea2d,
2018 0x7527bac7, 0xebe5f17b, 0x3d0739f7, 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08,
2019 0x56033046};
2020
2021 // Return 0 for outbound segment (hardware behavior).
2022 unsigned Idx = Shift >> 5;
2023 if (Idx + 2 >= std::size(TwoByPi)) {
2024 APFloat Zero = APFloat::getZero(II.getType()->getFltSemantics());
2025 return IC.replaceInstUsesWith(II, ConstantFP::get(II.getType(), Zero));
2026 }
2027
2028 unsigned BShift = Shift & 0x1f;
2029 uint64_t Thi = Make_64(TwoByPi[Idx], TwoByPi[Idx + 1]);
2030 uint64_t Tlo = Make_64(TwoByPi[Idx + 2], 0);
2031 if (BShift)
2032 Thi = (Thi << BShift) | (Tlo >> (64 - BShift));
2033 Thi = Thi >> 11;
2034 APFloat Result = APFloat((double)Thi);
2035
2036 int Scale = -53 - Shift;
2037 if (Exponent >= 1968)
2038 Scale += 128;
2039
2040 Result = scalbn(Result, Scale, RoundingMode::NearestTiesToEven);
2041 return IC.replaceInstUsesWith(II, ConstantFP::get(Src->getType(), Result));
2042 }
2043 case Intrinsic::amdgcn_sdot2:
2044 case Intrinsic::amdgcn_udot2:
2045 case Intrinsic::amdgcn_sdot4:
2046 case Intrinsic::amdgcn_udot4:
2047 case Intrinsic::amdgcn_sdot8:
2048 case Intrinsic::amdgcn_udot8: {
2049 Value *Src0 = II.getArgOperand(0);
2050 Value *Src1 = II.getArgOperand(1);
2051
2052 // Canonicalize the constant multiplicand to Src1.
2053 if (isa<Constant>(Src0) && !isa<Constant>(Src1)) {
2054 II.setArgOperand(0, Src1);
2055 II.setArgOperand(1, Src0);
2056 return &II;
2057 }
2058
2059 if (match(Src1, m_Zero()))
2060 return IC.replaceInstUsesWith(II, II.getArgOperand(2));
2061
2062 if (Instruction *I = foldAddIntoDotAccumulator(II, 2, 3, IC))
2063 return I;
2064
2065 break;
2066 }
2067 case Intrinsic::amdgcn_sudot4:
2068 case Intrinsic::amdgcn_sudot8: {
2069 Value *Src0 = II.getArgOperand(1);
2070 Value *Src1 = II.getArgOperand(3);
2071
2072 // Canonicalize the constant multiplicand to Src1, moving its sign flag with
2073 // it.
2074 if (isa<Constant>(Src0) && !isa<Constant>(Src1)) {
2075 Value *Sign0 = II.getArgOperand(0);
2076 Value *Sign1 = II.getArgOperand(2);
2077 II.setArgOperand(0, Sign1);
2078 II.setArgOperand(1, Src1);
2079 II.setArgOperand(2, Sign0);
2080 II.setArgOperand(3, Src0);
2081 return &II;
2082 }
2083
2084 if (match(Src1, m_Zero()))
2085 return IC.replaceInstUsesWith(II, II.getArgOperand(4));
2086
2087 if (Instruction *I = foldAddIntoDotAccumulator(II, 4, 5, IC))
2088 return I;
2089
2090 break;
2091 }
2092 case Intrinsic::amdgcn_fmul_legacy: {
2093 Value *Op0 = II.getArgOperand(0);
2094 Value *Op1 = II.getArgOperand(1);
2095
2096 for (Value *Src : {Op0, Op1}) {
2097 if (isa<PoisonValue>(Src))
2098 return IC.replaceInstUsesWith(II, Src);
2099 }
2100
2101 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
2102 // infinity, gives +0.0.
2103 // TODO: Move to InstSimplify?
2104 if (match(Op0, PatternMatch::m_AnyZeroFP()) ||
2106 return IC.replaceInstUsesWith(II, ConstantFP::getZero(II.getType()));
2107
2108 // If we can prove we don't have one of the special cases then we can use a
2109 // normal fmul instruction instead.
2110 if (canSimplifyLegacyMulToMul(II, Op0, Op1, IC)) {
2111 auto *FMul = IC.Builder.CreateFMulFMF(Op0, Op1, &II);
2112 FMul->takeName(&II);
2113 return IC.replaceInstUsesWith(II, FMul);
2114 }
2115 break;
2116 }
2117 case Intrinsic::amdgcn_fma_legacy: {
2118 Value *Op0 = II.getArgOperand(0);
2119 Value *Op1 = II.getArgOperand(1);
2120 Value *Op2 = II.getArgOperand(2);
2121
2122 for (Value *Src : {Op0, Op1, Op2}) {
2123 if (isa<PoisonValue>(Src))
2124 return IC.replaceInstUsesWith(II, Src);
2125 }
2126
2127 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
2128 // infinity, gives +0.0.
2129 // TODO: Move to InstSimplify?
2130 if (match(Op0, PatternMatch::m_AnyZeroFP()) ||
2132 // It's tempting to just return Op2 here, but that would give the wrong
2133 // result if Op2 was -0.0.
2134 auto *Zero = ConstantFP::getZero(II.getType());
2135 auto *FAdd = IC.Builder.CreateFAddFMF(Zero, Op2, &II);
2136 FAdd->takeName(&II);
2137 return IC.replaceInstUsesWith(II, FAdd);
2138 }
2139
2140 // If we can prove we don't have one of the special cases then we can use a
2141 // normal fma instead.
2142 if (canSimplifyLegacyMulToMul(II, Op0, Op1, IC)) {
2143 II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
2144 II.getModule(), Intrinsic::fma, II.getType()));
2145 return &II;
2146 }
2147 break;
2148 }
2149 case Intrinsic::amdgcn_is_shared:
2150 case Intrinsic::amdgcn_is_private: {
2151 Value *Src = II.getArgOperand(0);
2152 if (isa<PoisonValue>(Src))
2153 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
2154 if (isa<UndefValue>(Src))
2155 return IC.replaceInstUsesWith(II, UndefValue::get(II.getType()));
2156
2157 if (isa<ConstantPointerNull>(II.getArgOperand(0)))
2158 return IC.replaceInstUsesWith(II, ConstantInt::getFalse(II.getType()));
2159 break;
2160 }
2161 case Intrinsic::amdgcn_make_buffer_rsrc: {
2162 Value *Src = II.getArgOperand(0);
2163 if (isa<PoisonValue>(Src))
2164 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
2165
2166 // Normalize num_records to the correct width.
2167 std::optional<unsigned> Width = ST->getBufferResourceNumRecordsWidth();
2168 if (!Width)
2169 return std::nullopt;
2170 Type *NumRecordsTy = IC.Builder.getIntNTy(*Width);
2171 if (II.getArgOperand(2)->getType() == NumRecordsTy)
2172 return std::nullopt;
2173 SmallVector<Value *, 4> Args(II.args());
2174 Args[2] = IC.Builder.CreateZExtOrTrunc(Args[2], NumRecordsTy);
2176 Intrinsic::amdgcn_make_buffer_rsrc,
2177 {II.getType(), Src->getType(), NumRecordsTy}, Args);
2178 NewCall->copyMetadata(II);
2179 NewCall->setTailCallKind(II.getTailCallKind());
2180 // Copy over all attributes except those on num_records, which may no longer
2181 // be valid.
2182 NewCall->setAttributes(
2183 II.getAttributes().removeParamAttributes(II.getContext(), 2));
2184 NewCall->takeName(&II);
2185 return IC.replaceInstUsesWith(II, NewCall);
2186 }
2187 case Intrinsic::amdgcn_raw_buffer_store_format:
2188 case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
2189 case Intrinsic::amdgcn_struct_buffer_store_format:
2190 case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
2191 case Intrinsic::amdgcn_raw_tbuffer_store:
2192 case Intrinsic::amdgcn_raw_ptr_tbuffer_store:
2193 case Intrinsic::amdgcn_struct_tbuffer_store:
2194 case Intrinsic::amdgcn_struct_ptr_tbuffer_store:
2195 case Intrinsic::amdgcn_image_store_1d:
2196 case Intrinsic::amdgcn_image_store_1darray:
2197 case Intrinsic::amdgcn_image_store_2d:
2198 case Intrinsic::amdgcn_image_store_2darray:
2199 case Intrinsic::amdgcn_image_store_2darraymsaa:
2200 case Intrinsic::amdgcn_image_store_2dmsaa:
2201 case Intrinsic::amdgcn_image_store_3d:
2202 case Intrinsic::amdgcn_image_store_cube:
2203 case Intrinsic::amdgcn_image_store_mip_1d:
2204 case Intrinsic::amdgcn_image_store_mip_1darray:
2205 case Intrinsic::amdgcn_image_store_mip_2d:
2206 case Intrinsic::amdgcn_image_store_mip_2darray:
2207 case Intrinsic::amdgcn_image_store_mip_3d:
2208 case Intrinsic::amdgcn_image_store_mip_cube: {
2209 if (!isa<FixedVectorType>(II.getArgOperand(0)->getType()))
2210 break;
2211
2212 APInt DemandedElts;
2213 if (ST->hasDefaultComponentBroadcast())
2214 DemandedElts = defaultComponentBroadcast(II.getArgOperand(0));
2215 else if (ST->hasDefaultComponentZero())
2216 DemandedElts = trimTrailingZerosInVector(IC, II.getArgOperand(0), &II);
2217 else
2218 break;
2219
2220 if (simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts))
2221 return IC.eraseInstFromFunction(II);
2222
2223 break;
2224 }
2225 case Intrinsic::amdgcn_prng_b32: {
2226 auto *Src = II.getArgOperand(0);
2227 if (isa<UndefValue>(Src)) {
2228 return IC.replaceInstUsesWith(II, Src);
2229 }
2230 return std::nullopt;
2231 }
2232 case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
2233 case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
2234 Value *Src0 = II.getArgOperand(0);
2235 Value *Src1 = II.getArgOperand(1);
2236 uint64_t CBSZ = cast<ConstantInt>(II.getArgOperand(3))->getZExtValue();
2237 uint64_t BLGP = cast<ConstantInt>(II.getArgOperand(4))->getZExtValue();
2238 auto *Src0Ty = cast<FixedVectorType>(Src0->getType());
2239 auto *Src1Ty = cast<FixedVectorType>(Src1->getType());
2240
2241 auto getFormatNumRegs = [](uint64_t FormatVal) {
2242 switch (FormatVal) {
2245 return 6u;
2247 return 4u;
2250 return 8u;
2251 default:
2252 llvm_unreachable("invalid format value");
2253 }
2254 };
2255
2256 bool MadeChange = false;
2257 unsigned Src0NumElts = getFormatNumRegs(CBSZ);
2258 unsigned Src1NumElts = getFormatNumRegs(BLGP);
2259
2260 // Depending on the used format, fewer registers are required so shrink the
2261 // vector type.
2262 if (Src0Ty->getNumElements() > Src0NumElts) {
2263 Src0 = IC.Builder.CreateExtractVector(
2264 FixedVectorType::get(Src0Ty->getElementType(), Src0NumElts), Src0,
2265 uint64_t(0));
2266 MadeChange = true;
2267 }
2268
2269 if (Src1Ty->getNumElements() > Src1NumElts) {
2270 Src1 = IC.Builder.CreateExtractVector(
2271 FixedVectorType::get(Src1Ty->getElementType(), Src1NumElts), Src1,
2272 uint64_t(0));
2273 MadeChange = true;
2274 }
2275
2276 if (!MadeChange)
2277 return std::nullopt;
2278
2279 SmallVector<Value *, 10> Args(II.args());
2280 Args[0] = Src0;
2281 Args[1] = Src1;
2282
2283 Value *NewII = IC.Builder.CreateIntrinsic(
2284 IID, {Src0->getType(), Src1->getType()}, Args, &II);
2285 NewII->takeName(&II);
2286 return IC.replaceInstUsesWith(II, NewII);
2287 }
2288 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
2289 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
2290 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
2291 Value *Src0 = II.getArgOperand(1);
2292 Value *Src1 = II.getArgOperand(3);
2293 unsigned FmtA = cast<ConstantInt>(II.getArgOperand(0))->getZExtValue();
2294 uint64_t FmtB = cast<ConstantInt>(II.getArgOperand(2))->getZExtValue();
2295 auto *Src0Ty = cast<FixedVectorType>(Src0->getType());
2296 auto *Src1Ty = cast<FixedVectorType>(Src1->getType());
2297
2298 bool MadeChange = false;
2299 unsigned Src0NumElts = AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(FmtA);
2300 unsigned Src1NumElts = AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(FmtB);
2301
2302 // Depending on the used format, fewer registers are required so shrink the
2303 // vector type.
2304 if (Src0Ty->getNumElements() > Src0NumElts) {
2305 Src0 = IC.Builder.CreateExtractVector(
2306 FixedVectorType::get(Src0Ty->getElementType(), Src0NumElts), Src0,
2307 IC.Builder.getInt64(0));
2308 MadeChange = true;
2309 }
2310
2311 if (Src1Ty->getNumElements() > Src1NumElts) {
2312 Src1 = IC.Builder.CreateExtractVector(
2313 FixedVectorType::get(Src1Ty->getElementType(), Src1NumElts), Src1,
2314 IC.Builder.getInt64(0));
2315 MadeChange = true;
2316 }
2317
2318 if (!MadeChange)
2319 return std::nullopt;
2320
2321 SmallVector<Value *, 13> Args(II.args());
2322 Args[1] = Src0;
2323 Args[3] = Src1;
2324
2325 Value *NewII = IC.Builder.CreateIntrinsic(
2326 IID, {II.getArgOperand(5)->getType(), Src0->getType(), Src1->getType()},
2327 Args, &II);
2328 NewII->takeName(&II);
2329 return IC.replaceInstUsesWith(II, NewII);
2330 }
2331 }
2332 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
2333 AMDGPU::getImageDimIntrinsicInfo(II.getIntrinsicID())) {
2334 return simplifyAMDGCNImageIntrinsic(ST, ImageDimIntr, II, IC);
2335 }
2336 if (std::optional<unsigned> ScaleArgIdx =
2338 if (simplifyExponentOnlyScaleArg(IC, II, *ScaleArgIdx))
2339 return &II;
2340 }
2341 return std::nullopt;
2342}
2343
2344/// Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics.
2345///
2346/// The result of simplifying amdgcn image and buffer store intrinsics is updating
2347/// definitions of the intrinsics vector argument, not Uses of the result like
2348/// image and buffer loads.
2349/// Note: This only supports non-TFE/LWE image intrinsic calls; those have
2350/// struct returns.
2352 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts) const {
2353 // Only stores return void here; image stores take the dmask after the
2354 // stored value.
2355 const bool IsLoad = !II.getType()->isVoidTy();
2356 int DMaskIdx = -1;
2357 if (getAMDGPUImageDMaskIntrinsic(II.getIntrinsicID()))
2358 DMaskIdx = IsLoad ? 0 : 1;
2359
2360 auto *IIVTy = cast<FixedVectorType>(IsLoad ? II.getType()
2361 : II.getOperand(0)->getType());
2362 unsigned VWidth = IIVTy->getNumElements();
2363 if (VWidth == 1)
2364 return nullptr;
2365 Type *EltTy = IIVTy->getElementType();
2366
2369
2370 // Assume the arguments are unchanged and later override them, if needed.
2371 SmallVector<Value *, 16> Args(II.args());
2372 uint64_t OffsetAdd = 0;
2373
2374 if (DMaskIdx < 0) {
2375 // Buffer case.
2376
2377 const unsigned ActiveBits = DemandedElts.getActiveBits();
2378 const unsigned UnusedComponentsAtFront = DemandedElts.countr_zero();
2379
2380 // Start assuming the prefix of elements is demanded, but possibly clear
2381 // some other bits if there are trailing zeros (unused components at front)
2382 // and update offset.
2383 DemandedElts = (1 << ActiveBits) - 1;
2384
2385 if (UnusedComponentsAtFront > 0) {
2386 const AMDGPU::BufferIntrinsicInfo *BufInfo =
2387 AMDGPU::lookupBufferIntrinsic(II.getIntrinsicID());
2388 // buffer_load_format takes its format from the resource, which is
2389 // unknown here.
2390 bool CanAdjustOffset = BufInfo &&
2391 (BufInfo->Kind == AMDGPU::BufferKindLoad ||
2392 BufInfo->Kind == AMDGPU::BufferKindSLoad) &&
2393 !BufInfo->isFormatFromRsrc();
2394 // If resulting type is vec3, there is no point in trimming the scalar
2395 // load with updated offset, as the vec3 would most likely be widened to
2396 // vec4 anyway during lowering.
2397 if (CanAdjustOffset && BufInfo->Kind == AMDGPU::BufferKindSLoad &&
2398 ActiveBits == 4 && UnusedComponentsAtFront == 1)
2399 CanAdjustOffset = false;
2400
2401 // Set for tbuffer loads, whose component size comes from the format.
2402 unsigned FormatBitsPerComp = 0;
2403 if (CanAdjustOffset && BufInfo->isTyped()) {
2404 const unsigned FormatIdx = *BufInfo->getFormatArg();
2405 // The format describes the whole element, so also drop the leading
2406 // components from it to keep the element ending at the same byte.
2407 // Lanes past the end of the format are filled in by the hardware
2408 // rather than loaded; leave such loads alone.
2409 const AMDGPU::GcnBufferFormatInfo *OldFormat =
2411 cast<ConstantInt>(Args[FormatIdx])->getZExtValue(), *ST);
2412 const AMDGPU::GcnBufferFormatInfo *NewFormat = nullptr;
2413 if (OldFormat && ActiveBits <= OldFormat->NumComponents)
2415 OldFormat->BitsPerComp,
2416 OldFormat->NumComponents - UnusedComponentsAtFront,
2417 OldFormat->NumFormat, *ST);
2418 if (NewFormat) {
2419 Args[FormatIdx] =
2420 ConstantInt::get(Args[FormatIdx]->getType(), NewFormat->Format);
2421 FormatBitsPerComp = OldFormat->BitsPerComp;
2422 } else {
2423 CanAdjustOffset = false;
2424 }
2425 }
2426
2427 if (CanAdjustOffset) {
2428 unsigned OffsetIdx = BufInfo->getOffsetArg();
2429 // Clear demanded bits and update the offset.
2430 DemandedElts &= ~((1 << UnusedComponentsAtFront) - 1);
2431 auto *Offset = Args[OffsetIdx];
2432 uint64_t SingleComponentSizeInBits =
2433 FormatBitsPerComp
2434 ? FormatBitsPerComp
2436 OffsetAdd = UnusedComponentsAtFront * SingleComponentSizeInBits / 8;
2437 auto *OffsetAddVal = ConstantInt::get(Offset->getType(), OffsetAdd);
2438 Args[OffsetIdx] = IC.Builder.CreateAdd(Offset, OffsetAddVal);
2439 }
2440 }
2441 } else {
2442 // Image case.
2443
2444 ConstantInt *DMask = cast<ConstantInt>(Args[DMaskIdx]);
2445 unsigned DMaskVal = DMask->getZExtValue() & 0xf;
2446
2447 // dmask 0 has special semantics, do not simplify.
2448 if (DMaskVal == 0)
2449 return nullptr;
2450
2451 if (!IsLoad && !isMask_32(DMaskVal))
2452 return nullptr;
2453
2454 // Mask off values that are undefined because the dmask doesn't cover them
2455 DemandedElts &= (1 << llvm::popcount(DMaskVal)) - 1;
2456
2457 unsigned NewDMaskVal = 0;
2458 unsigned OrigLdStIdx = 0;
2459 for (unsigned SrcIdx = 0; SrcIdx < 4; ++SrcIdx) {
2460 const unsigned Bit = 1 << SrcIdx;
2461 if (!!(DMaskVal & Bit)) {
2462 if (!!DemandedElts[OrigLdStIdx])
2463 NewDMaskVal |= Bit;
2464 OrigLdStIdx++;
2465 }
2466 }
2467
2468 if (DMaskVal != NewDMaskVal)
2469 Args[DMaskIdx] = ConstantInt::get(DMask->getType(), NewDMaskVal);
2470 }
2471
2472 unsigned NewNumElts = DemandedElts.popcount();
2473 if (!NewNumElts)
2474 return PoisonValue::get(IIVTy);
2475
2476 if (NewNumElts >= VWidth && DemandedElts.isMask()) {
2477 if (DMaskIdx >= 0)
2478 II.setArgOperand(DMaskIdx, Args[DMaskIdx]);
2479 return nullptr;
2480 }
2481
2482 // Validate function argument and return types, extracting overloaded types
2483 // along the way.
2484 SmallVector<Type *, 6> OverloadTys;
2485 if (!Intrinsic::isSignatureValid(II.getCalledFunction(), OverloadTys))
2486 return nullptr;
2487
2488 Type *NewTy =
2489 (NewNumElts == 1) ? EltTy : FixedVectorType::get(EltTy, NewNumElts);
2490 OverloadTys[0] = NewTy;
2491
2492 if (!IsLoad) {
2493 SmallVector<int, 8> EltMask;
2494 for (unsigned OrigStoreIdx = 0; OrigStoreIdx < VWidth; ++OrigStoreIdx)
2495 if (DemandedElts[OrigStoreIdx])
2496 EltMask.push_back(OrigStoreIdx);
2497
2498 if (NewNumElts == 1)
2499 Args[0] = IC.Builder.CreateExtractElement(II.getOperand(0), EltMask[0]);
2500 else
2501 Args[0] = IC.Builder.CreateShuffleVector(II.getOperand(0), EltMask);
2502 }
2503
2505 II.getIntrinsicID(), OverloadTys, Args);
2506 NewCall->takeName(&II);
2507 NewCall->copyMetadata(II);
2508 AttributeList OldAttrList = II.getAttributes();
2509 NewCall->setAttributes(OldAttrList);
2510
2511 if (OffsetAdd != 0) {
2512 unsigned RsrcArg =
2513 AMDGPU::lookupRsrcIntrinsic(II.getIntrinsicID())->RsrcArg;
2514 if (MaybeAlign RsrcAlign = NewCall->getParamAlign(RsrcArg)) {
2515 NewCall->removeParamAttr(RsrcArg, Attribute::Alignment);
2516 NewCall->addParamAttr(
2517 RsrcArg,
2519 commonAlignment(*RsrcAlign, OffsetAdd)));
2520 }
2521 }
2522
2523 if (IsLoad) {
2524 if (NewNumElts == 1) {
2525 return IC.Builder.CreateInsertElement(PoisonValue::get(IIVTy), NewCall,
2526 DemandedElts.countr_zero());
2527 }
2528
2529 SmallVector<int, 8> EltMask;
2530 unsigned NewLoadIdx = 0;
2531 for (unsigned OrigLoadIdx = 0; OrigLoadIdx < VWidth; ++OrigLoadIdx) {
2532 if (!!DemandedElts[OrigLoadIdx])
2533 EltMask.push_back(NewLoadIdx++);
2534 else
2535 EltMask.push_back(NewNumElts);
2536 }
2537
2538 auto *Shuffle = IC.Builder.CreateShuffleVector(NewCall, EltMask);
2539
2540 return Shuffle;
2541 }
2542
2543 return NewCall;
2544}
2545
2547 InstCombiner &IC, IntrinsicInst &II, const APInt &DemandedElts,
2548 APInt &UndefElts) const {
2549 auto *VT = dyn_cast<FixedVectorType>(II.getType());
2550 if (!VT)
2551 return nullptr;
2552
2553 const unsigned FirstElt = DemandedElts.countr_zero();
2554 const unsigned LastElt = DemandedElts.getActiveBits() - 1;
2555 const unsigned MaskLen = LastElt - FirstElt + 1;
2556
2557 unsigned OldNumElts = VT->getNumElements();
2558 if (MaskLen == OldNumElts && MaskLen != 1)
2559 return nullptr;
2560
2561 Type *EltTy = VT->getElementType();
2562 Type *NewVT = MaskLen == 1 ? EltTy : FixedVectorType::get(EltTy, MaskLen);
2563
2564 // Theoretically we should support these intrinsics for any legal type. Avoid
2565 // introducing cases that aren't direct register types like v3i16.
2566 if (!isTypeLegal(NewVT))
2567 return nullptr;
2568
2569 Value *Src = II.getArgOperand(0);
2570
2571 // Make sure convergence tokens are preserved.
2572 // TODO: CreateIntrinsic should allow directly copying bundles
2574 II.getOperandBundlesAsDefs(OpBundles);
2575
2576 Module *M = IC.Builder.getModule();
2577 Function *Remangled =
2578 Intrinsic::getOrInsertDeclaration(M, II.getIntrinsicID(), {NewVT});
2579
2580 if (MaskLen == 1) {
2581 Value *Extract = IC.Builder.CreateExtractElement(Src, FirstElt);
2582
2583 // TODO: Preserve callsite attributes?
2584 CallInst *NewCall = IC.Builder.CreateCall(Remangled, {Extract}, OpBundles);
2585
2586 return IC.Builder.CreateInsertElement(PoisonValue::get(II.getType()),
2587 NewCall, FirstElt);
2588 }
2589
2590 SmallVector<int> ExtractMask(MaskLen, -1);
2591 for (unsigned I = 0; I != MaskLen; ++I) {
2592 if (DemandedElts[FirstElt + I])
2593 ExtractMask[I] = FirstElt + I;
2594 }
2595
2596 Value *Extract = IC.Builder.CreateShuffleVector(Src, ExtractMask);
2597
2598 // TODO: Preserve callsite attributes?
2599 CallInst *NewCall = IC.Builder.CreateCall(Remangled, {Extract}, OpBundles);
2600
2601 SmallVector<int> InsertMask(OldNumElts, -1);
2602 for (unsigned I = 0; I != MaskLen; ++I) {
2603 if (DemandedElts[FirstElt + I])
2604 InsertMask[FirstElt + I] = I;
2605 }
2606
2607 // FIXME: If the call has a convergence bundle, we end up leaving the dead
2608 // call behind.
2609 return IC.Builder.CreateShuffleVector(NewCall, InsertMask);
2610}
2611
2613 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
2614 APInt &UndefElts2, APInt &UndefElts3,
2615 std::function<void(Instruction *, unsigned, APInt, APInt &)>
2616 SimplifyAndSetOp) const {
2617 switch (II.getIntrinsicID()) {
2618 case Intrinsic::amdgcn_readfirstlane:
2619 SimplifyAndSetOp(&II, 0, DemandedElts, UndefElts);
2620 return simplifyAMDGCNLaneIntrinsicDemanded(IC, II, DemandedElts, UndefElts);
2621 default:
2622 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
2623 AMDGPU::lookupBufferIntrinsic(II.getIntrinsicID());
2624 BufInfo && (BufInfo->Kind == AMDGPU::BufferKindLoad ||
2625 BufInfo->Kind == AMDGPU::BufferKindSLoad))
2626 return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts);
2627 if (getAMDGPUImageDMaskIntrinsic(II.getIntrinsicID()))
2628 return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts);
2629 break;
2630 }
2631 return std::nullopt;
2632}
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
uint64_t Imm
static Value * createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo, uint32_t Hi)
Emit v_permlane16 with the precomputed lane-select halves.
static std::optional< unsigned > matchRowSharePattern(ArrayRef< uint8_t > Ids)
Match a row-share pattern: all 16 lanes of each row read the same source lane.
static bool matchMirrorPattern(ArrayRef< uint8_t > Ids)
Match an N-lane reversal (mirror) pattern.
static bool canSafelyConvertTo16Bit(Value &V, bool IsFloat, bool AllowI16SExt=false)
static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST, SmallVectorImpl< uint8_t > &Ids, const DataLayout &DL)
Build the per-lane shuffle map by evaluating Index for every lane in the wave.
static std::optional< unsigned > matchQuadPermPattern(ArrayRef< uint8_t > Ids)
Match a 4-lane (quad) permutation, encoded as the v_mov_b32_dpp QUAD_PERM control word: bits[1:0]=Ids...
static std::optional< unsigned > matchDsSwizzleRotatePattern(ArrayRef< uint8_t > Ids)
Match a GFX9+ DS_SWIZZLE rotate-mode permutation: a cyclic left-rotation of all 32 lanes within each ...
static std::optional< unsigned > matchHalfRowPermPattern(ArrayRef< uint8_t > Ids)
Match an 8-lane arbitrary permutation, encoded as the v_mov_b32_dpp8 24-bit selector (three bits per ...
static std::optional< unsigned > matchRowXMaskPattern(ArrayRef< uint8_t > Ids)
Match an XOR mask pattern within each 16-lane row: Ids[J] == Mask ^ J, with Mask in [1,...
static constexpr auto matchHalfRowMirrorPattern
static Value * createPermlaneX16(IRBuilderBase &B, Value *Val, uint32_t Lo, uint32_t Hi)
Emit v_permlanex16 with the precomputed lane-select halves.
static bool isRowPattern(ArrayRef< uint8_t > Ids)
Match an N-lane row pattern: each lane in [0, N) reads from a source lane in the same N-lane row,...
static bool canContractSqrtToRsq(const FPMathOperator *SqrtOp)
Return true if it's legal to contract llvm.amdgcn.rcp(llvm.sqrt)
static CallInst * rewriteCall(IRBuilderBase &B, CallInst &Old, Function &NewCallee, ArrayRef< Value * > Ops)
static Value * convertTo16Bit(Value &V, InstCombiner::BuilderTy &Builder)
static constexpr auto isFullRowPattern
static constexpr auto isQuadPattern
static APInt trimTrailingZerosInVector(InstCombiner &IC, Value *UseV, Instruction *I)
static uint64_t computePermlane16Masks(ArrayRef< uint8_t > Ids)
Pack a 16-lane permutation into a single 64-bit value: four bits per output lane, lane J in bits [J*4...
static bool matchHalfWaveSwapPattern(ArrayRef< uint8_t > Ids)
Match a half-wave swap: lane J reads from lane J ^ 32.
static bool hasPeriodicLayout(ArrayRef< uint8_t > Ids)
Lanes are partitioned into groups of Period; each group is a translated copy of the first: Ids[I] = I...
static std::optional< uint64_t > evalLaneExpr(Value *V, unsigned Lane, const GCNSubtarget &ST, const DataLayout &DL, unsigned Depth=0)
Evaluate V as a function of the lane ID and return its value on Lane, or std::nullopt if V is not a c...
static std::optional< Instruction * > tryOptimizeShufflePattern(InstCombiner &IC, IntrinsicInst &II, const GCNSubtarget &ST)
Try to fold a wave_shuffle/ds_bpermute whose lane index is a constant function of the lane ID into a ...
static constexpr auto isHalfRowPattern
static APInt defaultComponentBroadcast(Value *V)
static std::optional< unsigned > matchDsSwizzleBitmaskPattern(ArrayRef< uint8_t > Ids)
Match a DS_SWIZZLE bitmask-mode permutation: dst_lane = ((src_lane & AND) | OR) ^ XOR with each mask ...
static Value * createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset, const DataLayout &DL)
Emit ds_swizzle with the given immediate, bitcasting/converting between pointer/float types and i32 a...
static std::optional< Instruction * > modifyIntrinsicCall(IntrinsicInst &OldIntr, Instruction &InstToReplace, unsigned NewIntr, InstCombiner &IC, std::function< void(SmallVectorImpl< Value * > &, SmallVectorImpl< Type * > &)> Func)
Applies Func(OldIntr.Args, OldIntr.ArgTys), creates intrinsic call with modified arguments (based on ...
static Value * matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src, ArrayRef< uint8_t > Ids, const GCNSubtarget &ST, const DataLayout &DL)
Given a shuffle map, try to emit the best hardware intrinsic.
static std::optional< unsigned > matchRowRotatePattern(ArrayRef< uint8_t > Ids)
Match a 16-lane cyclic rotation; returns the rotation amount in [1, 15].
static bool isCrossRowPattern(ArrayRef< uint8_t > Ids)
Match a cross-row permutation suitable for v_permlanex16: every lane in the low 16-lane half reads fr...
static bool isThreadID(const GCNSubtarget &ST, Value *V)
static Value * createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl)
Emit v_mov_b32_dpp with the given control word, row/bank masks 0xF, and bound_ctrl=1 so out-of-bounds...
static APFloat fmed3AMDGCN(const APFloat &Src0, const APFloat &Src1, const APFloat &Src2)
static std::optional< Instruction * > simplifyAMDGCNImageIntrinsic(const GCNSubtarget *ST, const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr, IntrinsicInst &II, InstCombiner &IC)
static Value * createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector)
Emit v_mov_b32_dpp8 with the given 24-bit lane selector.
static Instruction * foldAddIntoDotAccumulator(IntrinsicInst &II, unsigned AccIdx, unsigned ClampIdx, InstCombiner &IC)
Try to fold an addition into the accumulator when saturation is disabled.
static Value * matchFPExtFromF16(Value *Arg)
Match an fpext from half to float, or a constant we can convert.
static constexpr auto matchFullRowMirrorPattern
static bool simplifyExponentOnlyScaleArg(InstCombiner &IC, IntrinsicInst &II, unsigned ScaleArgIdx)
Simplify the f32 scale operand of a V_CVT_SCALEF32_* conversion, of which only the exponent field is ...
static Value * createPermlane64(IRBuilderBase &B, Value *Val)
Emit v_permlane64 (swap of the two 32-lane halves of a wave64).
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
This file a TargetTransformInfoImplBase conforming object specific to the AMDGPU target machine.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Utilities for dealing with flags related to floating point properties and mode controls.
AMD GCN specific subclass of TargetSubtarget.
This file provides the interface for the instcombine pass implementation.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
uint64_t IntrinsicInst * II
if(PassOpts->AAPipeline)
int64_t C1
int64_t C0
MachineInstr *& Lo
This file contains some templates that are useful if you are working with the STL at all.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Value * RHS
Value * LHS
static const fltSemantics & IEEEsingle()
Definition APFloat.h:304
static constexpr roundingMode rmTowardZero
Definition APFloat.h:365
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:361
static const fltSemantics & IEEEhalf()
Definition APFloat.h:302
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
Definition APFloat.h:1224
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition APFloat.cpp:6045
bool bitwiseIsEqual(const APFloat &RHS) const
Definition APFloat.h:1548
bool isPosInfinity() const
Definition APFloat.h:1596
APFloat makeQuiet() const
Assuming this is an IEEE-754 NaN value, quiet its signaling bit.
Definition APFloat.h:1420
bool isNaN() const
Definition APFloat.h:1581
bool isSignaling() const
Definition APFloat.h:1585
APInt bitcastToAPInt() const
Definition APFloat.h:1475
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Definition APFloat.h:1183
bool isInfinity() const
Definition APFloat.h:1580
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition APInt.h:1426
unsigned popcount() const
Count the number of bits set.
Definition APInt.h:1690
LLVM_ABI uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
Definition APInt.cpp:517
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1532
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition APInt.h:1659
bool isMask(unsigned numBits) const
Definition APInt.h:484
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
Definition ArrayRef.h:218
size_t size() const
Get the array size.
Definition ArrayRef.h:141
static LLVM_ABI Attribute getWithDereferenceableBytes(LLVMContext &Context, uint64_t Bytes)
static LLVM_ABI Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
bool isTypeLegal(Type *Ty) const override
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Removes the attribute from the given argument.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
void setAttributes(AttributeList A)
Set the attributes for this call.
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
AttributeList getAttributes() const
Return the attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:420
const APFloat & getValueAPF() const
Definition Constants.h:463
static LLVM_ABI ConstantFP * getZero(Type *Ty, bool Negative=false)
static LLVM_ABI ConstantFP * getNaN(Type *Ty, bool Negative=false, uint64_t Payload=0)
static LLVM_ABI ConstantFP * getInfinity(Type *Ty, bool Negative=false)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static ConstantInt * getSigned(IntegerType *Ty, int64_t V, bool ImplicitTrunc=false)
Return a ConstantInt with the specified value for the specified type.
Definition Constants.h:135
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
This class represents a range of values.
LLVM_ABI ConstantRange add(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an addition of a value in this ran...
LLVM_ABI bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
LLVM_ABI ConstantRange intersectWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the intersection of this range with another range.
This is an important base class in LLVM.
Definition Constant.h:44
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition DataLayout.h:794
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Tagged union holding either a T or a Error.
Definition Error.h:485
This class represents an extension of floating point types.
Utility class for floating point operations which can have information about relaxed accuracy require...
Definition Operator.h:202
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags.
Definition Operator.h:291
bool hasApproxFunc() const
Test if this operation allows approximations of math library functions or intrinsics.
Definition Operator.h:288
LLVM_ABI float getFPAccuracy() const
Get the maximum error permitted by this operation in ULPs.
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
bool allowContract() const
Definition FMF.h:69
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
bool simplifyDemandedLaneMaskArg(InstCombiner &IC, IntrinsicInst &II, unsigned LaneAgIdx) const
Simplify a lane index operand (e.g.
std::optional< Instruction * > instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const override
Instruction * hoistLaneIntrinsicThroughOperand(InstCombiner &IC, IntrinsicInst &II) const
Value * simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts) const
Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics.
std::optional< Value * > simplifyDemandedVectorEltsIntrinsic(InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3, std::function< void(Instruction *, unsigned, APInt, APInt &)> SimplifyAndSetOp) const override
KnownIEEEMode fpenvIEEEMode(const Instruction &I) const
Return KnownIEEEMode::On if we know if the use context can assume "amdgpu-ieee"="true" and KnownIEEEM...
Value * simplifyAMDGCNLaneIntrinsicDemanded(InstCombiner &IC, IntrinsicInst &II, const APInt &DemandedElts, APInt &UndefElts) const
bool canSimplifyLegacyMulToMul(const Instruction &I, const Value *Op0, const Value *Op1, InstCombiner &IC) const
Common base class shared among various IRBuilders.
Definition IRBuilder.h:111
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Definition IRBuilder.h:446
LLVM_ABI CallInst * CreateIntrinsicWithoutFolding(Intrinsic::ID ID, ArrayRef< Type * > OverloadTypes, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="", ArrayRef< OperandBundleDef > OpBundles={})
Create a call to intrinsic ID with Args, mangled using OverloadTypes.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Definition IRBuilder.h:2748
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition IRBuilder.h:2736
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
Definition IRBuilder.h:541
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
Definition IRBuilder.h:2215
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition IRBuilder.h:1534
Value * CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx, const Twine &Name="")
Create a call to the vector.extract intrinsic.
Definition IRBuilder.h:1114
Module * getModule() const
Get the module.
Definition IRBuilder.h:190
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:2462
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition IRBuilder.h:476
Value * CreateMaxNum(Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create call to the maxnum intrinsic.
Definition IRBuilder.h:1045
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition IRBuilder.h:1513
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Definition IRBuilder.h:2197
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Definition IRBuilder.h:2770
LLVM_ABI Value * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > OverloadTypes, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="", ArrayRef< OperandBundleDef > OpBundles={}, function_ref< void(CallInst *)> SetFn=[](CallInst *) {})
Variant to create a possibly constant-folded intrinsic.
Value * CreateMaximumNum(Value *LHS, Value *RHS, const Twine &Name="")
Create call to the maximum intrinsic.
Definition IRBuilder.h:1073
Value * CreateMinNum(Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create call to the minnum intrinsic.
Definition IRBuilder.h:1033
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition IRBuilder.h:1424
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition IRBuilder.h:2640
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition IRBuilder.h:197
Value * CreateFAddFMF(Value *L, Value *R, FMFSource FMFSource, const Twine &Name="", MDNode *FPMD=nullptr)
Definition IRBuilder.h:1643
Value * CreateMinimumNum(Value *LHS, Value *RHS, const Twine &Name="")
Create call to the minimumnum intrinsic.
Definition IRBuilder.h:1067
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition IRBuilder.h:1553
Value * CreateFMulFMF(Value *L, Value *R, FMFSource FMFSource, const Twine &Name="", MDNode *FPMD=nullptr)
Definition IRBuilder.h:1681
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
The core instruction combiner logic.
const DataLayout & getDataLayout() const
virtual Instruction * eraseInstFromFunction(Instruction &I)=0
Combiner aware instruction erasure.
DominatorTree & getDominatorTree() const
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
Instruction * InsertNewInstWith(Instruction *New, BasicBlock::iterator Old)
Same as InsertNewInstBefore, but also sets the debug loc.
virtual bool SimplifyDemandedBits(Instruction *I, unsigned OpNo, const APInt &DemandedMask, KnownBits &Known, const SimplifyQuery &Q, unsigned Depth=0)=0
IRBuilder< TargetFolder, IRBuilderInstCombineInserter > BuilderTy
An IRBuilder that automatically inserts new instructions into the worklist.
static Value * stripSignOnlyFPOps(Value *Val)
Ignore all operations which only change the sign of a value, returning the underlying magnitude value...
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
const SimplifyQuery & getSimplifyQuery() const
LLVM_ABI void dropUBImplyingAttrsAndMetadata(ArrayRef< unsigned > Keep={})
Drop any attributes or metadata that can cause immediate undefined behavior.
LLVM_ABI Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
LLVM_ABI void copyFastMathFlags(FastMathFlags FMF)
Convenience function for transferring all fast-math flag values to this instruction,...
void dropPoisonGeneratingAnnotations()
Drops flags, attributes and metadata that may generate poison.
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
bool isKnownNeverInfOrNaN() const
Return true if it's known this can never be an infinity or nan.
LLVM_ABI bool isKnownNeverLogicalZero(DenormalMode Mode) const
Return true if it's known this can never be interpreted as a zero.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void resize(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:293
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
Definition Type.h:144
LLVM_ABI Type * getWithNewType(Type *EltTy) const
Given vector type, change the element type, whilst keeping the old number of elements.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:269
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
const Use & getOperandUse(unsigned i) const
Definition User.h:220
void setOperand(unsigned i, Value *Val)
Definition User.h:212
Value * getOperand(unsigned i) const
Definition User.h:207
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVM_ABI bool hasOneUser() const
Return true if there is exactly one user of this value.
Definition Value.cpp:158
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:395
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
const ParentTy * getParent() const
Definition ilist_node.h:34
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
APInt getExponentOnlyScaleDemandedBits()
LLVM_READONLY const MIMGOffsetMappingInfo * getMIMGOffsetMappingInfo(unsigned Offset)
uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt)
const ImageDimIntrinsicInfo * getImageDimIntrinsicByBaseOpcode(unsigned BaseOpcode, unsigned Dim)
std::optional< unsigned > getExponentOnlyScaleArgIdx(unsigned IntrID)
LLVM_READONLY const MIMGMIPMappingInfo * getMIMGMIPMappingInfo(unsigned MIP)
bool isTriviallyUniform(const Value *V, const Instruction *CtxI)
Return true if we can easily prove that V is uniform at CtxI.
const BufferIntrinsicInfo * lookupBufferIntrinsic(unsigned Intr)
LLVM_READONLY const MIMGBiasMappingInfo * getMIMGBiasMappingInfo(unsigned Bias)
std::optional< APFloat > evaluateRcp(const APFloat &Val)
Evaluate the constant-folded result of v_rcp for Val, accounting for the hardware's denormal flushing...
LLVM_READONLY const MIMGLZMappingInfo * getMIMGLZMappingInfo(unsigned L)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfo(unsigned DimEnum)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
const ImageDimIntrinsicInfo * getImageDimIntrinsicInfo(unsigned Intr)
const RsrcIntrinsic * lookupRsrcIntrinsic(unsigned Intr)
const GcnBufferFormatInfo * getGcnBufferFormatInfo(uint8_t BitsPerComp, uint8_t NumComponents, uint8_t NumFormat, const MCSubtargetInfo &STI)
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
match_bind< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
cstfp_pred_ty< is_any_zero_fp > m_AnyZeroFP()
Match a floating-point negative zero or positive zero.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
ap_match< APFloat > m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
auto m_Value()
Match an arbitrary value and ignore it.
CastInst_match< OpTy, FPExtInst > m_FPExt(const OpTy &Op)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
is_zero< true > m_Zero()
Match any null constant or a vector with all elements equal to 0.
BinaryOp_match< LHS, RHS, Instruction::Add, true > m_c_Add(const LHS &L, const RHS &R)
Matches a Add with LHS and RHS in either order.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
auto m_ConstantFP()
Match an arbitrary ConstantFP and ignore it.
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
auto m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
LLVM_ABI KnownFPClass computeKnownFPClass(const Value *V, const APInt &DemandedElts, FPClassTest InterestedClasses, const SimplifyQuery &SQ, unsigned Depth=0)
Determine which floating-point classes are valid for V, and return them in KnownFPClass bit sets.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
@ Known
Known to have no common set bits.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
Definition MathExtras.h:256
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
APFloat frexp(const APFloat &X, int &Exp, APFloat::roundingMode RM)
Equivalent of C standard library function.
Definition APFloat.h:1728
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2008 maxNum semantics.
Definition APFloat.h:1771
constexpr unsigned MaxAnalysisRecursionDepth
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
Returns: X * 2^Exp for integral exponents.
Definition APFloat.h:1716
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
Definition MathExtras.h:151
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
Definition MathExtras.h:156
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
constexpr int PoisonMaskElem
@ FMul
Product of floats.
@ FAdd
Sum of floats.
LLVM_ABI Value * findScalarElement(Value *V, unsigned EltNo)
Given a vector and an element number, see if the scalar value is already around as a register,...
@ NearestTiesToEven
roundTiesToEven.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:341
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition STLExtras.h:2182
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
LLVM_ABI Constant * ConstantFoldInstOperands(const Instruction *I, ArrayRef< Constant * > Ops, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, bool AllowNonDeterministic=true)
ConstantFoldInstOperands - Attempt to constant fold an instruction with the specified operands.
constexpr uint64_t Make_64(uint32_t High, uint32_t Low)
Make a 64-bit integer from a high / low pair of 32-bit integers.
Definition MathExtras.h:161
LLVM_ABI ConstantRange computeConstantRange(const Value *V, bool ForSigned, const SimplifyQuery &SQ, unsigned Depth=0)
Determine the possible constant range of an integer or vector of integer value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
The argument layout of a buffer intrinsic.
bool isFormatFromRsrc() const
Whether the data is converted according to the format in the resource.
unsigned getOffsetArg() const
The main offset: the voffset if there is one, and the soffset otherwise.
std::optional< unsigned > getFormatArg() const
Represent subnormal handling kind for floating point instruction inputs and outputs.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:111
SimplifyQuery getWithInstruction(const Instruction *I) const
LLVM_ABI bool isUndefValue(Value *V) const
If CanUseUndef is true, returns whether V is undef.