LLVM 24.0.0git
AMDGPUInstCombineIntrinsic.cpp
Go to the documentation of this file.
1//===- AMDGPInstCombineIntrinsic.cpp - AMDGPU specific InstCombine pass ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// This file implements a TargetTransformInfo analysis pass specific to the
11// AMDGPU target machine. It uses the target's detailed information to provide
12// more precise answers to certain TTI queries, while letting the target
13// independent and default TTI implementations handle the rest.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUInstrInfo.h"
19#include "GCNSubtarget.h"
20#include "SIDefines.h"
22#include "llvm/ADT/STLExtras.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/Dominators.h"
27#include "llvm/IR/IntrinsicsAMDGPU.h"
30#include <optional>
31
32using namespace llvm;
33using namespace llvm::PatternMatch;
34
35#define DEBUG_TYPE "AMDGPUtti"
36
37namespace {
38
39struct AMDGPUImageDMaskIntrinsic {
40 unsigned Intr;
41};
42
43#define GET_AMDGPUImageDMaskIntrinsicTable_IMPL
44#include "AMDGPUGenSearchableTables.inc"
45
46} // end anonymous namespace
47
48// Constant fold llvm.amdgcn.fmed3 intrinsics for standard inputs.
49//
50// A single NaN input is folded to minnum, so we rely on that folding for
51// handling NaNs.
52static APFloat fmed3AMDGCN(const APFloat &Src0, const APFloat &Src1,
53 const APFloat &Src2) {
54 assert(!Src0.isNaN() && !Src1.isNaN() && !Src2.isNaN() &&
55 "nans handled separately");
56 APFloat Max3 = maxnum(maxnum(Src0, Src1), Src2);
57
58 if (Max3.bitwiseIsEqual(Src0))
59 return maxnum(Src1, Src2);
60
61 if (Max3.bitwiseIsEqual(Src1))
62 return maxnum(Src0, Src2);
63
64 return maxnum(Src0, Src1);
65}
66
67// Check if a value can be converted to a 16-bit value without losing precision.
68// The value is expected to be either a float (IsFloat = true) or an unsigned
69// integer (IsFloat = false). When AllowI16SExt is set, a sext from i16 is also
70// accepted: for unsigned addresses sext and zext only differ for a negative
71// i16, which is out of bounds anyway (see caller).
72static bool canSafelyConvertTo16Bit(Value &V, bool IsFloat,
73 bool AllowI16SExt = false) {
74 Type *VTy = V.getType();
75 if (VTy->isHalfTy() || VTy->isIntegerTy(16)) {
76 // The value is already 16-bit, so we don't want to convert to 16-bit again!
77 return false;
78 }
79 if (IsFloat) {
80 if (ConstantFP *ConstFloat = dyn_cast<ConstantFP>(&V)) {
81 // We need to check that if we cast the index down to a half, we do not
82 // lose precision.
83 APFloat FloatValue(ConstFloat->getValueAPF());
84 bool LosesInfo = true;
86 &LosesInfo);
87 return !LosesInfo;
88 }
89 } else {
90 if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(&V)) {
91 // We need to check that if we cast the index down to an i16, we do not
92 // lose precision.
93 APInt IntValue(ConstInt->getValue());
94 return IntValue.getActiveBits() <= 16;
95 }
96 }
97
98 // Coordinates may arrive as extractelement((s|z|fp)ext Vec), Idx. The
99 // widening cast has one use per lane, so it is never sunk into the extract;
100 // strip the extract here so the cast check below is common to scalar and
101 // vector coords.
102 Value *CastCandidate;
103 if (!match(&V, m_ExtractElt(m_Value(CastCandidate), m_Value())))
104 CastCandidate = &V;
105
106 Value *CastSrc;
107 bool IsExt = IsFloat ? match(CastCandidate, m_FPExt(m_Value(CastSrc)))
108 : match(CastCandidate, m_ZExt(m_Value(CastSrc)));
109 if (!IsExt && !IsFloat && AllowI16SExt)
110 IsExt = match(CastCandidate, m_SExt(m_Value(CastSrc)));
111 if (IsExt) {
112 Type *CastSrcTy = CastSrc->getType()->getScalarType();
113 if (CastSrcTy->isHalfTy() || CastSrcTy->isIntegerTy(16))
114 return true;
115 }
116
117 return false;
118}
119
120// Convert a value to 16-bit.
122 Type *VTy = V.getType();
124 return cast<Instruction>(&V)->getOperand(0);
125 // Vector form: extractelement((s|z|fp)ext Vec), Idx -> extractelement(Vec,
126 // Idx), taking the narrow lane directly so the widening cast can be removed.
127 Instruction *VecCast;
128 Value *Idx;
129 if (match(&V, m_ExtractElt(m_Instruction(VecCast), m_Value(Idx))) &&
131 return Builder.CreateExtractElement(VecCast->getOperand(0), Idx);
132 if (VTy->isIntegerTy())
133 return Builder.CreateIntCast(&V, Type::getInt16Ty(V.getContext()), false);
134 if (VTy->isFloatingPointTy())
135 return Builder.CreateFPCast(&V, Type::getHalfTy(V.getContext()));
136
137 llvm_unreachable("Should never be called!");
138}
139
140/// Applies Func(OldIntr.Args, OldIntr.ArgTys), creates intrinsic call with
141/// modified arguments (based on OldIntr) and replaces InstToReplace with
142/// this newly created intrinsic call.
143static std::optional<Instruction *> modifyIntrinsicCall(
144 IntrinsicInst &OldIntr, Instruction &InstToReplace, unsigned NewIntr,
145 InstCombiner &IC,
146 std::function<void(SmallVectorImpl<Value *> &, SmallVectorImpl<Type *> &)>
147 Func) {
148 SmallVector<Type *, 4> OverloadTys;
149 if (!Intrinsic::isSignatureValid(OldIntr.getCalledFunction(), OverloadTys))
150 return std::nullopt;
151
152 SmallVector<Value *, 8> Args(OldIntr.args());
153
154 // Modify arguments and types
155 Func(Args, OverloadTys);
156
157 CallInst *NewCall =
158 IC.Builder.CreateIntrinsicWithoutFolding(NewIntr, OverloadTys, Args);
159 NewCall->takeName(&OldIntr);
160 NewCall->copyMetadata(OldIntr);
161 if (isa<FPMathOperator>(NewCall))
162 NewCall->copyFastMathFlags(&OldIntr);
163 // Copy attributes
164 AttributeList OldAttrList = OldIntr.getAttributes();
165 NewCall->setAttributes(OldAttrList);
166
167 // Erase and replace uses
168 if (!InstToReplace.getType()->isVoidTy())
169 IC.replaceInstUsesWith(InstToReplace, NewCall);
170
171 bool RemoveOldIntr = &OldIntr != &InstToReplace;
172
173 auto *RetValue = IC.eraseInstFromFunction(InstToReplace);
174 if (RemoveOldIntr)
175 IC.eraseInstFromFunction(OldIntr);
176
177 return RetValue;
178}
179
180static std::optional<Instruction *>
182 const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr,
184 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
186
187 // Optimize _L to _LZ when _L is zero
188 if (const auto *LZMappingInfo =
190 if (auto *ConstantLod =
191 dyn_cast<ConstantFP>(II.getOperand(ImageDimIntr->LodIndex))) {
192 if (ConstantLod->isZero() || ConstantLod->isNegative()) {
193 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
195 ImageDimIntr->Dim);
196 return modifyIntrinsicCall(
197 II, II, NewImageDimIntr->Intr, IC, [&](auto &Args, auto &ArgTys) {
198 Args.erase(Args.begin() + ImageDimIntr->LodIndex);
199 });
200 }
201 }
202 }
203
204 // Optimize _mip away, when 'lod' is zero
205 if (const auto *MIPMappingInfo =
207 if (auto *ConstantMip =
208 dyn_cast<ConstantInt>(II.getOperand(ImageDimIntr->MipIndex))) {
209 if (ConstantMip->isZero()) {
210 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
211 AMDGPU::getImageDimIntrinsicByBaseOpcode(MIPMappingInfo->NONMIP,
212 ImageDimIntr->Dim);
213 return modifyIntrinsicCall(
214 II, II, NewImageDimIntr->Intr, IC, [&](auto &Args, auto &ArgTys) {
215 Args.erase(Args.begin() + ImageDimIntr->MipIndex);
216 });
217 }
218 }
219 }
220
221 // Optimize _bias away when 'bias' is zero
222 if (const auto *BiasMappingInfo =
224 if (auto *ConstantBias =
225 dyn_cast<ConstantFP>(II.getOperand(ImageDimIntr->BiasIndex))) {
226 if (ConstantBias->isZero()) {
227 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
228 AMDGPU::getImageDimIntrinsicByBaseOpcode(BiasMappingInfo->NoBias,
229 ImageDimIntr->Dim);
230 return modifyIntrinsicCall(
231 II, II, NewImageDimIntr->Intr, IC, [&](auto &Args, auto &ArgTys) {
232 Args.erase(Args.begin() + ImageDimIntr->BiasIndex);
233 ArgTys.erase(ArgTys.begin() + ImageDimIntr->BiasTyArg);
234 });
235 }
236 }
237 }
238
239 // Optimize _offset away when 'offset' is zero
240 if (const auto *OffsetMappingInfo =
242 if (auto *ConstantOffset =
243 dyn_cast<ConstantInt>(II.getOperand(ImageDimIntr->OffsetIndex))) {
244 if (ConstantOffset->isZero()) {
245 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
247 OffsetMappingInfo->NoOffset, ImageDimIntr->Dim);
248 return modifyIntrinsicCall(
249 II, II, NewImageDimIntr->Intr, IC, [&](auto &Args, auto &ArgTys) {
250 Args.erase(Args.begin() + ImageDimIntr->OffsetIndex);
251 });
252 }
253 }
254 }
255
256 // Optimize the arrayed dim away when the array slice is zero, since slice 0
257 // is the base layer. Restricted to non-atomic, non-sampled image loads and
258 // stores for now.
259 const AMDGPU::MIMGDimInfo *DimInfo =
260 AMDGPU::getMIMGDimInfo(ImageDimIntr->Dim);
261 if (!BaseOpcode->Atomic && !BaseOpcode->Sampler && BaseOpcode->Coordinates &&
262 DimInfo->NonArrayDim != ImageDimIntr->Dim) {
263 // Address is [coords..., slice, (fragid)] plus an optional mip operand.
264 // The slice is the last coordinate, so index it from CoordStart.
265 unsigned SliceIndex = ImageDimIntr->CoordStart + DimInfo->NumCoords - 1 -
266 (DimInfo->MSAA ? 1 : 0);
267 auto *ConstantSlice = dyn_cast<ConstantInt>(II.getOperand(SliceIndex));
268 if (ConstantSlice && ConstantSlice->isZero()) {
269 if (const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
271 DimInfo->NonArrayDim)) {
272 return modifyIntrinsicCall(II, II, NewImageDimIntr->Intr, IC,
273 [&](auto &Args, auto &ArgTys) {
274 Args.erase(Args.begin() + SliceIndex);
275 });
276 }
277 }
278 }
279
280 // Try to use D16
281 if (ST->hasD16Images()) {
282 if (BaseOpcode->HasD16) {
283
284 // If the only use of image intrinsic is a fptrunc (with conversion to
285 // half) then both fptrunc and image intrinsic will be replaced with image
286 // intrinsic with D16 flag.
287 if (II.hasOneUse()) {
288 Instruction *User = II.user_back();
289
290 if (User->getOpcode() == Instruction::FPTrunc &&
292
293 return modifyIntrinsicCall(II, *User, ImageDimIntr->Intr, IC,
294 [&](auto &Args, auto &ArgTys) {
295 // Change return type of image intrinsic.
296 // Set it to return type of fptrunc.
297 ArgTys[0] = User->getType();
298 });
299 }
300 }
301
302 // Only perform D16 folding if every user of the image sample is
303 // an ExtractElementInst immediately followed by an FPTrunc to half.
305 ExtractTruncPairs;
306 bool AllHalfExtracts = true;
307
308 for (User *U : II.users()) {
309 auto *Ext = dyn_cast<ExtractElementInst>(U);
310 if (!Ext || !Ext->hasOneUse()) {
311 AllHalfExtracts = false;
312 break;
313 }
314
315 auto *Tr = dyn_cast<FPTruncInst>(*Ext->user_begin());
316 if (!Tr || !Tr->getType()->isHalfTy()) {
317 AllHalfExtracts = false;
318 break;
319 }
320
321 ExtractTruncPairs.emplace_back(Ext, Tr);
322 }
323
324 if (!ExtractTruncPairs.empty() && AllHalfExtracts) {
325 auto *VecTy = cast<VectorType>(II.getType());
326 Type *HalfVecTy =
327 VecTy->getWithNewType(Type::getHalfTy(II.getContext()));
328
329 // Obtain the original image sample intrinsic's signature
330 // and replace its return type with the half-vector for D16 folding
331 SmallVector<Type *, 8> OverloadTys;
332 if (!Intrinsic::isSignatureValid(II.getCalledFunction(), OverloadTys))
333 return std::nullopt;
334
335 OverloadTys[0] = HalfVecTy;
336 Module *M = II.getModule();
338 M, ImageDimIntr->Intr, OverloadTys);
339
340 II.mutateType(HalfVecTy);
341 II.setCalledFunction(HalfDecl);
342
343 IRBuilder<> Builder(*M);
344 for (auto &[Ext, Tr] : ExtractTruncPairs) {
345 Value *Idx = Ext->getIndexOperand();
346
347 Builder.SetInsertPoint(Tr);
348
349 Value *HalfExtract = Builder.CreateExtractElement(&II, Idx);
350 HalfExtract->takeName(Tr);
351
352 Tr->replaceAllUsesWith(HalfExtract);
353 }
354
355 for (auto &[Ext, Tr] : ExtractTruncPairs) {
356 IC.eraseInstFromFunction(*Tr);
357 IC.eraseInstFromFunction(*Ext);
358 }
359
360 return &II;
361 }
362 }
363 }
364
365 // Try to use A16 or G16
366 if (!ST->hasA16() && !ST->hasG16())
367 return std::nullopt;
368
369 // Address is interpreted as float if the instruction has a sampler or as
370 // unsigned int if there is no sampler.
371 bool HasSampler = BaseOpcode->Sampler;
372 bool FloatCoord = false;
373 // true means derivatives can be converted to 16 bit, coordinates not
374 bool OnlyDerivatives = false;
375
376 // Sampler-less addresses are unsigned, so a sext from i16 folds to a16 like a
377 // zext: they only disagree for a negative i16 (>= 0x8000), which is out of
378 // bounds while the max image dimension is <= 0x8000.
379 bool AllowI16SExt = !HasSampler;
380
381 for (unsigned OperandIndex = ImageDimIntr->GradientStart;
382 OperandIndex < ImageDimIntr->VAddrEnd; OperandIndex++) {
383 Value *Coord = II.getOperand(OperandIndex);
384 // If the values are not derived from 16-bit values, we cannot optimize.
385 if (!canSafelyConvertTo16Bit(*Coord, HasSampler, AllowI16SExt)) {
386 if (OperandIndex < ImageDimIntr->CoordStart ||
387 ImageDimIntr->GradientStart == ImageDimIntr->CoordStart) {
388 return std::nullopt;
389 }
390 // All gradients can be converted, so convert only them
391 OnlyDerivatives = true;
392 break;
393 }
394
395 assert(OperandIndex == ImageDimIntr->GradientStart ||
396 FloatCoord == Coord->getType()->isFloatingPointTy());
397 FloatCoord = Coord->getType()->isFloatingPointTy();
398 }
399
400 if (!OnlyDerivatives && !ST->hasA16())
401 OnlyDerivatives = true; // Only supports G16
402
403 // Check if there is a bias parameter and if it can be converted to f16
404 if (!OnlyDerivatives && ImageDimIntr->NumBiasArgs != 0) {
405 Value *Bias = II.getOperand(ImageDimIntr->BiasIndex);
406 assert(HasSampler &&
407 "Only image instructions with a sampler can have a bias");
408 if (!canSafelyConvertTo16Bit(*Bias, HasSampler))
409 OnlyDerivatives = true;
410 }
411
412 if (OnlyDerivatives && (!ST->hasG16() || ImageDimIntr->GradientStart ==
413 ImageDimIntr->CoordStart))
414 return std::nullopt;
415
416 Type *CoordType = FloatCoord ? Type::getHalfTy(II.getContext())
417 : Type::getInt16Ty(II.getContext());
418
419 return modifyIntrinsicCall(
420 II, II, II.getIntrinsicID(), IC, [&](auto &Args, auto &ArgTys) {
421 ArgTys[ImageDimIntr->GradientTyArg] = CoordType;
422 if (!OnlyDerivatives) {
423 ArgTys[ImageDimIntr->CoordTyArg] = CoordType;
424
425 // Change the bias type
426 if (ImageDimIntr->NumBiasArgs != 0)
427 ArgTys[ImageDimIntr->BiasTyArg] = Type::getHalfTy(II.getContext());
428 }
429
430 unsigned EndIndex =
431 OnlyDerivatives ? ImageDimIntr->CoordStart : ImageDimIntr->VAddrEnd;
432 for (unsigned OperandIndex = ImageDimIntr->GradientStart;
433 OperandIndex < EndIndex; OperandIndex++) {
434 Args[OperandIndex] =
435 convertTo16Bit(*II.getOperand(OperandIndex), IC.Builder);
436 }
437
438 // Convert the bias
439 if (!OnlyDerivatives && ImageDimIntr->NumBiasArgs != 0) {
440 Value *Bias = II.getOperand(ImageDimIntr->BiasIndex);
441 Args[ImageDimIntr->BiasIndex] = convertTo16Bit(*Bias, IC.Builder);
442 }
443 });
444}
445
447 const Value *Op0, const Value *Op1,
448 InstCombiner &IC) const {
449 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
450 // infinity, gives +0.0. If we can prove we don't have one of the special
451 // cases then we can use a normal multiply instead.
453 KnownFPClass Known0 =
455 DenormalMode Mode = I.getFunction()->getDenormalMode(APFloat::IEEEsingle());
456
457 // Bail early if Op0 may be zero and nsz is not set -- Op1 cannot help.
458 if (!Known0.isKnownNeverLogicalZero(Mode) && !I.hasNoSignedZeros())
459 return false;
460
461 KnownFPClass Known1 =
463
464 // Simplify if both operands are known non-zero.
465 if (Known0.isKnownNeverLogicalZero(Mode) &&
466 Known1.isKnownNeverLogicalZero(Mode))
467 return true;
468
469 // With nsz, two additional cases allow simplification:
470 // 1. One operand is not zero or infinity or NaN:
471 // Op0 NeverLogicalZero && NeverInfOrNaN, or symmetric for Op1.
472 // 2. Neither operand is infinity or NaN:
473 // Op0 NeverInfOrNaN && Op1 NeverInfOrNaN.
474 // The following condition captures both cases.
475 if (I.hasNoSignedZeros() &&
476 (Known0.isKnownNeverLogicalZero(Mode) || Known1.isKnownNeverInfOrNaN()) &&
477 (Known1.isKnownNeverLogicalZero(Mode) || Known0.isKnownNeverInfOrNaN()))
478 return true;
479
480 return false;
481}
482
483/// Match an fpext from half to float, or a constant we can convert.
485 Value *Src = nullptr;
486 ConstantFP *CFP = nullptr;
487 if (match(Arg, m_OneUse(m_FPExt(m_Value(Src))))) {
488 if (Src->getType()->isHalfTy())
489 return Src;
490 } else if (match(Arg, m_ConstantFP(CFP))) {
491 bool LosesInfo;
492 APFloat Val(CFP->getValueAPF());
494 if (!LosesInfo)
495 return ConstantFP::get(Type::getHalfTy(Arg->getContext()), Val);
496 }
497 return nullptr;
498}
499
500// Trim all zero components from the end of the vector \p UseV and return
501// an appropriate bitset with known elements.
503 Instruction *I) {
504 auto *VTy = cast<FixedVectorType>(UseV->getType());
505 unsigned VWidth = VTy->getNumElements();
506 APInt DemandedElts = APInt::getAllOnes(VWidth);
507
508 for (int i = VWidth - 1; i > 0; --i) {
509 auto *Elt = findScalarElement(UseV, i);
510 if (!Elt)
511 break;
512
513 if (auto *ConstElt = dyn_cast<Constant>(Elt)) {
514 if (!ConstElt->isNullValue() && !isa<UndefValue>(Elt))
515 break;
516 } else {
517 break;
518 }
519
520 DemandedElts.clearBit(i);
521 }
522
523 return DemandedElts;
524}
525
526// Trim elements of the end of the vector \p V, if they are
527// equal to the first element of the vector.
529 auto *VTy = cast<FixedVectorType>(V->getType());
530 unsigned VWidth = VTy->getNumElements();
531 APInt DemandedElts = APInt::getAllOnes(VWidth);
532 Value *FirstComponent = findScalarElement(V, 0);
533
534 SmallVector<int> ShuffleMask;
535 if (auto *SVI = dyn_cast<ShuffleVectorInst>(V))
536 SVI->getShuffleMask(ShuffleMask);
537
538 for (int I = VWidth - 1; I > 0; --I) {
539 if (ShuffleMask.empty()) {
540 auto *Elt = findScalarElement(V, I);
541 if (!Elt || (Elt != FirstComponent && !isa<UndefValue>(Elt)))
542 break;
543 } else {
544 // Detect identical elements in the shufflevector result, even though
545 // findScalarElement cannot tell us what that element is.
546 if (ShuffleMask[I] != ShuffleMask[0] && ShuffleMask[I] != PoisonMaskElem)
547 break;
548 }
549 DemandedElts.clearBit(I);
550 }
551
552 return DemandedElts;
553}
554
555static Value *
557 IntrinsicInst &II, APInt DemandedElts,
558 int DMaskIdx = -1, bool IsLoad = true);
559
560/// Return true if it's legal to contract llvm.amdgcn.rcp(llvm.sqrt)
561static bool canContractSqrtToRsq(const FPMathOperator *SqrtOp) {
562 return (SqrtOp->getType()->isFloatTy() &&
563 (SqrtOp->hasApproxFunc() || SqrtOp->getFPAccuracy() >= 1.0f)) ||
564 SqrtOp->getType()->isHalfTy();
565}
566
567/// Simplify a lane index operand (e.g. llvm.amdgcn.readlane src1).
568///
569/// The instruction only reads the low 5 bits for wave32, and 6 bits for wave64.
572 unsigned LaneArgIdx) const {
573 unsigned MaskBits = ST->getWavefrontSizeLog2();
574 APInt DemandedMask(32, maskTrailingOnes<unsigned>(MaskBits));
575
576 KnownBits Known(32);
577 if (IC.SimplifyDemandedBits(&II, LaneArgIdx, DemandedMask, Known))
578 return true;
579
580 if (!Known.isConstant())
581 return false;
582
583 // Out of bounds indexes may appear in wave64 code compiled for wave32.
584 // Unlike the DAG version, SimplifyDemandedBits does not change constants, so
585 // manually fix it up.
586
587 Value *LaneArg = II.getArgOperand(LaneArgIdx);
588 Constant *MaskedConst =
589 ConstantInt::get(LaneArg->getType(), Known.getConstant() & DemandedMask);
590 if (MaskedConst != LaneArg) {
591 II.getOperandUse(LaneArgIdx).set(MaskedConst);
592 return true;
593 }
594
595 return false;
596}
597
598/// Simplify the f32 scale operand of a V_CVT_SCALEF32_* conversion, of which
599/// only the exponent field is read.
601 unsigned ScaleArgIdx) {
602 Value *Scale = II.getArgOperand(ScaleArgIdx);
603 Value *StrippedSign = InstCombiner::stripSignOnlyFPOps(Scale);
604 if (StrippedSign != Scale) {
605 IC.replaceOperand(II, ScaleArgIdx, StrippedSign);
606 return true;
607 }
608
609 auto *BC = dyn_cast<BitCastInst>(Scale);
610 if (!BC || !BC->hasOneUse() || !BC->getSrcTy()->isIntegerTy(32))
611 return false;
612
613 KnownBits Known(32);
614 return IC.SimplifyDemandedBits(
616}
617
619 Function &NewCallee, ArrayRef<Value *> Ops) {
621 Old.getOperandBundlesAsDefs(OpBundles);
622
623 CallInst *NewCall = B.CreateCall(&NewCallee, Ops, OpBundles);
624 NewCall->takeName(&Old);
625 return NewCall;
626}
627
628// Return true for sequences of instructions that effectively assign
629// each lane to its thread ID
630static bool isThreadID(const GCNSubtarget &ST, Value *V) {
631 // Case 1:
632 // wave32: mbcnt_lo(-1, 0)
633 // wave64: mbcnt_hi(-1, mbcnt_lo(-1, 0))
639 if (ST.isWave32() && match(V, W32Pred))
640 return true;
641 if (ST.isWave64() && match(V, W64Pred))
642 return true;
643
644 // Case 2:
645 // workitem.id.x & (wavefront size - 1), or bare workitem.id.x when the
646 // workgroup is one wave long or less.
647 auto *I = dyn_cast<Instruction>(V);
648 if (!I || !ST.isWaveSizeKnown())
649 return false;
650
651 const Function &F = *I->getFunction();
652 if (!ST.hasWavefrontsEvenlySplittingXDim(F, /*RequiresUniformYZ=*/true))
653 return false;
654
655 unsigned WaveSize = ST.getWavefrontSize();
657 if (match(V, m_c_And(TidX, m_SpecificInt(WaveSize - 1))))
658 return true;
659
660 return match(V, TidX) && ST.getMaxWorkitemID(F, 0) < WaveSize;
661}
662
665 IntrinsicInst &II) const {
666 const auto IID = II.getIntrinsicID();
667 assert(IID == Intrinsic::amdgcn_readlane ||
668 IID == Intrinsic::amdgcn_readfirstlane ||
669 IID == Intrinsic::amdgcn_permlane64);
670
671 Instruction *OpInst = dyn_cast<Instruction>(II.getOperand(0));
672
673 // Only do this if both instructions are in the same block
674 // (so the exec mask won't change) and the readlane is the only user of its
675 // operand.
676 if (!OpInst || !OpInst->hasOneUser() || OpInst->getParent() != II.getParent())
677 return nullptr;
678
679 const bool IsReadLane = (IID == Intrinsic::amdgcn_readlane);
680
681 // If this is a readlane, check that the second operand is a constant, or is
682 // defined before OpInst so we know it's safe to move this intrinsic higher.
683 Value *LaneID = nullptr;
684 if (IsReadLane) {
685 LaneID = II.getOperand(1);
686
687 // readlane take an extra operand for the lane ID, so we must check if that
688 // LaneID value can be used at the point where we want to move the
689 // intrinsic.
690 if (auto *LaneIDInst = dyn_cast<Instruction>(LaneID)) {
691 if (!IC.getDominatorTree().dominates(LaneIDInst, OpInst))
692 return nullptr;
693 }
694 }
695
696 // Hoist the intrinsic (II) through OpInst.
697 //
698 // (II (OpInst x)) -> (OpInst (II x))
699 const auto DoIt = [&](unsigned OpIdx,
700 Function *NewIntrinsic) -> Instruction * {
701 SmallVector<Value *, 2> Ops{OpInst->getOperand(OpIdx)};
702 if (IsReadLane)
703 Ops.push_back(LaneID);
704
705 // Rewrite the intrinsic call.
706 CallInst *NewII = rewriteCall(IC.Builder, II, *NewIntrinsic, Ops);
707
708 // Rewrite OpInst so it takes the result of the intrinsic now.
709 Instruction &NewOp = *OpInst->clone();
710 NewOp.setOperand(OpIdx, NewII);
711 return &NewOp;
712 };
713
714 // TODO(?): Should we do more with permlane64?
715 if (IID == Intrinsic::amdgcn_permlane64 && !isa<BitCastInst>(OpInst))
716 return nullptr;
717
718 if (isa<UnaryOperator>(OpInst))
719 return DoIt(0, II.getCalledFunction());
720
721 if (isa<CastInst>(OpInst)) {
722 Value *Src = OpInst->getOperand(0);
723 Type *SrcTy = Src->getType();
724 if (!isTypeLegal(SrcTy))
725 return nullptr;
726
727 Function *Remangled =
728 Intrinsic::getOrInsertDeclaration(II.getModule(), IID, {SrcTy});
729 return DoIt(0, Remangled);
730 }
731
732 // We can also hoist through binary operators if the other operand is uniform.
733 if (isa<BinaryOperator>(OpInst)) {
734 // FIXME: If we had access to UniformityInfo here we could just check
735 // if the operand is uniform.
737 return DoIt(1, II.getCalledFunction());
739 return DoIt(0, II.getCalledFunction());
740 }
741
742 return nullptr;
743}
744
745/// Evaluate V as a function of the lane ID and return its value on Lane, or
746/// std::nullopt if V is not a closed-form expression of the lane ID.
747static std::optional<uint64_t> evalLaneExpr(Value *V, unsigned Lane,
748 const GCNSubtarget &ST,
749 const DataLayout &DL,
750 unsigned Depth = 0) {
752 return std::nullopt;
753
754 // Poison/undef in the index expression: bail and let InstCombine fold the
755 // intrinsic the usual way.
756 if (isa<UndefValue>(V))
757 return std::nullopt;
758
759 if (const ConstantInt *CI = dyn_cast<ConstantInt>(V))
760 return CI->getZExtValue();
761
762 if (isThreadID(ST, V))
763 return Lane;
764
766 if (!BO)
767 return std::nullopt;
768
769 std::optional<uint64_t> LHS =
770 evalLaneExpr(BO->getOperand(0), Lane, ST, DL, Depth + 1);
771 if (!LHS)
772 return std::nullopt;
773 std::optional<uint64_t> RHS =
774 evalLaneExpr(BO->getOperand(1), Lane, ST, DL, Depth + 1);
775 if (!RHS)
776 return std::nullopt;
777
778 Type *Ty = BO->getType();
779 Constant *Ops[] = {ConstantInt::get(Ty, *LHS), ConstantInt::get(Ty, *RHS)};
780 auto *CI =
782 return CI ? std::optional<uint64_t>(CI->getZExtValue()) : std::nullopt;
783}
784
785/// Build the per-lane shuffle map by evaluating Index for every lane in the
786/// wave. Returns false if any lane index is non-constant or out of range.
787static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST,
789 const DataLayout &DL) {
790 unsigned WaveSize = ST.getWavefrontSize();
791 Ids.resize(WaveSize);
792 for (unsigned Lane : seq(WaveSize)) {
793 std::optional<uint64_t> Val = evalLaneExpr(Index, Lane, ST, DL);
794 if (!Val || *Val >= WaveSize)
795 return false;
796 Ids[Lane] = static_cast<uint8_t>(*Val);
797 }
798 return true;
799}
800
801/// Lanes are partitioned into groups of Period; each group is a translated
802/// copy of the first: Ids[I] = Ids[I % Period] + (I & ~(Period - 1)).
803template <unsigned Period>
805 static_assert(isPowerOf2_32(Period), "Period must be a power of two");
806 for (size_t I = Period, E = Ids.size(); I < E; ++I)
807 if (Ids[I] != Ids[I % Period] + (I & ~(Period - 1)))
808 return false;
809 return true;
810}
811
812/// Match an N-lane row pattern: each lane in [0, N) reads from a source lane
813/// in the same N-lane row, and the pattern repeats periodically across rows.
814template <unsigned N> static bool isRowPattern(ArrayRef<uint8_t> Ids) {
815 for (unsigned I = 0; I < N; ++I)
816 if (Ids[I] >= N)
817 return false;
818 return hasPeriodicLayout<N>(Ids);
819}
820
821static constexpr auto isQuadPattern = isRowPattern<4>;
822static constexpr auto isHalfRowPattern = isRowPattern<8>;
823static constexpr auto isFullRowPattern = isRowPattern<16>;
824
825/// Match a 4-lane (quad) permutation, encoded as the v_mov_b32_dpp
826/// QUAD_PERM control word: bits[1:0]=Ids[0], [3:2]=Ids[1], [5:4]=Ids[2],
827/// [7:6]=Ids[3].
828static std::optional<unsigned> matchQuadPermPattern(ArrayRef<uint8_t> Ids) {
829 if (!isQuadPattern(Ids))
830 return std::nullopt;
831 return Ids[3] << 6 | Ids[2] << 4 | Ids[1] << 2 | Ids[0];
832}
833
834/// Match an N-lane reversal (mirror) pattern.
835template <unsigned N> static bool matchMirrorPattern(ArrayRef<uint8_t> Ids) {
836 if (!isRowPattern<N>(Ids))
837 return false;
838 for (unsigned J = 0; J < N; ++J)
839 if (Ids[J] != (N - 1) - J)
840 return false;
841 return true;
842}
843
846
847/// Match a 16-lane cyclic rotation; returns the rotation amount in [1, 15].
848static std::optional<unsigned> matchRowRotatePattern(ArrayRef<uint8_t> Ids) {
849 if (Ids[0] == 0 || !isFullRowPattern(Ids))
850 return std::nullopt;
851 for (unsigned J = 1; J < 16; ++J)
852 if (Ids[J] != (Ids[0] + J) % 16)
853 return std::nullopt;
854 return 16u - Ids[0];
855}
856
857/// Match a row-share pattern: all 16 lanes of each row read the same source
858/// lane. Returns the shared source lane index in [0, 16).
859static std::optional<unsigned> matchRowSharePattern(ArrayRef<uint8_t> Ids) {
860 if (!isFullRowPattern(Ids))
861 return std::nullopt;
862 if (!all_equal(Ids.take_front(16)))
863 return std::nullopt;
864 return Ids[0];
865}
866
867/// Match an XOR mask pattern within each 16-lane row: Ids[J] == Mask ^ J,
868/// with Mask in [1, 15].
869static std::optional<unsigned> matchRowXMaskPattern(ArrayRef<uint8_t> Ids) {
870 unsigned Mask = Ids[0];
871 if (Mask == 0 || !isFullRowPattern(Ids))
872 return std::nullopt;
873 for (unsigned J = 0; J < 16; ++J)
874 if (Ids[J] != (Mask ^ J))
875 return std::nullopt;
876 return Mask;
877}
878
879/// Match an 8-lane arbitrary permutation, encoded as the v_mov_b32_dpp8
880/// 24-bit selector (three bits per output lane).
881static std::optional<unsigned> matchHalfRowPermPattern(ArrayRef<uint8_t> Ids) {
882 if (!isHalfRowPattern(Ids))
883 return std::nullopt;
884 unsigned Selector = 0;
885 for (unsigned J = 0; J < 8; ++J)
886 Selector |= Ids[J] << (J * 3);
887 return Selector;
888}
889
890/// Pack a 16-lane permutation into a single 64-bit value: four bits per output
891/// lane, lane J in bits [J*4 + 3 : J*4]. The caller splits it into the low and
892/// high 32-bit selector operands of v_permlane16 / v_permlanex16.
894 uint64_t Sel = 0;
895 for (unsigned J = 0; J < 16; ++J)
896 Sel |= static_cast<uint64_t>(Ids[J] & 0xF) << (J * 4);
897 return Sel;
898}
899
900/// Match a half-wave swap: lane J reads from lane J ^ 32. Only meaningful on
901/// wave64 targets.
903 if (Ids.size() != 64)
904 return false;
905 for (unsigned J = 0; J < 64; ++J)
906 if (Ids[J] != (J ^ 32))
907 return false;
908 return true;
909}
910
911/// Match a cross-row permutation suitable for v_permlanex16: every lane in
912/// the low 16-lane half reads from the high half of its own row, and vice
913/// versa.
915 if (!hasPeriodicLayout<32>(Ids))
916 return false;
917 for (unsigned J = 0; J < 16; ++J) {
918 if (Ids[J] < 16 || Ids[J] >= 32)
919 return false;
920 if (Ids[J + 16] != Ids[J] - 16)
921 return false;
922 }
923 return true;
924}
925
926/// Match a DS_SWIZZLE bitmask-mode permutation:
927/// dst_lane = ((src_lane & AND) | OR) ^ XOR
928/// with each mask being five bits. Returns the encoded swizzle immediate.
929/// The hardware applies the formula independently within each 32-lane group,
930/// so on wave64 the high group must replicate the low one (translated by 32).
931static std::optional<unsigned>
933 if (!hasPeriodicLayout<32>(Ids))
934 return std::nullopt;
935
936 // The formula is per-bit: output bit B depends only on input bit B. Probe
937 // each bit with src=0 and src=(1<<B); if the output bit flipped, AND[B]=1
938 // and XOR[B] carries the constant offset; otherwise it is a constant bit
939 // encoded in OR (with AND[B]=0, XOR[B]=0).
940 unsigned AndMask = 0, OrMask = 0, XorMask = 0;
941 for (unsigned B = 0; B < 5; ++B) {
942 unsigned Bit0 = (Ids[0] >> B) & 1;
943 unsigned Bit1 = (Ids[1ull << B] >> B) & 1;
944 if (Bit0 != Bit1) {
945 AndMask |= 1u << B;
946 XorMask |= Bit0 << B;
947 } else {
948 OrMask |= Bit0 << B;
949 }
950 }
951
952 // The per-bit derivation assumes bit independence; verify the masks
953 // actually reproduce every lane in the 32-lane group.
954 for (unsigned I : seq(32u)) {
955 unsigned Expected = ((I & AndMask) | OrMask) ^ XorMask;
956 if (Ids[I] != Expected)
957 return std::nullopt;
958 }
959
964}
965
966/// Match a GFX9+ DS_SWIZZLE rotate-mode permutation: a cyclic left-rotation
967/// of all 32 lanes within each 32-lane group by a constant N in [0, 31],
968/// i.e. dst_lane = (src_lane + N) % 32. On wave64, hasPeriodicLayout<32>
969/// ensures both 32-lane groups rotate by the same amount.
970static std::optional<unsigned>
972 if (!hasPeriodicLayout<32>(Ids))
973 return std::nullopt;
974
975 // Determine the rotation amount from lane 0: every lane must read from
976 // lane (I + N) % 32 where N = Ids[0] and 0 <= N <= 31.
977 unsigned N = Ids[0];
978 if (N >= 32)
979 return std::nullopt;
980
981 for (unsigned I = 0; I < 32; ++I)
982 if (Ids[I] != (I + N) % 32)
983 return std::nullopt;
984
987}
988
989/// Emit v_mov_b32_dpp with the given control word, row/bank masks 0xF, and
990/// bound_ctrl=1 so out-of-bounds lanes are well-defined and the DPP mov can
991/// be folded into a consuming VALU op by GCNDPPCombine.
992static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl) {
993 Type *Ty = Val->getType();
994 return B.CreateIntrinsic(Intrinsic::amdgcn_update_dpp, {Ty},
995 {PoisonValue::get(Ty), Val, B.getInt32(Ctrl),
996 B.getInt32(0xF), B.getInt32(0xF), B.getTrue()});
997}
998
999/// Emit v_mov_b32_dpp8 with the given 24-bit lane selector.
1000static Value *createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector) {
1001 return B.CreateIntrinsic(Intrinsic::amdgcn_mov_dpp8, {Val->getType()},
1002 {Val, B.getInt32(Selector)});
1003}
1004
1005/// Emit v_permlane16 with the precomputed lane-select halves.
1007 uint32_t Hi) {
1008 Type *Ty = Val->getType();
1009 return B.CreateIntrinsic(Intrinsic::amdgcn_permlane16, {Ty},
1010 {PoisonValue::get(Ty), Val, B.getInt32(Lo),
1011 B.getInt32(Hi), B.getFalse(), B.getFalse()});
1012}
1013
1014/// Emit v_permlanex16 with the precomputed lane-select halves. Each output
1015/// lane reads from the other 16-lane half of the same row.
1017 uint32_t Hi) {
1018 Type *Ty = Val->getType();
1019 return B.CreateIntrinsic(Intrinsic::amdgcn_permlanex16, {Ty},
1020 {PoisonValue::get(Ty), Val, B.getInt32(Lo),
1021 B.getInt32(Hi), B.getFalse(), B.getFalse()});
1022}
1023
1024/// Emit ds_swizzle with the given immediate, bitcasting/converting between
1025/// pointer/float types and i32 as required by the intrinsic signature.
1027 const DataLayout &DL) {
1028 Type *OrigTy = Val->getType();
1029 assert(DL.getTypeSizeInBits(OrigTy) == 32 &&
1030 "ds_swizzle only supports 32-bit operands");
1031 IntegerType *I32Ty = B.getInt32Ty();
1032 Value *Src = Val;
1033 if (OrigTy->isPointerTy())
1034 Src = B.CreatePtrToInt(Src, I32Ty);
1035 else if (OrigTy != I32Ty)
1036 Src = B.CreateBitCast(Src, I32Ty);
1037 Value *Result = B.CreateIntrinsic(Intrinsic::amdgcn_ds_swizzle, {},
1038 {Src, B.getInt32(Offset)});
1039 if (OrigTy->isPointerTy())
1040 return B.CreateIntToPtr(Result, OrigTy);
1041 if (OrigTy != I32Ty)
1042 return B.CreateBitCast(Result, OrigTy);
1043 return Result;
1044}
1045
1046/// Emit v_permlane64 (swap of the two 32-lane halves of a wave64).
1048 return B.CreateIntrinsic(Intrinsic::amdgcn_permlane64, {Val->getType()},
1049 {Val});
1050}
1051
1052/// Given a shuffle map, try to emit the best hardware intrinsic.
1055 const GCNSubtarget &ST,
1056 const DataLayout &DL) {
1057 // Identity shuffle (every lane reads itself) folds to the source value.
1058 if (all_of(enumerate(Ids),
1059 [](const auto &E) { return E.value() == E.index(); }))
1060 return Src;
1061
1062 // Uniform shuffle (all lanes read the same value) is handled by cheaper
1063 // broadcast/readlane intrinsics.
1064 if (all_equal(Ids))
1065 return nullptr;
1066
1067 if (std::optional<unsigned> QP = matchQuadPermPattern(Ids)) {
1068 if (ST.hasDPP())
1069 return createUpdateDpp(B, Src, *QP);
1071 }
1072
1073 if (ST.hasDPP()) {
1078 if (std::optional<unsigned> Amt = matchRowRotatePattern(Ids))
1079 return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1);
1080 }
1081
1082 // row_share is supported on GFX90A and GFX10+; row_xmask is GFX10+ only.
1083 if (ST.hasDPPRowShare()) {
1084 if (std::optional<unsigned> Lane = matchRowSharePattern(Ids))
1085 return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_SHARE_FIRST + *Lane);
1086 }
1087
1088 if (ST.hasDPP() && ST.hasGFX10Insts()) {
1089 if (std::optional<unsigned> Mask = matchRowXMaskPattern(Ids))
1090 return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_XMASK_FIRST + *Mask);
1091 }
1092
1093 if (ST.hasDPP8()) {
1094 if (std::optional<unsigned> Sel = matchHalfRowPermPattern(Ids))
1095 return createMovDpp8(B, Src, *Sel);
1096 }
1097
1098 if (ST.hasPermlane16Insts()) {
1099 if (isFullRowPattern(Ids)) {
1101 return createPermlane16(B, Src, Lo_32(Sel), Hi_32(Sel));
1102 }
1103 // Cross-row shuffles (e.g. XOR 16..31) — covered by permlanex16.
1104 if (isCrossRowPattern(Ids)) {
1106 return createPermlaneX16(B, Src, Lo_32(Sel), Hi_32(Sel));
1107 }
1108 }
1109
1110 // Generic DS_SWIZZLE bitmask-mode fallback: handles any 32-lane shuffle that
1111 // can be expressed as dst = ((src & AND) | OR) ^ XOR with 5-bit masks. This
1112 // is available on every target that has ds_swizzle.
1113 if (std::optional<unsigned> Imm = matchDsSwizzleBitmaskPattern(Ids))
1114 return createDsSwizzle(B, Src, *Imm, DL);
1115
1116 // DS_SWIZZLE rotate mode (GFX9+): handles cyclic 32-lane rotations that
1117 // bitmask mode cannot express (e.g. +1 mod 32 requires inter-bit carry).
1118 if (ST.hasDsSwizzleRotateMode()) {
1119 if (std::optional<unsigned> Imm = matchDsSwizzleRotatePattern(Ids))
1120 return createDsSwizzle(B, Src, *Imm, DL);
1121 }
1122
1123 if (ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
1124 return createPermlane64(B, Src);
1125
1126 return nullptr;
1127}
1128
1129/// Try to fold a wave_shuffle/ds_bpermute whose lane index is a constant
1130/// function of the lane ID into a hardware-specific lane permutation intrinsic.
1131static std::optional<Instruction *>
1133 const GCNSubtarget &ST) {
1134 const DataLayout &DL = IC.getDataLayout();
1135 if (DL.getTypeSizeInBits(II.getType()) != 32)
1136 return std::nullopt;
1137
1138 if (!ST.isWaveSizeKnown())
1139 return std::nullopt;
1140
1141 unsigned WaveSize = ST.getWavefrontSize();
1142 bool IsBpermute = II.getIntrinsicID() == Intrinsic::amdgcn_ds_bpermute;
1143 Value *Src = II.getArgOperand(IsBpermute ? 1 : 0);
1144 Value *Index = II.getArgOperand(IsBpermute ? 0 : 1);
1145
1147 if (IsBpermute) {
1148 Ids.resize(WaveSize);
1149 for (unsigned Lane : seq(WaveSize)) {
1150 std::optional<uint64_t> Val = evalLaneExpr(Index, Lane, ST, DL);
1151 if (!Val || (*Val & 3) || (*Val >> 2) >= WaveSize)
1152 return std::nullopt;
1153 Ids[Lane] = static_cast<uint8_t>(*Val >> 2);
1154 }
1155 } else {
1156 if (!tryBuildShuffleMap(Index, ST, Ids, DL))
1157 return std::nullopt;
1158 }
1159
1160 Value *Result = matchShuffleToHWIntrinsic(IC.Builder, Src, Ids, ST, DL);
1161 if (!Result)
1162 return std::nullopt;
1163
1164 return IC.replaceInstUsesWith(II, Result);
1165}
1166
1167/// Try to fold an addition into the accumulator when saturation is disabled.
1169 unsigned AccIdx,
1170 unsigned ClampIdx,
1171 InstCombiner &IC) {
1172 // Reassociating across a saturating accumulation is not valid.
1173 if (!match(II.getArgOperand(ClampIdx), m_Zero()) || !II.hasOneUse())
1174 return nullptr;
1175
1176 const APInt *Acc = nullptr;
1177 if (!match(II.getArgOperand(AccIdx), m_APInt(Acc)))
1178 return nullptr;
1179
1180 auto *AccumUser = dyn_cast<BinaryOperator>(II.user_back());
1181 if (!AccumUser)
1182 return nullptr;
1183
1184 Value *AccumDelta = nullptr;
1185 if (!match(AccumUser, m_c_Add(m_Specific(&II), m_Value(AccumDelta))))
1186 return nullptr;
1187
1188 Value *NewAcc = nullptr;
1189 const APInt *AccumDeltaC = nullptr;
1190 if (Acc->isZero())
1191 NewAcc = AccumDelta;
1192 else if (match(AccumDelta, m_APInt(AccumDeltaC)))
1193 NewAcc = ConstantInt::get(II.getType(), *Acc + *AccumDeltaC);
1194 else
1195 return nullptr;
1196
1197 Instruction *NewII =
1198 IC.InsertNewInstWith(II.clone(), AccumUser->getIterator());
1199 NewII->setOperand(AccIdx, NewAcc);
1202 IC.replaceInstUsesWith(*AccumUser, NewII);
1203 IC.eraseInstFromFunction(*AccumUser);
1204 return &II;
1205}
1206
1207std::optional<Instruction *>
1209 Intrinsic::ID IID = II.getIntrinsicID();
1210 switch (IID) {
1211 case Intrinsic::amdgcn_implicitarg_ptr: {
1212 if (II.getFunction()->hasFnAttribute("amdgpu-no-implicitarg-ptr"))
1213 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
1214 uint64_t ImplicitArgBytes = ST->getImplicitArgNumBytes(*II.getFunction());
1215
1216 uint64_t CurrentOrNullBytes =
1217 II.getAttributes().getRetDereferenceableOrNullBytes();
1218 if (CurrentOrNullBytes != 0) {
1219 // Refine "dereferenceable (A) meets dereferenceable_or_null(B)"
1220 // into dereferenceable(max(A, B))
1221 uint64_t NewBytes = std::max(CurrentOrNullBytes, ImplicitArgBytes);
1222 II.addRetAttr(
1223 Attribute::getWithDereferenceableBytes(II.getContext(), NewBytes));
1224 II.removeRetAttr(Attribute::DereferenceableOrNull);
1225 return &II;
1226 }
1227
1228 uint64_t CurrentBytes = II.getAttributes().getRetDereferenceableBytes();
1229 uint64_t NewBytes = std::max(CurrentBytes, ImplicitArgBytes);
1230 if (NewBytes != CurrentBytes) {
1231 II.addRetAttr(
1232 Attribute::getWithDereferenceableBytes(II.getContext(), NewBytes));
1233 return &II;
1234 }
1235
1236 return std::nullopt;
1237 }
1238 case Intrinsic::amdgcn_rcp: {
1239 Value *Src = II.getArgOperand(0);
1240 if (isa<PoisonValue>(Src))
1241 return IC.replaceInstUsesWith(II, Src);
1242
1243 // TODO: Move to ConstantFolding/InstSimplify?
1244 if (isa<UndefValue>(Src)) {
1245 Type *Ty = II.getType();
1246 auto *QNaN = ConstantFP::get(Ty, APFloat::getQNaN(Ty->getFltSemantics()));
1247 return IC.replaceInstUsesWith(II, QNaN);
1248 }
1249
1250 if (II.isStrictFP())
1251 break;
1252
1253 if (const ConstantFP *C = dyn_cast<ConstantFP>(Src)) {
1254 std::optional<APFloat> Val = AMDGPU::evaluateRcp(C->getValueAPF());
1255 if (!Val)
1256 break;
1257
1258 return IC.replaceInstUsesWith(II, ConstantFP::get(II.getContext(), *Val));
1259 }
1260
1261 FastMathFlags FMF = cast<FPMathOperator>(II).getFastMathFlags();
1262 if (!FMF.allowContract())
1263 break;
1264 auto *SrcCI = dyn_cast<IntrinsicInst>(Src);
1265 if (!SrcCI)
1266 break;
1267
1268 auto IID = SrcCI->getIntrinsicID();
1269 // llvm.amdgcn.rcp(llvm.amdgcn.sqrt(x)) -> llvm.amdgcn.rsq(x) if contractable
1270 //
1271 // llvm.amdgcn.rcp(llvm.sqrt(x)) -> llvm.amdgcn.rsq(x) if contractable and
1272 // relaxed.
1273 if (IID == Intrinsic::amdgcn_sqrt || IID == Intrinsic::sqrt) {
1274 const FPMathOperator *SqrtOp = cast<FPMathOperator>(SrcCI);
1275 FastMathFlags InnerFMF = SqrtOp->getFastMathFlags();
1276 if (!InnerFMF.allowContract() || !SrcCI->hasOneUse())
1277 break;
1278
1279 if (IID == Intrinsic::sqrt && !canContractSqrtToRsq(SqrtOp))
1280 break;
1281
1283 SrcCI->getModule(), Intrinsic::amdgcn_rsq, {SrcCI->getType()});
1284
1285 InnerFMF |= FMF;
1286 II.setFastMathFlags(InnerFMF);
1287
1288 II.setCalledFunction(NewDecl);
1289 return IC.replaceOperand(II, 0, SrcCI->getArgOperand(0));
1290 }
1291
1292 break;
1293 }
1294 case Intrinsic::amdgcn_sqrt:
1295 case Intrinsic::amdgcn_rsq:
1296 case Intrinsic::amdgcn_tanh: {
1297 Value *Src = II.getArgOperand(0);
1298 if (isa<PoisonValue>(Src))
1299 return IC.replaceInstUsesWith(II, Src);
1300
1301 // TODO: Move to ConstantFolding/InstSimplify?
1302 if (isa<UndefValue>(Src)) {
1303 Type *Ty = II.getType();
1304 auto *QNaN = ConstantFP::get(Ty, APFloat::getQNaN(Ty->getFltSemantics()));
1305 return IC.replaceInstUsesWith(II, QNaN);
1306 }
1307
1308 // f16 amdgcn.sqrt is identical to regular sqrt.
1309 if (IID == Intrinsic::amdgcn_sqrt && Src->getType()->isHalfTy()) {
1311 II.getModule(), Intrinsic::sqrt, {II.getType()});
1312 II.setCalledFunction(NewDecl);
1313 return &II;
1314 }
1315
1316 break;
1317 }
1318 case Intrinsic::amdgcn_log:
1319 case Intrinsic::amdgcn_exp2: {
1320 const bool IsLog = IID == Intrinsic::amdgcn_log;
1321 const bool IsExp = IID == Intrinsic::amdgcn_exp2;
1322 Value *Src = II.getArgOperand(0);
1323 Type *Ty = II.getType();
1324
1325 if (isa<PoisonValue>(Src))
1326 return IC.replaceInstUsesWith(II, Src);
1327
1328 if (IC.getSimplifyQuery().isUndefValue(Src))
1330
1331 if (ConstantFP *C = dyn_cast<ConstantFP>(Src)) {
1332 if (C->isInfinity()) {
1333 // exp2(+inf) -> +inf
1334 // log2(+inf) -> +inf
1335 if (!C->isNegative())
1336 return IC.replaceInstUsesWith(II, C);
1337
1338 // exp2(-inf) -> 0
1339 if (IsExp && C->isNegative())
1341 }
1342
1343 if (II.isStrictFP())
1344 break;
1345
1346 if (C->isNaN()) {
1347 Constant *Quieted = ConstantFP::get(Ty, C->getValue().makeQuiet());
1348 return IC.replaceInstUsesWith(II, Quieted);
1349 }
1350
1351 // f32 instruction doesn't handle denormals, f16 does.
1352 if (C->isZero() || (C->getValue().isDenormal() && Ty->isFloatTy())) {
1353 Constant *FoldedValue = IsLog ? ConstantFP::getInfinity(Ty, true)
1354 : ConstantFP::get(Ty, 1.0);
1355 return IC.replaceInstUsesWith(II, FoldedValue);
1356 }
1357
1358 if (IsLog && C->isNegative())
1360
1361 // TODO: Full constant folding matching hardware behavior.
1362 }
1363
1364 break;
1365 }
1366 case Intrinsic::amdgcn_frexp_mant:
1367 case Intrinsic::amdgcn_frexp_exp: {
1368 Value *Src = II.getArgOperand(0);
1369 if (const ConstantFP *C = dyn_cast<ConstantFP>(Src)) {
1370 int Exp;
1371 APFloat Significand =
1372 frexp(C->getValueAPF(), Exp, APFloat::rmNearestTiesToEven);
1373
1374 if (IID == Intrinsic::amdgcn_frexp_mant) {
1375 return IC.replaceInstUsesWith(
1376 II, ConstantFP::get(II.getContext(), Significand));
1377 }
1378
1379 // Match instruction special case behavior.
1380 if (Exp == APFloat::IEK_NaN || Exp == APFloat::IEK_Inf)
1381 Exp = 0;
1382
1383 return IC.replaceInstUsesWith(II,
1384 ConstantInt::getSigned(II.getType(), Exp));
1385 }
1386
1387 if (isa<PoisonValue>(Src))
1388 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
1389
1390 if (isa<UndefValue>(Src)) {
1391 return IC.replaceInstUsesWith(II, UndefValue::get(II.getType()));
1392 }
1393
1394 break;
1395 }
1396 case Intrinsic::amdgcn_class: {
1397 Value *Src0 = II.getArgOperand(0);
1398 Value *Src1 = II.getArgOperand(1);
1399 const ConstantInt *CMask = dyn_cast<ConstantInt>(Src1);
1400 if (CMask) {
1401 II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
1402 II.getModule(), Intrinsic::is_fpclass, Src0->getType()));
1403
1404 // Clamp any excess bits, as they're illegal for the generic intrinsic.
1405 II.setArgOperand(1, ConstantInt::get(Src1->getType(),
1406 CMask->getZExtValue() & fcAllFlags));
1407 return &II;
1408 }
1409
1410 // Propagate poison.
1411 if (isa<PoisonValue>(Src0) || isa<PoisonValue>(Src1))
1412 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
1413
1414 // llvm.amdgcn.class(_, undef) -> false
1415 if (IC.getSimplifyQuery().isUndefValue(Src1))
1416 return IC.replaceInstUsesWith(II, ConstantInt::get(II.getType(), false));
1417
1418 // llvm.amdgcn.class(undef, mask) -> mask != 0
1419 if (IC.getSimplifyQuery().isUndefValue(Src0)) {
1420 Value *CmpMask = IC.Builder.CreateICmpNE(
1421 Src1, ConstantInt::getNullValue(Src1->getType()));
1422 return IC.replaceInstUsesWith(II, CmpMask);
1423 }
1424 break;
1425 }
1426 case Intrinsic::amdgcn_cvt_pkrtz: {
1427 auto foldFPTruncToF16RTZ = [](Value *Arg) -> Value * {
1428 Type *HalfTy = Type::getHalfTy(Arg->getContext());
1429
1430 if (isa<PoisonValue>(Arg))
1431 return PoisonValue::get(HalfTy);
1432 if (isa<UndefValue>(Arg))
1433 return UndefValue::get(HalfTy);
1434
1435 ConstantFP *CFP = nullptr;
1436 if (match(Arg, m_ConstantFP(CFP))) {
1437 bool LosesInfo;
1438 APFloat Val(CFP->getValueAPF());
1440 return ConstantFP::get(HalfTy, Val);
1441 }
1442
1443 Value *Src = nullptr;
1444 if (match(Arg, m_FPExt(m_Value(Src)))) {
1445 if (Src->getType()->isHalfTy())
1446 return Src;
1447 }
1448
1449 return nullptr;
1450 };
1451
1452 if (Value *Src0 = foldFPTruncToF16RTZ(II.getArgOperand(0))) {
1453 if (Value *Src1 = foldFPTruncToF16RTZ(II.getArgOperand(1))) {
1454 Value *V = PoisonValue::get(II.getType());
1455 V = IC.Builder.CreateInsertElement(V, Src0, (uint64_t)0);
1456 V = IC.Builder.CreateInsertElement(V, Src1, (uint64_t)1);
1457 return IC.replaceInstUsesWith(II, V);
1458 }
1459 }
1460
1461 break;
1462 }
1463 case Intrinsic::amdgcn_cvt_pknorm_i16:
1464 case Intrinsic::amdgcn_cvt_pknorm_u16:
1465 case Intrinsic::amdgcn_cvt_pk_i16:
1466 case Intrinsic::amdgcn_cvt_pk_u16: {
1467 Value *Src0 = II.getArgOperand(0);
1468 Value *Src1 = II.getArgOperand(1);
1469
1470 // TODO: Replace call with scalar operation if only one element is poison.
1471 if (isa<PoisonValue>(Src0) && isa<PoisonValue>(Src1))
1472 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
1473
1474 if (isa<UndefValue>(Src0) && isa<UndefValue>(Src1)) {
1475 return IC.replaceInstUsesWith(II, UndefValue::get(II.getType()));
1476 }
1477
1478 break;
1479 }
1480 case Intrinsic::amdgcn_cvt_off_f32_i4: {
1481 Value* Arg = II.getArgOperand(0);
1482 Type *Ty = II.getType();
1483
1484 if (isa<PoisonValue>(Arg))
1485 return IC.replaceInstUsesWith(II, PoisonValue::get(Ty));
1486
1487 if(IC.getSimplifyQuery().isUndefValue(Arg))
1489
1490 ConstantInt *CArg = dyn_cast<ConstantInt>(II.getArgOperand(0));
1491 if (!CArg)
1492 break;
1493
1494 // Tabulated 0.0625 * (sext (CArg & 0xf)).
1495 constexpr size_t ResValsSize = 16;
1496 static constexpr float ResVals[ResValsSize] = {
1497 0.0, 0.0625, 0.125, 0.1875, 0.25, 0.3125, 0.375, 0.4375,
1498 -0.5, -0.4375, -0.375, -0.3125, -0.25, -0.1875, -0.125, -0.0625};
1499 Constant *Res =
1500 ConstantFP::get(Ty, ResVals[CArg->getZExtValue() & (ResValsSize - 1)]);
1501 return IC.replaceInstUsesWith(II, Res);
1502 }
1503 case Intrinsic::amdgcn_ubfe:
1504 case Intrinsic::amdgcn_sbfe: {
1505 // Decompose simple cases into standard shifts.
1506 Value *Src = II.getArgOperand(0);
1507 if (isa<UndefValue>(Src)) {
1508 return IC.replaceInstUsesWith(II, Src);
1509 }
1510
1511 uint64_t Width;
1512 Type *Ty = II.getType();
1513 unsigned IntSize = Ty->getIntegerBitWidth();
1514
1515 ConstantInt *CWidth = dyn_cast<ConstantInt>(II.getArgOperand(2));
1516 if (CWidth) {
1517 Width = CWidth->getZExtValue();
1518 if ((Width & (IntSize - 1)) == 0) {
1520 }
1521
1522 // Hardware ignores high bits, so remove those.
1523 if (Width >= IntSize) {
1524 return IC.replaceOperand(
1525 II, 2, ConstantInt::get(CWidth->getType(), Width & (IntSize - 1)));
1526 }
1527 }
1528
1529 uint64_t Offset;
1530 ConstantInt *COffset = dyn_cast<ConstantInt>(II.getArgOperand(1));
1531 if (COffset) {
1532 Offset = COffset->getZExtValue();
1533 if (Offset >= IntSize) {
1534 return IC.replaceOperand(
1535 II, 1,
1536 ConstantInt::get(COffset->getType(), Offset & (IntSize - 1)));
1537 }
1538 }
1539
1540 bool Signed = IID == Intrinsic::amdgcn_sbfe;
1541
1542 if (!CWidth || !COffset)
1543 break;
1544
1545 // The case of Width == 0 is handled above, which makes this transformation
1546 // safe. If Width == 0, then the ashr and lshr instructions become poison
1547 // value since the shift amount would be equal to the bit size.
1548 assert(Width != 0);
1549
1550 // TODO: This allows folding to undef when the hardware has specific
1551 // behavior?
1552 if (Offset + Width < IntSize) {
1553 Value *Shl = IC.Builder.CreateShl(Src, IntSize - Offset - Width);
1554 Value *RightShift = Signed ? IC.Builder.CreateAShr(Shl, IntSize - Width)
1555 : IC.Builder.CreateLShr(Shl, IntSize - Width);
1556 RightShift->takeName(&II);
1557 return IC.replaceInstUsesWith(II, RightShift);
1558 }
1559
1560 Value *RightShift = Signed ? IC.Builder.CreateAShr(Src, Offset)
1561 : IC.Builder.CreateLShr(Src, Offset);
1562
1563 RightShift->takeName(&II);
1564 return IC.replaceInstUsesWith(II, RightShift);
1565 }
1566 case Intrinsic::amdgcn_exp:
1567 case Intrinsic::amdgcn_exp_row:
1568 case Intrinsic::amdgcn_exp_compr: {
1569 ConstantInt *En = cast<ConstantInt>(II.getArgOperand(1));
1570 uint64_t EnBits = En->getZExtValue();
1571 if (EnBits == 0xf)
1572 break; // All inputs enabled.
1573
1574 bool IsCompr = IID == Intrinsic::amdgcn_exp_compr;
1575 bool Changed = false;
1576 for (int I = 0; I < (IsCompr ? 2 : 4); ++I) {
1577 if ((!IsCompr && (EnBits & (1 << I)) == 0) ||
1578 (IsCompr && ((EnBits & (0x3 << (2 * I))) == 0))) {
1579 Value *Src = II.getArgOperand(I + 2);
1580 if (!isa<PoisonValue>(Src)) {
1581 IC.replaceOperand(II, I + 2, PoisonValue::get(Src->getType()));
1582 Changed = true;
1583 }
1584 }
1585 }
1586
1587 if (Changed) {
1588 return &II;
1589 }
1590
1591 break;
1592 }
1593 case Intrinsic::amdgcn_fmed3: {
1594 Value *Src0 = II.getArgOperand(0);
1595 Value *Src1 = II.getArgOperand(1);
1596 Value *Src2 = II.getArgOperand(2);
1597
1598 for (Value *Src : {Src0, Src1, Src2}) {
1599 if (isa<PoisonValue>(Src))
1600 return IC.replaceInstUsesWith(II, Src);
1601 }
1602
1603 if (II.isStrictFP())
1604 break;
1605
1606 // med3 with a nan input acts like
1607 // v_min_f32(v_min_f32(s0, s1), s2)
1608 //
1609 // Signalingness is ignored with ieee=0, so we fold to
1610 // minimumnum/maximumnum. With ieee=1, the v_min_f32 acts like llvm.minnum
1611 // with signaling nan handling. With ieee=0, like llvm.minimumnum except a
1612 // returned signaling nan will not be quieted.
1613
1614 // ieee=1
1615 // s0 snan: s2
1616 // s1 snan: s2
1617 // s2 snan: qnan
1618
1619 // s0 qnan: min(s1, s2)
1620 // s1 qnan: min(s0, s2)
1621 // s2 qnan: min(s0, s1)
1622
1623 // ieee=0
1624 // s0 _nan: min(s1, s2)
1625 // s1 _nan: min(s0, s2)
1626 // s2 _nan: min(s0, s1)
1627
1628 // med3 behavior with infinity
1629 // s0 +inf: max(s1, s2)
1630 // s1 +inf: max(s0, s2)
1631 // s2 +inf: max(s0, s1)
1632 // s0 -inf: min(s1, s2)
1633 // s1 -inf: min(s0, s2)
1634 // s2 -inf: min(s0, s1)
1635
1636 // Checking for NaN before canonicalization provides better fidelity when
1637 // mapping other operations onto fmed3 since the order of operands is
1638 // unchanged.
1639 Value *V = nullptr;
1640 const APFloat *ConstSrc0 = nullptr;
1641 const APFloat *ConstSrc1 = nullptr;
1642 const APFloat *ConstSrc2 = nullptr;
1643
1644 if ((match(Src0, m_APFloat(ConstSrc0)) &&
1645 (ConstSrc0->isNaN() || ConstSrc0->isInfinity())) ||
1646 isa<UndefValue>(Src0)) {
1647 const bool IsPosInfinity = ConstSrc0 && ConstSrc0->isPosInfinity();
1648 switch (fpenvIEEEMode(II)) {
1649 case KnownIEEEMode::On:
1650 // TODO: If Src2 is snan, does it need quieting?
1651 if (ConstSrc0 && ConstSrc0->isNaN() && ConstSrc0->isSignaling())
1652 return IC.replaceInstUsesWith(II, Src2);
1653
1654 V = IsPosInfinity ? IC.Builder.CreateMaxNum(Src1, Src2)
1655 : IC.Builder.CreateMinNum(Src1, Src2);
1656 break;
1657 case KnownIEEEMode::Off:
1658 V = IsPosInfinity ? IC.Builder.CreateMaximumNum(Src1, Src2)
1659 : IC.Builder.CreateMinimumNum(Src1, Src2);
1660 break;
1662 break;
1663 }
1664 } else if ((match(Src1, m_APFloat(ConstSrc1)) &&
1665 (ConstSrc1->isNaN() || ConstSrc1->isInfinity())) ||
1666 isa<UndefValue>(Src1)) {
1667 const bool IsPosInfinity = ConstSrc1 && ConstSrc1->isPosInfinity();
1668 switch (fpenvIEEEMode(II)) {
1669 case KnownIEEEMode::On:
1670 // TODO: If Src2 is snan, does it need quieting?
1671 if (ConstSrc1 && ConstSrc1->isNaN() && ConstSrc1->isSignaling())
1672 return IC.replaceInstUsesWith(II, Src2);
1673
1674 V = IsPosInfinity ? IC.Builder.CreateMaxNum(Src0, Src2)
1675 : IC.Builder.CreateMinNum(Src0, Src2);
1676 break;
1677 case KnownIEEEMode::Off:
1678 V = IsPosInfinity ? IC.Builder.CreateMaximumNum(Src0, Src2)
1679 : IC.Builder.CreateMinimumNum(Src0, Src2);
1680 break;
1682 break;
1683 }
1684 } else if ((match(Src2, m_APFloat(ConstSrc2)) &&
1685 (ConstSrc2->isNaN() || ConstSrc2->isInfinity())) ||
1686 isa<UndefValue>(Src2)) {
1687 switch (fpenvIEEEMode(II)) {
1688 case KnownIEEEMode::On:
1689 if (ConstSrc2 && ConstSrc2->isNaN() && ConstSrc2->isSignaling()) {
1690 auto *Quieted = ConstantFP::get(II.getType(), ConstSrc2->makeQuiet());
1691 return IC.replaceInstUsesWith(II, Quieted);
1692 }
1693
1694 V = (ConstSrc2 && ConstSrc2->isPosInfinity())
1695 ? IC.Builder.CreateMaxNum(Src0, Src1)
1696 : IC.Builder.CreateMinNum(Src0, Src1);
1697 break;
1698 case KnownIEEEMode::Off:
1699 V = (ConstSrc2 && ConstSrc2->isPosInfinity())
1700 ? IC.Builder.CreateMaximumNum(Src0, Src1)
1701 : IC.Builder.CreateMinimumNum(Src0, Src1);
1702 break;
1704 break;
1705 }
1706 }
1707
1708 if (V) {
1709 if (auto *CI = dyn_cast<CallInst>(V)) {
1710 CI->copyFastMathFlags(&II);
1711 CI->takeName(&II);
1712 }
1713 return IC.replaceInstUsesWith(II, V);
1714 }
1715
1716 bool Swap = false;
1717 // Canonicalize constants to RHS operands.
1718 //
1719 // fmed3(c0, x, c1) -> fmed3(x, c0, c1)
1720 if (isa<Constant>(Src0) && !isa<Constant>(Src1)) {
1721 std::swap(Src0, Src1);
1722 Swap = true;
1723 }
1724
1725 if (isa<Constant>(Src1) && !isa<Constant>(Src2)) {
1726 std::swap(Src1, Src2);
1727 Swap = true;
1728 }
1729
1730 if (isa<Constant>(Src0) && !isa<Constant>(Src1)) {
1731 std::swap(Src0, Src1);
1732 Swap = true;
1733 }
1734
1735 if (Swap) {
1736 II.setArgOperand(0, Src0);
1737 II.setArgOperand(1, Src1);
1738 II.setArgOperand(2, Src2);
1739 return &II;
1740 }
1741
1742 if (const ConstantFP *C0 = dyn_cast<ConstantFP>(Src0)) {
1743 if (const ConstantFP *C1 = dyn_cast<ConstantFP>(Src1)) {
1744 if (const ConstantFP *C2 = dyn_cast<ConstantFP>(Src2)) {
1745 APFloat Result = fmed3AMDGCN(C0->getValueAPF(), C1->getValueAPF(),
1746 C2->getValueAPF());
1747 return IC.replaceInstUsesWith(II,
1748 ConstantFP::get(II.getType(), Result));
1749 }
1750 }
1751 }
1752
1753 if (!ST->hasMed3_16())
1754 break;
1755
1756 // Repeat floating-point width reduction done for minnum/maxnum.
1757 // fmed3((fpext X), (fpext Y), (fpext Z)) -> fpext (fmed3(X, Y, Z))
1758 if (Value *X = matchFPExtFromF16(Src0)) {
1759 if (Value *Y = matchFPExtFromF16(Src1)) {
1760 if (Value *Z = matchFPExtFromF16(Src2)) {
1761 Value *NewCall = IC.Builder.CreateIntrinsic(
1762 IID, {X->getType()}, {X, Y, Z}, &II, II.getName());
1763 return new FPExtInst(NewCall, II.getType());
1764 }
1765 }
1766 }
1767
1768 break;
1769 }
1770 case Intrinsic::amdgcn_mbcnt_hi:
1771 // exec_hi is all 0, so this is just a copy.
1772 if (ST->isWave32())
1773 return IC.replaceInstUsesWith(II, II.getArgOperand(1));
1774 [[fallthrough]];
1775 case Intrinsic::amdgcn_mbcnt_lo: {
1776 ConstantRange AccRange =
1777 computeConstantRange(II.getArgOperand(1),
1778 /*ForSigned=*/false, IC.getSimplifyQuery());
1779 if (AccRange.isFullSet())
1780 return nullptr;
1781
1782 // TODO: Can raise lower bound by inspecting first argument.
1783 // Wave64 mbcnt_lo adds at most 32. Otherwise these add at most 31.
1784 unsigned MaxCount =
1785 IID == Intrinsic::amdgcn_mbcnt_lo && ST->isWave64() ? 32 : 31;
1786 ConstantRange MbcntRange(APInt(32, 0), APInt(32, MaxCount + 1));
1787 ConstantRange ComputedRange = AccRange.add(MbcntRange);
1788 if (ComputedRange.isFullSet())
1789 return nullptr;
1790
1791 if (std::optional<ConstantRange> ExistingRange = II.getRange()) {
1792 ComputedRange = ComputedRange.intersectWith(*ExistingRange);
1793 if (ComputedRange == *ExistingRange)
1794 return nullptr;
1795 }
1796
1797 II.addRangeRetAttr(ComputedRange);
1798 return nullptr;
1799 }
1800 case Intrinsic::amdgcn_ballot: {
1801 Value *Arg = II.getArgOperand(0);
1802 if (isa<PoisonValue>(Arg))
1803 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
1804
1805 if (auto *Src = dyn_cast<ConstantInt>(Arg)) {
1806 if (Src->isZero()) {
1807 // amdgcn.ballot(i1 0) is zero.
1808 return IC.replaceInstUsesWith(II, Constant::getNullValue(II.getType()));
1809 }
1810 }
1811 if (ST->isWave32() && II.getType()->getIntegerBitWidth() == 64) {
1812 // %b64 = call i64 ballot.i64(...)
1813 // =>
1814 // %b32 = call i32 ballot.i32(...)
1815 // %b64 = zext i32 %b32 to i64
1817 IC.Builder.CreateIntrinsic(Intrinsic::amdgcn_ballot,
1818 {IC.Builder.getInt32Ty()},
1819 {II.getArgOperand(0)}),
1820 II.getType());
1821 Call->takeName(&II);
1822 return IC.replaceInstUsesWith(II, Call);
1823 }
1824 break;
1825 }
1826 case Intrinsic::amdgcn_wavefrontsize: {
1827 if (ST->isWaveSizeKnown())
1828 return IC.replaceInstUsesWith(
1829 II, ConstantInt::get(II.getType(), ST->getWavefrontSize()));
1830 break;
1831 }
1832 case Intrinsic::amdgcn_wqm_vote: {
1833 // wqm_vote is identity when the argument is constant.
1834 if (!isa<Constant>(II.getArgOperand(0)))
1835 break;
1836
1837 return IC.replaceInstUsesWith(II, II.getArgOperand(0));
1838 }
1839 case Intrinsic::amdgcn_kill: {
1840 const ConstantInt *C = dyn_cast<ConstantInt>(II.getArgOperand(0));
1841 if (!C || !C->getZExtValue())
1842 break;
1843
1844 // amdgcn.kill(i1 1) is a no-op
1845 return IC.eraseInstFromFunction(II);
1846 }
1847 case Intrinsic::amdgcn_s_sendmsg:
1848 case Intrinsic::amdgcn_s_sendmsghalt: {
1849 // The second operand is copied to m0, but is only actually used for
1850 // certain message types. For message types that are known to not use m0,
1851 // fold it to poison.
1852 using namespace AMDGPU::SendMsg;
1853
1854 Value *M0Val = II.getArgOperand(1);
1855 if (isa<PoisonValue>(M0Val))
1856 break;
1857
1858 auto *MsgImm = cast<ConstantInt>(II.getArgOperand(0));
1859 uint16_t MsgId, OpId, StreamId;
1860 decodeMsg(static_cast<unsigned>(MsgImm->getZExtValue()), MsgId, OpId,
1861 StreamId, *ST);
1862
1863 if (!msgDoesNotUseM0(MsgId, *ST))
1864 break;
1865
1866 // Drop UB-implying attributes since we're replacing with poison.
1867 II.dropUBImplyingAttrsAndMetadata();
1868 IC.replaceOperand(II, 1, PoisonValue::get(M0Val->getType()));
1869 return nullptr;
1870 }
1871 case Intrinsic::amdgcn_update_dpp: {
1872 Value *Old = II.getArgOperand(0);
1873
1874 auto *BC = cast<ConstantInt>(II.getArgOperand(5));
1875 auto *RM = cast<ConstantInt>(II.getArgOperand(3));
1876 auto *BM = cast<ConstantInt>(II.getArgOperand(4));
1877 if (BC->isNullValue() || RM->getZExtValue() != 0xF ||
1878 BM->getZExtValue() != 0xF || isa<PoisonValue>(Old))
1879 break;
1880
1881 // If bound_ctrl = 1, row mask = bank mask = 0xf we can omit old value.
1882 return IC.replaceOperand(II, 0, PoisonValue::get(Old->getType()));
1883 }
1884 case Intrinsic::amdgcn_permlane16:
1885 case Intrinsic::amdgcn_permlane16_var:
1886 case Intrinsic::amdgcn_permlanex16:
1887 case Intrinsic::amdgcn_permlanex16_var: {
1888 // Discard vdst_in if it's not going to be read.
1889 Value *VDstIn = II.getArgOperand(0);
1890 if (isa<PoisonValue>(VDstIn))
1891 break;
1892
1893 // FetchInvalid operand idx.
1894 unsigned int FiIdx = (IID == Intrinsic::amdgcn_permlane16 ||
1895 IID == Intrinsic::amdgcn_permlanex16)
1896 ? 4 /* for permlane16 and permlanex16 */
1897 : 3; /* for permlane16_var and permlanex16_var */
1898
1899 // BoundCtrl operand idx.
1900 // For permlane16 and permlanex16 it should be 5
1901 // For Permlane16_var and permlanex16_var it should be 4
1902 unsigned int BcIdx = FiIdx + 1;
1903
1904 ConstantInt *FetchInvalid = cast<ConstantInt>(II.getArgOperand(FiIdx));
1905 ConstantInt *BoundCtrl = cast<ConstantInt>(II.getArgOperand(BcIdx));
1906 if (!FetchInvalid->getZExtValue() && !BoundCtrl->getZExtValue())
1907 break;
1908
1909 return IC.replaceOperand(II, 0, PoisonValue::get(VDstIn->getType()));
1910 }
1911 case Intrinsic::amdgcn_wave_shuffle:
1912 return tryOptimizeShufflePattern(IC, II, *ST);
1913 case Intrinsic::amdgcn_permlane64:
1914 case Intrinsic::amdgcn_readfirstlane:
1915 case Intrinsic::amdgcn_readlane:
1916 case Intrinsic::amdgcn_ds_bpermute: {
1917 // If the data argument is uniform these intrinsics return it unchanged.
1918 unsigned SrcIdx = IID == Intrinsic::amdgcn_ds_bpermute ? 1 : 0;
1919 const Use &Src = II.getArgOperandUse(SrcIdx);
1921 return IC.replaceInstUsesWith(II, Src.get());
1922
1923 if (IID == Intrinsic::amdgcn_readlane &&
1925 return &II;
1926
1927 // If the lane argument of bpermute is uniform, change it to readlane. This
1928 // generates better code and can enable further optimizations because
1929 // readlane is AlwaysUniform.
1930 if (IID == Intrinsic::amdgcn_ds_bpermute) {
1931 const Use &Lane = II.getArgOperandUse(0);
1932 if (AMDGPU::isTriviallyUniform(Lane)) {
1933 Value *NewLane = IC.Builder.CreateLShr(Lane, 2);
1935 II.getModule(), Intrinsic::amdgcn_readlane, II.getType());
1936 II.setCalledFunction(NewDecl);
1937 II.setOperand(0, Src);
1938 II.setOperand(1, NewLane);
1939 return &II;
1940 }
1941 }
1942
1943 if (IID == Intrinsic::amdgcn_ds_bpermute)
1944 return tryOptimizeShufflePattern(IC, II, *ST);
1945
1947 return Res;
1948
1949 return std::nullopt;
1950 }
1951 case Intrinsic::amdgcn_wave_match_b32: {
1952 const Use &Src0 = II.getArgOperandUse(0);
1953 const Use &Src1 = II.getArgOperandUse(1);
1954 if (Src0.get() == Src1.get() && AMDGPU::isTriviallyUniform(Src0)) {
1956 II.getModule(), Intrinsic::amdgcn_ballot, II.getType());
1957 CallInst *NewCall =
1958 IC.Builder.CreateCall(NewF, {IC.Builder.getInt1(true)});
1959 return IC.replaceInstUsesWith(II, NewCall);
1960 }
1961 break;
1962 }
1963 case Intrinsic::amdgcn_writelane: {
1964 // TODO: Fold bitcast like readlane.
1965 if (simplifyDemandedLaneMaskArg(IC, II, 1))
1966 return &II;
1967 return std::nullopt;
1968 }
1969 case Intrinsic::amdgcn_trig_preop: {
1970 // The intrinsic is declared with name mangling, but currently the
1971 // instruction only exists for f64
1972 if (!II.getType()->isDoubleTy())
1973 break;
1974
1975 Value *Src = II.getArgOperand(0);
1976 Value *Segment = II.getArgOperand(1);
1977 if (isa<PoisonValue>(Src) || isa<PoisonValue>(Segment))
1978 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
1979
1980 if (isa<UndefValue>(Segment))
1981 return IC.replaceInstUsesWith(II, ConstantFP::getZero(II.getType()));
1982
1983 // Sign bit is not used.
1984 Value *StrippedSign = InstCombiner::stripSignOnlyFPOps(Src);
1985 if (StrippedSign != Src)
1986 return IC.replaceOperand(II, 0, StrippedSign);
1987
1988 if (II.isStrictFP())
1989 break;
1990
1991 const ConstantFP *CSrc = dyn_cast<ConstantFP>(Src);
1992 if (!CSrc && !isa<UndefValue>(Src))
1993 break;
1994
1995 // The instruction ignores special cases, and literally just extracts the
1996 // exponents. Fold undef to nan, and index the table as normal.
1997 APInt FSrcInt = CSrc ? CSrc->getValueAPF().bitcastToAPInt()
1998 : APFloat::getQNaN(II.getType()->getFltSemantics())
1999 .bitcastToAPInt();
2000
2001 const ConstantInt *Cseg = dyn_cast<ConstantInt>(Segment);
2002 if (!Cseg) {
2003 if (isa<UndefValue>(Src))
2004 return IC.replaceInstUsesWith(II, ConstantFP::getZero(II.getType()));
2005 break;
2006 }
2007
2008 unsigned Exponent =
2009 static_cast<unsigned>(FSrcInt.extractBitsAsZExtValue(11, 52));
2010 unsigned SegmentVal =
2011 static_cast<unsigned>(Cseg->getValue().trunc(5).getZExtValue());
2012 unsigned Shift = SegmentVal * 53;
2013 if (Exponent > 1077)
2014 Shift += Exponent - 1077;
2015
2016 // 2.0/PI table.
2017 static const uint32_t TwoByPi[] = {
2018 0xa2f9836e, 0x4e441529, 0xfc2757d1, 0xf534ddc0, 0xdb629599, 0x3c439041,
2019 0xfe5163ab, 0xdebbc561, 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c,
2020 0xfe1deb1c, 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41,
2021 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, 0xde05980f,
2022 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, 0x4f463f66, 0x9e5fea2d,
2023 0x7527bac7, 0xebe5f17b, 0x3d0739f7, 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08,
2024 0x56033046};
2025
2026 // Return 0 for outbound segment (hardware behavior).
2027 unsigned Idx = Shift >> 5;
2028 if (Idx + 2 >= std::size(TwoByPi)) {
2029 APFloat Zero = APFloat::getZero(II.getType()->getFltSemantics());
2030 return IC.replaceInstUsesWith(II, ConstantFP::get(II.getType(), Zero));
2031 }
2032
2033 unsigned BShift = Shift & 0x1f;
2034 uint64_t Thi = Make_64(TwoByPi[Idx], TwoByPi[Idx + 1]);
2035 uint64_t Tlo = Make_64(TwoByPi[Idx + 2], 0);
2036 if (BShift)
2037 Thi = (Thi << BShift) | (Tlo >> (64 - BShift));
2038 Thi = Thi >> 11;
2039 APFloat Result = APFloat((double)Thi);
2040
2041 int Scale = -53 - Shift;
2042 if (Exponent >= 1968)
2043 Scale += 128;
2044
2045 Result = scalbn(Result, Scale, RoundingMode::NearestTiesToEven);
2046 return IC.replaceInstUsesWith(II, ConstantFP::get(Src->getType(), Result));
2047 }
2048 case Intrinsic::amdgcn_sdot2:
2049 case Intrinsic::amdgcn_udot2:
2050 case Intrinsic::amdgcn_sdot4:
2051 case Intrinsic::amdgcn_udot4:
2052 case Intrinsic::amdgcn_sdot8:
2053 case Intrinsic::amdgcn_udot8: {
2054 Value *Src0 = II.getArgOperand(0);
2055 Value *Src1 = II.getArgOperand(1);
2056
2057 // Canonicalize the constant multiplicand to Src1.
2058 if (isa<Constant>(Src0) && !isa<Constant>(Src1)) {
2059 II.setArgOperand(0, Src1);
2060 II.setArgOperand(1, Src0);
2061 return &II;
2062 }
2063
2064 if (match(Src1, m_Zero()))
2065 return IC.replaceInstUsesWith(II, II.getArgOperand(2));
2066
2067 if (Instruction *I = foldAddIntoDotAccumulator(II, 2, 3, IC))
2068 return I;
2069
2070 break;
2071 }
2072 case Intrinsic::amdgcn_sudot4:
2073 case Intrinsic::amdgcn_sudot8: {
2074 Value *Src0 = II.getArgOperand(1);
2075 Value *Src1 = II.getArgOperand(3);
2076
2077 // Canonicalize the constant multiplicand to Src1, moving its sign flag with
2078 // it.
2079 if (isa<Constant>(Src0) && !isa<Constant>(Src1)) {
2080 Value *Sign0 = II.getArgOperand(0);
2081 Value *Sign1 = II.getArgOperand(2);
2082 II.setArgOperand(0, Sign1);
2083 II.setArgOperand(1, Src1);
2084 II.setArgOperand(2, Sign0);
2085 II.setArgOperand(3, Src0);
2086 return &II;
2087 }
2088
2089 if (match(Src1, m_Zero()))
2090 return IC.replaceInstUsesWith(II, II.getArgOperand(4));
2091
2092 if (Instruction *I = foldAddIntoDotAccumulator(II, 4, 5, IC))
2093 return I;
2094
2095 break;
2096 }
2097 case Intrinsic::amdgcn_fmul_legacy: {
2098 Value *Op0 = II.getArgOperand(0);
2099 Value *Op1 = II.getArgOperand(1);
2100
2101 for (Value *Src : {Op0, Op1}) {
2102 if (isa<PoisonValue>(Src))
2103 return IC.replaceInstUsesWith(II, Src);
2104 }
2105
2106 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
2107 // infinity, gives +0.0.
2108 // TODO: Move to InstSimplify?
2109 if (match(Op0, PatternMatch::m_AnyZeroFP()) ||
2111 return IC.replaceInstUsesWith(II, ConstantFP::getZero(II.getType()));
2112
2113 // If we can prove we don't have one of the special cases then we can use a
2114 // normal fmul instruction instead.
2115 if (canSimplifyLegacyMulToMul(II, Op0, Op1, IC)) {
2116 auto *FMul = IC.Builder.CreateFMulFMF(Op0, Op1, &II);
2117 FMul->takeName(&II);
2118 return IC.replaceInstUsesWith(II, FMul);
2119 }
2120 break;
2121 }
2122 case Intrinsic::amdgcn_fma_legacy: {
2123 Value *Op0 = II.getArgOperand(0);
2124 Value *Op1 = II.getArgOperand(1);
2125 Value *Op2 = II.getArgOperand(2);
2126
2127 for (Value *Src : {Op0, Op1, Op2}) {
2128 if (isa<PoisonValue>(Src))
2129 return IC.replaceInstUsesWith(II, Src);
2130 }
2131
2132 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
2133 // infinity, gives +0.0.
2134 // TODO: Move to InstSimplify?
2135 if (match(Op0, PatternMatch::m_AnyZeroFP()) ||
2137 // It's tempting to just return Op2 here, but that would give the wrong
2138 // result if Op2 was -0.0.
2139 auto *Zero = ConstantFP::getZero(II.getType());
2140 auto *FAdd = IC.Builder.CreateFAddFMF(Zero, Op2, &II);
2141 FAdd->takeName(&II);
2142 return IC.replaceInstUsesWith(II, FAdd);
2143 }
2144
2145 // If we can prove we don't have one of the special cases then we can use a
2146 // normal fma instead.
2147 if (canSimplifyLegacyMulToMul(II, Op0, Op1, IC)) {
2148 II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
2149 II.getModule(), Intrinsic::fma, II.getType()));
2150 return &II;
2151 }
2152 break;
2153 }
2154 case Intrinsic::amdgcn_is_shared:
2155 case Intrinsic::amdgcn_is_private: {
2156 Value *Src = II.getArgOperand(0);
2157 if (isa<PoisonValue>(Src))
2158 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
2159 if (isa<UndefValue>(Src))
2160 return IC.replaceInstUsesWith(II, UndefValue::get(II.getType()));
2161
2162 if (isa<ConstantPointerNull>(II.getArgOperand(0)))
2163 return IC.replaceInstUsesWith(II, ConstantInt::getFalse(II.getType()));
2164 break;
2165 }
2166 case Intrinsic::amdgcn_make_buffer_rsrc: {
2167 Value *Src = II.getArgOperand(0);
2168 if (isa<PoisonValue>(Src))
2169 return IC.replaceInstUsesWith(II, PoisonValue::get(II.getType()));
2170
2171 // Normalize num_records to the correct width.
2172 std::optional<unsigned> Width = ST->getBufferResourceNumRecordsWidth();
2173 if (!Width)
2174 return std::nullopt;
2175 Type *NumRecordsTy = IC.Builder.getIntNTy(*Width);
2176 if (II.getArgOperand(2)->getType() == NumRecordsTy)
2177 return std::nullopt;
2178 SmallVector<Value *, 4> Args(II.args());
2179 Args[2] = IC.Builder.CreateZExtOrTrunc(Args[2], NumRecordsTy);
2181 Intrinsic::amdgcn_make_buffer_rsrc,
2182 {II.getType(), Src->getType(), NumRecordsTy}, Args);
2183 NewCall->copyMetadata(II);
2184 NewCall->setTailCallKind(II.getTailCallKind());
2185 // Copy over all attributes except those on num_records, which may no longer
2186 // be valid.
2187 NewCall->setAttributes(
2188 II.getAttributes().removeParamAttributes(II.getContext(), 2));
2189 NewCall->takeName(&II);
2190 return IC.replaceInstUsesWith(II, NewCall);
2191 }
2192 case Intrinsic::amdgcn_raw_buffer_store_format:
2193 case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
2194 case Intrinsic::amdgcn_struct_buffer_store_format:
2195 case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
2196 case Intrinsic::amdgcn_raw_tbuffer_store:
2197 case Intrinsic::amdgcn_raw_ptr_tbuffer_store:
2198 case Intrinsic::amdgcn_struct_tbuffer_store:
2199 case Intrinsic::amdgcn_struct_ptr_tbuffer_store:
2200 case Intrinsic::amdgcn_image_store_1d:
2201 case Intrinsic::amdgcn_image_store_1darray:
2202 case Intrinsic::amdgcn_image_store_2d:
2203 case Intrinsic::amdgcn_image_store_2darray:
2204 case Intrinsic::amdgcn_image_store_2darraymsaa:
2205 case Intrinsic::amdgcn_image_store_2dmsaa:
2206 case Intrinsic::amdgcn_image_store_3d:
2207 case Intrinsic::amdgcn_image_store_cube:
2208 case Intrinsic::amdgcn_image_store_mip_1d:
2209 case Intrinsic::amdgcn_image_store_mip_1darray:
2210 case Intrinsic::amdgcn_image_store_mip_2d:
2211 case Intrinsic::amdgcn_image_store_mip_2darray:
2212 case Intrinsic::amdgcn_image_store_mip_3d:
2213 case Intrinsic::amdgcn_image_store_mip_cube: {
2214 if (!isa<FixedVectorType>(II.getArgOperand(0)->getType()))
2215 break;
2216
2217 APInt DemandedElts;
2218 if (ST->hasDefaultComponentBroadcast())
2219 DemandedElts = defaultComponentBroadcast(II.getArgOperand(0));
2220 else if (ST->hasDefaultComponentZero())
2221 DemandedElts = trimTrailingZerosInVector(IC, II.getArgOperand(0), &II);
2222 else
2223 break;
2224
2225 int DMaskIdx = getAMDGPUImageDMaskIntrinsic(II.getIntrinsicID()) ? 1 : -1;
2226 if (simplifyAMDGCNMemoryIntrinsicDemanded(*ST, IC, II, DemandedElts,
2227 DMaskIdx, false)) {
2228 return IC.eraseInstFromFunction(II);
2229 }
2230
2231 break;
2232 }
2233 case Intrinsic::amdgcn_prng_b32: {
2234 auto *Src = II.getArgOperand(0);
2235 if (isa<UndefValue>(Src)) {
2236 return IC.replaceInstUsesWith(II, Src);
2237 }
2238 return std::nullopt;
2239 }
2240 case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
2241 case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
2242 Value *Src0 = II.getArgOperand(0);
2243 Value *Src1 = II.getArgOperand(1);
2244 uint64_t CBSZ = cast<ConstantInt>(II.getArgOperand(3))->getZExtValue();
2245 uint64_t BLGP = cast<ConstantInt>(II.getArgOperand(4))->getZExtValue();
2246 auto *Src0Ty = cast<FixedVectorType>(Src0->getType());
2247 auto *Src1Ty = cast<FixedVectorType>(Src1->getType());
2248
2249 auto getFormatNumRegs = [](uint64_t FormatVal) {
2250 switch (FormatVal) {
2253 return 6u;
2255 return 4u;
2258 return 8u;
2259 default:
2260 llvm_unreachable("invalid format value");
2261 }
2262 };
2263
2264 bool MadeChange = false;
2265 unsigned Src0NumElts = getFormatNumRegs(CBSZ);
2266 unsigned Src1NumElts = getFormatNumRegs(BLGP);
2267
2268 // Depending on the used format, fewer registers are required so shrink the
2269 // vector type.
2270 if (Src0Ty->getNumElements() > Src0NumElts) {
2271 Src0 = IC.Builder.CreateExtractVector(
2272 FixedVectorType::get(Src0Ty->getElementType(), Src0NumElts), Src0,
2273 uint64_t(0));
2274 MadeChange = true;
2275 }
2276
2277 if (Src1Ty->getNumElements() > Src1NumElts) {
2278 Src1 = IC.Builder.CreateExtractVector(
2279 FixedVectorType::get(Src1Ty->getElementType(), Src1NumElts), Src1,
2280 uint64_t(0));
2281 MadeChange = true;
2282 }
2283
2284 if (!MadeChange)
2285 return std::nullopt;
2286
2287 SmallVector<Value *, 10> Args(II.args());
2288 Args[0] = Src0;
2289 Args[1] = Src1;
2290
2291 Value *NewII = IC.Builder.CreateIntrinsic(
2292 IID, {Src0->getType(), Src1->getType()}, Args, &II);
2293 NewII->takeName(&II);
2294 return IC.replaceInstUsesWith(II, NewII);
2295 }
2296 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
2297 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
2298 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
2299 Value *Src0 = II.getArgOperand(1);
2300 Value *Src1 = II.getArgOperand(3);
2301 unsigned FmtA = cast<ConstantInt>(II.getArgOperand(0))->getZExtValue();
2302 uint64_t FmtB = cast<ConstantInt>(II.getArgOperand(2))->getZExtValue();
2303 auto *Src0Ty = cast<FixedVectorType>(Src0->getType());
2304 auto *Src1Ty = cast<FixedVectorType>(Src1->getType());
2305
2306 bool MadeChange = false;
2307 unsigned Src0NumElts = AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(FmtA);
2308 unsigned Src1NumElts = AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(FmtB);
2309
2310 // Depending on the used format, fewer registers are required so shrink the
2311 // vector type.
2312 if (Src0Ty->getNumElements() > Src0NumElts) {
2313 Src0 = IC.Builder.CreateExtractVector(
2314 FixedVectorType::get(Src0Ty->getElementType(), Src0NumElts), Src0,
2315 IC.Builder.getInt64(0));
2316 MadeChange = true;
2317 }
2318
2319 if (Src1Ty->getNumElements() > Src1NumElts) {
2320 Src1 = IC.Builder.CreateExtractVector(
2321 FixedVectorType::get(Src1Ty->getElementType(), Src1NumElts), Src1,
2322 IC.Builder.getInt64(0));
2323 MadeChange = true;
2324 }
2325
2326 if (!MadeChange)
2327 return std::nullopt;
2328
2329 SmallVector<Value *, 13> Args(II.args());
2330 Args[1] = Src0;
2331 Args[3] = Src1;
2332
2333 Value *NewII = IC.Builder.CreateIntrinsic(
2334 IID, {II.getArgOperand(5)->getType(), Src0->getType(), Src1->getType()},
2335 Args, &II);
2336 NewII->takeName(&II);
2337 return IC.replaceInstUsesWith(II, NewII);
2338 }
2339 }
2340 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
2341 AMDGPU::getImageDimIntrinsicInfo(II.getIntrinsicID())) {
2342 return simplifyAMDGCNImageIntrinsic(ST, ImageDimIntr, II, IC);
2343 }
2344 if (std::optional<unsigned> ScaleArgIdx =
2346 if (simplifyExponentOnlyScaleArg(IC, II, *ScaleArgIdx))
2347 return &II;
2348 }
2349 return std::nullopt;
2350}
2351
2352/// Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics.
2353///
2354/// The result of simplifying amdgcn image and buffer store intrinsics is updating
2355/// definitions of the intrinsics vector argument, not Uses of the result like
2356/// image and buffer loads.
2357/// Note: This only supports non-TFE/LWE image intrinsic calls; those have
2358/// struct returns.
2360 InstCombiner &IC,
2362 APInt DemandedElts,
2363 int DMaskIdx, bool IsLoad) {
2364
2365 auto *IIVTy = cast<FixedVectorType>(IsLoad ? II.getType()
2366 : II.getOperand(0)->getType());
2367 unsigned VWidth = IIVTy->getNumElements();
2368 if (VWidth == 1)
2369 return nullptr;
2370 Type *EltTy = IIVTy->getElementType();
2371
2374
2375 // Assume the arguments are unchanged and later override them, if needed.
2376 SmallVector<Value *, 16> Args(II.args());
2377 uint64_t OffsetAdd = 0;
2378
2379 if (DMaskIdx < 0) {
2380 // Buffer case.
2381
2382 const unsigned ActiveBits = DemandedElts.getActiveBits();
2383 const unsigned UnusedComponentsAtFront = DemandedElts.countr_zero();
2384
2385 // Start assuming the prefix of elements is demanded, but possibly clear
2386 // some other bits if there are trailing zeros (unused components at front)
2387 // and update offset.
2388 DemandedElts = (1 << ActiveBits) - 1;
2389
2390 if (UnusedComponentsAtFront > 0) {
2391 static const unsigned InvalidOffsetIdx = 0xf;
2392
2393 unsigned OffsetIdx;
2394 // Set for tbuffer loads, whose component size comes from the format.
2395 unsigned FormatBitsPerComp = 0;
2396 switch (II.getIntrinsicID()) {
2397 case Intrinsic::amdgcn_raw_buffer_load:
2398 case Intrinsic::amdgcn_raw_ptr_buffer_load:
2399 OffsetIdx = 1;
2400 break;
2401 case Intrinsic::amdgcn_s_buffer_load:
2402 case Intrinsic::amdgcn_ptr_s_buffer_load:
2403 // If resulting type is vec3, there is no point in trimming the
2404 // load with updated offset, as the vec3 would most likely be widened to
2405 // vec4 anyway during lowering.
2406 if (ActiveBits == 4 && UnusedComponentsAtFront == 1)
2407 OffsetIdx = InvalidOffsetIdx;
2408 else
2409 OffsetIdx = 1;
2410 break;
2411 case Intrinsic::amdgcn_struct_buffer_load:
2412 case Intrinsic::amdgcn_struct_ptr_buffer_load:
2413 OffsetIdx = 2;
2414 break;
2415 case Intrinsic::amdgcn_raw_tbuffer_load:
2416 case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
2417 case Intrinsic::amdgcn_struct_tbuffer_load:
2418 case Intrinsic::amdgcn_struct_ptr_tbuffer_load: {
2419 // The struct intrinsic variants add a vindex operand before the offset.
2420 const bool HasVIndex =
2421 II.getIntrinsicID() == Intrinsic::amdgcn_struct_tbuffer_load ||
2422 II.getIntrinsicID() == Intrinsic::amdgcn_struct_ptr_tbuffer_load;
2423 OffsetIdx = HasVIndex ? 2 : 1;
2424 const unsigned FormatIdx = OffsetIdx + 2;
2425
2426 // The format describes the whole element, so also drop the leading
2427 // components from it to keep the element ending at the same byte.
2428 // Lanes past the end of the format are filled in by the hardware
2429 // rather than loaded; leave such loads alone.
2430 const AMDGPU::GcnBufferFormatInfo *OldFormat =
2432 cast<ConstantInt>(Args[FormatIdx])->getZExtValue(), ST);
2433 const AMDGPU::GcnBufferFormatInfo *NewFormat = nullptr;
2434 if (OldFormat && ActiveBits <= OldFormat->NumComponents)
2436 OldFormat->BitsPerComp,
2437 OldFormat->NumComponents - UnusedComponentsAtFront,
2438 OldFormat->NumFormat, ST);
2439 if (!NewFormat) {
2440 OffsetIdx = InvalidOffsetIdx;
2441 break;
2442 }
2443 Args[FormatIdx] =
2444 ConstantInt::get(Args[FormatIdx]->getType(), NewFormat->Format);
2445 FormatBitsPerComp = OldFormat->BitsPerComp;
2446 break;
2447 }
2448 default:
2449 // buffer_load_format takes its format from the resource, which is
2450 // unknown here.
2451 OffsetIdx = InvalidOffsetIdx;
2452 break;
2453 }
2454
2455 if (OffsetIdx != InvalidOffsetIdx) {
2456 // Clear demanded bits and update the offset.
2457 DemandedElts &= ~((1 << UnusedComponentsAtFront) - 1);
2458 auto *Offset = Args[OffsetIdx];
2459 uint64_t SingleComponentSizeInBits =
2460 FormatBitsPerComp
2461 ? FormatBitsPerComp
2463 OffsetAdd = UnusedComponentsAtFront * SingleComponentSizeInBits / 8;
2464 auto *OffsetAddVal = ConstantInt::get(Offset->getType(), OffsetAdd);
2465 Args[OffsetIdx] = IC.Builder.CreateAdd(Offset, OffsetAddVal);
2466 }
2467 }
2468 } else {
2469 // Image case.
2470
2471 ConstantInt *DMask = cast<ConstantInt>(Args[DMaskIdx]);
2472 unsigned DMaskVal = DMask->getZExtValue() & 0xf;
2473
2474 // dmask 0 has special semantics, do not simplify.
2475 if (DMaskVal == 0)
2476 return nullptr;
2477
2478 if (!IsLoad && !isMask_32(DMaskVal))
2479 return nullptr;
2480
2481 // Mask off values that are undefined because the dmask doesn't cover them
2482 DemandedElts &= (1 << llvm::popcount(DMaskVal)) - 1;
2483
2484 unsigned NewDMaskVal = 0;
2485 unsigned OrigLdStIdx = 0;
2486 for (unsigned SrcIdx = 0; SrcIdx < 4; ++SrcIdx) {
2487 const unsigned Bit = 1 << SrcIdx;
2488 if (!!(DMaskVal & Bit)) {
2489 if (!!DemandedElts[OrigLdStIdx])
2490 NewDMaskVal |= Bit;
2491 OrigLdStIdx++;
2492 }
2493 }
2494
2495 if (DMaskVal != NewDMaskVal)
2496 Args[DMaskIdx] = ConstantInt::get(DMask->getType(), NewDMaskVal);
2497 }
2498
2499 unsigned NewNumElts = DemandedElts.popcount();
2500 if (!NewNumElts)
2501 return PoisonValue::get(IIVTy);
2502
2503 if (NewNumElts >= VWidth && DemandedElts.isMask()) {
2504 if (DMaskIdx >= 0)
2505 II.setArgOperand(DMaskIdx, Args[DMaskIdx]);
2506 return nullptr;
2507 }
2508
2509 // Validate function argument and return types, extracting overloaded types
2510 // along the way.
2511 SmallVector<Type *, 6> OverloadTys;
2512 if (!Intrinsic::isSignatureValid(II.getCalledFunction(), OverloadTys))
2513 return nullptr;
2514
2515 Type *NewTy =
2516 (NewNumElts == 1) ? EltTy : FixedVectorType::get(EltTy, NewNumElts);
2517 OverloadTys[0] = NewTy;
2518
2519 if (!IsLoad) {
2520 SmallVector<int, 8> EltMask;
2521 for (unsigned OrigStoreIdx = 0; OrigStoreIdx < VWidth; ++OrigStoreIdx)
2522 if (DemandedElts[OrigStoreIdx])
2523 EltMask.push_back(OrigStoreIdx);
2524
2525 if (NewNumElts == 1)
2526 Args[0] = IC.Builder.CreateExtractElement(II.getOperand(0), EltMask[0]);
2527 else
2528 Args[0] = IC.Builder.CreateShuffleVector(II.getOperand(0), EltMask);
2529 }
2530
2532 II.getIntrinsicID(), OverloadTys, Args);
2533 NewCall->takeName(&II);
2534 NewCall->copyMetadata(II);
2535 AttributeList OldAttrList = II.getAttributes();
2536 NewCall->setAttributes(OldAttrList);
2537
2538 if (OffsetAdd != 0) {
2539 unsigned RsrcArg =
2540 AMDGPU::lookupRsrcIntrinsic(II.getIntrinsicID())->RsrcArg;
2541 if (MaybeAlign RsrcAlign = NewCall->getParamAlign(RsrcArg)) {
2542 NewCall->removeParamAttr(RsrcArg, Attribute::Alignment);
2543 NewCall->addParamAttr(
2544 RsrcArg,
2546 commonAlignment(*RsrcAlign, OffsetAdd)));
2547 }
2548 }
2549
2550 if (IsLoad) {
2551 if (NewNumElts == 1) {
2552 return IC.Builder.CreateInsertElement(PoisonValue::get(IIVTy), NewCall,
2553 DemandedElts.countr_zero());
2554 }
2555
2556 SmallVector<int, 8> EltMask;
2557 unsigned NewLoadIdx = 0;
2558 for (unsigned OrigLoadIdx = 0; OrigLoadIdx < VWidth; ++OrigLoadIdx) {
2559 if (!!DemandedElts[OrigLoadIdx])
2560 EltMask.push_back(NewLoadIdx++);
2561 else
2562 EltMask.push_back(NewNumElts);
2563 }
2564
2565 auto *Shuffle = IC.Builder.CreateShuffleVector(NewCall, EltMask);
2566
2567 return Shuffle;
2568 }
2569
2570 return NewCall;
2571}
2572
2574 InstCombiner &IC, IntrinsicInst &II, const APInt &DemandedElts,
2575 APInt &UndefElts) const {
2576 auto *VT = dyn_cast<FixedVectorType>(II.getType());
2577 if (!VT)
2578 return nullptr;
2579
2580 const unsigned FirstElt = DemandedElts.countr_zero();
2581 const unsigned LastElt = DemandedElts.getActiveBits() - 1;
2582 const unsigned MaskLen = LastElt - FirstElt + 1;
2583
2584 unsigned OldNumElts = VT->getNumElements();
2585 if (MaskLen == OldNumElts && MaskLen != 1)
2586 return nullptr;
2587
2588 Type *EltTy = VT->getElementType();
2589 Type *NewVT = MaskLen == 1 ? EltTy : FixedVectorType::get(EltTy, MaskLen);
2590
2591 // Theoretically we should support these intrinsics for any legal type. Avoid
2592 // introducing cases that aren't direct register types like v3i16.
2593 if (!isTypeLegal(NewVT))
2594 return nullptr;
2595
2596 Value *Src = II.getArgOperand(0);
2597
2598 // Make sure convergence tokens are preserved.
2599 // TODO: CreateIntrinsic should allow directly copying bundles
2601 II.getOperandBundlesAsDefs(OpBundles);
2602
2603 Module *M = IC.Builder.getModule();
2604 Function *Remangled =
2605 Intrinsic::getOrInsertDeclaration(M, II.getIntrinsicID(), {NewVT});
2606
2607 if (MaskLen == 1) {
2608 Value *Extract = IC.Builder.CreateExtractElement(Src, FirstElt);
2609
2610 // TODO: Preserve callsite attributes?
2611 CallInst *NewCall = IC.Builder.CreateCall(Remangled, {Extract}, OpBundles);
2612
2613 return IC.Builder.CreateInsertElement(PoisonValue::get(II.getType()),
2614 NewCall, FirstElt);
2615 }
2616
2617 SmallVector<int> ExtractMask(MaskLen, -1);
2618 for (unsigned I = 0; I != MaskLen; ++I) {
2619 if (DemandedElts[FirstElt + I])
2620 ExtractMask[I] = FirstElt + I;
2621 }
2622
2623 Value *Extract = IC.Builder.CreateShuffleVector(Src, ExtractMask);
2624
2625 // TODO: Preserve callsite attributes?
2626 CallInst *NewCall = IC.Builder.CreateCall(Remangled, {Extract}, OpBundles);
2627
2628 SmallVector<int> InsertMask(OldNumElts, -1);
2629 for (unsigned I = 0; I != MaskLen; ++I) {
2630 if (DemandedElts[FirstElt + I])
2631 InsertMask[FirstElt + I] = I;
2632 }
2633
2634 // FIXME: If the call has a convergence bundle, we end up leaving the dead
2635 // call behind.
2636 return IC.Builder.CreateShuffleVector(NewCall, InsertMask);
2637}
2638
2640 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
2641 APInt &UndefElts2, APInt &UndefElts3,
2642 std::function<void(Instruction *, unsigned, APInt, APInt &)>
2643 SimplifyAndSetOp) const {
2644 switch (II.getIntrinsicID()) {
2645 case Intrinsic::amdgcn_readfirstlane:
2646 SimplifyAndSetOp(&II, 0, DemandedElts, UndefElts);
2647 return simplifyAMDGCNLaneIntrinsicDemanded(IC, II, DemandedElts, UndefElts);
2648 case Intrinsic::amdgcn_raw_buffer_load:
2649 case Intrinsic::amdgcn_raw_ptr_buffer_load:
2650 case Intrinsic::amdgcn_raw_buffer_load_format:
2651 case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
2652 case Intrinsic::amdgcn_raw_tbuffer_load:
2653 case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
2654 case Intrinsic::amdgcn_s_buffer_load:
2655 case Intrinsic::amdgcn_ptr_s_buffer_load:
2656 case Intrinsic::amdgcn_struct_buffer_load:
2657 case Intrinsic::amdgcn_struct_ptr_buffer_load:
2658 case Intrinsic::amdgcn_struct_buffer_load_format:
2659 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
2660 case Intrinsic::amdgcn_struct_tbuffer_load:
2661 case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
2662 return simplifyAMDGCNMemoryIntrinsicDemanded(*ST, IC, II, DemandedElts);
2663 default: {
2664 if (getAMDGPUImageDMaskIntrinsic(II.getIntrinsicID())) {
2665 return simplifyAMDGCNMemoryIntrinsicDemanded(*ST, IC, II, DemandedElts,
2666 0);
2667 }
2668 break;
2669 }
2670 }
2671 return std::nullopt;
2672}
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
uint64_t Imm
static Value * createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo, uint32_t Hi)
Emit v_permlane16 with the precomputed lane-select halves.
static std::optional< unsigned > matchRowSharePattern(ArrayRef< uint8_t > Ids)
Match a row-share pattern: all 16 lanes of each row read the same source lane.
static bool matchMirrorPattern(ArrayRef< uint8_t > Ids)
Match an N-lane reversal (mirror) pattern.
static bool canSafelyConvertTo16Bit(Value &V, bool IsFloat, bool AllowI16SExt=false)
static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST, SmallVectorImpl< uint8_t > &Ids, const DataLayout &DL)
Build the per-lane shuffle map by evaluating Index for every lane in the wave.
static std::optional< unsigned > matchQuadPermPattern(ArrayRef< uint8_t > Ids)
Match a 4-lane (quad) permutation, encoded as the v_mov_b32_dpp QUAD_PERM control word: bits[1:0]=Ids...
static std::optional< unsigned > matchDsSwizzleRotatePattern(ArrayRef< uint8_t > Ids)
Match a GFX9+ DS_SWIZZLE rotate-mode permutation: a cyclic left-rotation of all 32 lanes within each ...
static std::optional< unsigned > matchHalfRowPermPattern(ArrayRef< uint8_t > Ids)
Match an 8-lane arbitrary permutation, encoded as the v_mov_b32_dpp8 24-bit selector (three bits per ...
static std::optional< unsigned > matchRowXMaskPattern(ArrayRef< uint8_t > Ids)
Match an XOR mask pattern within each 16-lane row: Ids[J] == Mask ^ J, with Mask in [1,...
static constexpr auto matchHalfRowMirrorPattern
static Value * createPermlaneX16(IRBuilderBase &B, Value *Val, uint32_t Lo, uint32_t Hi)
Emit v_permlanex16 with the precomputed lane-select halves.
static bool isRowPattern(ArrayRef< uint8_t > Ids)
Match an N-lane row pattern: each lane in [0, N) reads from a source lane in the same N-lane row,...
static bool canContractSqrtToRsq(const FPMathOperator *SqrtOp)
Return true if it's legal to contract llvm.amdgcn.rcp(llvm.sqrt)
static CallInst * rewriteCall(IRBuilderBase &B, CallInst &Old, Function &NewCallee, ArrayRef< Value * > Ops)
static Value * convertTo16Bit(Value &V, InstCombiner::BuilderTy &Builder)
static constexpr auto isFullRowPattern
static constexpr auto isQuadPattern
static APInt trimTrailingZerosInVector(InstCombiner &IC, Value *UseV, Instruction *I)
static uint64_t computePermlane16Masks(ArrayRef< uint8_t > Ids)
Pack a 16-lane permutation into a single 64-bit value: four bits per output lane, lane J in bits [J*4...
static bool matchHalfWaveSwapPattern(ArrayRef< uint8_t > Ids)
Match a half-wave swap: lane J reads from lane J ^ 32.
static bool hasPeriodicLayout(ArrayRef< uint8_t > Ids)
Lanes are partitioned into groups of Period; each group is a translated copy of the first: Ids[I] = I...
static std::optional< uint64_t > evalLaneExpr(Value *V, unsigned Lane, const GCNSubtarget &ST, const DataLayout &DL, unsigned Depth=0)
Evaluate V as a function of the lane ID and return its value on Lane, or std::nullopt if V is not a c...
static std::optional< Instruction * > tryOptimizeShufflePattern(InstCombiner &IC, IntrinsicInst &II, const GCNSubtarget &ST)
Try to fold a wave_shuffle/ds_bpermute whose lane index is a constant function of the lane ID into a ...
static constexpr auto isHalfRowPattern
static Value * simplifyAMDGCNMemoryIntrinsicDemanded(const GCNSubtarget &ST, InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, int DMaskIdx=-1, bool IsLoad=true)
Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics.
static APInt defaultComponentBroadcast(Value *V)
static std::optional< unsigned > matchDsSwizzleBitmaskPattern(ArrayRef< uint8_t > Ids)
Match a DS_SWIZZLE bitmask-mode permutation: dst_lane = ((src_lane & AND) | OR) ^ XOR with each mask ...
static Value * createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset, const DataLayout &DL)
Emit ds_swizzle with the given immediate, bitcasting/converting between pointer/float types and i32 a...
static std::optional< Instruction * > modifyIntrinsicCall(IntrinsicInst &OldIntr, Instruction &InstToReplace, unsigned NewIntr, InstCombiner &IC, std::function< void(SmallVectorImpl< Value * > &, SmallVectorImpl< Type * > &)> Func)
Applies Func(OldIntr.Args, OldIntr.ArgTys), creates intrinsic call with modified arguments (based on ...
static Value * matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src, ArrayRef< uint8_t > Ids, const GCNSubtarget &ST, const DataLayout &DL)
Given a shuffle map, try to emit the best hardware intrinsic.
static std::optional< unsigned > matchRowRotatePattern(ArrayRef< uint8_t > Ids)
Match a 16-lane cyclic rotation; returns the rotation amount in [1, 15].
static bool isCrossRowPattern(ArrayRef< uint8_t > Ids)
Match a cross-row permutation suitable for v_permlanex16: every lane in the low 16-lane half reads fr...
static bool isThreadID(const GCNSubtarget &ST, Value *V)
static Value * createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl)
Emit v_mov_b32_dpp with the given control word, row/bank masks 0xF, and bound_ctrl=1 so out-of-bounds...
static APFloat fmed3AMDGCN(const APFloat &Src0, const APFloat &Src1, const APFloat &Src2)
static std::optional< Instruction * > simplifyAMDGCNImageIntrinsic(const GCNSubtarget *ST, const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr, IntrinsicInst &II, InstCombiner &IC)
static Value * createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector)
Emit v_mov_b32_dpp8 with the given 24-bit lane selector.
static Instruction * foldAddIntoDotAccumulator(IntrinsicInst &II, unsigned AccIdx, unsigned ClampIdx, InstCombiner &IC)
Try to fold an addition into the accumulator when saturation is disabled.
static Value * matchFPExtFromF16(Value *Arg)
Match an fpext from half to float, or a constant we can convert.
static constexpr auto matchFullRowMirrorPattern
static bool simplifyExponentOnlyScaleArg(InstCombiner &IC, IntrinsicInst &II, unsigned ScaleArgIdx)
Simplify the f32 scale operand of a V_CVT_SCALEF32_* conversion, of which only the exponent field is ...
static Value * createPermlane64(IRBuilderBase &B, Value *Val)
Emit v_permlane64 (swap of the two 32-lane halves of a wave64).
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
This file a TargetTransformInfoImplBase conforming object specific to the AMDGPU target machine.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Utilities for dealing with flags related to floating point properties and mode controls.
AMD GCN specific subclass of TargetSubtarget.
This file provides the interface for the instcombine pass implementation.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
uint64_t IntrinsicInst * II
if(PassOpts->AAPipeline)
int64_t C1
int64_t C0
MachineInstr *& Lo
This file contains some templates that are useful if you are working with the STL at all.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Value * RHS
Value * LHS
static const fltSemantics & IEEEsingle()
Definition APFloat.h:304
static constexpr roundingMode rmTowardZero
Definition APFloat.h:365
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:361
static const fltSemantics & IEEEhalf()
Definition APFloat.h:302
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
Definition APFloat.h:1224
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition APFloat.cpp:6045
bool bitwiseIsEqual(const APFloat &RHS) const
Definition APFloat.h:1548
bool isPosInfinity() const
Definition APFloat.h:1596
APFloat makeQuiet() const
Assuming this is an IEEE-754 NaN value, quiet its signaling bit.
Definition APFloat.h:1420
bool isNaN() const
Definition APFloat.h:1581
bool isSignaling() const
Definition APFloat.h:1585
APInt bitcastToAPInt() const
Definition APFloat.h:1475
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Definition APFloat.h:1183
bool isInfinity() const
Definition APFloat.h:1580
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition APInt.h:1426
unsigned popcount() const
Count the number of bits set.
Definition APInt.h:1690
LLVM_ABI uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
Definition APInt.cpp:517
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1532
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition APInt.h:1659
bool isMask(unsigned numBits) const
Definition APInt.h:484
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
Definition ArrayRef.h:218
size_t size() const
Get the array size.
Definition ArrayRef.h:141
static LLVM_ABI Attribute getWithDereferenceableBytes(LLVMContext &Context, uint64_t Bytes)
static LLVM_ABI Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
bool isTypeLegal(Type *Ty) const override
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Removes the attribute from the given argument.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
void setAttributes(AttributeList A)
Set the attributes for this call.
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
AttributeList getAttributes() const
Return the attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:420
const APFloat & getValueAPF() const
Definition Constants.h:463
static LLVM_ABI ConstantFP * getZero(Type *Ty, bool Negative=false)
static LLVM_ABI ConstantFP * getNaN(Type *Ty, bool Negative=false, uint64_t Payload=0)
static LLVM_ABI ConstantFP * getInfinity(Type *Ty, bool Negative=false)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static ConstantInt * getSigned(IntegerType *Ty, int64_t V, bool ImplicitTrunc=false)
Return a ConstantInt with the specified value for the specified type.
Definition Constants.h:135
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
This class represents a range of values.
LLVM_ABI ConstantRange add(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an addition of a value in this ran...
LLVM_ABI bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
LLVM_ABI ConstantRange intersectWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the intersection of this range with another range.
This is an important base class in LLVM.
Definition Constant.h:44
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition DataLayout.h:794
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Tagged union holding either a T or a Error.
Definition Error.h:485
This class represents an extension of floating point types.
Utility class for floating point operations which can have information about relaxed accuracy require...
Definition Operator.h:202
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags.
Definition Operator.h:291
bool hasApproxFunc() const
Test if this operation allows approximations of math library functions or intrinsics.
Definition Operator.h:288
LLVM_ABI float getFPAccuracy() const
Get the maximum error permitted by this operation in ULPs.
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
bool allowContract() const
Definition FMF.h:69
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
bool simplifyDemandedLaneMaskArg(InstCombiner &IC, IntrinsicInst &II, unsigned LaneAgIdx) const
Simplify a lane index operand (e.g.
std::optional< Instruction * > instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const override
Instruction * hoistLaneIntrinsicThroughOperand(InstCombiner &IC, IntrinsicInst &II) const
std::optional< Value * > simplifyDemandedVectorEltsIntrinsic(InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3, std::function< void(Instruction *, unsigned, APInt, APInt &)> SimplifyAndSetOp) const override
KnownIEEEMode fpenvIEEEMode(const Instruction &I) const
Return KnownIEEEMode::On if we know if the use context can assume "amdgpu-ieee"="true" and KnownIEEEM...
Value * simplifyAMDGCNLaneIntrinsicDemanded(InstCombiner &IC, IntrinsicInst &II, const APInt &DemandedElts, APInt &UndefElts) const
bool canSimplifyLegacyMulToMul(const Instruction &I, const Value *Op0, const Value *Op1, InstCombiner &IC) const
Common base class shared among various IRBuilders.
Definition IRBuilder.h:111
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Definition IRBuilder.h:446
LLVM_ABI CallInst * CreateIntrinsicWithoutFolding(Intrinsic::ID ID, ArrayRef< Type * > OverloadTypes, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="", ArrayRef< OperandBundleDef > OpBundles={})
Create a call to intrinsic ID with Args, mangled using OverloadTypes.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Definition IRBuilder.h:2748
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition IRBuilder.h:2736
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
Definition IRBuilder.h:541
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
Definition IRBuilder.h:2215
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition IRBuilder.h:1534
Value * CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx, const Twine &Name="")
Create a call to the vector.extract intrinsic.
Definition IRBuilder.h:1114
Module * getModule() const
Get the module.
Definition IRBuilder.h:190
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:2462
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition IRBuilder.h:476
Value * CreateMaxNum(Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create call to the maxnum intrinsic.
Definition IRBuilder.h:1045
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition IRBuilder.h:1513
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Definition IRBuilder.h:2197
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Definition IRBuilder.h:2770
LLVM_ABI Value * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > OverloadTypes, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="", ArrayRef< OperandBundleDef > OpBundles={}, function_ref< void(CallInst *)> SetFn=[](CallInst *) {})
Variant to create a possibly constant-folded intrinsic.
Value * CreateMaximumNum(Value *LHS, Value *RHS, const Twine &Name="")
Create call to the maximum intrinsic.
Definition IRBuilder.h:1073
Value * CreateMinNum(Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create call to the minnum intrinsic.
Definition IRBuilder.h:1033
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition IRBuilder.h:1424
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition IRBuilder.h:2640
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition IRBuilder.h:197
Value * CreateFAddFMF(Value *L, Value *R, FMFSource FMFSource, const Twine &Name="", MDNode *FPMD=nullptr)
Definition IRBuilder.h:1643
Value * CreateMinimumNum(Value *LHS, Value *RHS, const Twine &Name="")
Create call to the minimumnum intrinsic.
Definition IRBuilder.h:1067
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition IRBuilder.h:1553
Value * CreateFMulFMF(Value *L, Value *R, FMFSource FMFSource, const Twine &Name="", MDNode *FPMD=nullptr)
Definition IRBuilder.h:1681
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
The core instruction combiner logic.
const DataLayout & getDataLayout() const
virtual Instruction * eraseInstFromFunction(Instruction &I)=0
Combiner aware instruction erasure.
DominatorTree & getDominatorTree() const
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
Instruction * InsertNewInstWith(Instruction *New, BasicBlock::iterator Old)
Same as InsertNewInstBefore, but also sets the debug loc.
virtual bool SimplifyDemandedBits(Instruction *I, unsigned OpNo, const APInt &DemandedMask, KnownBits &Known, const SimplifyQuery &Q, unsigned Depth=0)=0
IRBuilder< TargetFolder, IRBuilderInstCombineInserter > BuilderTy
An IRBuilder that automatically inserts new instructions into the worklist.
static Value * stripSignOnlyFPOps(Value *Val)
Ignore all operations which only change the sign of a value, returning the underlying magnitude value...
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
const SimplifyQuery & getSimplifyQuery() const
LLVM_ABI void dropUBImplyingAttrsAndMetadata(ArrayRef< unsigned > Keep={})
Drop any attributes or metadata that can cause immediate undefined behavior.
LLVM_ABI Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
LLVM_ABI void copyFastMathFlags(FastMathFlags FMF)
Convenience function for transferring all fast-math flag values to this instruction,...
void dropPoisonGeneratingAnnotations()
Drops flags, attributes and metadata that may generate poison.
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
bool isKnownNeverInfOrNaN() const
Return true if it's known this can never be an infinity or nan.
LLVM_ABI bool isKnownNeverLogicalZero(DenormalMode Mode) const
Return true if it's known this can never be interpreted as a zero.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void resize(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:293
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
Definition Type.h:144
LLVM_ABI Type * getWithNewType(Type *EltTy) const
Given vector type, change the element type, whilst keeping the old number of elements.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:269
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
const Use & getOperandUse(unsigned i) const
Definition User.h:220
void setOperand(unsigned i, Value *Val)
Definition User.h:212
Value * getOperand(unsigned i) const
Definition User.h:207
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVM_ABI bool hasOneUser() const
Return true if there is exactly one user of this value.
Definition Value.cpp:158
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:395
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
const ParentTy * getParent() const
Definition ilist_node.h:34
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
APInt getExponentOnlyScaleDemandedBits()
LLVM_READONLY const MIMGOffsetMappingInfo * getMIMGOffsetMappingInfo(unsigned Offset)
uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt)
const ImageDimIntrinsicInfo * getImageDimIntrinsicByBaseOpcode(unsigned BaseOpcode, unsigned Dim)
std::optional< unsigned > getExponentOnlyScaleArgIdx(unsigned IntrID)
LLVM_READONLY const MIMGMIPMappingInfo * getMIMGMIPMappingInfo(unsigned MIP)
bool isTriviallyUniform(const Value *V, const Instruction *CtxI)
Return true if we can easily prove that V is uniform at CtxI.
LLVM_READONLY const MIMGBiasMappingInfo * getMIMGBiasMappingInfo(unsigned Bias)
std::optional< APFloat > evaluateRcp(const APFloat &Val)
Evaluate the constant-folded result of v_rcp for Val, accounting for the hardware's denormal flushing...
LLVM_READONLY const MIMGLZMappingInfo * getMIMGLZMappingInfo(unsigned L)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfo(unsigned DimEnum)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
const ImageDimIntrinsicInfo * getImageDimIntrinsicInfo(unsigned Intr)
const RsrcIntrinsic * lookupRsrcIntrinsic(unsigned Intr)
const GcnBufferFormatInfo * getGcnBufferFormatInfo(uint8_t BitsPerComp, uint8_t NumComponents, uint8_t NumFormat, const MCSubtargetInfo &STI)
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
match_bind< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
cstfp_pred_ty< is_any_zero_fp > m_AnyZeroFP()
Match a floating-point negative zero or positive zero.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
ap_match< APFloat > m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
auto m_Value()
Match an arbitrary value and ignore it.
CastInst_match< OpTy, FPExtInst > m_FPExt(const OpTy &Op)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
is_zero< true > m_Zero()
Match any null constant or a vector with all elements equal to 0.
BinaryOp_match< LHS, RHS, Instruction::Add, true > m_c_Add(const LHS &L, const RHS &R)
Matches a Add with LHS and RHS in either order.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
auto m_ConstantFP()
Match an arbitrary ConstantFP and ignore it.
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
auto m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
LLVM_ABI KnownFPClass computeKnownFPClass(const Value *V, const APInt &DemandedElts, FPClassTest InterestedClasses, const SimplifyQuery &SQ, unsigned Depth=0)
Determine which floating-point classes are valid for V, and return them in KnownFPClass bit sets.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
@ Known
Known to have no common set bits.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
Definition MathExtras.h:256
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
APFloat frexp(const APFloat &X, int &Exp, APFloat::roundingMode RM)
Equivalent of C standard library function.
Definition APFloat.h:1728
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2008 maxNum semantics.
Definition APFloat.h:1771
constexpr unsigned MaxAnalysisRecursionDepth
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
Returns: X * 2^Exp for integral exponents.
Definition APFloat.h:1716
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
Definition MathExtras.h:151
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
Definition MathExtras.h:156
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
constexpr int PoisonMaskElem
@ FMul
Product of floats.
@ FAdd
Sum of floats.
LLVM_ABI Value * findScalarElement(Value *V, unsigned EltNo)
Given a vector and an element number, see if the scalar value is already around as a register,...
@ NearestTiesToEven
roundTiesToEven.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:341
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition STLExtras.h:2182
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
LLVM_ABI Constant * ConstantFoldInstOperands(const Instruction *I, ArrayRef< Constant * > Ops, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, bool AllowNonDeterministic=true)
ConstantFoldInstOperands - Attempt to constant fold an instruction with the specified operands.
constexpr uint64_t Make_64(uint32_t High, uint32_t Low)
Make a 64-bit integer from a high / low pair of 32-bit integers.
Definition MathExtras.h:161
LLVM_ABI ConstantRange computeConstantRange(const Value *V, bool ForSigned, const SimplifyQuery &SQ, unsigned Depth=0)
Determine the possible constant range of an integer or vector of integer value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
Represent subnormal handling kind for floating point instruction inputs and outputs.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:111
SimplifyQuery getWithInstruction(const Instruction *I) const
LLVM_ABI bool isUndefValue(Value *V) const
If CanUseUndef is true, returns whether V is undef.