LLVM 24.0.0git
ScalarizeMaskedMemIntrin.cpp
Go to the documentation of this file.
1//===- ScalarizeMaskedMemIntrin.cpp - Scalarize unsupported masked mem ----===//
2// intrinsics
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass replaces masked memory intrinsics - when unsupported by the target
11// - with a chain of basic blocks, that deal with the elements one-by-one if the
12// appropriate mask bit is set.
13//
14//===----------------------------------------------------------------------===//
15
17#include "llvm/ADT/Twine.h"
21#include "llvm/IR/BasicBlock.h"
22#include "llvm/IR/Constant.h"
23#include "llvm/IR/Constants.h"
25#include "llvm/IR/Dominators.h"
26#include "llvm/IR/Function.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/Instruction.h"
32#include "llvm/IR/Metadata.h"
34#include "llvm/IR/Type.h"
35#include "llvm/IR/Value.h"
37#include "llvm/Pass.h"
41#include <cassert>
42#include <optional>
43
44using namespace llvm;
45
46#define DEBUG_TYPE "scalarize-masked-mem-intrin"
47
48namespace {
49
50class ScalarizeMaskedMemIntrinLegacyPass : public FunctionPass {
51public:
52 static char ID; // Pass identification, replacement for typeid
53
54 explicit ScalarizeMaskedMemIntrinLegacyPass() : FunctionPass(ID) {
57 }
58
59 bool runOnFunction(Function &F) override;
60
61 StringRef getPassName() const override {
62 return "Scalarize Masked Memory Intrinsics";
63 }
64
65 void getAnalysisUsage(AnalysisUsage &AU) const override {
68 }
69};
70
71} // end anonymous namespace
72
73static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT,
74 const TargetTransformInfo &TTI, const DataLayout &DL,
75 bool HasBranchDivergence, DomTreeUpdater *DTU);
76static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
78 const DataLayout &DL, bool HasBranchDivergence,
79 DomTreeUpdater *DTU);
80
81char ScalarizeMaskedMemIntrinLegacyPass::ID = 0;
82
83INITIALIZE_PASS_BEGIN(ScalarizeMaskedMemIntrinLegacyPass, DEBUG_TYPE,
84 "Scalarize unsupported masked memory intrinsics", false,
85 false)
88INITIALIZE_PASS_END(ScalarizeMaskedMemIntrinLegacyPass, DEBUG_TYPE,
89 "Scalarize unsupported masked memory intrinsics", false,
90 false)
91
93 return new ScalarizeMaskedMemIntrinLegacyPass();
94}
95
96static bool isConstantIntVector(Value *Mask) {
98 if (!C)
99 return false;
100
101 unsigned NumElts = cast<FixedVectorType>(Mask->getType())->getNumElements();
102 for (unsigned i = 0; i != NumElts; ++i) {
103 Constant *CElt = C->getAggregateElement(i);
104 if (!CElt || !isa<ConstantInt>(CElt))
105 return false;
106 }
107
108 return true;
109}
110
111static unsigned adjustForEndian(const DataLayout &DL, unsigned VectorWidth,
112 unsigned Idx) {
113 return DL.isBigEndian() ? VectorWidth - 1 - Idx : Idx;
114}
115
117 Instruction &Dest, const Instruction &Source, const DataLayout &DL,
118 unsigned SourcePtrOperand, unsigned DestPtrOperand, Type *AccessType,
119 bool IsWholeAccess, std::optional<size_t> ByteOffset) {
120 // Only propagate metadata that is valid on each constituent memory access.
121 // In particular, do not copy metadata whose meaning is tied to the call,
122 // such as !prof or !callsite.
123 Dest.copyMetadata(Source,
124 {LLVMContext::MD_nontemporal,
125 LLVMContext::MD_mem_parallel_loop_access,
126 LLVMContext::MD_access_group, LLVMContext::MD_annotation,
127 LLVMContext::MD_nosanitize, LLVMContext::MD_mmra});
128
129 AAMDNodes AANodes = Source.getAAMetadata();
130 if (IsWholeAccess)
131 Dest.setAAMetadata(AANodes);
132 else if (ByteOffset)
133 Dest.setAAMetadata(AANodes.adjustForAccess(*ByteOffset, AccessType, DL));
134 else {
135 // The packed address is runtime-dependent. The other AA metadata remains
136 // applicable, but !tbaa.struct cannot be adjusted to a known byte range.
137 AANodes.TBAAStruct = nullptr;
138 Dest.setAAMetadata(AANodes);
139 }
140 setMemCacheHint(Dest, DestPtrOperand,
141 getMemCacheHint(Source, SourcePtrOperand));
142}
143
145 const Instruction &Source,
146 const DataLayout &DL,
147 unsigned SourcePtrOperand,
148 std::optional<size_t> ByteOffset) {
149 copyMetadataForMemoryAccess(Dest, Source, DL, SourcePtrOperand,
150 Dest.getPointerOperandIndex(), Dest.getType(),
151 Dest.getType() == Source.getType(), ByteOffset);
152
153 // !range applies element-wise to vectors, so the same range describes each
154 // scalar result. The other metadata here also describes the loaded result.
155 Dest.copyMetadata(Source, {LLVMContext::MD_fpmath, LLVMContext::MD_range,
156 LLVMContext::MD_invariant_load});
157}
158
160 const Instruction &Source,
161 const DataLayout &DL,
162 unsigned SourcePtrOperand,
163 std::optional<size_t> ByteOffset) {
165 Dest, Source, DL, SourcePtrOperand, Dest.getPointerOperandIndex(),
166 Dest.getValueOperand()->getType(),
167 Dest.getValueOperand()->getType() == Source.getOperand(0)->getType(),
168 ByteOffset);
169}
170
171// Translate a masked load intrinsic like
172// <16 x i32 > @llvm.masked.load( <16 x i32>* %addr,
173// <16 x i1> %mask, <16 x i32> %passthru)
174// to a chain of basic blocks, with loading element one-by-one if
175// the appropriate mask bit is set
176//
177// %1 = bitcast i8* %addr to i32*
178// %2 = extractelement <16 x i1> %mask, i32 0
179// br i1 %2, label %cond.load, label %else
180//
181// cond.load: ; preds = %0
182// %3 = getelementptr i32* %1, i32 0
183// %4 = load i32* %3
184// %5 = insertelement <16 x i32> %passthru, i32 %4, i32 0
185// br label %else
186//
187// else: ; preds = %0, %cond.load
188// %res.phi.else = phi <16 x i32> [ %5, %cond.load ], [ poison, %0 ]
189// %6 = extractelement <16 x i1> %mask, i32 1
190// br i1 %6, label %cond.load1, label %else2
191//
192// cond.load1: ; preds = %else
193// %7 = getelementptr i32* %1, i32 1
194// %8 = load i32* %7
195// %9 = insertelement <16 x i32> %res.phi.else, i32 %8, i32 1
196// br label %else2
197//
198// else2: ; preds = %else, %cond.load1
199// %res.phi.else3 = phi <16 x i32> [ %9, %cond.load1 ], [ %res.phi.else, %else
200// ] %10 = extractelement <16 x i1> %mask, i32 2 br i1 %10, label %cond.load4,
201// label %else5
202//
203static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
204 CallInst *CI, DomTreeUpdater *DTU,
205 bool &ModifiedDT) {
206 Value *Ptr = CI->getArgOperand(0);
207 Value *Mask = CI->getArgOperand(1);
208 Value *Src0 = CI->getArgOperand(2);
209
210 const Align AlignVal = CI->getParamAlign(0).valueOrOne();
211 VectorType *VecType = cast<FixedVectorType>(CI->getType());
212
213 Type *EltTy = VecType->getElementType();
214
215 Instruction *InsertPt = CI;
216 IRBuilder<> Builder(InsertPt);
217 Builder.disableGEPCanonicalization();
218 BasicBlock *IfBlock = CI->getParent();
219
220 // Short-cut if the mask is all-true.
221 if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
222 LoadInst *NewI = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal);
223 copyMetadataForScalarizedLoad(*NewI, *CI, DL, /*SourcePtrOperand=*/0,
224 std::nullopt);
225 NewI->takeName(CI);
226 CI->replaceAllUsesWith(NewI);
227 CI->eraseFromParent();
228 return;
229 }
230
231 // Adjust alignment for the scalar instruction.
232 const Align AdjustedAlignVal =
233 commonAlignment(AlignVal, EltTy->getPrimitiveSizeInBits() / 8);
234 unsigned VectorWidth = cast<FixedVectorType>(VecType)->getNumElements();
235
236 // The result vector
237 Value *VResult = Src0;
238
239 if (isConstantIntVector(Mask)) {
240 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
241 if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue())
242 continue;
243 Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
244 LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);
246 *Load, *CI, DL, /*SourcePtrOperand=*/0,
247 Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
248 VResult = Builder.CreateInsertElement(VResult, Load, Idx);
249 }
250 CI->replaceAllUsesWith(VResult);
251 CI->eraseFromParent();
252 return;
253 }
254
255 // Optimize the case where the "masked load" is a predicated load - that is,
256 // where the mask is the splat of a non-constant scalar boolean. In that case,
257 // use that splated value as the guard on a conditional vector load.
258 if (isSplatValue(Mask, /*Index=*/0)) {
259 Value *Predicate = Builder.CreateExtractElement(Mask, uint64_t(0ull),
260 Mask->getName() + ".first");
261 // We mark the branch weights as explicitly unknown given they would only
262 // be derivable from the mask which we do not have VP information for.
263 Instruction *ThenTerm =
264 SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
266 *CI->getFunction(), DEBUG_TYPE),
267 DTU);
268
269 BasicBlock *CondBlock = ThenTerm->getParent();
270 CondBlock->setName("cond.load");
271 Builder.SetInsertPoint(CondBlock->getTerminator());
272 LoadInst *Load = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal,
273 CI->getName() + ".cond.load");
274 copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
275 std::nullopt);
276
277 BasicBlock *PostLoad = ThenTerm->getSuccessor(0);
278 Builder.SetInsertPoint(PostLoad->begin());
279 PHINode *Phi = Builder.CreatePHI(VecType, /*NumReservedValues=*/2);
280 Phi->addIncoming(Load, CondBlock);
281 Phi->addIncoming(Src0, IfBlock);
282 Phi->takeName(CI);
283
284 CI->replaceAllUsesWith(Phi);
285 CI->eraseFromParent();
286 ModifiedDT = true;
287 return;
288 }
289 // If the mask is not v1i1, use scalar bit test operations. This generates
290 // better results on X86 at least. However, don't do this on GPUs and other
291 // machines with divergence, as there each i1 needs a vector register.
292 Value *SclrMask = nullptr;
293 if (VectorWidth != 1 && !HasBranchDivergence) {
294 Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);
295 SclrMask = Builder.CreateBitCast(Mask, SclrMaskTy, "scalar_mask");
296 }
297
298 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
299 // Fill the "else" block, created in the previous iteration
300 //
301 // %res.phi.else3 = phi <16 x i32> [ %11, %cond.load1 ], [ %res.phi.else,
302 // %else ] %mask_1 = and i16 %scalar_mask, i32 1 << Idx %cond = icmp ne i16
303 // %mask_1, 0 br i1 %mask_1, label %cond.load, label %else
304 //
305 // On GPUs, use
306 // %cond = extrectelement %mask, Idx
307 // instead
309 if (SclrMask != nullptr) {
310 Value *Mask = Builder.getInt(APInt::getOneBitSet(
311 VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
312 Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
313 Builder.getIntN(VectorWidth, 0));
314 } else {
315 Predicate = Builder.CreateExtractElement(Mask, Idx);
316 }
317
318 // Create "cond" block
319 //
320 // %EltAddr = getelementptr i32* %1, i32 0
321 // %Elt = load i32* %EltAddr
322 // VResult = insertelement <16 x i32> VResult, i32 %Elt, i32 Idx
323 //
324 // We mark the branch weights as explicitly unknown given they would only
325 // be derivable from the mask which we do not have VP information for.
326 Instruction *ThenTerm =
327 SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
329 *CI->getFunction(), DEBUG_TYPE),
330 DTU);
331
332 BasicBlock *CondBlock = ThenTerm->getParent();
333 CondBlock->setName("cond.load");
334
335 Builder.SetInsertPoint(CondBlock->getTerminator());
336 Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
337 LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);
339 *Load, *CI, DL, /*SourcePtrOperand=*/0,
340 Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
341 Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);
342
343 // Create "else" block, fill it in the next iteration
344 BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
345 NewIfBlock->setName("else");
346 BasicBlock *PrevIfBlock = IfBlock;
347 IfBlock = NewIfBlock;
348
349 // Create the phi to join the new and previous value.
350 Builder.SetInsertPoint(NewIfBlock->begin());
351 PHINode *Phi = Builder.CreatePHI(VecType, 2, "res.phi.else");
352 Phi->addIncoming(NewVResult, CondBlock);
353 Phi->addIncoming(VResult, PrevIfBlock);
354 VResult = Phi;
355 }
356
357 CI->replaceAllUsesWith(VResult);
358 CI->eraseFromParent();
359
360 ModifiedDT = true;
361}
362
363// Translate a masked store intrinsic, like
364// void @llvm.masked.store(<16 x i32> %src, <16 x i32>* %addr,
365// <16 x i1> %mask)
366// to a chain of basic blocks, that stores element one-by-one if
367// the appropriate mask bit is set
368//
369// %1 = bitcast i8* %addr to i32*
370// %2 = extractelement <16 x i1> %mask, i32 0
371// br i1 %2, label %cond.store, label %else
372//
373// cond.store: ; preds = %0
374// %3 = extractelement <16 x i32> %val, i32 0
375// %4 = getelementptr i32* %1, i32 0
376// store i32 %3, i32* %4
377// br label %else
378//
379// else: ; preds = %0, %cond.store
380// %5 = extractelement <16 x i1> %mask, i32 1
381// br i1 %5, label %cond.store1, label %else2
382//
383// cond.store1: ; preds = %else
384// %6 = extractelement <16 x i32> %val, i32 1
385// %7 = getelementptr i32* %1, i32 1
386// store i32 %6, i32* %7
387// br label %else2
388// . . .
389static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
390 CallInst *CI, DomTreeUpdater *DTU,
391 bool &ModifiedDT) {
392 Value *Src = CI->getArgOperand(0);
393 Value *Ptr = CI->getArgOperand(1);
394 Value *Mask = CI->getArgOperand(2);
395
396 const Align AlignVal = CI->getParamAlign(1).valueOrOne();
397 auto *VecType = cast<VectorType>(Src->getType());
398
399 Type *EltTy = VecType->getElementType();
400
401 Instruction *InsertPt = CI;
402 IRBuilder<> Builder(InsertPt);
403 Builder.disableGEPCanonicalization();
404
405 // Short-cut if the mask is all-true.
406 if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
407 StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
408 Store->takeName(CI);
409 copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
410 std::nullopt);
411 // This is a one-to-one replacement, so the assignment link remains valid.
412 Store->copyMetadata(*CI, LLVMContext::MD_DIAssignID);
413 CI->eraseFromParent();
414 return;
415 }
416
417 // Adjust alignment for the scalar instruction.
418 const Align AdjustedAlignVal =
419 commonAlignment(AlignVal, EltTy->getPrimitiveSizeInBits() / 8);
420 unsigned VectorWidth = cast<FixedVectorType>(VecType)->getNumElements();
421
422 if (isConstantIntVector(Mask)) {
423 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
424 if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue())
425 continue;
426 Value *OneElt = Builder.CreateExtractElement(Src, Idx);
427 Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
429 Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
431 *Store, *CI, DL, /*SourcePtrOperand=*/1,
432 Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
433 }
434 CI->eraseFromParent();
435 return;
436 }
437
438 // Optimize the case where the "masked store" is a predicated store - that is,
439 // when the mask is the splat of a non-constant scalar boolean. In that case,
440 // optimize to a conditional store.
441 if (isSplatValue(Mask, /*Index=*/0)) {
442 Value *Predicate = Builder.CreateExtractElement(Mask, uint64_t(0ull),
443 Mask->getName() + ".first");
444 // We mark the branch weights as explicitly unknown given they would only
445 // be derivable from the mask which we do not have VP information for.
446 Instruction *ThenTerm =
447 SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
449 *CI->getFunction(), DEBUG_TYPE),
450 DTU);
451 BasicBlock *CondBlock = ThenTerm->getParent();
452 CondBlock->setName("cond.store");
453 Builder.SetInsertPoint(CondBlock->getTerminator());
454
455 StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
456 Store->takeName(CI);
457 copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
458 std::nullopt);
459 // This is a one-to-one replacement, so the assignment link remains valid.
460 Store->copyMetadata(*CI, LLVMContext::MD_DIAssignID);
461
462 CI->eraseFromParent();
463 ModifiedDT = true;
464 return;
465 }
466
467 // If the mask is not v1i1, use scalar bit test operations. This generates
468 // better results on X86 at least. However, don't do this on GPUs or other
469 // machines with branch divergence, as there each i1 takes up a register.
470 Value *SclrMask = nullptr;
471 if (VectorWidth != 1 && !HasBranchDivergence) {
472 Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);
473 SclrMask = Builder.CreateBitCast(Mask, SclrMaskTy, "scalar_mask");
474 }
475
476 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
477 // Fill the "else" block, created in the previous iteration
478 //
479 // %mask_1 = and i16 %scalar_mask, i32 1 << Idx
480 // %cond = icmp ne i16 %mask_1, 0
481 // br i1 %mask_1, label %cond.store, label %else
482 //
483 // On GPUs, use
484 // %cond = extrectelement %mask, Idx
485 // instead
487 if (SclrMask != nullptr) {
488 Value *Mask = Builder.getInt(APInt::getOneBitSet(
489 VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
490 Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
491 Builder.getIntN(VectorWidth, 0));
492 } else {
493 Predicate = Builder.CreateExtractElement(Mask, Idx);
494 }
495
496 // Create "cond" block
497 //
498 // %OneElt = extractelement <16 x i32> %Src, i32 Idx
499 // %EltAddr = getelementptr i32* %1, i32 0
500 // %store i32 %OneElt, i32* %EltAddr
501 //
502 // We mark the branch weights as explicitly unknown given they would only
503 // be derivable from the mask which we do not have VP information for.
504 Instruction *ThenTerm =
505 SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
507 *CI->getFunction(), DEBUG_TYPE),
508 DTU);
509
510 BasicBlock *CondBlock = ThenTerm->getParent();
511 CondBlock->setName("cond.store");
512
513 Builder.SetInsertPoint(CondBlock->getTerminator());
514 Value *OneElt = Builder.CreateExtractElement(Src, Idx);
515 Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
517 Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
519 *Store, *CI, DL, /*SourcePtrOperand=*/1,
520 Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
521
522 // Create "else" block, fill it in the next iteration
523 BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
524 NewIfBlock->setName("else");
525
526 Builder.SetInsertPoint(NewIfBlock->begin());
527 }
528 CI->eraseFromParent();
529
530 ModifiedDT = true;
531}
532
533// Translate a masked gather intrinsic like
534// <16 x i32 > @llvm.masked.gather.v16i32( <16 x i32*> %Ptrs, i32 4,
535// <16 x i1> %Mask, <16 x i32> %Src)
536// to a chain of basic blocks, with loading element one-by-one if
537// the appropriate mask bit is set
538//
539// %Ptrs = getelementptr i32, i32* %base, <16 x i64> %ind
540// %Mask0 = extractelement <16 x i1> %Mask, i32 0
541// br i1 %Mask0, label %cond.load, label %else
542//
543// cond.load:
544// %Ptr0 = extractelement <16 x i32*> %Ptrs, i32 0
545// %Load0 = load i32, i32* %Ptr0, align 4
546// %Res0 = insertelement <16 x i32> poison, i32 %Load0, i32 0
547// br label %else
548//
549// else:
550// %res.phi.else = phi <16 x i32>[%Res0, %cond.load], [poison, %0]
551// %Mask1 = extractelement <16 x i1> %Mask, i32 1
552// br i1 %Mask1, label %cond.load1, label %else2
553//
554// cond.load1:
555// %Ptr1 = extractelement <16 x i32*> %Ptrs, i32 1
556// %Load1 = load i32, i32* %Ptr1, align 4
557// %Res1 = insertelement <16 x i32> %res.phi.else, i32 %Load1, i32 1
558// br label %else2
559// . . .
560// %Result = select <16 x i1> %Mask, <16 x i32> %res.phi.select, <16 x i32> %Src
561// ret <16 x i32> %Result
563 bool HasBranchDivergence, CallInst *CI,
564 DomTreeUpdater *DTU, bool &ModifiedDT) {
565 Value *Ptrs = CI->getArgOperand(0);
566 Value *Mask = CI->getArgOperand(1);
567 Value *Src0 = CI->getArgOperand(2);
568
569 auto *VecType = cast<FixedVectorType>(CI->getType());
570 Type *EltTy = VecType->getElementType();
571
572 Instruction *InsertPt = CI;
573 IRBuilder<> Builder(InsertPt);
574 BasicBlock *IfBlock = CI->getParent();
575 Align AlignVal = CI->getParamAlign(0).valueOrOne();
576
577 // The result vector
578 Value *VResult = Src0;
579 unsigned VectorWidth = VecType->getNumElements();
580
581 // Shorten the way if the mask is a vector of constants.
582 if (isConstantIntVector(Mask)) {
583 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
584 if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue())
585 continue;
586 Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
587 LoadInst *Load =
588 Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
589 copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
590 /*ByteOffset=*/0);
591 VResult =
592 Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
593 }
594 CI->replaceAllUsesWith(VResult);
595 CI->eraseFromParent();
596 return;
597 }
598
599 // If the mask is not v1i1, use scalar bit test operations. This generates
600 // better results on X86 at least. However, don't do this on GPUs or other
601 // machines with branch divergence, as there, each i1 takes up a register.
602 Value *SclrMask = nullptr;
603 if (VectorWidth != 1 && !HasBranchDivergence) {
604 Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);
605 SclrMask = Builder.CreateBitCast(Mask, SclrMaskTy, "scalar_mask");
606 }
607
608 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
609 // Fill the "else" block, created in the previous iteration
610 //
611 // %Mask1 = and i16 %scalar_mask, i32 1 << Idx
612 // %cond = icmp ne i16 %mask_1, 0
613 // br i1 %Mask1, label %cond.load, label %else
614 //
615 // On GPUs, use
616 // %cond = extrectelement %mask, Idx
617 // instead
618
620 if (SclrMask != nullptr) {
621 Value *Mask = Builder.getInt(APInt::getOneBitSet(
622 VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
623 Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
624 Builder.getIntN(VectorWidth, 0));
625 } else {
626 Predicate = Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));
627 }
628
629 // Create "cond" block
630 //
631 // %EltAddr = getelementptr i32* %1, i32 0
632 // %Elt = load i32* %EltAddr
633 // VResult = insertelement <16 x i32> VResult, i32 %Elt, i32 Idx
634 //
635 // We mark the branch weights as explicitly unknown given they would only
636 // be derivable from the mask which we do not have VP information for.
637 Instruction *ThenTerm =
638 SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
640 *CI->getFunction(), DEBUG_TYPE),
641 DTU);
642
643 BasicBlock *CondBlock = ThenTerm->getParent();
644 CondBlock->setName("cond.load");
645
646 Builder.SetInsertPoint(CondBlock->getTerminator());
647 Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
648 LoadInst *Load =
649 Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
650 copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
651 /*ByteOffset=*/0);
652 Value *NewVResult =
653 Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
654
655 // Create "else" block, fill it in the next iteration
656 BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
657 NewIfBlock->setName("else");
658 BasicBlock *PrevIfBlock = IfBlock;
659 IfBlock = NewIfBlock;
660
661 // Create the phi to join the new and previous value.
662 Builder.SetInsertPoint(NewIfBlock->begin());
663 PHINode *Phi = Builder.CreatePHI(VecType, 2, "res.phi.else");
664 Phi->addIncoming(NewVResult, CondBlock);
665 Phi->addIncoming(VResult, PrevIfBlock);
666 VResult = Phi;
667 }
668
669 CI->replaceAllUsesWith(VResult);
670 CI->eraseFromParent();
671
672 ModifiedDT = true;
673}
674
675// Translate a masked scatter intrinsic, like
676// void @llvm.masked.scatter.v16i32(<16 x i32> %Src, <16 x i32*>* %Ptrs, i32 4,
677// <16 x i1> %Mask)
678// to a chain of basic blocks, that stores element one-by-one if
679// the appropriate mask bit is set.
680//
681// %Ptrs = getelementptr i32, i32* %ptr, <16 x i64> %ind
682// %Mask0 = extractelement <16 x i1> %Mask, i32 0
683// br i1 %Mask0, label %cond.store, label %else
684//
685// cond.store:
686// %Elt0 = extractelement <16 x i32> %Src, i32 0
687// %Ptr0 = extractelement <16 x i32*> %Ptrs, i32 0
688// store i32 %Elt0, i32* %Ptr0, align 4
689// br label %else
690//
691// else:
692// %Mask1 = extractelement <16 x i1> %Mask, i32 1
693// br i1 %Mask1, label %cond.store1, label %else2
694//
695// cond.store1:
696// %Elt1 = extractelement <16 x i32> %Src, i32 1
697// %Ptr1 = extractelement <16 x i32*> %Ptrs, i32 1
698// store i32 %Elt1, i32* %Ptr1, align 4
699// br label %else2
700// . . .
702 bool HasBranchDivergence, CallInst *CI,
703 DomTreeUpdater *DTU, bool &ModifiedDT) {
704 Value *Src = CI->getArgOperand(0);
705 Value *Ptrs = CI->getArgOperand(1);
706 Value *Mask = CI->getArgOperand(2);
707
708 auto *SrcFVTy = cast<FixedVectorType>(Src->getType());
709
710 assert(
711 isa<VectorType>(Ptrs->getType()) &&
712 isa<PointerType>(cast<VectorType>(Ptrs->getType())->getElementType()) &&
713 "Vector of pointers is expected in masked scatter intrinsic");
714
715 Instruction *InsertPt = CI;
716 IRBuilder<> Builder(InsertPt);
717
718 Align AlignVal = CI->getParamAlign(1).valueOrOne();
719 unsigned VectorWidth = SrcFVTy->getNumElements();
720
721 // Shorten the way if the mask is a vector of constants.
722 if (isConstantIntVector(Mask)) {
723 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
724 if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue())
725 continue;
726 Value *OneElt =
727 Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
728 Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
729 StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
731 /*SourcePtrOperand=*/1,
732 /*ByteOffset=*/0);
733 }
734 CI->eraseFromParent();
735 return;
736 }
737
738 // If the mask is not v1i1, use scalar bit test operations. This generates
739 // better results on X86 at least.
740 Value *SclrMask = nullptr;
741 if (VectorWidth != 1 && !HasBranchDivergence) {
742 Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);
743 SclrMask = Builder.CreateBitCast(Mask, SclrMaskTy, "scalar_mask");
744 }
745
746 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
747 // Fill the "else" block, created in the previous iteration
748 //
749 // %Mask1 = and i16 %scalar_mask, i32 1 << Idx
750 // %cond = icmp ne i16 %mask_1, 0
751 // br i1 %Mask1, label %cond.store, label %else
752 //
753 // On GPUs, use
754 // %cond = extrectelement %mask, Idx
755 // instead
757 if (SclrMask != nullptr) {
758 Value *Mask = Builder.getInt(APInt::getOneBitSet(
759 VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
760 Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
761 Builder.getIntN(VectorWidth, 0));
762 } else {
763 Predicate = Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));
764 }
765
766 // Create "cond" block
767 //
768 // %Elt1 = extractelement <16 x i32> %Src, i32 1
769 // %Ptr1 = extractelement <16 x i32*> %Ptrs, i32 1
770 // %store i32 %Elt1, i32* %Ptr1
771 //
772 // We mark the branch weights as explicitly unknown given they would only
773 // be derivable from the mask which we do not have VP information for.
774 Instruction *ThenTerm =
775 SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
777 *CI->getFunction(), DEBUG_TYPE),
778 DTU);
779
780 BasicBlock *CondBlock = ThenTerm->getParent();
781 CondBlock->setName("cond.store");
782
783 Builder.SetInsertPoint(CondBlock->getTerminator());
784 Value *OneElt = Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
785 Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
786 StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
788 /*SourcePtrOperand=*/1,
789 /*ByteOffset=*/0);
790
791 // Create "else" block, fill it in the next iteration
792 BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
793 NewIfBlock->setName("else");
794
795 Builder.SetInsertPoint(NewIfBlock->begin());
796 }
797 CI->eraseFromParent();
798
799 ModifiedDT = true;
800}
801
803 bool HasBranchDivergence, CallInst *CI,
804 DomTreeUpdater *DTU, bool &ModifiedDT) {
805 Value *Ptr = CI->getArgOperand(0);
806 Value *Mask = CI->getArgOperand(1);
807 Value *PassThru = CI->getArgOperand(2);
808 Align Alignment = CI->getParamAlign(0).valueOrOne();
809
810 auto *VecType = cast<FixedVectorType>(CI->getType());
811
812 Type *EltTy = VecType->getElementType();
813
814 Instruction *InsertPt = CI;
815 IRBuilder<> Builder(InsertPt);
816 Builder.disableGEPCanonicalization();
817 BasicBlock *IfBlock = CI->getParent();
818
819 unsigned VectorWidth = VecType->getNumElements();
820
821 // The result vector
822 Value *VResult = PassThru;
823
824 // Adjust alignment for the scalar instruction.
825 const Align AdjustedAlignment =
826 commonAlignment(Alignment, EltTy->getPrimitiveSizeInBits() / 8);
827
828 // Shorten the way if the mask is a vector of constants.
829 // Create a build_vector pattern, with loads/poisons as necessary and then
830 // shuffle blend with the pass through value.
831 if (isConstantIntVector(Mask)) {
832 unsigned MemIndex = 0;
833 VResult = PoisonValue::get(VecType);
834 SmallVector<int, 16> ShuffleMask(VectorWidth, PoisonMaskElem);
835 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
836 Value *InsertElt;
837 if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue()) {
838 InsertElt = PoisonValue::get(EltTy);
839 ShuffleMask[Idx] = Idx + VectorWidth;
840 } else {
841 Value *NewPtr =
842 Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);
843 LoadInst *Load = Builder.CreateAlignedLoad(
844 EltTy, NewPtr, AdjustedAlignment, "Load" + Twine(Idx));
846 *Load, *CI, DL, /*SourcePtrOperand=*/0,
847 MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
848 InsertElt = Load;
849 ShuffleMask[Idx] = Idx;
850 ++MemIndex;
851 }
852 VResult = Builder.CreateInsertElement(VResult, InsertElt, Idx,
853 "Res" + Twine(Idx));
854 }
855 VResult = Builder.CreateShuffleVector(VResult, PassThru, ShuffleMask);
856 CI->replaceAllUsesWith(VResult);
857 CI->eraseFromParent();
858 return;
859 }
860
861 // If the mask is not v1i1, use scalar bit test operations. This generates
862 // better results on X86 at least. However, don't do this on GPUs or other
863 // machines with branch divergence, as there, each i1 takes up a register.
864 Value *SclrMask = nullptr;
865 if (VectorWidth != 1 && !HasBranchDivergence) {
866 Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);
867 SclrMask = Builder.CreateBitCast(Mask, SclrMaskTy, "scalar_mask");
868 }
869
870 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
871 // Fill the "else" block, created in the previous iteration
872 //
873 // %res.phi.else3 = phi <16 x i32> [ %11, %cond.load1 ], [ %res.phi.else,
874 // %else ] %mask_1 = extractelement <16 x i1> %mask, i32 Idx br i1 %mask_1,
875 // label %cond.load, label %else
876 //
877 // On GPUs, use
878 // %cond = extrectelement %mask, Idx
879 // instead
880
882 if (SclrMask != nullptr) {
883 Value *Mask = Builder.getInt(APInt::getOneBitSet(
884 VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
885 Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
886 Builder.getIntN(VectorWidth, 0));
887 } else {
888 Predicate = Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));
889 }
890
891 // Create "cond" block
892 //
893 // %EltAddr = getelementptr i32* %1, i32 0
894 // %Elt = load i32* %EltAddr
895 // VResult = insertelement <16 x i32> VResult, i32 %Elt, i32 Idx
896 //
897 // We mark the branch weights as explicitly unknown given they would only
898 // be derivable from the mask which we do not have VP information for.
899 Instruction *ThenTerm =
900 SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
902 *CI->getFunction(), DEBUG_TYPE),
903 DTU);
904
905 BasicBlock *CondBlock = ThenTerm->getParent();
906 CondBlock->setName("cond.load");
907
908 Builder.SetInsertPoint(CondBlock->getTerminator());
909 LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Ptr, AdjustedAlignment);
910 copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
911 std::nullopt);
912 Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);
913
914 // Move the pointer if there are more blocks to come.
915 Value *NewPtr;
916 if ((Idx + 1) != VectorWidth)
917 NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, 1);
918
919 // Create "else" block, fill it in the next iteration
920 BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
921 NewIfBlock->setName("else");
922 BasicBlock *PrevIfBlock = IfBlock;
923 IfBlock = NewIfBlock;
924
925 // Create the phi to join the new and previous value.
926 Builder.SetInsertPoint(NewIfBlock->begin());
927 PHINode *ResultPhi = Builder.CreatePHI(VecType, 2, "res.phi.else");
928 ResultPhi->addIncoming(NewVResult, CondBlock);
929 ResultPhi->addIncoming(VResult, PrevIfBlock);
930 VResult = ResultPhi;
931
932 // Add a PHI for the pointer if this isn't the last iteration.
933 if ((Idx + 1) != VectorWidth) {
934 PHINode *PtrPhi = Builder.CreatePHI(Ptr->getType(), 2, "ptr.phi.else");
935 PtrPhi->addIncoming(NewPtr, CondBlock);
936 PtrPhi->addIncoming(Ptr, PrevIfBlock);
937 Ptr = PtrPhi;
938 }
939 }
940
941 CI->replaceAllUsesWith(VResult);
942 CI->eraseFromParent();
943
944 ModifiedDT = true;
945}
946
948 bool HasBranchDivergence, CallInst *CI,
949 DomTreeUpdater *DTU,
950 bool &ModifiedDT) {
951 Value *Src = CI->getArgOperand(0);
952 Value *Ptr = CI->getArgOperand(1);
953 Value *Mask = CI->getArgOperand(2);
954 Align Alignment = CI->getParamAlign(1).valueOrOne();
955
956 auto *VecType = cast<FixedVectorType>(Src->getType());
957
958 Instruction *InsertPt = CI;
959 IRBuilder<> Builder(InsertPt);
960 Builder.disableGEPCanonicalization();
961 BasicBlock *IfBlock = CI->getParent();
962
963 Type *EltTy = VecType->getElementType();
964
965 // Adjust alignment for the scalar instruction.
966 const Align AdjustedAlignment =
967 commonAlignment(Alignment, EltTy->getPrimitiveSizeInBits() / 8);
968
969 unsigned VectorWidth = VecType->getNumElements();
970
971 // Shorten the way if the mask is a vector of constants.
972 if (isConstantIntVector(Mask)) {
973 unsigned MemIndex = 0;
974 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
975 if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue())
976 continue;
977 Value *OneElt =
978 Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
979 Value *NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);
981 Builder.CreateAlignedStore(OneElt, NewPtr, AdjustedAlignment);
983 *Store, *CI, DL, /*SourcePtrOperand=*/1,
984 MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
985 ++MemIndex;
986 }
987 CI->eraseFromParent();
988 return;
989 }
990
991 // If the mask is not v1i1, use scalar bit test operations. This generates
992 // better results on X86 at least. However, don't do this on GPUs or other
993 // machines with branch divergence, as there, each i1 takes up a register.
994 Value *SclrMask = nullptr;
995 if (VectorWidth != 1 && !HasBranchDivergence) {
996 Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);
997 SclrMask = Builder.CreateBitCast(Mask, SclrMaskTy, "scalar_mask");
998 }
999
1000 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1001 // Fill the "else" block, created in the previous iteration
1002 //
1003 // %mask_1 = extractelement <16 x i1> %mask, i32 Idx
1004 // br i1 %mask_1, label %cond.store, label %else
1005 //
1006 // On GPUs, use
1007 // %cond = extrectelement %mask, Idx
1008 // instead
1010 if (SclrMask != nullptr) {
1011 Value *Mask = Builder.getInt(APInt::getOneBitSet(
1012 VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
1013 Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
1014 Builder.getIntN(VectorWidth, 0));
1015 } else {
1016 Predicate = Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));
1017 }
1018
1019 // Create "cond" block
1020 //
1021 // %OneElt = extractelement <16 x i32> %Src, i32 Idx
1022 // %EltAddr = getelementptr i32* %1, i32 0
1023 // %store i32 %OneElt, i32* %EltAddr
1024 //
1025 // We mark the branch weights as explicitly unknown given they would only
1026 // be derivable from the mask which we do not have VP information for.
1027 Instruction *ThenTerm =
1028 SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
1030 *CI->getFunction(), DEBUG_TYPE),
1031 DTU);
1032
1033 BasicBlock *CondBlock = ThenTerm->getParent();
1034 CondBlock->setName("cond.store");
1035
1036 Builder.SetInsertPoint(CondBlock->getTerminator());
1037 Value *OneElt = Builder.CreateExtractElement(Src, Idx);
1038 StoreInst *Store =
1039 Builder.CreateAlignedStore(OneElt, Ptr, AdjustedAlignment);
1040 copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
1041 std::nullopt);
1042
1043 // Move the pointer if there are more blocks to come.
1044 Value *NewPtr;
1045 if ((Idx + 1) != VectorWidth)
1046 NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, 1);
1047
1048 // Create "else" block, fill it in the next iteration
1049 BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
1050 NewIfBlock->setName("else");
1051 BasicBlock *PrevIfBlock = IfBlock;
1052 IfBlock = NewIfBlock;
1053
1054 Builder.SetInsertPoint(NewIfBlock->begin());
1055
1056 // Add a PHI for the pointer if this isn't the last iteration.
1057 if ((Idx + 1) != VectorWidth) {
1058 PHINode *PtrPhi = Builder.CreatePHI(Ptr->getType(), 2, "ptr.phi.else");
1059 PtrPhi->addIncoming(NewPtr, CondBlock);
1060 PtrPhi->addIncoming(Ptr, PrevIfBlock);
1061 Ptr = PtrPhi;
1062 }
1063 }
1064 CI->eraseFromParent();
1065
1066 ModifiedDT = true;
1067}
1068
1070 DomTreeUpdater *DTU,
1071 bool &ModifiedDT) {
1072 // If we extend histogram to return a result someday (like the updated vector)
1073 // then we'll need to support it here.
1074 assert(CI->getType()->isVoidTy() && "Histogram with non-void return.");
1075 Value *Ptrs = CI->getArgOperand(0);
1076 Value *Inc = CI->getArgOperand(1);
1077 Value *Mask = CI->getArgOperand(2);
1078
1079 auto *AddrType = cast<FixedVectorType>(Ptrs->getType());
1080 Type *EltTy = Inc->getType();
1081
1082 Instruction *InsertPt = CI;
1083 IRBuilder<> Builder(InsertPt);
1084
1085 // FIXME: Do we need to add an alignment parameter to the intrinsic?
1086 unsigned VectorWidth = AddrType->getNumElements();
1087 auto CreateHistogramUpdateValue = [&](IntrinsicInst *CI, Value *Load,
1088 Value *Inc) -> Value * {
1089 Value *UpdateOp;
1090 switch (CI->getIntrinsicID()) {
1091 case Intrinsic::experimental_vector_histogram_add:
1092 UpdateOp = Builder.CreateAdd(Load, Inc);
1093 break;
1094 case Intrinsic::experimental_vector_histogram_uadd_sat:
1095 UpdateOp =
1096 Builder.CreateIntrinsic(Intrinsic::uadd_sat, {EltTy}, {Load, Inc});
1097 break;
1098 case Intrinsic::experimental_vector_histogram_umin:
1099 UpdateOp = Builder.CreateIntrinsic(Intrinsic::umin, {EltTy}, {Load, Inc});
1100 break;
1101 case Intrinsic::experimental_vector_histogram_umax:
1102 UpdateOp = Builder.CreateIntrinsic(Intrinsic::umax, {EltTy}, {Load, Inc});
1103 break;
1104
1105 default:
1106 llvm_unreachable("Unexpected histogram intrinsic");
1107 }
1108 return UpdateOp;
1109 };
1110
1111 // Shorten the way if the mask is a vector of constants.
1112 if (isConstantIntVector(Mask)) {
1113 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1114 if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue())
1115 continue;
1116 Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
1117 LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
1118 copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
1119 /*ByteOffset=*/0);
1120 Value *Update =
1121 CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
1122 StoreInst *Store = Builder.CreateStore(Update, Ptr);
1124 /*SourcePtrOperand=*/0,
1125 /*ByteOffset=*/0);
1126 }
1127 CI->eraseFromParent();
1128 return;
1129 }
1130
1131 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1132 Value *Predicate =
1133 Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));
1134
1135 // We mark the branch weights as explicitly unknown given they would only
1136 // be derivable from the mask which we do not have VP information for.
1137 Instruction *ThenTerm =
1138 SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
1140 *CI->getFunction(), DEBUG_TYPE),
1141 DTU);
1142
1143 BasicBlock *CondBlock = ThenTerm->getParent();
1144 CondBlock->setName("cond.histogram.update");
1145
1146 Builder.SetInsertPoint(CondBlock->getTerminator());
1147 Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
1148 LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
1149 copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
1150 /*ByteOffset=*/0);
1151 Value *UpdateOp =
1152 CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
1153 StoreInst *Store = Builder.CreateStore(UpdateOp, Ptr);
1155 /*SourcePtrOperand=*/0,
1156 /*ByteOffset=*/0);
1157
1158 // Create "else" block, fill it in the next iteration
1159 BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
1160 NewIfBlock->setName("else");
1161 Builder.SetInsertPoint(NewIfBlock->begin());
1162 }
1163
1164 CI->eraseFromParent();
1165 ModifiedDT = true;
1166}
1167
1169 DominatorTree *DT) {
1170 std::optional<DomTreeUpdater> DTU;
1171 if (DT)
1172 DTU.emplace(DT, DomTreeUpdater::UpdateStrategy::Lazy);
1173
1174 bool EverMadeChange = false;
1175 bool MadeChange = true;
1176 auto &DL = F.getDataLayout();
1177 bool HasBranchDivergence = TTI.hasBranchDivergence(&F);
1178 while (MadeChange) {
1179 MadeChange = false;
1181 bool ModifiedDTOnIteration = false;
1182 MadeChange |= optimizeBlock(BB, ModifiedDTOnIteration, TTI, DL,
1183 HasBranchDivergence, DTU ? &*DTU : nullptr);
1184
1185 // Restart BB iteration if the dominator tree of the Function was changed
1186 if (ModifiedDTOnIteration)
1187 break;
1188 }
1189
1190 EverMadeChange |= MadeChange;
1191 }
1192 return EverMadeChange;
1193}
1194
1195bool ScalarizeMaskedMemIntrinLegacyPass::runOnFunction(Function &F) {
1196 auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
1197 DominatorTree *DT = nullptr;
1198 if (auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>())
1199 DT = &DTWP->getDomTree();
1200 return runImpl(F, TTI, DT);
1201}
1202
1203PreservedAnalyses
1214
1215static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT,
1216 const TargetTransformInfo &TTI, const DataLayout &DL,
1217 bool HasBranchDivergence, DomTreeUpdater *DTU) {
1218 bool MadeChange = false;
1219
1220 BasicBlock::iterator CurInstIterator = BB.begin();
1221 while (CurInstIterator != BB.end()) {
1222 if (CallInst *CI = dyn_cast<CallInst>(&*CurInstIterator++))
1223 MadeChange |=
1224 optimizeCallInst(CI, ModifiedDT, TTI, DL, HasBranchDivergence, DTU);
1225 if (ModifiedDT)
1226 return true;
1227 }
1228
1229 return MadeChange;
1230}
1231
1232static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
1233 const TargetTransformInfo &TTI,
1234 const DataLayout &DL, bool HasBranchDivergence,
1235 DomTreeUpdater *DTU) {
1237 if (II) {
1238 // The scalarization code below does not work for scalable vectors.
1239 if (isa<ScalableVectorType>(II->getType()) ||
1240 any_of(II->args(),
1241 [](Value *V) { return isa<ScalableVectorType>(V->getType()); }))
1242 return false;
1243 switch (II->getIntrinsicID()) {
1244 default:
1245 break;
1246 case Intrinsic::experimental_vector_histogram_add:
1247 case Intrinsic::experimental_vector_histogram_uadd_sat:
1248 case Intrinsic::experimental_vector_histogram_umin:
1249 case Intrinsic::experimental_vector_histogram_umax:
1250 if (TTI.isLegalMaskedVectorHistogram(CI->getArgOperand(0)->getType(),
1251 CI->getArgOperand(1)->getType()))
1252 return false;
1253 scalarizeMaskedVectorHistogram(DL, CI, DTU, ModifiedDT);
1254 return true;
1255 case Intrinsic::masked_load:
1256 // Scalarize unsupported vector masked load
1257 if (TTI.isLegalMaskedLoad(
1258 CI->getType(), CI->getParamAlign(0).valueOrOne(),
1260 ->getAddressSpace(),
1264 return false;
1265 scalarizeMaskedLoad(DL, HasBranchDivergence, CI, DTU, ModifiedDT);
1266 return true;
1267 case Intrinsic::masked_store:
1268 if (TTI.isLegalMaskedStore(
1269 CI->getArgOperand(0)->getType(),
1270 CI->getParamAlign(1).valueOrOne(),
1272 ->getAddressSpace(),
1276 return false;
1277 scalarizeMaskedStore(DL, HasBranchDivergence, CI, DTU, ModifiedDT);
1278 return true;
1279 case Intrinsic::masked_gather: {
1280 Align Alignment = CI->getParamAlign(0).valueOrOne();
1281 Type *LoadTy = CI->getType();
1282 if (TTI.isLegalMaskedGather(LoadTy, Alignment) &&
1283 !TTI.forceScalarizeMaskedGather(cast<VectorType>(LoadTy), Alignment))
1284 return false;
1285 scalarizeMaskedGather(DL, HasBranchDivergence, CI, DTU, ModifiedDT);
1286 return true;
1287 }
1288 case Intrinsic::masked_scatter: {
1289 Align Alignment = CI->getParamAlign(1).valueOrOne();
1290 Type *StoreTy = CI->getArgOperand(0)->getType();
1291 if (TTI.isLegalMaskedScatter(StoreTy, Alignment) &&
1292 !TTI.forceScalarizeMaskedScatter(cast<VectorType>(StoreTy),
1293 Alignment))
1294 return false;
1295 scalarizeMaskedScatter(DL, HasBranchDivergence, CI, DTU, ModifiedDT);
1296 return true;
1297 }
1298 case Intrinsic::masked_expandload:
1299 if (TTI.isLegalMaskedExpandLoad(
1300 CI->getType(),
1301 CI->getAttributes().getParamAttrs(0).getAlignment().valueOrOne()))
1302 return false;
1303 scalarizeMaskedExpandLoad(DL, HasBranchDivergence, CI, DTU, ModifiedDT);
1304 return true;
1305 case Intrinsic::masked_compressstore:
1306 if (TTI.isLegalMaskedCompressStore(
1307 CI->getArgOperand(0)->getType(),
1308 CI->getAttributes().getParamAttrs(1).getAlignment().valueOrOne()))
1309 return false;
1310 scalarizeMaskedCompressStore(DL, HasBranchDivergence, CI, DTU,
1311 ModifiedDT);
1312 return true;
1313 }
1314 }
1315
1316 return false;
1317}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static bool runImpl(MachineFunction &MF)
Definition CFIFixup.cpp:304
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool runOnFunction(Function &F, bool PostInlining)
#define DEBUG_TYPE
#define F(x, y, z)
Definition MD5.cpp:54
This file contains the declarations for metadata subclasses.
uint64_t IntrinsicInst * II
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
This file contains the declarations for profiling metadata utility functions.
static void scalarizeMaskedExpandLoad(const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void copyMetadataForScalarizedStore(StoreInst &Dest, const Instruction &Source, const DataLayout &DL, unsigned SourcePtrOperand, std::optional< size_t > ByteOffset)
static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, bool HasBranchDivergence, DomTreeUpdater *DTU)
static void scalarizeMaskedScatter(const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static unsigned adjustForEndian(const DataLayout &DL, unsigned VectorWidth, unsigned Idx)
static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, bool HasBranchDivergence, DomTreeUpdater *DTU)
static void copyMetadataForScalarizedLoad(LoadInst &Dest, const Instruction &Source, const DataLayout &DL, unsigned SourcePtrOperand, std::optional< size_t > ByteOffset)
static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void scalarizeMaskedCompressStore(const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void scalarizeMaskedGather(const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void copyMetadataForMemoryAccess(Instruction &Dest, const Instruction &Source, const DataLayout &DL, unsigned SourcePtrOperand, unsigned DestPtrOperand, Type *AccessType, bool IsWholeAccess, std::optional< size_t > ByteOffset)
static bool runImpl(Function &F, const TargetTransformInfo &TTI, DominatorTree *DT)
static bool isConstantIntVector(Value *Mask)
static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
This pass exposes codegen information to IR-level passes.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition APInt.h:235
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator end()
Definition BasicBlock.h:453
iterator begin()
Instruction iterator methods.
Definition BasicBlock.h:440
InstListType::iterator iterator
Instruction iterators...
Definition BasicBlock.h:170
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
Value * getArgOperand(unsigned i) const
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
AttributeList getAttributes() const
Return the attributes for this call.
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
Definition Constant.h:44
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
Analysis pass which computes a DominatorTree.
Definition Dominators.h:241
Legacy analysis pass which computes a DominatorTree.
Definition Dominators.h:277
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:312
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
LLVM_ABI void setAAMetadata(const AAMDNodes &N)
Sets the AA metadata on this instruction from the AAMDNodes structure.
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
LLVM_ABI BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
A wrapper class for inspecting calls to intrinsic functions.
An instruction for reading from memory.
static unsigned getPointerOperandIndex()
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static LLVM_ABI PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalyses & preserve()
Mark an analysis as preserved.
Definition Analysis.h:133
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Value * getValueOperand()
static unsigned getPointerOperandIndex()
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
Analysis pass providing the TargetTransformInfo.
Wrapper pass for TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:298
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:389
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:395
const ParentTy * getParent() const
Definition ilist_node.h:34
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
This is an optimization pass for GlobalISel generic memory operations.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI void setMemCacheHint(Instruction &I, unsigned OperandNo, MDNode *Hint)
Set the hint payload for operand OperandNo of I, preserving other operands' hints.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
LLVM_ABI FunctionPass * createScalarizeMaskedMemIntrinLegacyPass()
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
LLVM_ABI bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
LLVM_ABI MDNode * getMemCacheHint(const Instruction &I, unsigned OperandNo=0)
Return the hint payload for operand OperandNo of I, or nullptr if absent.
LLVM_ABI void initializeScalarizeMaskedMemIntrinLegacyPassPass(PassRegistry &)
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI MDNode * getExplicitlyUnknownBranchWeightsIfProfiled(Function &F, StringRef PassName)
Returns a metadata node containing unknown branch weights if the function has an entry count,...
constexpr int PoisonMaskElem
TargetTransformInfo TTI
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition Metadata.h:775
MDNode * TBAAStruct
The tag for type-based alias analysis (tbaa struct).
Definition Metadata.h:795
LLVM_ABI AAMDNodes adjustForAccess(unsigned AccessSize)
Create a new AAMDNode for accessing AccessSize bytes of this AAMDNode.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition Alignment.h:135
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)