LLVM 17.0.0git
IRBuilder.cpp
Go to the documentation of this file.
1//===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/IR/IRBuilder.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/IR/Constant.h"
17#include "llvm/IR/Constants.h"
20#include "llvm/IR/Function.h"
21#include "llvm/IR/GlobalValue.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/LLVMContext.h"
26#include "llvm/IR/NoFolder.h"
27#include "llvm/IR/Operator.h"
28#include "llvm/IR/Statepoint.h"
29#include "llvm/IR/Type.h"
30#include "llvm/IR/Value.h"
32#include <cassert>
33#include <cstdint>
34#include <optional>
35#include <vector>
36
37using namespace llvm;
38
39/// CreateGlobalString - Make a new global variable with an initializer that
40/// has array of i8 type filled in with the nul terminated string value
41/// specified. If Name is specified, it is the name of the global variable
42/// created.
44 const Twine &Name,
45 unsigned AddressSpace,
46 Module *M) {
48 if (!M)
49 M = BB->getParent()->getParent();
50 auto *GV = new GlobalVariable(
51 *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
52 StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
53 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
54 GV->setAlignment(Align(1));
55 return GV;
56}
57
59 assert(BB && BB->getParent() && "No current function!");
60 return BB->getParent()->getReturnType();
61}
62
63Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
64 auto *PT = cast<PointerType>(Ptr->getType());
65 if (PT->isOpaqueOrPointeeTypeMatches(getInt8Ty()))
66 return Ptr;
67
68 // Otherwise, we need to insert a bitcast.
69 return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace()));
70}
71
73 for (auto &KV : MetadataToCopy)
74 if (KV.first == LLVMContext::MD_dbg)
75 return {cast<DILocation>(KV.second)};
76
77 return {};
78}
80 for (const auto &KV : MetadataToCopy)
81 if (KV.first == LLVMContext::MD_dbg) {
82 I->setDebugLoc(DebugLoc(KV.second));
83 return;
84 }
85}
86
88IRBuilderBase::createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
89 const Twine &Name, Instruction *FMFSource,
91 CallInst *CI = CreateCall(Callee, Ops, OpBundles, Name);
92 if (FMFSource)
93 CI->copyFastMathFlags(FMFSource);
94 return CI;
95}
96
98 assert(isa<ConstantInt>(Scaling) && "Expected constant integer");
99 if (cast<ConstantInt>(Scaling)->isZero())
100 return Scaling;
102 Function *TheFn =
103 Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()});
104 CallInst *CI = CreateCall(TheFn, {}, {}, Name);
105 return cast<ConstantInt>(Scaling)->getSExtValue() == 1
106 ? CI
107 : CreateMul(CI, Scaling);
108}
109
111 Type *STy = DstType->getScalarType();
112 if (isa<ScalableVectorType>(DstType)) {
113 Type *StepVecType = DstType;
114 // TODO: We expect this special case (element type < 8 bits) to be
115 // temporary - once the intrinsic properly supports < 8 bits this code
116 // can be removed.
117 if (STy->getScalarSizeInBits() < 8)
118 StepVecType =
119 VectorType::get(getInt8Ty(), cast<ScalableVectorType>(DstType));
120 Value *Res = CreateIntrinsic(Intrinsic::experimental_stepvector,
121 {StepVecType}, {}, nullptr, Name);
122 if (StepVecType != DstType)
123 Res = CreateTrunc(Res, DstType);
124 return Res;
125 }
126
127 unsigned NumEls = cast<FixedVectorType>(DstType)->getNumElements();
128
129 // Create a vector of consecutive numbers from zero to VF.
131 for (unsigned i = 0; i < NumEls; ++i)
132 Indices.push_back(ConstantInt::get(STy, i));
133
134 // Add the consecutive indices to the vector value.
135 return ConstantVector::get(Indices);
136}
137
139 MaybeAlign Align, bool isVolatile,
140 MDNode *TBAATag, MDNode *ScopeTag,
141 MDNode *NoAliasTag) {
142 Ptr = getCastedInt8PtrValue(Ptr);
143 Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
144 Type *Tys[] = { Ptr->getType(), Size->getType() };
145 Module *M = BB->getParent()->getParent();
146 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
147
148 CallInst *CI = CreateCall(TheFn, Ops);
149
150 if (Align)
151 cast<MemSetInst>(CI)->setDestAlignment(*Align);
152
153 // Set the TBAA info if present.
154 if (TBAATag)
155 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
156
157 if (ScopeTag)
158 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
159
160 if (NoAliasTag)
161 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
162
163 return CI;
164}
165
167 Value *Val, Value *Size,
168 bool IsVolatile, MDNode *TBAATag,
169 MDNode *ScopeTag,
170 MDNode *NoAliasTag) {
171 Dst = getCastedInt8PtrValue(Dst);
172 Value *Ops[] = {Dst, Val, Size, getInt1(IsVolatile)};
173 Type *Tys[] = {Dst->getType(), Size->getType()};
174 Module *M = BB->getParent()->getParent();
175 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset_inline, Tys);
176
177 CallInst *CI = CreateCall(TheFn, Ops);
178
179 if (DstAlign)
180 cast<MemSetInlineInst>(CI)->setDestAlignment(*DstAlign);
181
182 // Set the TBAA info if present.
183 if (TBAATag)
184 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
185
186 if (ScopeTag)
187 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
188
189 if (NoAliasTag)
190 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
191
192 return CI;
193}
194
196 Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
197 MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
198
199 Ptr = getCastedInt8PtrValue(Ptr);
200 Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
201 Type *Tys[] = {Ptr->getType(), Size->getType()};
202 Module *M = BB->getParent()->getParent();
204 M, Intrinsic::memset_element_unordered_atomic, Tys);
205
206 CallInst *CI = CreateCall(TheFn, Ops);
207
208 cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
209
210 // Set the TBAA info if present.
211 if (TBAATag)
212 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
213
214 if (ScopeTag)
215 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
216
217 if (NoAliasTag)
218 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
219
220 return CI;
221}
222
224 Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
225 MaybeAlign SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag,
226 MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
227 Dst = getCastedInt8PtrValue(Dst);
228 Src = getCastedInt8PtrValue(Src);
229
230 Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
231 Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
232 Module *M = BB->getParent()->getParent();
233 Function *TheFn = Intrinsic::getDeclaration(M, IntrID, Tys);
234
235 CallInst *CI = CreateCall(TheFn, Ops);
236
237 auto* MCI = cast<MemTransferInst>(CI);
238 if (DstAlign)
239 MCI->setDestAlignment(*DstAlign);
240 if (SrcAlign)
241 MCI->setSourceAlignment(*SrcAlign);
242
243 // Set the TBAA info if present.
244 if (TBAATag)
245 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
246
247 // Set the TBAA Struct info if present.
248 if (TBAAStructTag)
249 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
250
251 if (ScopeTag)
252 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
253
254 if (NoAliasTag)
255 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
256
257 return CI;
258}
259
261 Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign,
262 Value *Size, bool IsVolatile, MDNode *TBAATag, MDNode *TBAAStructTag,
263 MDNode *ScopeTag, MDNode *NoAliasTag) {
264 Dst = getCastedInt8PtrValue(Dst);
265 Src = getCastedInt8PtrValue(Src);
266
267 Value *Ops[] = {Dst, Src, Size, getInt1(IsVolatile)};
268 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
269 Function *F = BB->getParent();
270 Module *M = F->getParent();
271 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy_inline, Tys);
272
273 CallInst *CI = CreateCall(TheFn, Ops);
274
275 auto *MCI = cast<MemCpyInlineInst>(CI);
276 if (DstAlign)
277 MCI->setDestAlignment(*DstAlign);
278 if (SrcAlign)
279 MCI->setSourceAlignment(*SrcAlign);
280
281 // Set the TBAA info if present.
282 if (TBAATag)
283 MCI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
284
285 // Set the TBAA Struct info if present.
286 if (TBAAStructTag)
287 MCI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
288
289 if (ScopeTag)
290 MCI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
291
292 if (NoAliasTag)
293 MCI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
294
295 return CI;
296}
297
299 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
300 uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
301 MDNode *ScopeTag, MDNode *NoAliasTag) {
302 assert(DstAlign >= ElementSize &&
303 "Pointer alignment must be at least element size");
304 assert(SrcAlign >= ElementSize &&
305 "Pointer alignment must be at least element size");
306 Dst = getCastedInt8PtrValue(Dst);
307 Src = getCastedInt8PtrValue(Src);
308
309 Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
310 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
311 Module *M = BB->getParent()->getParent();
313 M, Intrinsic::memcpy_element_unordered_atomic, Tys);
314
315 CallInst *CI = CreateCall(TheFn, Ops);
316
317 // Set the alignment of the pointer args.
318 auto *AMCI = cast<AtomicMemCpyInst>(CI);
319 AMCI->setDestAlignment(DstAlign);
320 AMCI->setSourceAlignment(SrcAlign);
321
322 // Set the TBAA info if present.
323 if (TBAATag)
324 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
325
326 // Set the TBAA Struct info if present.
327 if (TBAAStructTag)
328 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
329
330 if (ScopeTag)
331 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
332
333 if (NoAliasTag)
334 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
335
336 return CI;
337}
338
340 Value *Src, MaybeAlign SrcAlign,
341 Value *Size, bool isVolatile,
342 MDNode *TBAATag, MDNode *ScopeTag,
343 MDNode *NoAliasTag) {
344 Dst = getCastedInt8PtrValue(Dst);
345 Src = getCastedInt8PtrValue(Src);
346
347 Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
348 Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
349 Module *M = BB->getParent()->getParent();
350 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
351
352 CallInst *CI = CreateCall(TheFn, Ops);
353
354 auto *MMI = cast<MemMoveInst>(CI);
355 if (DstAlign)
356 MMI->setDestAlignment(*DstAlign);
357 if (SrcAlign)
358 MMI->setSourceAlignment(*SrcAlign);
359
360 // Set the TBAA info if present.
361 if (TBAATag)
362 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
363
364 if (ScopeTag)
365 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
366
367 if (NoAliasTag)
368 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
369
370 return CI;
371}
372
374 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
375 uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
376 MDNode *ScopeTag, MDNode *NoAliasTag) {
377 assert(DstAlign >= ElementSize &&
378 "Pointer alignment must be at least element size");
379 assert(SrcAlign >= ElementSize &&
380 "Pointer alignment must be at least element size");
381 Dst = getCastedInt8PtrValue(Dst);
382 Src = getCastedInt8PtrValue(Src);
383
384 Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
385 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
386 Module *M = BB->getParent()->getParent();
388 M, Intrinsic::memmove_element_unordered_atomic, Tys);
389
390 CallInst *CI = CreateCall(TheFn, Ops);
391
392 // Set the alignment of the pointer args.
393 CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
394 CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
395
396 // Set the TBAA info if present.
397 if (TBAATag)
398 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
399
400 // Set the TBAA Struct info if present.
401 if (TBAAStructTag)
402 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
403
404 if (ScopeTag)
405 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
406
407 if (NoAliasTag)
408 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
409
410 return CI;
411}
412
413CallInst *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
415 Value *Ops[] = {Src};
416 Type *Tys[] = { Src->getType() };
417 auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
418 return CreateCall(Decl, Ops);
419}
420
423 Value *Ops[] = {Acc, Src};
424 auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fadd,
425 {Src->getType()});
426 return CreateCall(Decl, Ops);
427}
428
431 Value *Ops[] = {Acc, Src};
432 auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fmul,
433 {Src->getType()});
434 return CreateCall(Decl, Ops);
435}
436
438 return getReductionIntrinsic(Intrinsic::vector_reduce_add, Src);
439}
440
442 return getReductionIntrinsic(Intrinsic::vector_reduce_mul, Src);
443}
444
446 return getReductionIntrinsic(Intrinsic::vector_reduce_and, Src);
447}
448
450 return getReductionIntrinsic(Intrinsic::vector_reduce_or, Src);
451}
452
454 return getReductionIntrinsic(Intrinsic::vector_reduce_xor, Src);
455}
456
458 auto ID =
459 IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
460 return getReductionIntrinsic(ID, Src);
461}
462
464 auto ID =
465 IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
466 return getReductionIntrinsic(ID, Src);
467}
468
470 return getReductionIntrinsic(Intrinsic::vector_reduce_fmax, Src);
471}
472
474 return getReductionIntrinsic(Intrinsic::vector_reduce_fmin, Src);
475}
476
478 assert(isa<PointerType>(Ptr->getType()) &&
479 "lifetime.start only applies to pointers.");
480 Ptr = getCastedInt8PtrValue(Ptr);
481 if (!Size)
482 Size = getInt64(-1);
483 else
484 assert(Size->getType() == getInt64Ty() &&
485 "lifetime.start requires the size to be an i64");
486 Value *Ops[] = { Size, Ptr };
487 Module *M = BB->getParent()->getParent();
488 Function *TheFn =
489 Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
490 return CreateCall(TheFn, Ops);
491}
492
494 assert(isa<PointerType>(Ptr->getType()) &&
495 "lifetime.end only applies to pointers.");
496 Ptr = getCastedInt8PtrValue(Ptr);
497 if (!Size)
498 Size = getInt64(-1);
499 else
500 assert(Size->getType() == getInt64Ty() &&
501 "lifetime.end requires the size to be an i64");
502 Value *Ops[] = { Size, Ptr };
503 Module *M = BB->getParent()->getParent();
504 Function *TheFn =
505 Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
506 return CreateCall(TheFn, Ops);
507}
508
510
511 assert(isa<PointerType>(Ptr->getType()) &&
512 "invariant.start only applies to pointers.");
513 Ptr = getCastedInt8PtrValue(Ptr);
514 if (!Size)
515 Size = getInt64(-1);
516 else
517 assert(Size->getType() == getInt64Ty() &&
518 "invariant.start requires the size to be an i64");
519
520 Value *Ops[] = {Size, Ptr};
521 // Fill in the single overloaded type: memory object type.
522 Type *ObjectPtr[1] = {Ptr->getType()};
523 Module *M = BB->getParent()->getParent();
524 Function *TheFn =
525 Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
526 return CreateCall(TheFn, Ops);
527}
528
530 if (auto *O = dyn_cast<GlobalObject>(Ptr))
531 return O->getAlign();
532 if (auto *A = dyn_cast<GlobalAlias>(Ptr))
533 return A->getAliaseeObject()->getAlign();
534 return {};
535}
536
538#ifndef NDEBUG
539 // Handle specially for constexpr cast. This is possible when
540 // opaque pointers not enabled since constant could be sinked
541 // directly by the design of llvm. This could be eliminated
542 // after we eliminate the abuse of constexpr.
543 auto *V = Ptr;
544 if (auto *CE = dyn_cast<ConstantExpr>(V))
545 if (CE->isCast())
546 V = CE->getOperand(0);
547
548 assert(isa<GlobalValue>(V) && cast<GlobalValue>(V)->isThreadLocal() &&
549 "threadlocal_address only applies to thread local variables.");
550#endif
551 CallInst *CI = CreateIntrinsic(llvm::Intrinsic::threadlocal_address,
552 {Ptr->getType()}, {Ptr});
553 if (MaybeAlign A = getAlign(Ptr)) {
556 }
557 return CI;
558}
559
560CallInst *
562 ArrayRef<OperandBundleDef> OpBundles) {
563 assert(Cond->getType() == getInt1Ty() &&
564 "an assumption condition must be of type i1");
565
566 Value *Ops[] = { Cond };
567 Module *M = BB->getParent()->getParent();
568 Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
569 return CreateCall(FnAssume, Ops, OpBundles);
570}
571
573 Module *M = BB->getModule();
574 auto *FnIntrinsic = Intrinsic::getDeclaration(
575 M, Intrinsic::experimental_noalias_scope_decl, {});
576 return CreateCall(FnIntrinsic, {Scope});
577}
578
579/// Create a call to a Masked Load intrinsic.
580/// \p Ty - vector type to load
581/// \p Ptr - base pointer for the load
582/// \p Alignment - alignment of the source location
583/// \p Mask - vector of booleans which indicates what vector lanes should
584/// be accessed in memory
585/// \p PassThru - pass-through value that is used to fill the masked-off lanes
586/// of the result
587/// \p Name - name of the result variable
589 Value *Mask, Value *PassThru,
590 const Twine &Name) {
591 auto *PtrTy = cast<PointerType>(Ptr->getType());
592 assert(Ty->isVectorTy() && "Type should be vector");
593 assert(PtrTy->isOpaqueOrPointeeTypeMatches(Ty) && "Wrong element type");
594 assert(Mask && "Mask should not be all-ones (null)");
595 if (!PassThru)
596 PassThru = PoisonValue::get(Ty);
597 Type *OverloadedTypes[] = { Ty, PtrTy };
598 Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
599 return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
600 OverloadedTypes, Name);
601}
602
603/// Create a call to a Masked Store intrinsic.
604/// \p Val - data to be stored,
605/// \p Ptr - base pointer for the store
606/// \p Alignment - alignment of the destination location
607/// \p Mask - vector of booleans which indicates what vector lanes should
608/// be accessed in memory
610 Align Alignment, Value *Mask) {
611 auto *PtrTy = cast<PointerType>(Ptr->getType());
612 Type *DataTy = Val->getType();
613 assert(DataTy->isVectorTy() && "Val should be a vector");
614 assert(PtrTy->isOpaqueOrPointeeTypeMatches(DataTy) && "Wrong element type");
615 assert(Mask && "Mask should not be all-ones (null)");
616 Type *OverloadedTypes[] = { DataTy, PtrTy };
617 Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask};
618 return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
619}
620
621/// Create a call to a Masked intrinsic, with given intrinsic Id,
622/// an array of operands - Ops, and an array of overloaded types -
623/// OverloadedTypes.
624CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
626 ArrayRef<Type *> OverloadedTypes,
627 const Twine &Name) {
628 Module *M = BB->getParent()->getParent();
629 Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
630 return CreateCall(TheFn, Ops, {}, Name);
631}
632
633/// Create a call to a Masked Gather intrinsic.
634/// \p Ty - vector type to gather
635/// \p Ptrs - vector of pointers for loading
636/// \p Align - alignment for one element
637/// \p Mask - vector of booleans which indicates what vector lanes should
638/// be accessed in memory
639/// \p PassThru - pass-through value that is used to fill the masked-off lanes
640/// of the result
641/// \p Name - name of the result variable
643 Align Alignment, Value *Mask,
644 Value *PassThru,
645 const Twine &Name) {
646 auto *VecTy = cast<VectorType>(Ty);
647 ElementCount NumElts = VecTy->getElementCount();
648 auto *PtrsTy = cast<VectorType>(Ptrs->getType());
649 assert(cast<PointerType>(PtrsTy->getElementType())
650 ->isOpaqueOrPointeeTypeMatches(
651 cast<VectorType>(Ty)->getElementType()) &&
652 "Element type mismatch");
653 assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");
654
655 if (!Mask)
658
659 if (!PassThru)
660 PassThru = PoisonValue::get(Ty);
661
662 Type *OverloadedTypes[] = {Ty, PtrsTy};
663 Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
664
665 // We specify only one type when we create this intrinsic. Types of other
666 // arguments are derived from this type.
667 return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
668 Name);
669}
670
671/// Create a call to a Masked Scatter intrinsic.
672/// \p Data - data to be stored,
673/// \p Ptrs - the vector of pointers, where the \p Data elements should be
674/// stored
675/// \p Align - alignment for one element
676/// \p Mask - vector of booleans which indicates what vector lanes should
677/// be accessed in memory
679 Align Alignment, Value *Mask) {
680 auto *PtrsTy = cast<VectorType>(Ptrs->getType());
681 auto *DataTy = cast<VectorType>(Data->getType());
682 ElementCount NumElts = PtrsTy->getElementCount();
683
684#ifndef NDEBUG
685 auto *PtrTy = cast<PointerType>(PtrsTy->getElementType());
686 assert(NumElts == DataTy->getElementCount() &&
687 PtrTy->isOpaqueOrPointeeTypeMatches(DataTy->getElementType()) &&
688 "Incompatible pointer and data types");
689#endif
690
691 if (!Mask)
694
695 Type *OverloadedTypes[] = {DataTy, PtrsTy};
696 Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask};
697
698 // We specify only one type when we create this intrinsic. Types of other
699 // arguments are derived from this type.
700 return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
701}
702
703/// Create a call to Masked Expand Load intrinsic
704/// \p Ty - vector type to load
705/// \p Ptr - base pointer for the load
706/// \p Mask - vector of booleans which indicates what vector lanes should
707/// be accessed in memory
708/// \p PassThru - pass-through value that is used to fill the masked-off lanes
709/// of the result
710/// \p Name - name of the result variable
712 Value *Mask, Value *PassThru,
713 const Twine &Name) {
714 auto *PtrTy = cast<PointerType>(Ptr->getType());
715 assert(Ty->isVectorTy() && "Type should be vector");
716 assert(PtrTy->isOpaqueOrPointeeTypeMatches(
717 cast<FixedVectorType>(Ty)->getElementType()) &&
718 "Wrong element type");
719 (void)PtrTy;
720 assert(Mask && "Mask should not be all-ones (null)");
721 if (!PassThru)
722 PassThru = PoisonValue::get(Ty);
723 Type *OverloadedTypes[] = {Ty};
724 Value *Ops[] = {Ptr, Mask, PassThru};
725 return CreateMaskedIntrinsic(Intrinsic::masked_expandload, Ops,
726 OverloadedTypes, Name);
727}
728
729/// Create a call to Masked Compress Store intrinsic
730/// \p Val - data to be stored,
731/// \p Ptr - base pointer for the store
732/// \p Mask - vector of booleans which indicates what vector lanes should
733/// be accessed in memory
735 Value *Mask) {
736 auto *PtrTy = cast<PointerType>(Ptr->getType());
737 Type *DataTy = Val->getType();
738 assert(DataTy->isVectorTy() && "Val should be a vector");
739 assert(PtrTy->isOpaqueOrPointeeTypeMatches(
740 cast<FixedVectorType>(DataTy)->getElementType()) &&
741 "Wrong element type");
742 (void)PtrTy;
743 assert(Mask && "Mask should not be all-ones (null)");
744 Type *OverloadedTypes[] = {DataTy};
745 Value *Ops[] = {Val, Ptr, Mask};
746 return CreateMaskedIntrinsic(Intrinsic::masked_compressstore, Ops,
747 OverloadedTypes);
748}
749
750template <typename T0>
751static std::vector<Value *>
753 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
754 std::vector<Value *> Args;
755 Args.push_back(B.getInt64(ID));
756 Args.push_back(B.getInt32(NumPatchBytes));
757 Args.push_back(ActualCallee);
758 Args.push_back(B.getInt32(CallArgs.size()));
759 Args.push_back(B.getInt32(Flags));
760 llvm::append_range(Args, CallArgs);
761 // GC Transition and Deopt args are now always handled via operand bundle.
762 // They will be removed from the signature of gc.statepoint shortly.
763 Args.push_back(B.getInt32(0));
764 Args.push_back(B.getInt32(0));
765 // GC args are now encoded in the gc-live operand bundle
766 return Args;
767}
768
769template<typename T1, typename T2, typename T3>
770static std::vector<OperandBundleDef>
771getStatepointBundles(std::optional<ArrayRef<T1>> TransitionArgs,
772 std::optional<ArrayRef<T2>> DeoptArgs,
773 ArrayRef<T3> GCArgs) {
774 std::vector<OperandBundleDef> Rval;
775 if (DeoptArgs) {
776 SmallVector<Value*, 16> DeoptValues;
777 llvm::append_range(DeoptValues, *DeoptArgs);
778 Rval.emplace_back("deopt", DeoptValues);
779 }
780 if (TransitionArgs) {
781 SmallVector<Value*, 16> TransitionValues;
782 llvm::append_range(TransitionValues, *TransitionArgs);
783 Rval.emplace_back("gc-transition", TransitionValues);
784 }
785 if (GCArgs.size()) {
786 SmallVector<Value*, 16> LiveValues;
787 llvm::append_range(LiveValues, GCArgs);
788 Rval.emplace_back("gc-live", LiveValues);
789 }
790 return Rval;
791}
792
793template <typename T0, typename T1, typename T2, typename T3>
795 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
796 FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
797 std::optional<ArrayRef<T1>> TransitionArgs,
798 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
799 const Twine &Name) {
800 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
801 // Fill in the one generic type'd argument (the function is also vararg)
802 Function *FnStatepoint =
803 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
804 {ActualCallee.getCallee()->getType()});
805
806 std::vector<Value *> Args = getStatepointArgs(
807 *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs);
808
809 CallInst *CI = Builder->CreateCall(
810 FnStatepoint, Args,
811 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
812 CI->addParamAttr(2,
813 Attribute::get(Builder->getContext(), Attribute::ElementType,
814 ActualCallee.getFunctionType()));
815 return CI;
816}
817
819 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
820 ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
821 ArrayRef<Value *> GCArgs, const Twine &Name) {
822 return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
823 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
824 CallArgs, std::nullopt /* No Transition Args */, DeoptArgs, GCArgs, Name);
825}
826
828 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
829 uint32_t Flags, ArrayRef<Value *> CallArgs,
830 std::optional<ArrayRef<Use>> TransitionArgs,
831 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
832 const Twine &Name) {
833 return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
834 this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
835 DeoptArgs, GCArgs, Name);
836}
837
839 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
840 ArrayRef<Use> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
841 ArrayRef<Value *> GCArgs, const Twine &Name) {
842 return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
843 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
844 CallArgs, std::nullopt, DeoptArgs, GCArgs, Name);
845}
846
847template <typename T0, typename T1, typename T2, typename T3>
849 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
850 FunctionCallee ActualInvokee, BasicBlock *NormalDest,
851 BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<T0> InvokeArgs,
852 std::optional<ArrayRef<T1>> TransitionArgs,
853 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
854 const Twine &Name) {
855 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
856 // Fill in the one generic type'd argument (the function is also vararg)
857 Function *FnStatepoint =
858 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
859 {ActualInvokee.getCallee()->getType()});
860
861 std::vector<Value *> Args =
862 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(),
863 Flags, InvokeArgs);
864
865 InvokeInst *II = Builder->CreateInvoke(
866 FnStatepoint, NormalDest, UnwindDest, Args,
867 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
868 II->addParamAttr(2,
869 Attribute::get(Builder->getContext(), Attribute::ElementType,
870 ActualInvokee.getFunctionType()));
871 return II;
872}
873
875 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
876 BasicBlock *NormalDest, BasicBlock *UnwindDest,
877 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
878 ArrayRef<Value *> GCArgs, const Twine &Name) {
879 return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
880 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
881 uint32_t(StatepointFlags::None), InvokeArgs,
882 std::nullopt /* No Transition Args*/, DeoptArgs, GCArgs, Name);
883}
884
886 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
887 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
888 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Use>> TransitionArgs,
889 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
890 const Twine &Name) {
891 return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
892 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
893 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
894}
895
897 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
898 BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
899 std::optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs,
900 const Twine &Name) {
901 return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
902 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
903 uint32_t(StatepointFlags::None), InvokeArgs, std::nullopt, DeoptArgs,
904 GCArgs, Name);
905}
906
908 Type *ResultType, const Twine &Name) {
909 Intrinsic::ID ID = Intrinsic::experimental_gc_result;
910 Module *M = BB->getParent()->getParent();
911 Type *Types[] = {ResultType};
912 Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
913
914 Value *Args[] = {Statepoint};
915 return CreateCall(FnGCResult, Args, {}, Name);
916}
917
919 int BaseOffset, int DerivedOffset,
920 Type *ResultType, const Twine &Name) {
921 Module *M = BB->getParent()->getParent();
922 Type *Types[] = {ResultType};
923 Function *FnGCRelocate =
924 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
925
926 Value *Args[] = {Statepoint, getInt32(BaseOffset), getInt32(DerivedOffset)};
927 return CreateCall(FnGCRelocate, Args, {}, Name);
928}
929
931 const Twine &Name) {
932 Module *M = BB->getParent()->getParent();
933 Type *PtrTy = DerivedPtr->getType();
934 Function *FnGCFindBase = Intrinsic::getDeclaration(
935 M, Intrinsic::experimental_gc_get_pointer_base, {PtrTy, PtrTy});
936 return CreateCall(FnGCFindBase, {DerivedPtr}, {}, Name);
937}
938
940 const Twine &Name) {
941 Module *M = BB->getParent()->getParent();
942 Type *PtrTy = DerivedPtr->getType();
943 Function *FnGCGetOffset = Intrinsic::getDeclaration(
944 M, Intrinsic::experimental_gc_get_pointer_offset, {PtrTy});
945 return CreateCall(FnGCGetOffset, {DerivedPtr}, {}, Name);
946}
947
949 Instruction *FMFSource,
950 const Twine &Name) {
951 Module *M = BB->getModule();
953 return createCallHelper(Fn, {V}, Name, FMFSource);
954}
955
957 Value *RHS,
958 Instruction *FMFSource,
959 const Twine &Name) {
960 Module *M = BB->getModule();
962 return createCallHelper(Fn, {LHS, RHS}, Name, FMFSource);
963}
964
966 ArrayRef<Type *> Types,
968 Instruction *FMFSource,
969 const Twine &Name) {
970 Module *M = BB->getModule();
971 Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
972 return createCallHelper(Fn, Args, Name, FMFSource);
973}
974
977 Instruction *FMFSource,
978 const Twine &Name) {
979 Module *M = BB->getModule();
980
984
985 SmallVector<Type *> ArgTys;
986 ArgTys.reserve(Args.size());
987 for (auto &I : Args)
988 ArgTys.push_back(I->getType());
989 FunctionType *FTy = FunctionType::get(RetTy, ArgTys, false);
990 SmallVector<Type *> OverloadTys;
992 matchIntrinsicSignature(FTy, TableRef, OverloadTys);
993 (void)Res;
995 "Wrong types for intrinsic!");
996 // TODO: Handle varargs intrinsics.
997
998 Function *Fn = Intrinsic::getDeclaration(M, ID, OverloadTys);
999 return createCallHelper(Fn, Args, Name, FMFSource);
1000}
1001
1003 Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource,
1004 const Twine &Name, MDNode *FPMathTag,
1005 std::optional<RoundingMode> Rounding,
1006 std::optional<fp::ExceptionBehavior> Except) {
1007 Value *RoundingV = getConstrainedFPRounding(Rounding);
1008 Value *ExceptV = getConstrainedFPExcept(Except);
1009
1010 FastMathFlags UseFMF = FMF;
1011 if (FMFSource)
1012 UseFMF = FMFSource->getFastMathFlags();
1013
1014 CallInst *C = CreateIntrinsic(ID, {L->getType()},
1015 {L, R, RoundingV, ExceptV}, nullptr, Name);
1017 setFPAttrs(C, FPMathTag, UseFMF);
1018 return C;
1019}
1020
1022 const Twine &Name, MDNode *FPMathTag) {
1023 if (Instruction::isBinaryOp(Opc)) {
1024 assert(Ops.size() == 2 && "Invalid number of operands!");
1025 return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc),
1026 Ops[0], Ops[1], Name, FPMathTag);
1027 }
1028 if (Instruction::isUnaryOp(Opc)) {
1029 assert(Ops.size() == 1 && "Invalid number of operands!");
1030 return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc),
1031 Ops[0], Name, FPMathTag);
1032 }
1033 llvm_unreachable("Unexpected opcode!");
1034}
1035
1037 Intrinsic::ID ID, Value *V, Type *DestTy,
1038 Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag,
1039 std::optional<RoundingMode> Rounding,
1040 std::optional<fp::ExceptionBehavior> Except) {
1041 Value *ExceptV = getConstrainedFPExcept(Except);
1042
1043 FastMathFlags UseFMF = FMF;
1044 if (FMFSource)
1045 UseFMF = FMFSource->getFastMathFlags();
1046
1047 CallInst *C;
1048 bool HasRoundingMD = false;
1049 switch (ID) {
1050 default:
1051 break;
1052#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
1053 case Intrinsic::INTRINSIC: \
1054 HasRoundingMD = ROUND_MODE; \
1055 break;
1056#include "llvm/IR/ConstrainedOps.def"
1057 }
1058 if (HasRoundingMD) {
1059 Value *RoundingV = getConstrainedFPRounding(Rounding);
1060 C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
1061 nullptr, Name);
1062 } else
1063 C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,
1064 Name);
1065
1067
1068 if (isa<FPMathOperator>(C))
1069 setFPAttrs(C, FPMathTag, UseFMF);
1070 return C;
1071}
1072
1073Value *IRBuilderBase::CreateFCmpHelper(
1074 CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name,
1075 MDNode *FPMathTag, bool IsSignaling) {
1076 if (IsFPConstrained) {
1077 auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
1078 : Intrinsic::experimental_constrained_fcmp;
1079 return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name);
1080 }
1081
1082 if (auto *LC = dyn_cast<Constant>(LHS))
1083 if (auto *RC = dyn_cast<Constant>(RHS))
1084 return Insert(Folder.CreateFCmp(P, LC, RC), Name);
1085 return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name);
1086}
1087
1090 const Twine &Name, std::optional<fp::ExceptionBehavior> Except) {
1091 Value *PredicateV = getConstrainedFPPredicate(P);
1092 Value *ExceptV = getConstrainedFPExcept(Except);
1093
1094 CallInst *C = CreateIntrinsic(ID, {L->getType()},
1095 {L, R, PredicateV, ExceptV}, nullptr, Name);
1097 return C;
1098}
1099
1102 std::optional<RoundingMode> Rounding,
1103 std::optional<fp::ExceptionBehavior> Except) {
1105
1106 append_range(UseArgs, Args);
1107 bool HasRoundingMD = false;
1108 switch (Callee->getIntrinsicID()) {
1109 default:
1110 break;
1111#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
1112 case Intrinsic::INTRINSIC: \
1113 HasRoundingMD = ROUND_MODE; \
1114 break;
1115#include "llvm/IR/ConstrainedOps.def"
1116 }
1117 if (HasRoundingMD)
1118 UseArgs.push_back(getConstrainedFPRounding(Rounding));
1119 UseArgs.push_back(getConstrainedFPExcept(Except));
1120
1121 CallInst *C = CreateCall(Callee, UseArgs, Name);
1123 return C;
1124}
1125
1127 const Twine &Name, Instruction *MDFrom) {
1128 if (auto *V = Folder.FoldSelect(C, True, False))
1129 return V;
1130
1131 SelectInst *Sel = SelectInst::Create(C, True, False);
1132 if (MDFrom) {
1133 MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof);
1134 MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable);
1135 Sel = addBranchMetadata(Sel, Prof, Unpred);
1136 }
1137 if (isa<FPMathOperator>(Sel))
1138 setFPAttrs(Sel, nullptr /* MDNode* */, FMF);
1139 return Insert(Sel, Name);
1140}
1141
1143 const Twine &Name) {
1144 assert(LHS->getType() == RHS->getType() &&
1145 "Pointer subtraction operand types must match!");
1146 assert(cast<PointerType>(LHS->getType())
1147 ->isOpaqueOrPointeeTypeMatches(ElemTy) &&
1148 "Pointer type must match element type");
1151 Value *Difference = CreateSub(LHS_int, RHS_int);
1152 return CreateExactSDiv(Difference, ConstantExpr::getSizeOf(ElemTy),
1153 Name);
1154}
1155
1157 assert(isa<PointerType>(Ptr->getType()) &&
1158 "launder.invariant.group only applies to pointers.");
1159 // FIXME: we could potentially avoid casts to/from i8*.
1160 auto *PtrType = Ptr->getType();
1161 auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
1162 if (PtrType != Int8PtrTy)
1163 Ptr = CreateBitCast(Ptr, Int8PtrTy);
1164 Module *M = BB->getParent()->getParent();
1165 Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration(
1166 M, Intrinsic::launder_invariant_group, {Int8PtrTy});
1167
1168 assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy &&
1169 FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
1170 Int8PtrTy &&
1171 "LaunderInvariantGroup should take and return the same type");
1172
1173 CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr});
1174
1175 if (PtrType != Int8PtrTy)
1176 return CreateBitCast(Fn, PtrType);
1177 return Fn;
1178}
1179
1181 assert(isa<PointerType>(Ptr->getType()) &&
1182 "strip.invariant.group only applies to pointers.");
1183
1184 // FIXME: we could potentially avoid casts to/from i8*.
1185 auto *PtrType = Ptr->getType();
1186 auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
1187 if (PtrType != Int8PtrTy)
1188 Ptr = CreateBitCast(Ptr, Int8PtrTy);
1189 Module *M = BB->getParent()->getParent();
1190 Function *FnStripInvariantGroup = Intrinsic::getDeclaration(
1191 M, Intrinsic::strip_invariant_group, {Int8PtrTy});
1192
1193 assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy &&
1194 FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
1195 Int8PtrTy &&
1196 "StripInvariantGroup should take and return the same type");
1197
1198 CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr});
1199
1200 if (PtrType != Int8PtrTy)
1201 return CreateBitCast(Fn, PtrType);
1202 return Fn;
1203}
1204
1206 auto *Ty = cast<VectorType>(V->getType());
1207 if (isa<ScalableVectorType>(Ty)) {
1208 Module *M = BB->getParent()->getParent();
1210 M, Intrinsic::experimental_vector_reverse, Ty);
1211 return Insert(CallInst::Create(F, V), Name);
1212 }
1213 // Keep the original behaviour for fixed vector
1214 SmallVector<int, 8> ShuffleMask;
1215 int NumElts = Ty->getElementCount().getKnownMinValue();
1216 for (int i = 0; i < NumElts; ++i)
1217 ShuffleMask.push_back(NumElts - i - 1);
1218 return CreateShuffleVector(V, ShuffleMask, Name);
1219}
1220
1222 const Twine &Name) {
1223 assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1224 assert(V1->getType() == V2->getType() &&
1225 "Splice expects matching operand types!");
1226
1227 if (auto *VTy = dyn_cast<ScalableVectorType>(V1->getType())) {
1228 Module *M = BB->getParent()->getParent();
1230 M, Intrinsic::experimental_vector_splice, VTy);
1231
1232 Value *Ops[] = {V1, V2, getInt32(Imm)};
1233 return Insert(CallInst::Create(F, Ops), Name);
1234 }
1235
1236 unsigned NumElts = cast<FixedVectorType>(V1->getType())->getNumElements();
1237 assert(((-Imm <= NumElts) || (Imm < NumElts)) &&
1238 "Invalid immediate for vector splice!");
1239
1240 // Keep the original behaviour for fixed vector
1241 unsigned Idx = (NumElts + Imm) % NumElts;
1243 for (unsigned I = 0; I < NumElts; ++I)
1244 Mask.push_back(Idx + I);
1245
1246 return CreateShuffleVector(V1, V2, Mask);
1247}
1248
1250 const Twine &Name) {
1251 auto EC = ElementCount::getFixed(NumElts);
1252 return CreateVectorSplat(EC, V, Name);
1253}
1254
1256 const Twine &Name) {
1257 assert(EC.isNonZero() && "Cannot splat to an empty vector!");
1258
1259 // First insert it into a poison vector so we can shuffle it.
1260 Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));
1261 V = CreateInsertElement(Poison, V, getInt64(0), Name + ".splatinsert");
1262
1263 // Shuffle the value across the desired number of elements.
1265 Zeros.resize(EC.getKnownMinValue());
1266 return CreateShuffleVector(V, Zeros, Name + ".splat");
1267}
1268
1270 const DataLayout &DL, Value *From, IntegerType *ExtractedTy,
1271 uint64_t Offset, const Twine &Name) {
1272 auto *IntTy = cast<IntegerType>(From->getType());
1273 assert(DL.getTypeStoreSize(ExtractedTy) + Offset <=
1274 DL.getTypeStoreSize(IntTy) &&
1275 "Element extends past full value");
1276 uint64_t ShAmt = 8 * Offset;
1277 Value *V = From;
1278 if (DL.isBigEndian())
1279 ShAmt = 8 * (DL.getTypeStoreSize(IntTy) -
1280 DL.getTypeStoreSize(ExtractedTy) - Offset);
1281 if (ShAmt) {
1282 V = CreateLShr(V, ShAmt, Name + ".shift");
1283 }
1284 assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() &&
1285 "Cannot extract to a larger integer!");
1286 if (ExtractedTy != IntTy) {
1287 V = CreateTrunc(V, ExtractedTy, Name + ".trunc");
1288 }
1289 return V;
1290}
1291
1293 Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex,
1294 MDNode *DbgInfo) {
1295 auto *BaseType = Base->getType();
1296 assert(isa<PointerType>(BaseType) &&
1297 "Invalid Base ptr type for preserve.array.access.index.");
1298 assert(cast<PointerType>(BaseType)->isOpaqueOrPointeeTypeMatches(ElTy) &&
1299 "Pointer element type mismatch");
1300
1301 Value *LastIndexV = getInt32(LastIndex);
1303 SmallVector<Value *, 4> IdxList(Dimension, Zero);
1304 IdxList.push_back(LastIndexV);
1305
1306 Type *ResultType =
1308
1309 Module *M = BB->getParent()->getParent();
1310 Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration(
1311 M, Intrinsic::preserve_array_access_index, {ResultType, BaseType});
1312
1313 Value *DimV = getInt32(Dimension);
1314 CallInst *Fn =
1315 CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV});
1316 Fn->addParamAttr(
1317 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
1318 if (DbgInfo)
1319 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1320
1321 return Fn;
1322}
1323
1325 Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1326 assert(isa<PointerType>(Base->getType()) &&
1327 "Invalid Base ptr type for preserve.union.access.index.");
1328 auto *BaseType = Base->getType();
1329
1330 Module *M = BB->getParent()->getParent();
1331 Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration(
1332 M, Intrinsic::preserve_union_access_index, {BaseType, BaseType});
1333
1334 Value *DIIndex = getInt32(FieldIndex);
1335 CallInst *Fn =
1336 CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex});
1337 if (DbgInfo)
1338 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1339
1340 return Fn;
1341}
1342
1344 Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1345 MDNode *DbgInfo) {
1346 auto *BaseType = Base->getType();
1347 assert(isa<PointerType>(BaseType) &&
1348 "Invalid Base ptr type for preserve.struct.access.index.");
1349 assert(cast<PointerType>(BaseType)->isOpaqueOrPointeeTypeMatches(ElTy) &&
1350 "Pointer element type mismatch");
1351
1352 Value *GEPIndex = getInt32(Index);
1354 Type *ResultType =
1355 GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex});
1356
1357 Module *M = BB->getParent()->getParent();
1358 Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration(
1359 M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType});
1360
1361 Value *DIIndex = getInt32(FieldIndex);
1362 CallInst *Fn = CreateCall(FnPreserveStructAccessIndex,
1363 {Base, GEPIndex, DIIndex});
1364 Fn->addParamAttr(
1365 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
1366 if (DbgInfo)
1367 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1368
1369 return Fn;
1370}
1371
1372CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1373 Value *PtrValue,
1374 Value *AlignValue,
1375 Value *OffsetValue) {
1376 SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1377 if (OffsetValue)
1378 Vals.push_back(OffsetValue);
1379 OperandBundleDefT<Value *> AlignOpB("align", Vals);
1380 return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB});
1381}
1382
1384 Value *PtrValue,
1385 unsigned Alignment,
1386 Value *OffsetValue) {
1387 assert(isa<PointerType>(PtrValue->getType()) &&
1388 "trying to create an alignment assumption on a non-pointer?");
1389 assert(Alignment != 0 && "Invalid Alignment");
1390 auto *PtrTy = cast<PointerType>(PtrValue->getType());
1391 Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
1392 Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment);
1393 return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
1394}
1395
1397 Value *PtrValue,
1398 Value *Alignment,
1399 Value *OffsetValue) {
1400 assert(isa<PointerType>(PtrValue->getType()) &&
1401 "trying to create an alignment assumption on a non-pointer?");
1402 return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
1403}
1404
1408void ConstantFolder::anchor() {}
1409void NoFolder::anchor() {}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Callee
ArrayRef< TableEntry > TableRef
assume Assume Builder
SmallVector< MachineOperand, 4 > Cond
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
std::string Name
uint64_t Size
static InvokeInst * CreateGCStatepointInvokeCommon(IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags, ArrayRef< T0 > InvokeArgs, std::optional< ArrayRef< T1 > > TransitionArgs, std::optional< ArrayRef< T2 > > DeoptArgs, ArrayRef< T3 > GCArgs, const Twine &Name)
Definition: IRBuilder.cpp:848
static CallInst * CreateGCStatepointCallCommon(IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee, uint32_t Flags, ArrayRef< T0 > CallArgs, std::optional< ArrayRef< T1 > > TransitionArgs, std::optional< ArrayRef< T2 > > DeoptArgs, ArrayRef< T3 > GCArgs, const Twine &Name)
Definition: IRBuilder.cpp:794
static std::vector< OperandBundleDef > getStatepointBundles(std::optional< ArrayRef< T1 > > TransitionArgs, std::optional< ArrayRef< T2 > > DeoptArgs, ArrayRef< T3 > GCArgs)
Definition: IRBuilder.cpp:771
static std::vector< Value * > getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags, ArrayRef< T0 > CallArgs)
Definition: IRBuilder.cpp:752
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:524
static M68kRelType getType(unsigned Kind, MCSymbolRefExpr::VariantKind &Modifier, bool &IsPCRel)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define P(N)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Value * RHS
Value * LHS
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:158
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:91
static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
Definition: Attributes.cpp:167
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:112
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Definition: BasicBlock.cpp:146
void addRetAttr(Attribute::AttrKind Kind)
Adds the attribute to the return value.
Definition: InstrTypes.h:1526
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Definition: InstrTypes.h:1536
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:718
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2994
static Constant * getSizeOf(Type *Ty)
getSizeOf constant expr - computes the (alloc) size of a type (in address-units, not bits) in a targe...
Definition: Constants.cpp:2378
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:835
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:887
static Constant * get(ArrayRef< Constant * > V)
Definition: Constants.cpp:1356
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:403
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:114
A debug info location.
Definition: DebugLoc.h:33
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:291
This instruction compares its operands according to the predicate given to the constructor.
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:21
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
FunctionType * getFunctionType()
Definition: DerivedTypes.h:182
Class to represent function types.
Definition: DerivedTypes.h:103
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:174
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:179
static Type * getGEPReturnType(Type *ElTy, Value *Ptr, ArrayRef< Value * > IdxList)
Returns the pointer type returned by the GEP instruction, which may be a vector of pointers.
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
Value * CreateExactSDiv(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1315
CallInst * CreateElementUnorderedAtomicMemCpy(Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, uint32_t ElementSize, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert an element unordered-atomic memcpy between the specified pointers.
Definition: IRBuilder.cpp:298
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Definition: IRBuilder.h:447
CallInst * CreateMaskedCompressStore(Value *Val, Value *Ptr, Value *Mask=nullptr)
Create a call to Masked Compress Store intrinsic.
Definition: IRBuilder.cpp:734
BasicBlock * BB
Definition: IRBuilder.h:119
CallInst * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
Definition: IRBuilder.cpp:948
Value * CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS, const Twine &Name="")
Return the i64 difference between two pointer values, dividing out the size of the pointed-to objects...
Definition: IRBuilder.cpp:1142
CallInst * CreateMulReduce(Value *Src)
Create a vector int mul reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:441
CallInst * CreateFAddReduce(Value *Acc, Value *Src)
Create a sequential vector fadd reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:421
CallInst * CreateMaskedExpandLoad(Type *Ty, Value *Ptr, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Expand Load intrinsic.
Definition: IRBuilder.cpp:711
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1912
Value * CreateVScale(Constant *Scaling, const Twine &Name="")
Create a call to llvm.vscale, multiplied by Scaling.
Definition: IRBuilder.cpp:97
Value * CreateLaunderInvariantGroup(Value *Ptr)
Create a launder.invariant.group intrinsic call.
Definition: IRBuilder.cpp:1156
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2353
CallInst * CreateThreadLocalAddress(Value *Ptr)
Create a call to llvm.threadlocal.address intrinsic.
Definition: IRBuilder.cpp:537
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Definition: IRBuilder.h:497
Type * getCurrentFunctionReturnType() const
Get the return type of the current function that we're emitting into.
Definition: IRBuilder.cpp:58
CallInst * CreateGCGetPointerBase(Value *DerivedPtr, const Twine &Name="")
Create a call to the experimental.gc.pointer.base intrinsic to get the base pointer for the specified...
Definition: IRBuilder.cpp:930
CallInst * CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee, ArrayRef< Value * > CallArgs, std::optional< ArrayRef< Value * > > DeoptArgs, ArrayRef< Value * > GCArgs, const Twine &Name="")
Create a call to the experimental.gc.statepoint intrinsic to start a new statepoint sequence.
Definition: IRBuilder.cpp:818
GlobalVariable * CreateGlobalString(StringRef Str, const Twine &Name="", unsigned AddressSpace=0, Module *M=nullptr)
Make a new global variable with initializer type i8*.
Definition: IRBuilder.cpp:43
CallInst * CreateLifetimeStart(Value *Ptr, ConstantInt *Size=nullptr)
Create a lifetime.start intrinsic.
Definition: IRBuilder.cpp:477
CallInst * CreateConstrainedFPCmp(Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R, const Twine &Name="", std::optional< fp::ExceptionBehavior > Except=std::nullopt)
Definition: IRBuilder.cpp:1088
CallInst * CreateAndReduce(Value *Src)
Create a vector int AND reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:445
Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")
Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.
Definition: IRBuilder.cpp:1221
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Definition: IRBuilder.cpp:1249
CallInst * CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, Value *Size, bool IsVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Definition: IRBuilder.cpp:260
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Definition: IRBuilder.cpp:965
Value * CreatePreserveStructAccessIndex(Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex, MDNode *DbgInfo)
Definition: IRBuilder.cpp:1343
CallInst * CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue, unsigned Alignment, Value *OffsetValue=nullptr)
Create an assume intrinsic call that represents an alignment assumption on the provided pointer.
Definition: IRBuilder.cpp:1383
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
Definition: IRBuilder.cpp:588
CallInst * CreateConstrainedFPCall(Function *Callee, ArrayRef< Value * > Args, const Twine &Name="", std::optional< RoundingMode > Rounding=std::nullopt, std::optional< fp::ExceptionBehavior > Except=std::nullopt)
Definition: IRBuilder.cpp:1100
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, MaybeAlign Align, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memset to the specified pointer and the specified value.
Definition: IRBuilder.h:580
LLVMContext & Context
Definition: IRBuilder.h:121
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.cpp:1126
CallInst * CreateGCGetPointerOffset(Value *DerivedPtr, const Twine &Name="")
Create a call to the experimental.gc.get.pointer.offset intrinsic to get the offset of the specified ...
Definition: IRBuilder.cpp:939
CallInst * CreateAddReduce(Value *Src)
Create a vector int add reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:437
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1352
IntegerType * getIntPtrTy(const DataLayout &DL, unsigned AddrSpace=0)
Fetch the type of an integer with size at least as big as that of a pointer in the given address spac...
Definition: IRBuilder.h:566
BasicBlock * GetInsertBlock() const
Definition: IRBuilder.h:174
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Definition: IRBuilder.h:517
CallInst * CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, Value *Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Definition: IRBuilder.cpp:223
Value * CreateVectorReverse(Value *V, const Twine &Name="")
Return a vector value that contains the vector V reversed.
Definition: IRBuilder.cpp:1205
CallInst * CreateXorReduce(Value *Src)
Create a vector int XOR reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:453
FastMathFlags FMF
Definition: IRBuilder.h:126
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition: IRBuilder.h:477
Value * CreateUnOp(Instruction::UnaryOps Opc, Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1668
CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:449
CallInst * CreateFPMinReduce(Value *Src)
Create a vector float min reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:473
CallInst * CreateFPMaxReduce(Value *Src)
Create a vector float max reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:469
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:472
Value * CreateExtractInteger(const DataLayout &DL, Value *From, IntegerType *ExtractedTy, uint64_t Offset, const Twine &Name)
Return a value that has been extracted from a larger integer type.
Definition: IRBuilder.cpp:1269
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
Definition: IRBuilder.h:145
CallInst * CreateConstrainedFPBinOp(Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource=nullptr, const Twine &Name="", MDNode *FPMathTag=nullptr, std::optional< RoundingMode > Rounding=std::nullopt, std::optional< fp::ExceptionBehavior > Except=std::nullopt)
Definition: IRBuilder.cpp:1002
DebugLoc getCurrentDebugLocation() const
Get location information used by debugging information.
Definition: IRBuilder.cpp:72
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1259
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2008
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
Definition: IRBuilder.h:560
Value * CreateNAryOp(unsigned Opc, ArrayRef< Value * > Ops, const Twine &Name="", MDNode *FPMathTag=nullptr)
Create either a UnaryOperator or BinaryOperator depending on Opc.
Definition: IRBuilder.cpp:1021
CallInst * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Definition: IRBuilder.cpp:956
CallInst * CreateAssumption(Value *Cond, ArrayRef< OperandBundleDef > OpBundles=std::nullopt)
Create an assume intrinsic call that allows the optimizer to assume that the provided condition will ...
Definition: IRBuilder.cpp:561
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Definition: IRBuilder.h:2375
LLVMContext & getContext() const
Definition: IRBuilder.h:176
Value * CreatePreserveUnionAccessIndex(Value *Base, unsigned FieldIndex, MDNode *DbgInfo)
Definition: IRBuilder.cpp:1324
CallInst * CreateIntMaxReduce(Value *Src, bool IsSigned=false)
Create a vector integer max reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:457
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Definition: IRBuilder.cpp:609
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1998
CallInst * CreateGCResult(Instruction *Statepoint, Type *ResultType, const Twine &Name="")
Create a call to the experimental.gc.result intrinsic to extract the result from a call wrapped in a ...
Definition: IRBuilder.cpp:907
CallInst * CreateLifetimeEnd(Value *Ptr, ConstantInt *Size=nullptr)
Create a lifetime.end intrinsic.
Definition: IRBuilder.cpp:493
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1581
CallInst * CreateConstrainedFPCast(Intrinsic::ID ID, Value *V, Type *DestTy, Instruction *FMFSource=nullptr, const Twine &Name="", MDNode *FPMathTag=nullptr, std::optional< RoundingMode > Rounding=std::nullopt, std::optional< fp::ExceptionBehavior > Except=std::nullopt)
Definition: IRBuilder.cpp:1036
CallInst * CreateElementUnorderedAtomicMemMove(Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, uint32_t ElementSize, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert an element unordered-atomic memmove between the specified pointers.
Definition: IRBuilder.cpp:373
CallInst * CreateIntMinReduce(Value *Src, bool IsSigned=false)
Create a vector integer min reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:463
void setConstrainedFPCallAttr(CallBase *I)
Definition: IRBuilder.h:347
InvokeInst * CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef< Value * > InvokeArgs, std::optional< ArrayRef< Value * > > DeoptArgs, ArrayRef< Value * > GCArgs, const Twine &Name="")
Create an invoke to the experimental.gc.statepoint intrinsic to start a new statepoint sequence.
Definition: IRBuilder.cpp:874
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2293
const IRBuilderFolder & Folder
Definition: IRBuilder.h:122
CallInst * CreateMemSetInline(Value *Dst, MaybeAlign DstAlign, Value *Val, Value *Size, bool IsVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Definition: IRBuilder.cpp:166
CallInst * CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val, uint64_t Size, Align Alignment, uint32_t ElementSize, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert an element unordered-atomic memset of the region of memory starting at the given po...
Definition: IRBuilder.h:605
CallInst * CreateFMulReduce(Value *Acc, Value *Src)
Create a sequential vector fmul reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:429
void SetInstDebugLocation(Instruction *I) const
If this builder has a current debug location, set it on the specified instruction.
Definition: IRBuilder.cpp:79
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Definition: IRBuilder.h:502
CallInst * CreateGCRelocate(Instruction *Statepoint, int BaseOffset, int DerivedOffset, Type *ResultType, const Twine &Name="")
Create a call to the experimental.gc.relocate intrinsics to project the relocated value of one pointe...
Definition: IRBuilder.cpp:918
Value * CreateStepVector(Type *DstType, const Twine &Name="")
Creates a vector of type DstType with the linear sequence <0, 1, ...>
Definition: IRBuilder.cpp:110
CallInst * CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, uint64_t Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Definition: IRBuilder.h:676
Value * CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex, MDNode *DbgInfo)
Definition: IRBuilder.cpp:1292
CallInst * CreateInvariantStart(Value *Ptr, ConstantInt *Size=nullptr)
Create a call to invariant.start intrinsic.
Definition: IRBuilder.cpp:509
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1276
Instruction * CreateNoAliasScopeDeclaration(Value *Scope)
Create a llvm.experimental.noalias.scope.decl intrinsic call.
Definition: IRBuilder.cpp:572
CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
Definition: IRBuilder.cpp:678
Value * CreateStripInvariantGroup(Value *Ptr)
Create a strip.invariant.group intrinsic call.
Definition: IRBuilder.cpp:1180
CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
Definition: IRBuilder.cpp:642
virtual Value * FoldSelect(Value *C, Value *True, Value *False) const =0
virtual Value * CreateFCmp(CmpInst::Predicate P, Constant *LHS, Constant *RHS) const =0
virtual ~IRBuilderFolder()
void copyFastMathFlags(FastMathFlags FMF)
Convenience function for transferring all fast-math flag values to this instruction,...
bool isBinaryOp() const
Definition: Instruction.h:173
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:275
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1455
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
bool isUnaryOp() const
Definition: Instruction.h:172
Class to represent integer types.
Definition: DerivedTypes.h:40
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
Definition: DerivedTypes.h:72
Invoke instruction.
Metadata node.
Definition: Metadata.h:943
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
A container for an operand bundle being viewed as a set of values rather than a set of uses.
Definition: InstrTypes.h:1137
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1759
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", Instruction *InsertBefore=nullptr, Instruction *MDFrom=nullptr)
void reserve(size_type N)
Definition: SmallVector.h:667
void resize(size_type N)
Definition: SmallVector.h:642
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:258
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:341
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:994
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:682
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl< IITDescriptor > &T)
Return the IIT table descriptor for the specified intrinsic into an array of IITDescriptors.
Definition: Function.cpp:1332
@ MatchIntrinsicTypes_Match
Definition: Intrinsics.h:215
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1502
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:406
AddressSpace
Definition: NVPTXBaseInfo.h:21
bool getAlign(const Function &F, unsigned index, unsigned &align)
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:2014
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117