LLVM 18.0.0git
IRBuilder.cpp
Go to the documentation of this file.
1//===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/IR/IRBuilder.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/IR/Constant.h"
17#include "llvm/IR/Constants.h"
20#include "llvm/IR/Function.h"
21#include "llvm/IR/GlobalValue.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/LLVMContext.h"
26#include "llvm/IR/NoFolder.h"
27#include "llvm/IR/Operator.h"
28#include "llvm/IR/Statepoint.h"
29#include "llvm/IR/Type.h"
30#include "llvm/IR/Value.h"
32#include <cassert>
33#include <cstdint>
34#include <optional>
35#include <vector>
36
37using namespace llvm;
38
39/// CreateGlobalString - Make a new global variable with an initializer that
40/// has array of i8 type filled in with the nul terminated string value
41/// specified. If Name is specified, it is the name of the global variable
42/// created.
44 const Twine &Name,
45 unsigned AddressSpace,
46 Module *M) {
48 if (!M)
49 M = BB->getParent()->getParent();
50 auto *GV = new GlobalVariable(
51 *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
52 StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
53 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
54 GV->setAlignment(Align(1));
55 return GV;
56}
57
59 assert(BB && BB->getParent() && "No current function!");
60 return BB->getParent()->getReturnType();
61}
62
64 for (auto &KV : MetadataToCopy)
65 if (KV.first == LLVMContext::MD_dbg)
66 return {cast<DILocation>(KV.second)};
67
68 return {};
69}
71 for (const auto &KV : MetadataToCopy)
72 if (KV.first == LLVMContext::MD_dbg) {
73 I->setDebugLoc(DebugLoc(KV.second));
74 return;
75 }
76}
77
79IRBuilderBase::createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
80 const Twine &Name, Instruction *FMFSource,
82 CallInst *CI = CreateCall(Callee, Ops, OpBundles, Name);
83 if (FMFSource)
84 CI->copyFastMathFlags(FMFSource);
85 return CI;
86}
87
89 assert(isa<ConstantInt>(Scaling) && "Expected constant integer");
90 if (cast<ConstantInt>(Scaling)->isZero())
91 return Scaling;
93 Function *TheFn =
94 Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()});
95 CallInst *CI = CreateCall(TheFn, {}, {}, Name);
96 return cast<ConstantInt>(Scaling)->isOne() ? CI : CreateMul(CI, Scaling);
97}
98
100 Constant *MinEC = ConstantInt::get(DstType, EC.getKnownMinValue());
101 return EC.isScalable() ? CreateVScale(MinEC) : MinEC;
102}
103
105 Constant *MinSize = ConstantInt::get(DstType, Size.getKnownMinValue());
106 return Size.isScalable() ? CreateVScale(MinSize) : MinSize;
107}
108
110 Type *STy = DstType->getScalarType();
111 if (isa<ScalableVectorType>(DstType)) {
112 Type *StepVecType = DstType;
113 // TODO: We expect this special case (element type < 8 bits) to be
114 // temporary - once the intrinsic properly supports < 8 bits this code
115 // can be removed.
116 if (STy->getScalarSizeInBits() < 8)
117 StepVecType =
118 VectorType::get(getInt8Ty(), cast<ScalableVectorType>(DstType));
119 Value *Res = CreateIntrinsic(Intrinsic::experimental_stepvector,
120 {StepVecType}, {}, nullptr, Name);
121 if (StepVecType != DstType)
122 Res = CreateTrunc(Res, DstType);
123 return Res;
124 }
125
126 unsigned NumEls = cast<FixedVectorType>(DstType)->getNumElements();
127
128 // Create a vector of consecutive numbers from zero to VF.
130 for (unsigned i = 0; i < NumEls; ++i)
131 Indices.push_back(ConstantInt::get(STy, i));
132
133 // Add the consecutive indices to the vector value.
134 return ConstantVector::get(Indices);
135}
136
138 MaybeAlign Align, bool isVolatile,
139 MDNode *TBAATag, MDNode *ScopeTag,
140 MDNode *NoAliasTag) {
141 Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
142 Type *Tys[] = { Ptr->getType(), Size->getType() };
143 Module *M = BB->getParent()->getParent();
144 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
145
146 CallInst *CI = CreateCall(TheFn, Ops);
147
148 if (Align)
149 cast<MemSetInst>(CI)->setDestAlignment(*Align);
150
151 // Set the TBAA info if present.
152 if (TBAATag)
153 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
154
155 if (ScopeTag)
156 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
157
158 if (NoAliasTag)
159 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
160
161 return CI;
162}
163
165 Value *Val, Value *Size,
166 bool IsVolatile, MDNode *TBAATag,
167 MDNode *ScopeTag,
168 MDNode *NoAliasTag) {
169 Value *Ops[] = {Dst, Val, Size, getInt1(IsVolatile)};
170 Type *Tys[] = {Dst->getType(), Size->getType()};
171 Module *M = BB->getParent()->getParent();
172 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset_inline, Tys);
173
174 CallInst *CI = CreateCall(TheFn, Ops);
175
176 if (DstAlign)
177 cast<MemSetInlineInst>(CI)->setDestAlignment(*DstAlign);
178
179 // Set the TBAA info if present.
180 if (TBAATag)
181 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
182
183 if (ScopeTag)
184 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
185
186 if (NoAliasTag)
187 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
188
189 return CI;
190}
191
193 Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
194 MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
195
196 Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
197 Type *Tys[] = {Ptr->getType(), Size->getType()};
198 Module *M = BB->getParent()->getParent();
200 M, Intrinsic::memset_element_unordered_atomic, Tys);
201
202 CallInst *CI = CreateCall(TheFn, Ops);
203
204 cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
205
206 // Set the TBAA info if present.
207 if (TBAATag)
208 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
209
210 if (ScopeTag)
211 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
212
213 if (NoAliasTag)
214 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
215
216 return CI;
217}
218
220 Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
221 MaybeAlign SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag,
222 MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
223 assert((IntrID == Intrinsic::memcpy || IntrID == Intrinsic::memcpy_inline ||
224 IntrID == Intrinsic::memmove) &&
225 "Unexpected intrinsic ID");
226 Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
227 Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
228 Module *M = BB->getParent()->getParent();
229 Function *TheFn = Intrinsic::getDeclaration(M, IntrID, Tys);
230
231 CallInst *CI = CreateCall(TheFn, Ops);
232
233 auto* MCI = cast<MemTransferInst>(CI);
234 if (DstAlign)
235 MCI->setDestAlignment(*DstAlign);
236 if (SrcAlign)
237 MCI->setSourceAlignment(*SrcAlign);
238
239 // Set the TBAA info if present.
240 if (TBAATag)
241 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
242
243 // Set the TBAA Struct info if present.
244 if (TBAAStructTag)
245 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
246
247 if (ScopeTag)
248 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
249
250 if (NoAliasTag)
251 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
252
253 return CI;
254}
255
257 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
258 uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
259 MDNode *ScopeTag, MDNode *NoAliasTag) {
260 assert(DstAlign >= ElementSize &&
261 "Pointer alignment must be at least element size");
262 assert(SrcAlign >= ElementSize &&
263 "Pointer alignment must be at least element size");
264 Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
265 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
266 Module *M = BB->getParent()->getParent();
268 M, Intrinsic::memcpy_element_unordered_atomic, Tys);
269
270 CallInst *CI = CreateCall(TheFn, Ops);
271
272 // Set the alignment of the pointer args.
273 auto *AMCI = cast<AtomicMemCpyInst>(CI);
274 AMCI->setDestAlignment(DstAlign);
275 AMCI->setSourceAlignment(SrcAlign);
276
277 // Set the TBAA info if present.
278 if (TBAATag)
279 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
280
281 // Set the TBAA Struct info if present.
282 if (TBAAStructTag)
283 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
284
285 if (ScopeTag)
286 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
287
288 if (NoAliasTag)
289 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
290
291 return CI;
292}
293
294/// isConstantOne - Return true only if val is constant int 1
295static bool isConstantOne(const Value *Val) {
296 assert(Val && "isConstantOne does not work with nullptr Val");
297 const ConstantInt *CVal = dyn_cast<ConstantInt>(Val);
298 return CVal && CVal->isOne();
299}
300
302 Value *AllocSize, Value *ArraySize,
304 Function *MallocF, const Twine &Name) {
305 // malloc(type) becomes:
306 // i8* malloc(typeSize)
307 // malloc(type, arraySize) becomes:
308 // i8* malloc(typeSize*arraySize)
309 if (!ArraySize)
310 ArraySize = ConstantInt::get(IntPtrTy, 1);
311 else if (ArraySize->getType() != IntPtrTy)
312 ArraySize = CreateIntCast(ArraySize, IntPtrTy, false);
313
314 if (!isConstantOne(ArraySize)) {
315 if (isConstantOne(AllocSize)) {
316 AllocSize = ArraySize; // Operand * 1 = Operand
317 } else {
318 // Multiply type size by the array size...
319 AllocSize = CreateMul(ArraySize, AllocSize, "mallocsize");
320 }
321 }
322
323 assert(AllocSize->getType() == IntPtrTy && "malloc arg is wrong size");
324 // Create the call to Malloc.
325 Module *M = BB->getParent()->getParent();
327 FunctionCallee MallocFunc = MallocF;
328 if (!MallocFunc)
329 // prototype malloc as "void *malloc(size_t)"
330 MallocFunc = M->getOrInsertFunction("malloc", BPTy, IntPtrTy);
331 CallInst *MCall = CreateCall(MallocFunc, AllocSize, OpB, Name);
332
333 MCall->setTailCall();
334 if (Function *F = dyn_cast<Function>(MallocFunc.getCallee())) {
335 MCall->setCallingConv(F->getCallingConv());
336 F->setReturnDoesNotAlias();
337 }
338
339 assert(!MCall->getType()->isVoidTy() && "Malloc has void return type");
340
341 return MCall;
342}
343
345 Value *AllocSize, Value *ArraySize,
346 Function *MallocF, const Twine &Name) {
347
348 return CreateMalloc(IntPtrTy, AllocTy, AllocSize, ArraySize, std::nullopt,
349 MallocF, Name);
350}
351
352/// CreateFree - Generate the IR for a call to the builtin free function.
355 assert(Source->getType()->isPointerTy() &&
356 "Can not free something of nonpointer type!");
357
358 Module *M = BB->getParent()->getParent();
359
360 Type *VoidTy = Type::getVoidTy(M->getContext());
361 Type *VoidPtrTy = PointerType::getUnqual(M->getContext());
362 // prototype free as "void free(void*)"
363 FunctionCallee FreeFunc = M->getOrInsertFunction("free", VoidTy, VoidPtrTy);
364 CallInst *Result = CreateCall(FreeFunc, Source, Bundles, "");
365 Result->setTailCall();
366 if (Function *F = dyn_cast<Function>(FreeFunc.getCallee()))
367 Result->setCallingConv(F->getCallingConv());
368
369 return Result;
370}
371
373 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
374 uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
375 MDNode *ScopeTag, MDNode *NoAliasTag) {
376 assert(DstAlign >= ElementSize &&
377 "Pointer alignment must be at least element size");
378 assert(SrcAlign >= ElementSize &&
379 "Pointer alignment must be at least element size");
380 Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
381 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
382 Module *M = BB->getParent()->getParent();
384 M, Intrinsic::memmove_element_unordered_atomic, Tys);
385
386 CallInst *CI = CreateCall(TheFn, Ops);
387
388 // Set the alignment of the pointer args.
389 CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
390 CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
391
392 // Set the TBAA info if present.
393 if (TBAATag)
394 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
395
396 // Set the TBAA Struct info if present.
397 if (TBAAStructTag)
398 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
399
400 if (ScopeTag)
401 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
402
403 if (NoAliasTag)
404 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
405
406 return CI;
407}
408
409CallInst *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
411 Value *Ops[] = {Src};
412 Type *Tys[] = { Src->getType() };
413 auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
414 return CreateCall(Decl, Ops);
415}
416
419 Value *Ops[] = {Acc, Src};
420 auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fadd,
421 {Src->getType()});
422 return CreateCall(Decl, Ops);
423}
424
427 Value *Ops[] = {Acc, Src};
428 auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fmul,
429 {Src->getType()});
430 return CreateCall(Decl, Ops);
431}
432
434 return getReductionIntrinsic(Intrinsic::vector_reduce_add, Src);
435}
436
438 return getReductionIntrinsic(Intrinsic::vector_reduce_mul, Src);
439}
440
442 return getReductionIntrinsic(Intrinsic::vector_reduce_and, Src);
443}
444
446 return getReductionIntrinsic(Intrinsic::vector_reduce_or, Src);
447}
448
450 return getReductionIntrinsic(Intrinsic::vector_reduce_xor, Src);
451}
452
454 auto ID =
455 IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
456 return getReductionIntrinsic(ID, Src);
457}
458
460 auto ID =
461 IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
462 return getReductionIntrinsic(ID, Src);
463}
464
466 return getReductionIntrinsic(Intrinsic::vector_reduce_fmax, Src);
467}
468
470 return getReductionIntrinsic(Intrinsic::vector_reduce_fmin, Src);
471}
472
474 return getReductionIntrinsic(Intrinsic::vector_reduce_fmaximum, Src);
475}
476
478 return getReductionIntrinsic(Intrinsic::vector_reduce_fminimum, Src);
479}
480
482 assert(isa<PointerType>(Ptr->getType()) &&
483 "lifetime.start only applies to pointers.");
484 if (!Size)
485 Size = getInt64(-1);
486 else
487 assert(Size->getType() == getInt64Ty() &&
488 "lifetime.start requires the size to be an i64");
489 Value *Ops[] = { Size, Ptr };
490 Module *M = BB->getParent()->getParent();
491 Function *TheFn =
492 Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
493 return CreateCall(TheFn, Ops);
494}
495
497 assert(isa<PointerType>(Ptr->getType()) &&
498 "lifetime.end only applies to pointers.");
499 if (!Size)
500 Size = getInt64(-1);
501 else
502 assert(Size->getType() == getInt64Ty() &&
503 "lifetime.end requires the size to be an i64");
504 Value *Ops[] = { Size, Ptr };
505 Module *M = BB->getParent()->getParent();
506 Function *TheFn =
507 Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
508 return CreateCall(TheFn, Ops);
509}
510
512
513 assert(isa<PointerType>(Ptr->getType()) &&
514 "invariant.start only applies to pointers.");
515 if (!Size)
516 Size = getInt64(-1);
517 else
518 assert(Size->getType() == getInt64Ty() &&
519 "invariant.start requires the size to be an i64");
520
521 Value *Ops[] = {Size, Ptr};
522 // Fill in the single overloaded type: memory object type.
523 Type *ObjectPtr[1] = {Ptr->getType()};
524 Module *M = BB->getParent()->getParent();
525 Function *TheFn =
526 Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
527 return CreateCall(TheFn, Ops);
528}
529
531 if (auto *O = dyn_cast<GlobalObject>(Ptr))
532 return O->getAlign();
533 if (auto *A = dyn_cast<GlobalAlias>(Ptr))
534 return A->getAliaseeObject()->getAlign();
535 return {};
536}
537
539#ifndef NDEBUG
540 // Handle specially for constexpr cast. This is possible when
541 // opaque pointers not enabled since constant could be sinked
542 // directly by the design of llvm. This could be eliminated
543 // after we eliminate the abuse of constexpr.
544 auto *V = Ptr;
545 if (auto *CE = dyn_cast<ConstantExpr>(V))
546 if (CE->isCast())
547 V = CE->getOperand(0);
548
549 assert(isa<GlobalValue>(V) && cast<GlobalValue>(V)->isThreadLocal() &&
550 "threadlocal_address only applies to thread local variables.");
551#endif
552 CallInst *CI = CreateIntrinsic(llvm::Intrinsic::threadlocal_address,
553 {Ptr->getType()}, {Ptr});
554 if (MaybeAlign A = getAlign(Ptr)) {
557 }
558 return CI;
559}
560
561CallInst *
563 ArrayRef<OperandBundleDef> OpBundles) {
564 assert(Cond->getType() == getInt1Ty() &&
565 "an assumption condition must be of type i1");
566
567 Value *Ops[] = { Cond };
568 Module *M = BB->getParent()->getParent();
569 Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
570 return CreateCall(FnAssume, Ops, OpBundles);
571}
572
574 Module *M = BB->getModule();
575 auto *FnIntrinsic = Intrinsic::getDeclaration(
576 M, Intrinsic::experimental_noalias_scope_decl, {});
577 return CreateCall(FnIntrinsic, {Scope});
578}
579
580/// Create a call to a Masked Load intrinsic.
581/// \p Ty - vector type to load
582/// \p Ptr - base pointer for the load
583/// \p Alignment - alignment of the source location
584/// \p Mask - vector of booleans which indicates what vector lanes should
585/// be accessed in memory
586/// \p PassThru - pass-through value that is used to fill the masked-off lanes
587/// of the result
588/// \p Name - name of the result variable
590 Value *Mask, Value *PassThru,
591 const Twine &Name) {
592 auto *PtrTy = cast<PointerType>(Ptr->getType());
593 assert(Ty->isVectorTy() && "Type should be vector");
594 assert(Mask && "Mask should not be all-ones (null)");
595 if (!PassThru)
596 PassThru = PoisonValue::get(Ty);
597 Type *OverloadedTypes[] = { Ty, PtrTy };
598 Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
599 return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
600 OverloadedTypes, Name);
601}
602
603/// Create a call to a Masked Store intrinsic.
604/// \p Val - data to be stored,
605/// \p Ptr - base pointer for the store
606/// \p Alignment - alignment of the destination location
607/// \p Mask - vector of booleans which indicates what vector lanes should
608/// be accessed in memory
610 Align Alignment, Value *Mask) {
611 auto *PtrTy = cast<PointerType>(Ptr->getType());
612 Type *DataTy = Val->getType();
613 assert(DataTy->isVectorTy() && "Val should be a vector");
614 assert(Mask && "Mask should not be all-ones (null)");
615 Type *OverloadedTypes[] = { DataTy, PtrTy };
616 Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask};
617 return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
618}
619
620/// Create a call to a Masked intrinsic, with given intrinsic Id,
621/// an array of operands - Ops, and an array of overloaded types -
622/// OverloadedTypes.
623CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
625 ArrayRef<Type *> OverloadedTypes,
626 const Twine &Name) {
627 Module *M = BB->getParent()->getParent();
628 Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
629 return CreateCall(TheFn, Ops, {}, Name);
630}
631
632/// Create a call to a Masked Gather intrinsic.
633/// \p Ty - vector type to gather
634/// \p Ptrs - vector of pointers for loading
635/// \p Align - alignment for one element
636/// \p Mask - vector of booleans which indicates what vector lanes should
637/// be accessed in memory
638/// \p PassThru - pass-through value that is used to fill the masked-off lanes
639/// of the result
640/// \p Name - name of the result variable
642 Align Alignment, Value *Mask,
643 Value *PassThru,
644 const Twine &Name) {
645 auto *VecTy = cast<VectorType>(Ty);
646 ElementCount NumElts = VecTy->getElementCount();
647 auto *PtrsTy = cast<VectorType>(Ptrs->getType());
648 assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");
649
650 if (!Mask)
651 Mask = getAllOnesMask(NumElts);
652
653 if (!PassThru)
654 PassThru = PoisonValue::get(Ty);
655
656 Type *OverloadedTypes[] = {Ty, PtrsTy};
657 Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
658
659 // We specify only one type when we create this intrinsic. Types of other
660 // arguments are derived from this type.
661 return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
662 Name);
663}
664
665/// Create a call to a Masked Scatter intrinsic.
666/// \p Data - data to be stored,
667/// \p Ptrs - the vector of pointers, where the \p Data elements should be
668/// stored
669/// \p Align - alignment for one element
670/// \p Mask - vector of booleans which indicates what vector lanes should
671/// be accessed in memory
673 Align Alignment, Value *Mask) {
674 auto *PtrsTy = cast<VectorType>(Ptrs->getType());
675 auto *DataTy = cast<VectorType>(Data->getType());
676 ElementCount NumElts = PtrsTy->getElementCount();
677
678 if (!Mask)
679 Mask = getAllOnesMask(NumElts);
680
681 Type *OverloadedTypes[] = {DataTy, PtrsTy};
682 Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask};
683
684 // We specify only one type when we create this intrinsic. Types of other
685 // arguments are derived from this type.
686 return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
687}
688
689/// Create a call to Masked Expand Load intrinsic
690/// \p Ty - vector type to load
691/// \p Ptr - base pointer for the load
692/// \p Mask - vector of booleans which indicates what vector lanes should
693/// be accessed in memory
694/// \p PassThru - pass-through value that is used to fill the masked-off lanes
695/// of the result
696/// \p Name - name of the result variable
698 Value *Mask, Value *PassThru,
699 const Twine &Name) {
700 assert(Ty->isVectorTy() && "Type should be vector");
701 assert(Mask && "Mask should not be all-ones (null)");
702 if (!PassThru)
703 PassThru = PoisonValue::get(Ty);
704 Type *OverloadedTypes[] = {Ty};
705 Value *Ops[] = {Ptr, Mask, PassThru};
706 return CreateMaskedIntrinsic(Intrinsic::masked_expandload, Ops,
707 OverloadedTypes, Name);
708}
709
710/// Create a call to Masked Compress Store intrinsic
711/// \p Val - data to be stored,
712/// \p Ptr - base pointer for the store
713/// \p Mask - vector of booleans which indicates what vector lanes should
714/// be accessed in memory
716 Value *Mask) {
717 Type *DataTy = Val->getType();
718 assert(DataTy->isVectorTy() && "Val should be a vector");
719 assert(Mask && "Mask should not be all-ones (null)");
720 Type *OverloadedTypes[] = {DataTy};
721 Value *Ops[] = {Val, Ptr, Mask};
722 return CreateMaskedIntrinsic(Intrinsic::masked_compressstore, Ops,
723 OverloadedTypes);
724}
725
726template <typename T0>
727static std::vector<Value *>
729 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
730 std::vector<Value *> Args;
731 Args.push_back(B.getInt64(ID));
732 Args.push_back(B.getInt32(NumPatchBytes));
733 Args.push_back(ActualCallee);
734 Args.push_back(B.getInt32(CallArgs.size()));
735 Args.push_back(B.getInt32(Flags));
736 llvm::append_range(Args, CallArgs);
737 // GC Transition and Deopt args are now always handled via operand bundle.
738 // They will be removed from the signature of gc.statepoint shortly.
739 Args.push_back(B.getInt32(0));
740 Args.push_back(B.getInt32(0));
741 // GC args are now encoded in the gc-live operand bundle
742 return Args;
743}
744
745template<typename T1, typename T2, typename T3>
746static std::vector<OperandBundleDef>
747getStatepointBundles(std::optional<ArrayRef<T1>> TransitionArgs,
748 std::optional<ArrayRef<T2>> DeoptArgs,
749 ArrayRef<T3> GCArgs) {
750 std::vector<OperandBundleDef> Rval;
751 if (DeoptArgs) {
752 SmallVector<Value*, 16> DeoptValues;
753 llvm::append_range(DeoptValues, *DeoptArgs);
754 Rval.emplace_back("deopt", DeoptValues);
755 }
756 if (TransitionArgs) {
757 SmallVector<Value*, 16> TransitionValues;
758 llvm::append_range(TransitionValues, *TransitionArgs);
759 Rval.emplace_back("gc-transition", TransitionValues);
760 }
761 if (GCArgs.size()) {
762 SmallVector<Value*, 16> LiveValues;
763 llvm::append_range(LiveValues, GCArgs);
764 Rval.emplace_back("gc-live", LiveValues);
765 }
766 return Rval;
767}
768
769template <typename T0, typename T1, typename T2, typename T3>
771 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
772 FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
773 std::optional<ArrayRef<T1>> TransitionArgs,
774 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
775 const Twine &Name) {
776 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
777 // Fill in the one generic type'd argument (the function is also vararg)
778 Function *FnStatepoint =
779 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
780 {ActualCallee.getCallee()->getType()});
781
782 std::vector<Value *> Args = getStatepointArgs(
783 *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs);
784
785 CallInst *CI = Builder->CreateCall(
786 FnStatepoint, Args,
787 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
788 CI->addParamAttr(2,
789 Attribute::get(Builder->getContext(), Attribute::ElementType,
790 ActualCallee.getFunctionType()));
791 return CI;
792}
793
795 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
796 ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
797 ArrayRef<Value *> GCArgs, const Twine &Name) {
798 return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
799 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
800 CallArgs, std::nullopt /* No Transition Args */, DeoptArgs, GCArgs, Name);
801}
802
804 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
805 uint32_t Flags, ArrayRef<Value *> CallArgs,
806 std::optional<ArrayRef<Use>> TransitionArgs,
807 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
808 const Twine &Name) {
809 return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
810 this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
811 DeoptArgs, GCArgs, Name);
812}
813
815 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
816 ArrayRef<Use> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
817 ArrayRef<Value *> GCArgs, const Twine &Name) {
818 return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
819 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
820 CallArgs, std::nullopt, DeoptArgs, GCArgs, Name);
821}
822
823template <typename T0, typename T1, typename T2, typename T3>
825 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
826 FunctionCallee ActualInvokee, BasicBlock *NormalDest,
827 BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<T0> InvokeArgs,
828 std::optional<ArrayRef<T1>> TransitionArgs,
829 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
830 const Twine &Name) {
831 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
832 // Fill in the one generic type'd argument (the function is also vararg)
833 Function *FnStatepoint =
834 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
835 {ActualInvokee.getCallee()->getType()});
836
837 std::vector<Value *> Args =
838 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(),
839 Flags, InvokeArgs);
840
841 InvokeInst *II = Builder->CreateInvoke(
842 FnStatepoint, NormalDest, UnwindDest, Args,
843 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
844 II->addParamAttr(2,
845 Attribute::get(Builder->getContext(), Attribute::ElementType,
846 ActualInvokee.getFunctionType()));
847 return II;
848}
849
851 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
852 BasicBlock *NormalDest, BasicBlock *UnwindDest,
853 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
854 ArrayRef<Value *> GCArgs, const Twine &Name) {
855 return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
856 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
857 uint32_t(StatepointFlags::None), InvokeArgs,
858 std::nullopt /* No Transition Args*/, DeoptArgs, GCArgs, Name);
859}
860
862 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
863 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
864 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Use>> TransitionArgs,
865 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
866 const Twine &Name) {
867 return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
868 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
869 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
870}
871
873 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
874 BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
875 std::optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs,
876 const Twine &Name) {
877 return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
878 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
879 uint32_t(StatepointFlags::None), InvokeArgs, std::nullopt, DeoptArgs,
880 GCArgs, Name);
881}
882
884 Type *ResultType, const Twine &Name) {
885 Intrinsic::ID ID = Intrinsic::experimental_gc_result;
886 Module *M = BB->getParent()->getParent();
887 Type *Types[] = {ResultType};
888 Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
889
890 Value *Args[] = {Statepoint};
891 return CreateCall(FnGCResult, Args, {}, Name);
892}
893
895 int BaseOffset, int DerivedOffset,
896 Type *ResultType, const Twine &Name) {
897 Module *M = BB->getParent()->getParent();
898 Type *Types[] = {ResultType};
899 Function *FnGCRelocate =
900 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
901
902 Value *Args[] = {Statepoint, getInt32(BaseOffset), getInt32(DerivedOffset)};
903 return CreateCall(FnGCRelocate, Args, {}, Name);
904}
905
907 const Twine &Name) {
908 Module *M = BB->getParent()->getParent();
909 Type *PtrTy = DerivedPtr->getType();
910 Function *FnGCFindBase = Intrinsic::getDeclaration(
911 M, Intrinsic::experimental_gc_get_pointer_base, {PtrTy, PtrTy});
912 return CreateCall(FnGCFindBase, {DerivedPtr}, {}, Name);
913}
914
916 const Twine &Name) {
917 Module *M = BB->getParent()->getParent();
918 Type *PtrTy = DerivedPtr->getType();
919 Function *FnGCGetOffset = Intrinsic::getDeclaration(
920 M, Intrinsic::experimental_gc_get_pointer_offset, {PtrTy});
921 return CreateCall(FnGCGetOffset, {DerivedPtr}, {}, Name);
922}
923
925 Instruction *FMFSource,
926 const Twine &Name) {
927 Module *M = BB->getModule();
928 Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
929 return createCallHelper(Fn, {V}, Name, FMFSource);
930}
931
933 Value *RHS,
934 Instruction *FMFSource,
935 const Twine &Name) {
936 Module *M = BB->getModule();
938 return createCallHelper(Fn, {LHS, RHS}, Name, FMFSource);
939}
940
942 ArrayRef<Type *> Types,
944 Instruction *FMFSource,
945 const Twine &Name) {
946 Module *M = BB->getModule();
947 Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
948 return createCallHelper(Fn, Args, Name, FMFSource);
949}
950
953 Instruction *FMFSource,
954 const Twine &Name) {
955 Module *M = BB->getModule();
956
960
961 SmallVector<Type *> ArgTys;
962 ArgTys.reserve(Args.size());
963 for (auto &I : Args)
964 ArgTys.push_back(I->getType());
965 FunctionType *FTy = FunctionType::get(RetTy, ArgTys, false);
966 SmallVector<Type *> OverloadTys;
968 matchIntrinsicSignature(FTy, TableRef, OverloadTys);
969 (void)Res;
971 "Wrong types for intrinsic!");
972 // TODO: Handle varargs intrinsics.
973
974 Function *Fn = Intrinsic::getDeclaration(M, ID, OverloadTys);
975 return createCallHelper(Fn, Args, Name, FMFSource);
976}
977
979 Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource,
980 const Twine &Name, MDNode *FPMathTag,
981 std::optional<RoundingMode> Rounding,
982 std::optional<fp::ExceptionBehavior> Except) {
983 Value *RoundingV = getConstrainedFPRounding(Rounding);
984 Value *ExceptV = getConstrainedFPExcept(Except);
985
986 FastMathFlags UseFMF = FMF;
987 if (FMFSource)
988 UseFMF = FMFSource->getFastMathFlags();
989
990 CallInst *C = CreateIntrinsic(ID, {L->getType()},
991 {L, R, RoundingV, ExceptV}, nullptr, Name);
993 setFPAttrs(C, FPMathTag, UseFMF);
994 return C;
995}
996
998 Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource,
999 const Twine &Name, MDNode *FPMathTag,
1000 std::optional<fp::ExceptionBehavior> Except) {
1001 Value *ExceptV = getConstrainedFPExcept(Except);
1002
1003 FastMathFlags UseFMF = FMF;
1004 if (FMFSource)
1005 UseFMF = FMFSource->getFastMathFlags();
1006
1007 CallInst *C =
1008 CreateIntrinsic(ID, {L->getType()}, {L, R, ExceptV}, nullptr, Name);
1010 setFPAttrs(C, FPMathTag, UseFMF);
1011 return C;
1012}
1013
1015 const Twine &Name, MDNode *FPMathTag) {
1016 if (Instruction::isBinaryOp(Opc)) {
1017 assert(Ops.size() == 2 && "Invalid number of operands!");
1018 return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc),
1019 Ops[0], Ops[1], Name, FPMathTag);
1020 }
1021 if (Instruction::isUnaryOp(Opc)) {
1022 assert(Ops.size() == 1 && "Invalid number of operands!");
1023 return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc),
1024 Ops[0], Name, FPMathTag);
1025 }
1026 llvm_unreachable("Unexpected opcode!");
1027}
1028
1030 Intrinsic::ID ID, Value *V, Type *DestTy,
1031 Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag,
1032 std::optional<RoundingMode> Rounding,
1033 std::optional<fp::ExceptionBehavior> Except) {
1034 Value *ExceptV = getConstrainedFPExcept(Except);
1035
1036 FastMathFlags UseFMF = FMF;
1037 if (FMFSource)
1038 UseFMF = FMFSource->getFastMathFlags();
1039
1040 CallInst *C;
1041 bool HasRoundingMD = false;
1042 switch (ID) {
1043 default:
1044 break;
1045#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
1046 case Intrinsic::INTRINSIC: \
1047 HasRoundingMD = ROUND_MODE; \
1048 break;
1049#include "llvm/IR/ConstrainedOps.def"
1050 }
1051 if (HasRoundingMD) {
1052 Value *RoundingV = getConstrainedFPRounding(Rounding);
1053 C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
1054 nullptr, Name);
1055 } else
1056 C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,
1057 Name);
1058
1060
1061 if (isa<FPMathOperator>(C))
1062 setFPAttrs(C, FPMathTag, UseFMF);
1063 return C;
1064}
1065
1066Value *IRBuilderBase::CreateFCmpHelper(
1067 CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name,
1068 MDNode *FPMathTag, bool IsSignaling) {
1069 if (IsFPConstrained) {
1070 auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
1071 : Intrinsic::experimental_constrained_fcmp;
1072 return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name);
1073 }
1074
1075 if (auto *LC = dyn_cast<Constant>(LHS))
1076 if (auto *RC = dyn_cast<Constant>(RHS))
1077 return Insert(Folder.CreateFCmp(P, LC, RC), Name);
1078 return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name);
1079}
1080
1083 const Twine &Name, std::optional<fp::ExceptionBehavior> Except) {
1084 Value *PredicateV = getConstrainedFPPredicate(P);
1085 Value *ExceptV = getConstrainedFPExcept(Except);
1086
1087 CallInst *C = CreateIntrinsic(ID, {L->getType()},
1088 {L, R, PredicateV, ExceptV}, nullptr, Name);
1090 return C;
1091}
1092
1094 Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
1095 std::optional<RoundingMode> Rounding,
1096 std::optional<fp::ExceptionBehavior> Except) {
1098
1099 append_range(UseArgs, Args);
1100 bool HasRoundingMD = false;
1101 switch (Callee->getIntrinsicID()) {
1102 default:
1103 break;
1104#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
1105 case Intrinsic::INTRINSIC: \
1106 HasRoundingMD = ROUND_MODE; \
1107 break;
1108#include "llvm/IR/ConstrainedOps.def"
1109 }
1110 if (HasRoundingMD)
1111 UseArgs.push_back(getConstrainedFPRounding(Rounding));
1112 UseArgs.push_back(getConstrainedFPExcept(Except));
1113
1114 CallInst *C = CreateCall(Callee, UseArgs, Name);
1116 return C;
1117}
1118
1120 const Twine &Name, Instruction *MDFrom) {
1121 if (auto *V = Folder.FoldSelect(C, True, False))
1122 return V;
1123
1124 SelectInst *Sel = SelectInst::Create(C, True, False);
1125 if (MDFrom) {
1126 MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof);
1127 MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable);
1128 Sel = addBranchMetadata(Sel, Prof, Unpred);
1129 }
1130 if (isa<FPMathOperator>(Sel))
1131 setFPAttrs(Sel, nullptr /* MDNode* */, FMF);
1132 return Insert(Sel, Name);
1133}
1134
1136 const Twine &Name) {
1137 assert(LHS->getType() == RHS->getType() &&
1138 "Pointer subtraction operand types must match!");
1141 Value *Difference = CreateSub(LHS_int, RHS_int);
1142 return CreateExactSDiv(Difference, ConstantExpr::getSizeOf(ElemTy),
1143 Name);
1144}
1145
1147 assert(isa<PointerType>(Ptr->getType()) &&
1148 "launder.invariant.group only applies to pointers.");
1149 auto *PtrType = Ptr->getType();
1150 Module *M = BB->getParent()->getParent();
1151 Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration(
1152 M, Intrinsic::launder_invariant_group, {PtrType});
1153
1154 assert(FnLaunderInvariantGroup->getReturnType() == PtrType &&
1155 FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
1156 PtrType &&
1157 "LaunderInvariantGroup should take and return the same type");
1158
1159 return CreateCall(FnLaunderInvariantGroup, {Ptr});
1160}
1161
1163 assert(isa<PointerType>(Ptr->getType()) &&
1164 "strip.invariant.group only applies to pointers.");
1165
1166 auto *PtrType = Ptr->getType();
1167 Module *M = BB->getParent()->getParent();
1168 Function *FnStripInvariantGroup = Intrinsic::getDeclaration(
1169 M, Intrinsic::strip_invariant_group, {PtrType});
1170
1171 assert(FnStripInvariantGroup->getReturnType() == PtrType &&
1172 FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
1173 PtrType &&
1174 "StripInvariantGroup should take and return the same type");
1175
1176 return CreateCall(FnStripInvariantGroup, {Ptr});
1177}
1178
1180 auto *Ty = cast<VectorType>(V->getType());
1181 if (isa<ScalableVectorType>(Ty)) {
1182 Module *M = BB->getParent()->getParent();
1184 M, Intrinsic::experimental_vector_reverse, Ty);
1185 return Insert(CallInst::Create(F, V), Name);
1186 }
1187 // Keep the original behaviour for fixed vector
1188 SmallVector<int, 8> ShuffleMask;
1189 int NumElts = Ty->getElementCount().getKnownMinValue();
1190 for (int i = 0; i < NumElts; ++i)
1191 ShuffleMask.push_back(NumElts - i - 1);
1192 return CreateShuffleVector(V, ShuffleMask, Name);
1193}
1194
1196 const Twine &Name) {
1197 assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1198 assert(V1->getType() == V2->getType() &&
1199 "Splice expects matching operand types!");
1200
1201 if (auto *VTy = dyn_cast<ScalableVectorType>(V1->getType())) {
1202 Module *M = BB->getParent()->getParent();
1204 M, Intrinsic::experimental_vector_splice, VTy);
1205
1206 Value *Ops[] = {V1, V2, getInt32(Imm)};
1207 return Insert(CallInst::Create(F, Ops), Name);
1208 }
1209
1210 unsigned NumElts = cast<FixedVectorType>(V1->getType())->getNumElements();
1211 assert(((-Imm <= NumElts) || (Imm < NumElts)) &&
1212 "Invalid immediate for vector splice!");
1213
1214 // Keep the original behaviour for fixed vector
1215 unsigned Idx = (NumElts + Imm) % NumElts;
1217 for (unsigned I = 0; I < NumElts; ++I)
1218 Mask.push_back(Idx + I);
1219
1220 return CreateShuffleVector(V1, V2, Mask);
1221}
1222
1224 const Twine &Name) {
1225 auto EC = ElementCount::getFixed(NumElts);
1226 return CreateVectorSplat(EC, V, Name);
1227}
1228
1230 const Twine &Name) {
1231 assert(EC.isNonZero() && "Cannot splat to an empty vector!");
1232
1233 // First insert it into a poison vector so we can shuffle it.
1234 Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));
1235 V = CreateInsertElement(Poison, V, getInt64(0), Name + ".splatinsert");
1236
1237 // Shuffle the value across the desired number of elements.
1239 Zeros.resize(EC.getKnownMinValue());
1240 return CreateShuffleVector(V, Zeros, Name + ".splat");
1241}
1242
1244 Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex,
1245 MDNode *DbgInfo) {
1246 auto *BaseType = Base->getType();
1247 assert(isa<PointerType>(BaseType) &&
1248 "Invalid Base ptr type for preserve.array.access.index.");
1249
1250 Value *LastIndexV = getInt32(LastIndex);
1252 SmallVector<Value *, 4> IdxList(Dimension, Zero);
1253 IdxList.push_back(LastIndexV);
1254
1255 Type *ResultType = GetElementPtrInst::getGEPReturnType(Base, IdxList);
1256
1257 Module *M = BB->getParent()->getParent();
1258 Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration(
1259 M, Intrinsic::preserve_array_access_index, {ResultType, BaseType});
1260
1261 Value *DimV = getInt32(Dimension);
1262 CallInst *Fn =
1263 CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV});
1264 Fn->addParamAttr(
1265 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
1266 if (DbgInfo)
1267 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1268
1269 return Fn;
1270}
1271
1273 Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1274 assert(isa<PointerType>(Base->getType()) &&
1275 "Invalid Base ptr type for preserve.union.access.index.");
1276 auto *BaseType = Base->getType();
1277
1278 Module *M = BB->getParent()->getParent();
1279 Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration(
1280 M, Intrinsic::preserve_union_access_index, {BaseType, BaseType});
1281
1282 Value *DIIndex = getInt32(FieldIndex);
1283 CallInst *Fn =
1284 CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex});
1285 if (DbgInfo)
1286 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1287
1288 return Fn;
1289}
1290
1292 Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1293 MDNode *DbgInfo) {
1294 auto *BaseType = Base->getType();
1295 assert(isa<PointerType>(BaseType) &&
1296 "Invalid Base ptr type for preserve.struct.access.index.");
1297
1298 Value *GEPIndex = getInt32(Index);
1300 Type *ResultType =
1301 GetElementPtrInst::getGEPReturnType(Base, {Zero, GEPIndex});
1302
1303 Module *M = BB->getParent()->getParent();
1304 Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration(
1305 M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType});
1306
1307 Value *DIIndex = getInt32(FieldIndex);
1308 CallInst *Fn = CreateCall(FnPreserveStructAccessIndex,
1309 {Base, GEPIndex, DIIndex});
1310 Fn->addParamAttr(
1311 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
1312 if (DbgInfo)
1313 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1314
1315 return Fn;
1316}
1317
1319 ConstantInt *TestV = getInt32(Test);
1320 Module *M = BB->getParent()->getParent();
1321 Function *FnIsFPClass =
1322 Intrinsic::getDeclaration(M, Intrinsic::is_fpclass, {FPNum->getType()});
1323 return CreateCall(FnIsFPClass, {FPNum, TestV});
1324}
1325
1326CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1327 Value *PtrValue,
1328 Value *AlignValue,
1329 Value *OffsetValue) {
1330 SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1331 if (OffsetValue)
1332 Vals.push_back(OffsetValue);
1333 OperandBundleDefT<Value *> AlignOpB("align", Vals);
1334 return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB});
1335}
1336
1338 Value *PtrValue,
1339 unsigned Alignment,
1340 Value *OffsetValue) {
1341 assert(isa<PointerType>(PtrValue->getType()) &&
1342 "trying to create an alignment assumption on a non-pointer?");
1343 assert(Alignment != 0 && "Invalid Alignment");
1344 auto *PtrTy = cast<PointerType>(PtrValue->getType());
1345 Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
1346 Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment);
1347 return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
1348}
1349
1351 Value *PtrValue,
1352 Value *Alignment,
1353 Value *OffsetValue) {
1354 assert(isa<PointerType>(PtrValue->getType()) &&
1355 "trying to create an alignment assumption on a non-pointer?");
1356 return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
1357}
1358
1362void ConstantFolder::anchor() {}
1363void NoFolder::anchor() {}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
ArrayRef< TableEntry > TableRef
assume Assume Builder
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
std::string Name
uint64_t Size
static bool isConstantOne(const Value *Val)
isConstantOne - Return true only if val is constant int 1
Definition: IRBuilder.cpp:295
static InvokeInst * CreateGCStatepointInvokeCommon(IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags, ArrayRef< T0 > InvokeArgs, std::optional< ArrayRef< T1 > > TransitionArgs, std::optional< ArrayRef< T2 > > DeoptArgs, ArrayRef< T3 > GCArgs, const Twine &Name)
Definition: IRBuilder.cpp:824
static CallInst * CreateGCStatepointCallCommon(IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee, uint32_t Flags, ArrayRef< T0 > CallArgs, std::optional< ArrayRef< T1 > > TransitionArgs, std::optional< ArrayRef< T2 > > DeoptArgs, ArrayRef< T3 > GCArgs, const Twine &Name)
Definition: IRBuilder.cpp:770
static std::vector< OperandBundleDef > getStatepointBundles(std::optional< ArrayRef< T1 > > TransitionArgs, std::optional< ArrayRef< T2 > > DeoptArgs, ArrayRef< T3 > GCArgs)
Definition: IRBuilder.cpp:747
static std::vector< Value * > getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags, ArrayRef< T0 > CallArgs)
Definition: IRBuilder.cpp:728
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:526
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define P(N)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
Value * RHS
Value * LHS
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:92
static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
Definition: Attributes.cpp:168
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:112
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Definition: BasicBlock.cpp:145
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1474
void addRetAttr(Attribute::AttrKind Kind)
Adds the attribute to the return value.
Definition: InstrTypes.h:1531
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Definition: InstrTypes.h:1541
This class represents a function call, abstracting a target machine's calling convention.
void setTailCall(bool IsTc=true)
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:711
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2890
static Constant * getSizeOf(Type *Ty)
getSizeOf constant expr - computes the (alloc) size of a type (in address-units, not bits) in a targe...
Definition: Constants.cpp:2334
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Definition: Constants.h:203
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:833
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
static Constant * get(ArrayRef< Constant * > V)
Definition: Constants.cpp:1342
This is an important base class in LLVM.
Definition: Constant.h:41
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
A debug info location.
Definition: DebugLoc.h:33
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:291
This instruction compares its operands according to the predicate given to the constructor.
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:20
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
FunctionType * getFunctionType()
Definition: DerivedTypes.h:182
Class to represent function types.
Definition: DerivedTypes.h:103
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:176
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:181
static Type * getGEPReturnType(Value *Ptr, ArrayRef< Value * > IdxList)
Returns the pointer type returned by the GEP instruction, which may be a vector of pointers.
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
Value * CreateExactSDiv(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1391
CallInst * CreateElementUnorderedAtomicMemCpy(Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, uint32_t ElementSize, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert an element unordered-atomic memcpy between the specified pointers.
Definition: IRBuilder.cpp:256
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Definition: IRBuilder.h:447
CallInst * CreateMaskedCompressStore(Value *Val, Value *Ptr, Value *Mask=nullptr)
Create a call to Masked Compress Store intrinsic.
Definition: IRBuilder.cpp:715
BasicBlock * BB
Definition: IRBuilder.h:119
CallInst * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
Definition: IRBuilder.cpp:924
Value * CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS, const Twine &Name="")
Return the i64 difference between two pointer values, dividing out the size of the pointed-to objects...
Definition: IRBuilder.cpp:1135
CallInst * CreateMulReduce(Value *Src)
Create a vector int mul reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:437
CallInst * CreateFAddReduce(Value *Acc, Value *Src)
Create a sequential vector fadd reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:417
CallInst * CreateMaskedExpandLoad(Type *Ty, Value *Ptr, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Expand Load intrinsic.
Definition: IRBuilder.cpp:697
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1993
Value * CreateVScale(Constant *Scaling, const Twine &Name="")
Create a call to llvm.vscale, multiplied by Scaling.
Definition: IRBuilder.cpp:88
Value * CreateLaunderInvariantGroup(Value *Ptr)
Create a launder.invariant.group intrinsic call.
Definition: IRBuilder.cpp:1146
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2434
CallInst * CreateThreadLocalAddress(Value *Ptr)
Create a call to llvm.threadlocal.address intrinsic.
Definition: IRBuilder.cpp:538
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Definition: IRBuilder.h:497
Type * getCurrentFunctionReturnType() const
Get the return type of the current function that we're emitting into.
Definition: IRBuilder.cpp:58
CallInst * CreateGCGetPointerBase(Value *DerivedPtr, const Twine &Name="")
Create a call to the experimental.gc.pointer.base intrinsic to get the base pointer for the specified...
Definition: IRBuilder.cpp:906
CallInst * CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee, ArrayRef< Value * > CallArgs, std::optional< ArrayRef< Value * > > DeoptArgs, ArrayRef< Value * > GCArgs, const Twine &Name="")
Create a call to the experimental.gc.statepoint intrinsic to start a new statepoint sequence.
Definition: IRBuilder.cpp:794
GlobalVariable * CreateGlobalString(StringRef Str, const Twine &Name="", unsigned AddressSpace=0, Module *M=nullptr)
Make a new global variable with initializer type i8*.
Definition: IRBuilder.cpp:43
CallInst * CreateLifetimeStart(Value *Ptr, ConstantInt *Size=nullptr)
Create a lifetime.start intrinsic.
Definition: IRBuilder.cpp:481
CallInst * CreateConstrainedFPCmp(Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R, const Twine &Name="", std::optional< fp::ExceptionBehavior > Except=std::nullopt)
Definition: IRBuilder.cpp:1081
CallInst * CreateFree(Value *Source, ArrayRef< OperandBundleDef > Bundles=std::nullopt)
Generate the IR for a call to the builtin free function.
Definition: IRBuilder.cpp:353
CallInst * CreateAndReduce(Value *Src)
Create a vector int AND reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:441
Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")
Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.
Definition: IRBuilder.cpp:1195
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Definition: IRBuilder.cpp:1223
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Definition: IRBuilder.cpp:941
Value * CreatePreserveStructAccessIndex(Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex, MDNode *DbgInfo)
Definition: IRBuilder.cpp:1291
CallInst * CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue, unsigned Alignment, Value *OffsetValue=nullptr)
Create an assume intrinsic call that represents an alignment assumption on the provided pointer.
Definition: IRBuilder.cpp:1337
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
Definition: IRBuilder.cpp:589
CallInst * CreateConstrainedFPCall(Function *Callee, ArrayRef< Value * > Args, const Twine &Name="", std::optional< RoundingMode > Rounding=std::nullopt, std::optional< fp::ExceptionBehavior > Except=std::nullopt)
Definition: IRBuilder.cpp:1093
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, MaybeAlign Align, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memset to the specified pointer and the specified value.
Definition: IRBuilder.h:586
LLVMContext & Context
Definition: IRBuilder.h:121
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.cpp:1119
CallInst * CreateGCGetPointerOffset(Value *DerivedPtr, const Twine &Name="")
Create a call to the experimental.gc.get.pointer.offset intrinsic to get the offset of the specified ...
Definition: IRBuilder.cpp:915
Value * CreateTypeSize(Type *DstType, TypeSize Size)
Create an expression which evaluates to the number of units in Size at runtime.
Definition: IRBuilder.cpp:104
CallInst * CreateAddReduce(Value *Src)
Create a vector int add reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:433
IntegerType * getIntPtrTy(const DataLayout &DL, unsigned AddrSpace=0)
Fetch the type of an integer with size at least as big as that of a pointer in the given address spac...
Definition: IRBuilder.h:566
CallInst * CreateConstrainedFPUnroundedBinOp(Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource=nullptr, const Twine &Name="", MDNode *FPMathTag=nullptr, std::optional< fp::ExceptionBehavior > Except=std::nullopt)
Definition: IRBuilder.cpp:997
BasicBlock * GetInsertBlock() const
Definition: IRBuilder.h:174
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Definition: IRBuilder.h:517
CallInst * CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, Value *Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Definition: IRBuilder.cpp:219
Value * CreateVectorReverse(Value *V, const Twine &Name="")
Return a vector value that contains the vector V reversed.
Definition: IRBuilder.cpp:1179
CallInst * CreateXorReduce(Value *Src)
Create a vector int XOR reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:449
FastMathFlags FMF
Definition: IRBuilder.h:126
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition: IRBuilder.h:477
Value * getAllOnesMask(ElementCount NumElts)
Return an all true boolean vector (mask) with NumElts lanes.
Definition: IRBuilder.h:839
Value * CreateUnOp(Instruction::UnaryOps Opc, Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1749
CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:445
CallInst * CreateMalloc(Type *IntPtrTy, Type *AllocTy, Value *AllocSize, Value *ArraySize, ArrayRef< OperandBundleDef > OpB, Function *MallocF=nullptr, const Twine &Name="")
Definition: IRBuilder.cpp:301
CallInst * CreateFPMinReduce(Value *Src)
Create a vector float min reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:469
CallInst * CreateFPMaximumReduce(Value *Src)
Create a vector float maximum reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:473
Value * createIsFPClass(Value *FPNum, unsigned Test)
Definition: IRBuilder.cpp:1318
CallInst * CreateFPMaxReduce(Value *Src)
Create a vector float max reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:465
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:472
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
Definition: IRBuilder.h:145
CallInst * CreateConstrainedFPBinOp(Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource=nullptr, const Twine &Name="", MDNode *FPMathTag=nullptr, std::optional< RoundingMode > Rounding=std::nullopt, std::optional< fp::ExceptionBehavior > Except=std::nullopt)
Definition: IRBuilder.cpp:978
DebugLoc getCurrentDebugLocation() const
Get location information used by debugging information.
Definition: IRBuilder.cpp:63
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1335
Value * CreateNAryOp(unsigned Opc, ArrayRef< Value * > Ops, const Twine &Name="", MDNode *FPMathTag=nullptr)
Create either a UnaryOperator or BinaryOperator depending on Opc.
Definition: IRBuilder.cpp:1014
CallInst * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Definition: IRBuilder.cpp:932
CallInst * CreateAssumption(Value *Cond, ArrayRef< OperandBundleDef > OpBundles=std::nullopt)
Create an assume intrinsic call that allows the optimizer to assume that the provided condition will ...
Definition: IRBuilder.cpp:562
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Definition: IRBuilder.h:2456
LLVMContext & getContext() const
Definition: IRBuilder.h:176
Value * CreatePreserveUnionAccessIndex(Value *Base, unsigned FieldIndex, MDNode *DbgInfo)
Definition: IRBuilder.cpp:1272
CallInst * CreateIntMaxReduce(Value *Src, bool IsSigned=false)
Create a vector integer max reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:453
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Definition: IRBuilder.cpp:609
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2079
CallInst * CreateGCResult(Instruction *Statepoint, Type *ResultType, const Twine &Name="")
Create a call to the experimental.gc.result intrinsic to extract the result from a call wrapped in a ...
Definition: IRBuilder.cpp:883
CallInst * CreateLifetimeEnd(Value *Ptr, ConstantInt *Size=nullptr)
Create a lifetime.end intrinsic.
Definition: IRBuilder.cpp:496
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1657
Value * CreateElementCount(Type *DstType, ElementCount EC)
Create an expression which evaluates to the number of elements in EC at runtime.
Definition: IRBuilder.cpp:99
CallInst * CreateConstrainedFPCast(Intrinsic::ID ID, Value *V, Type *DestTy, Instruction *FMFSource=nullptr, const Twine &Name="", MDNode *FPMathTag=nullptr, std::optional< RoundingMode > Rounding=std::nullopt, std::optional< fp::ExceptionBehavior > Except=std::nullopt)
Definition: IRBuilder.cpp:1029
CallInst * CreateElementUnorderedAtomicMemMove(Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, uint32_t ElementSize, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert an element unordered-atomic memmove between the specified pointers.
Definition: IRBuilder.cpp:372
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition: IRBuilder.h:2158
CallInst * CreateIntMinReduce(Value *Src, bool IsSigned=false)
Create a vector integer min reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:459
void setConstrainedFPCallAttr(CallBase *I)
Definition: IRBuilder.h:347
InvokeInst * CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef< Value * > InvokeArgs, std::optional< ArrayRef< Value * > > DeoptArgs, ArrayRef< Value * > GCArgs, const Twine &Name="")
Create an invoke to the experimental.gc.statepoint intrinsic to start a new statepoint sequence.
Definition: IRBuilder.cpp:850
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2374
const IRBuilderFolder & Folder
Definition: IRBuilder.h:122
CallInst * CreateMemSetInline(Value *Dst, MaybeAlign DstAlign, Value *Val, Value *Size, bool IsVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Definition: IRBuilder.cpp:164
CallInst * CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val, uint64_t Size, Align Alignment, uint32_t ElementSize, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert an element unordered-atomic memset of the region of memory starting at the given po...
Definition: IRBuilder.h:611
CallInst * CreateFMulReduce(Value *Acc, Value *Src)
Create a sequential vector fmul reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:425
void SetInstDebugLocation(Instruction *I) const
If this builder has a current debug location, set it on the specified instruction.
Definition: IRBuilder.cpp:70
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Definition: IRBuilder.h:502
CallInst * CreateGCRelocate(Instruction *Statepoint, int BaseOffset, int DerivedOffset, Type *ResultType, const Twine &Name="")
Create a call to the experimental.gc.relocate intrinsics to project the relocated value of one pointe...
Definition: IRBuilder.cpp:894
Value * CreateStepVector(Type *DstType, const Twine &Name="")
Creates a vector of type DstType with the linear sequence <0, 1, ...>
Definition: IRBuilder.cpp:109
Value * CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex, MDNode *DbgInfo)
Definition: IRBuilder.cpp:1243
CallInst * CreateInvariantStart(Value *Ptr, ConstantInt *Size=nullptr)
Create a call to invariant.start intrinsic.
Definition: IRBuilder.cpp:511
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1352
Instruction * CreateNoAliasScopeDeclaration(Value *Scope)
Create a llvm.experimental.noalias.scope.decl intrinsic call.
Definition: IRBuilder.cpp:573
CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
Definition: IRBuilder.cpp:672
CallInst * CreateFPMinimumReduce(Value *Src)
Create a vector float minimum reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:477
Value * CreateStripInvariantGroup(Value *Ptr)
Create a strip.invariant.group intrinsic call.
Definition: IRBuilder.cpp:1162
CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
Definition: IRBuilder.cpp:641
virtual Value * FoldSelect(Value *C, Value *True, Value *False) const =0
virtual Value * CreateFCmp(CmpInst::Predicate P, Constant *LHS, Constant *RHS) const =0
virtual ~IRBuilderFolder()
void copyFastMathFlags(FastMathFlags FMF)
Convenience function for transferring all fast-math flag values to this instruction,...
bool isBinaryOp() const
Definition: Instruction.h:200
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:302
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1521
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
bool isUnaryOp() const
Definition: Instruction.h:199
Invoke instruction.
Metadata node.
Definition: Metadata.h:950
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
A container for an operand bundle being viewed as a set of values rather than a set of uses.
Definition: InstrTypes.h:1143
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:659
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1743
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", Instruction *InsertBefore=nullptr, Instruction *MDFrom=nullptr)
void reserve(size_type N)
Definition: SmallVector.h:667
void resize(size_type N)
Definition: SmallVector.h:642
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:265
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:348
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1069
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:677
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl< IITDescriptor > &T)
Return the IIT table descriptor for the specified intrinsic into an array of IITDescriptors.
Definition: Function.cpp:1266
@ MatchIntrinsicTypes_Match
Definition: Intrinsics.h:209
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1422
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
AddressSpace
Definition: NVPTXBaseInfo.h:21
bool getAlign(const Function &F, unsigned index, unsigned &align)
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:2037
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117