LLVM 17.0.0git
IRBuilder.cpp
Go to the documentation of this file.
1//===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/IR/IRBuilder.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/IR/Constant.h"
17#include "llvm/IR/Constants.h"
20#include "llvm/IR/Function.h"
21#include "llvm/IR/GlobalValue.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/LLVMContext.h"
26#include "llvm/IR/NoFolder.h"
27#include "llvm/IR/Operator.h"
28#include "llvm/IR/Statepoint.h"
29#include "llvm/IR/Type.h"
30#include "llvm/IR/Value.h"
32#include <cassert>
33#include <cstdint>
34#include <optional>
35#include <vector>
36
37using namespace llvm;
38
39/// CreateGlobalString - Make a new global variable with an initializer that
40/// has array of i8 type filled in with the nul terminated string value
41/// specified. If Name is specified, it is the name of the global variable
42/// created.
44 const Twine &Name,
45 unsigned AddressSpace,
46 Module *M) {
48 if (!M)
49 M = BB->getParent()->getParent();
50 auto *GV = new GlobalVariable(
51 *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
52 StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
53 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
54 GV->setAlignment(Align(1));
55 return GV;
56}
57
59 assert(BB && BB->getParent() && "No current function!");
60 return BB->getParent()->getReturnType();
61}
62
63Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
64 auto *PT = cast<PointerType>(Ptr->getType());
65 if (PT->isOpaqueOrPointeeTypeMatches(getInt8Ty()))
66 return Ptr;
67
68 // Otherwise, we need to insert a bitcast.
69 return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace()));
70}
71
73 for (auto &KV : MetadataToCopy)
74 if (KV.first == LLVMContext::MD_dbg)
75 return {cast<DILocation>(KV.second)};
76
77 return {};
78}
80 for (const auto &KV : MetadataToCopy)
81 if (KV.first == LLVMContext::MD_dbg) {
82 I->setDebugLoc(DebugLoc(KV.second));
83 return;
84 }
85}
86
88IRBuilderBase::createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
89 const Twine &Name, Instruction *FMFSource,
91 CallInst *CI = CreateCall(Callee, Ops, OpBundles, Name);
92 if (FMFSource)
93 CI->copyFastMathFlags(FMFSource);
94 return CI;
95}
96
98 assert(isa<ConstantInt>(Scaling) && "Expected constant integer");
99 if (cast<ConstantInt>(Scaling)->isZero())
100 return Scaling;
102 Function *TheFn =
103 Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()});
104 CallInst *CI = CreateCall(TheFn, {}, {}, Name);
105 return cast<ConstantInt>(Scaling)->isOne() ? CI : CreateMul(CI, Scaling);
106}
107
109 Constant *MinEC = ConstantInt::get(DstType, EC.getKnownMinValue());
110 return EC.isScalable() ? CreateVScale(MinEC) : MinEC;
111}
112
114 Constant *MinSize = ConstantInt::get(DstType, Size.getKnownMinValue());
115 return Size.isScalable() ? CreateVScale(MinSize) : MinSize;
116}
117
119 Type *STy = DstType->getScalarType();
120 if (isa<ScalableVectorType>(DstType)) {
121 Type *StepVecType = DstType;
122 // TODO: We expect this special case (element type < 8 bits) to be
123 // temporary - once the intrinsic properly supports < 8 bits this code
124 // can be removed.
125 if (STy->getScalarSizeInBits() < 8)
126 StepVecType =
127 VectorType::get(getInt8Ty(), cast<ScalableVectorType>(DstType));
128 Value *Res = CreateIntrinsic(Intrinsic::experimental_stepvector,
129 {StepVecType}, {}, nullptr, Name);
130 if (StepVecType != DstType)
131 Res = CreateTrunc(Res, DstType);
132 return Res;
133 }
134
135 unsigned NumEls = cast<FixedVectorType>(DstType)->getNumElements();
136
137 // Create a vector of consecutive numbers from zero to VF.
139 for (unsigned i = 0; i < NumEls; ++i)
140 Indices.push_back(ConstantInt::get(STy, i));
141
142 // Add the consecutive indices to the vector value.
143 return ConstantVector::get(Indices);
144}
145
147 MaybeAlign Align, bool isVolatile,
148 MDNode *TBAATag, MDNode *ScopeTag,
149 MDNode *NoAliasTag) {
150 Ptr = getCastedInt8PtrValue(Ptr);
151 Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
152 Type *Tys[] = { Ptr->getType(), Size->getType() };
153 Module *M = BB->getParent()->getParent();
154 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
155
156 CallInst *CI = CreateCall(TheFn, Ops);
157
158 if (Align)
159 cast<MemSetInst>(CI)->setDestAlignment(*Align);
160
161 // Set the TBAA info if present.
162 if (TBAATag)
163 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
164
165 if (ScopeTag)
166 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
167
168 if (NoAliasTag)
169 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
170
171 return CI;
172}
173
175 Value *Val, Value *Size,
176 bool IsVolatile, MDNode *TBAATag,
177 MDNode *ScopeTag,
178 MDNode *NoAliasTag) {
179 Dst = getCastedInt8PtrValue(Dst);
180 Value *Ops[] = {Dst, Val, Size, getInt1(IsVolatile)};
181 Type *Tys[] = {Dst->getType(), Size->getType()};
182 Module *M = BB->getParent()->getParent();
183 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset_inline, Tys);
184
185 CallInst *CI = CreateCall(TheFn, Ops);
186
187 if (DstAlign)
188 cast<MemSetInlineInst>(CI)->setDestAlignment(*DstAlign);
189
190 // Set the TBAA info if present.
191 if (TBAATag)
192 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
193
194 if (ScopeTag)
195 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
196
197 if (NoAliasTag)
198 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
199
200 return CI;
201}
202
204 Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
205 MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
206
207 Ptr = getCastedInt8PtrValue(Ptr);
208 Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
209 Type *Tys[] = {Ptr->getType(), Size->getType()};
210 Module *M = BB->getParent()->getParent();
212 M, Intrinsic::memset_element_unordered_atomic, Tys);
213
214 CallInst *CI = CreateCall(TheFn, Ops);
215
216 cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
217
218 // Set the TBAA info if present.
219 if (TBAATag)
220 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
221
222 if (ScopeTag)
223 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
224
225 if (NoAliasTag)
226 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
227
228 return CI;
229}
230
232 Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
233 MaybeAlign SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag,
234 MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
235 Dst = getCastedInt8PtrValue(Dst);
236 Src = getCastedInt8PtrValue(Src);
237
238 Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
239 Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
240 Module *M = BB->getParent()->getParent();
241 Function *TheFn = Intrinsic::getDeclaration(M, IntrID, Tys);
242
243 CallInst *CI = CreateCall(TheFn, Ops);
244
245 auto* MCI = cast<MemTransferInst>(CI);
246 if (DstAlign)
247 MCI->setDestAlignment(*DstAlign);
248 if (SrcAlign)
249 MCI->setSourceAlignment(*SrcAlign);
250
251 // Set the TBAA info if present.
252 if (TBAATag)
253 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
254
255 // Set the TBAA Struct info if present.
256 if (TBAAStructTag)
257 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
258
259 if (ScopeTag)
260 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
261
262 if (NoAliasTag)
263 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
264
265 return CI;
266}
267
269 Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign,
270 Value *Size, bool IsVolatile, MDNode *TBAATag, MDNode *TBAAStructTag,
271 MDNode *ScopeTag, MDNode *NoAliasTag) {
272 Dst = getCastedInt8PtrValue(Dst);
273 Src = getCastedInt8PtrValue(Src);
274
275 Value *Ops[] = {Dst, Src, Size, getInt1(IsVolatile)};
276 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
277 Function *F = BB->getParent();
278 Module *M = F->getParent();
279 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy_inline, Tys);
280
281 CallInst *CI = CreateCall(TheFn, Ops);
282
283 auto *MCI = cast<MemCpyInlineInst>(CI);
284 if (DstAlign)
285 MCI->setDestAlignment(*DstAlign);
286 if (SrcAlign)
287 MCI->setSourceAlignment(*SrcAlign);
288
289 // Set the TBAA info if present.
290 if (TBAATag)
291 MCI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
292
293 // Set the TBAA Struct info if present.
294 if (TBAAStructTag)
295 MCI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
296
297 if (ScopeTag)
298 MCI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
299
300 if (NoAliasTag)
301 MCI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
302
303 return CI;
304}
305
307 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
308 uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
309 MDNode *ScopeTag, MDNode *NoAliasTag) {
310 assert(DstAlign >= ElementSize &&
311 "Pointer alignment must be at least element size");
312 assert(SrcAlign >= ElementSize &&
313 "Pointer alignment must be at least element size");
314 Dst = getCastedInt8PtrValue(Dst);
315 Src = getCastedInt8PtrValue(Src);
316
317 Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
318 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
319 Module *M = BB->getParent()->getParent();
321 M, Intrinsic::memcpy_element_unordered_atomic, Tys);
322
323 CallInst *CI = CreateCall(TheFn, Ops);
324
325 // Set the alignment of the pointer args.
326 auto *AMCI = cast<AtomicMemCpyInst>(CI);
327 AMCI->setDestAlignment(DstAlign);
328 AMCI->setSourceAlignment(SrcAlign);
329
330 // Set the TBAA info if present.
331 if (TBAATag)
332 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
333
334 // Set the TBAA Struct info if present.
335 if (TBAAStructTag)
336 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
337
338 if (ScopeTag)
339 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
340
341 if (NoAliasTag)
342 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
343
344 return CI;
345}
346
348 Value *Src, MaybeAlign SrcAlign,
349 Value *Size, bool isVolatile,
350 MDNode *TBAATag, MDNode *ScopeTag,
351 MDNode *NoAliasTag) {
352 Dst = getCastedInt8PtrValue(Dst);
353 Src = getCastedInt8PtrValue(Src);
354
355 Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
356 Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
357 Module *M = BB->getParent()->getParent();
358 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
359
360 CallInst *CI = CreateCall(TheFn, Ops);
361
362 auto *MMI = cast<MemMoveInst>(CI);
363 if (DstAlign)
364 MMI->setDestAlignment(*DstAlign);
365 if (SrcAlign)
366 MMI->setSourceAlignment(*SrcAlign);
367
368 // Set the TBAA info if present.
369 if (TBAATag)
370 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
371
372 if (ScopeTag)
373 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
374
375 if (NoAliasTag)
376 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
377
378 return CI;
379}
380
382 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
383 uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
384 MDNode *ScopeTag, MDNode *NoAliasTag) {
385 assert(DstAlign >= ElementSize &&
386 "Pointer alignment must be at least element size");
387 assert(SrcAlign >= ElementSize &&
388 "Pointer alignment must be at least element size");
389 Dst = getCastedInt8PtrValue(Dst);
390 Src = getCastedInt8PtrValue(Src);
391
392 Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
393 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
394 Module *M = BB->getParent()->getParent();
396 M, Intrinsic::memmove_element_unordered_atomic, Tys);
397
398 CallInst *CI = CreateCall(TheFn, Ops);
399
400 // Set the alignment of the pointer args.
401 CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
402 CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
403
404 // Set the TBAA info if present.
405 if (TBAATag)
406 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
407
408 // Set the TBAA Struct info if present.
409 if (TBAAStructTag)
410 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
411
412 if (ScopeTag)
413 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
414
415 if (NoAliasTag)
416 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
417
418 return CI;
419}
420
421CallInst *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
423 Value *Ops[] = {Src};
424 Type *Tys[] = { Src->getType() };
425 auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
426 return CreateCall(Decl, Ops);
427}
428
431 Value *Ops[] = {Acc, Src};
432 auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fadd,
433 {Src->getType()});
434 return CreateCall(Decl, Ops);
435}
436
439 Value *Ops[] = {Acc, Src};
440 auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fmul,
441 {Src->getType()});
442 return CreateCall(Decl, Ops);
443}
444
446 return getReductionIntrinsic(Intrinsic::vector_reduce_add, Src);
447}
448
450 return getReductionIntrinsic(Intrinsic::vector_reduce_mul, Src);
451}
452
454 return getReductionIntrinsic(Intrinsic::vector_reduce_and, Src);
455}
456
458 return getReductionIntrinsic(Intrinsic::vector_reduce_or, Src);
459}
460
462 return getReductionIntrinsic(Intrinsic::vector_reduce_xor, Src);
463}
464
466 auto ID =
467 IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
468 return getReductionIntrinsic(ID, Src);
469}
470
472 auto ID =
473 IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
474 return getReductionIntrinsic(ID, Src);
475}
476
478 return getReductionIntrinsic(Intrinsic::vector_reduce_fmax, Src);
479}
480
482 return getReductionIntrinsic(Intrinsic::vector_reduce_fmin, Src);
483}
484
486 assert(isa<PointerType>(Ptr->getType()) &&
487 "lifetime.start only applies to pointers.");
488 Ptr = getCastedInt8PtrValue(Ptr);
489 if (!Size)
490 Size = getInt64(-1);
491 else
492 assert(Size->getType() == getInt64Ty() &&
493 "lifetime.start requires the size to be an i64");
494 Value *Ops[] = { Size, Ptr };
495 Module *M = BB->getParent()->getParent();
496 Function *TheFn =
497 Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
498 return CreateCall(TheFn, Ops);
499}
500
502 assert(isa<PointerType>(Ptr->getType()) &&
503 "lifetime.end only applies to pointers.");
504 Ptr = getCastedInt8PtrValue(Ptr);
505 if (!Size)
506 Size = getInt64(-1);
507 else
508 assert(Size->getType() == getInt64Ty() &&
509 "lifetime.end requires the size to be an i64");
510 Value *Ops[] = { Size, Ptr };
511 Module *M = BB->getParent()->getParent();
512 Function *TheFn =
513 Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
514 return CreateCall(TheFn, Ops);
515}
516
518
519 assert(isa<PointerType>(Ptr->getType()) &&
520 "invariant.start only applies to pointers.");
521 Ptr = getCastedInt8PtrValue(Ptr);
522 if (!Size)
523 Size = getInt64(-1);
524 else
525 assert(Size->getType() == getInt64Ty() &&
526 "invariant.start requires the size to be an i64");
527
528 Value *Ops[] = {Size, Ptr};
529 // Fill in the single overloaded type: memory object type.
530 Type *ObjectPtr[1] = {Ptr->getType()};
531 Module *M = BB->getParent()->getParent();
532 Function *TheFn =
533 Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
534 return CreateCall(TheFn, Ops);
535}
536
538 if (auto *O = dyn_cast<GlobalObject>(Ptr))
539 return O->getAlign();
540 if (auto *A = dyn_cast<GlobalAlias>(Ptr))
541 return A->getAliaseeObject()->getAlign();
542 return {};
543}
544
546#ifndef NDEBUG
547 // Handle specially for constexpr cast. This is possible when
548 // opaque pointers not enabled since constant could be sinked
549 // directly by the design of llvm. This could be eliminated
550 // after we eliminate the abuse of constexpr.
551 auto *V = Ptr;
552 if (auto *CE = dyn_cast<ConstantExpr>(V))
553 if (CE->isCast())
554 V = CE->getOperand(0);
555
556 assert(isa<GlobalValue>(V) && cast<GlobalValue>(V)->isThreadLocal() &&
557 "threadlocal_address only applies to thread local variables.");
558#endif
559 CallInst *CI = CreateIntrinsic(llvm::Intrinsic::threadlocal_address,
560 {Ptr->getType()}, {Ptr});
561 if (MaybeAlign A = getAlign(Ptr)) {
564 }
565 return CI;
566}
567
568CallInst *
570 ArrayRef<OperandBundleDef> OpBundles) {
571 assert(Cond->getType() == getInt1Ty() &&
572 "an assumption condition must be of type i1");
573
574 Value *Ops[] = { Cond };
575 Module *M = BB->getParent()->getParent();
576 Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
577 return CreateCall(FnAssume, Ops, OpBundles);
578}
579
581 Module *M = BB->getModule();
582 auto *FnIntrinsic = Intrinsic::getDeclaration(
583 M, Intrinsic::experimental_noalias_scope_decl, {});
584 return CreateCall(FnIntrinsic, {Scope});
585}
586
587/// Create a call to a Masked Load intrinsic.
588/// \p Ty - vector type to load
589/// \p Ptr - base pointer for the load
590/// \p Alignment - alignment of the source location
591/// \p Mask - vector of booleans which indicates what vector lanes should
592/// be accessed in memory
593/// \p PassThru - pass-through value that is used to fill the masked-off lanes
594/// of the result
595/// \p Name - name of the result variable
597 Value *Mask, Value *PassThru,
598 const Twine &Name) {
599 auto *PtrTy = cast<PointerType>(Ptr->getType());
600 assert(Ty->isVectorTy() && "Type should be vector");
601 assert(PtrTy->isOpaqueOrPointeeTypeMatches(Ty) && "Wrong element type");
602 assert(Mask && "Mask should not be all-ones (null)");
603 if (!PassThru)
604 PassThru = PoisonValue::get(Ty);
605 Type *OverloadedTypes[] = { Ty, PtrTy };
606 Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
607 return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
608 OverloadedTypes, Name);
609}
610
611/// Create a call to a Masked Store intrinsic.
612/// \p Val - data to be stored,
613/// \p Ptr - base pointer for the store
614/// \p Alignment - alignment of the destination location
615/// \p Mask - vector of booleans which indicates what vector lanes should
616/// be accessed in memory
618 Align Alignment, Value *Mask) {
619 auto *PtrTy = cast<PointerType>(Ptr->getType());
620 Type *DataTy = Val->getType();
621 assert(DataTy->isVectorTy() && "Val should be a vector");
622 assert(PtrTy->isOpaqueOrPointeeTypeMatches(DataTy) && "Wrong element type");
623 assert(Mask && "Mask should not be all-ones (null)");
624 Type *OverloadedTypes[] = { DataTy, PtrTy };
625 Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask};
626 return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
627}
628
629/// Create a call to a Masked intrinsic, with given intrinsic Id,
630/// an array of operands - Ops, and an array of overloaded types -
631/// OverloadedTypes.
632CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
634 ArrayRef<Type *> OverloadedTypes,
635 const Twine &Name) {
636 Module *M = BB->getParent()->getParent();
637 Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
638 return CreateCall(TheFn, Ops, {}, Name);
639}
640
641/// Create a call to a Masked Gather intrinsic.
642/// \p Ty - vector type to gather
643/// \p Ptrs - vector of pointers for loading
644/// \p Align - alignment for one element
645/// \p Mask - vector of booleans which indicates what vector lanes should
646/// be accessed in memory
647/// \p PassThru - pass-through value that is used to fill the masked-off lanes
648/// of the result
649/// \p Name - name of the result variable
651 Align Alignment, Value *Mask,
652 Value *PassThru,
653 const Twine &Name) {
654 auto *VecTy = cast<VectorType>(Ty);
655 ElementCount NumElts = VecTy->getElementCount();
656 auto *PtrsTy = cast<VectorType>(Ptrs->getType());
657 assert(cast<PointerType>(PtrsTy->getElementType())
658 ->isOpaqueOrPointeeTypeMatches(
659 cast<VectorType>(Ty)->getElementType()) &&
660 "Element type mismatch");
661 assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");
662
663 if (!Mask)
666
667 if (!PassThru)
668 PassThru = PoisonValue::get(Ty);
669
670 Type *OverloadedTypes[] = {Ty, PtrsTy};
671 Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
672
673 // We specify only one type when we create this intrinsic. Types of other
674 // arguments are derived from this type.
675 return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
676 Name);
677}
678
679/// Create a call to a Masked Scatter intrinsic.
680/// \p Data - data to be stored,
681/// \p Ptrs - the vector of pointers, where the \p Data elements should be
682/// stored
683/// \p Align - alignment for one element
684/// \p Mask - vector of booleans which indicates what vector lanes should
685/// be accessed in memory
687 Align Alignment, Value *Mask) {
688 auto *PtrsTy = cast<VectorType>(Ptrs->getType());
689 auto *DataTy = cast<VectorType>(Data->getType());
690 ElementCount NumElts = PtrsTy->getElementCount();
691
692#ifndef NDEBUG
693 auto *PtrTy = cast<PointerType>(PtrsTy->getElementType());
694 assert(NumElts == DataTy->getElementCount() &&
695 PtrTy->isOpaqueOrPointeeTypeMatches(DataTy->getElementType()) &&
696 "Incompatible pointer and data types");
697#endif
698
699 if (!Mask)
702
703 Type *OverloadedTypes[] = {DataTy, PtrsTy};
704 Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask};
705
706 // We specify only one type when we create this intrinsic. Types of other
707 // arguments are derived from this type.
708 return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
709}
710
711/// Create a call to Masked Expand Load intrinsic
712/// \p Ty - vector type to load
713/// \p Ptr - base pointer for the load
714/// \p Mask - vector of booleans which indicates what vector lanes should
715/// be accessed in memory
716/// \p PassThru - pass-through value that is used to fill the masked-off lanes
717/// of the result
718/// \p Name - name of the result variable
720 Value *Mask, Value *PassThru,
721 const Twine &Name) {
722 auto *PtrTy = cast<PointerType>(Ptr->getType());
723 assert(Ty->isVectorTy() && "Type should be vector");
724 assert(PtrTy->isOpaqueOrPointeeTypeMatches(
725 cast<FixedVectorType>(Ty)->getElementType()) &&
726 "Wrong element type");
727 (void)PtrTy;
728 assert(Mask && "Mask should not be all-ones (null)");
729 if (!PassThru)
730 PassThru = PoisonValue::get(Ty);
731 Type *OverloadedTypes[] = {Ty};
732 Value *Ops[] = {Ptr, Mask, PassThru};
733 return CreateMaskedIntrinsic(Intrinsic::masked_expandload, Ops,
734 OverloadedTypes, Name);
735}
736
737/// Create a call to Masked Compress Store intrinsic
738/// \p Val - data to be stored,
739/// \p Ptr - base pointer for the store
740/// \p Mask - vector of booleans which indicates what vector lanes should
741/// be accessed in memory
743 Value *Mask) {
744 auto *PtrTy = cast<PointerType>(Ptr->getType());
745 Type *DataTy = Val->getType();
746 assert(DataTy->isVectorTy() && "Val should be a vector");
747 assert(PtrTy->isOpaqueOrPointeeTypeMatches(
748 cast<FixedVectorType>(DataTy)->getElementType()) &&
749 "Wrong element type");
750 (void)PtrTy;
751 assert(Mask && "Mask should not be all-ones (null)");
752 Type *OverloadedTypes[] = {DataTy};
753 Value *Ops[] = {Val, Ptr, Mask};
754 return CreateMaskedIntrinsic(Intrinsic::masked_compressstore, Ops,
755 OverloadedTypes);
756}
757
758template <typename T0>
759static std::vector<Value *>
761 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
762 std::vector<Value *> Args;
763 Args.push_back(B.getInt64(ID));
764 Args.push_back(B.getInt32(NumPatchBytes));
765 Args.push_back(ActualCallee);
766 Args.push_back(B.getInt32(CallArgs.size()));
767 Args.push_back(B.getInt32(Flags));
768 llvm::append_range(Args, CallArgs);
769 // GC Transition and Deopt args are now always handled via operand bundle.
770 // They will be removed from the signature of gc.statepoint shortly.
771 Args.push_back(B.getInt32(0));
772 Args.push_back(B.getInt32(0));
773 // GC args are now encoded in the gc-live operand bundle
774 return Args;
775}
776
777template<typename T1, typename T2, typename T3>
778static std::vector<OperandBundleDef>
779getStatepointBundles(std::optional<ArrayRef<T1>> TransitionArgs,
780 std::optional<ArrayRef<T2>> DeoptArgs,
781 ArrayRef<T3> GCArgs) {
782 std::vector<OperandBundleDef> Rval;
783 if (DeoptArgs) {
784 SmallVector<Value*, 16> DeoptValues;
785 llvm::append_range(DeoptValues, *DeoptArgs);
786 Rval.emplace_back("deopt", DeoptValues);
787 }
788 if (TransitionArgs) {
789 SmallVector<Value*, 16> TransitionValues;
790 llvm::append_range(TransitionValues, *TransitionArgs);
791 Rval.emplace_back("gc-transition", TransitionValues);
792 }
793 if (GCArgs.size()) {
794 SmallVector<Value*, 16> LiveValues;
795 llvm::append_range(LiveValues, GCArgs);
796 Rval.emplace_back("gc-live", LiveValues);
797 }
798 return Rval;
799}
800
801template <typename T0, typename T1, typename T2, typename T3>
803 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
804 FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
805 std::optional<ArrayRef<T1>> TransitionArgs,
806 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
807 const Twine &Name) {
808 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
809 // Fill in the one generic type'd argument (the function is also vararg)
810 Function *FnStatepoint =
811 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
812 {ActualCallee.getCallee()->getType()});
813
814 std::vector<Value *> Args = getStatepointArgs(
815 *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs);
816
817 CallInst *CI = Builder->CreateCall(
818 FnStatepoint, Args,
819 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
820 CI->addParamAttr(2,
821 Attribute::get(Builder->getContext(), Attribute::ElementType,
822 ActualCallee.getFunctionType()));
823 return CI;
824}
825
827 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
828 ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
829 ArrayRef<Value *> GCArgs, const Twine &Name) {
830 return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
831 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
832 CallArgs, std::nullopt /* No Transition Args */, DeoptArgs, GCArgs, Name);
833}
834
836 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
837 uint32_t Flags, ArrayRef<Value *> CallArgs,
838 std::optional<ArrayRef<Use>> TransitionArgs,
839 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
840 const Twine &Name) {
841 return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
842 this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
843 DeoptArgs, GCArgs, Name);
844}
845
847 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
848 ArrayRef<Use> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
849 ArrayRef<Value *> GCArgs, const Twine &Name) {
850 return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
851 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
852 CallArgs, std::nullopt, DeoptArgs, GCArgs, Name);
853}
854
855template <typename T0, typename T1, typename T2, typename T3>
857 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
858 FunctionCallee ActualInvokee, BasicBlock *NormalDest,
859 BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<T0> InvokeArgs,
860 std::optional<ArrayRef<T1>> TransitionArgs,
861 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
862 const Twine &Name) {
863 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
864 // Fill in the one generic type'd argument (the function is also vararg)
865 Function *FnStatepoint =
866 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
867 {ActualInvokee.getCallee()->getType()});
868
869 std::vector<Value *> Args =
870 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(),
871 Flags, InvokeArgs);
872
873 InvokeInst *II = Builder->CreateInvoke(
874 FnStatepoint, NormalDest, UnwindDest, Args,
875 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
876 II->addParamAttr(2,
877 Attribute::get(Builder->getContext(), Attribute::ElementType,
878 ActualInvokee.getFunctionType()));
879 return II;
880}
881
883 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
884 BasicBlock *NormalDest, BasicBlock *UnwindDest,
885 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
886 ArrayRef<Value *> GCArgs, const Twine &Name) {
887 return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
888 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
889 uint32_t(StatepointFlags::None), InvokeArgs,
890 std::nullopt /* No Transition Args*/, DeoptArgs, GCArgs, Name);
891}
892
894 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
895 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
896 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Use>> TransitionArgs,
897 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
898 const Twine &Name) {
899 return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
900 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
901 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
902}
903
905 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
906 BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
907 std::optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs,
908 const Twine &Name) {
909 return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
910 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
911 uint32_t(StatepointFlags::None), InvokeArgs, std::nullopt, DeoptArgs,
912 GCArgs, Name);
913}
914
916 Type *ResultType, const Twine &Name) {
917 Intrinsic::ID ID = Intrinsic::experimental_gc_result;
918 Module *M = BB->getParent()->getParent();
919 Type *Types[] = {ResultType};
920 Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
921
922 Value *Args[] = {Statepoint};
923 return CreateCall(FnGCResult, Args, {}, Name);
924}
925
927 int BaseOffset, int DerivedOffset,
928 Type *ResultType, const Twine &Name) {
929 Module *M = BB->getParent()->getParent();
930 Type *Types[] = {ResultType};
931 Function *FnGCRelocate =
932 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
933
934 Value *Args[] = {Statepoint, getInt32(BaseOffset), getInt32(DerivedOffset)};
935 return CreateCall(FnGCRelocate, Args, {}, Name);
936}
937
939 const Twine &Name) {
940 Module *M = BB->getParent()->getParent();
941 Type *PtrTy = DerivedPtr->getType();
942 Function *FnGCFindBase = Intrinsic::getDeclaration(
943 M, Intrinsic::experimental_gc_get_pointer_base, {PtrTy, PtrTy});
944 return CreateCall(FnGCFindBase, {DerivedPtr}, {}, Name);
945}
946
948 const Twine &Name) {
949 Module *M = BB->getParent()->getParent();
950 Type *PtrTy = DerivedPtr->getType();
951 Function *FnGCGetOffset = Intrinsic::getDeclaration(
952 M, Intrinsic::experimental_gc_get_pointer_offset, {PtrTy});
953 return CreateCall(FnGCGetOffset, {DerivedPtr}, {}, Name);
954}
955
957 Instruction *FMFSource,
958 const Twine &Name) {
959 Module *M = BB->getModule();
960 Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
961 return createCallHelper(Fn, {V}, Name, FMFSource);
962}
963
965 Value *RHS,
966 Instruction *FMFSource,
967 const Twine &Name) {
968 Module *M = BB->getModule();
970 return createCallHelper(Fn, {LHS, RHS}, Name, FMFSource);
971}
972
974 ArrayRef<Type *> Types,
976 Instruction *FMFSource,
977 const Twine &Name) {
978 Module *M = BB->getModule();
979 Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
980 return createCallHelper(Fn, Args, Name, FMFSource);
981}
982
985 Instruction *FMFSource,
986 const Twine &Name) {
987 Module *M = BB->getModule();
988
992
993 SmallVector<Type *> ArgTys;
994 ArgTys.reserve(Args.size());
995 for (auto &I : Args)
996 ArgTys.push_back(I->getType());
997 FunctionType *FTy = FunctionType::get(RetTy, ArgTys, false);
998 SmallVector<Type *> OverloadTys;
1000 matchIntrinsicSignature(FTy, TableRef, OverloadTys);
1001 (void)Res;
1003 "Wrong types for intrinsic!");
1004 // TODO: Handle varargs intrinsics.
1005
1006 Function *Fn = Intrinsic::getDeclaration(M, ID, OverloadTys);
1007 return createCallHelper(Fn, Args, Name, FMFSource);
1008}
1009
1011 Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource,
1012 const Twine &Name, MDNode *FPMathTag,
1013 std::optional<RoundingMode> Rounding,
1014 std::optional<fp::ExceptionBehavior> Except) {
1015 Value *RoundingV = getConstrainedFPRounding(Rounding);
1016 Value *ExceptV = getConstrainedFPExcept(Except);
1017
1018 FastMathFlags UseFMF = FMF;
1019 if (FMFSource)
1020 UseFMF = FMFSource->getFastMathFlags();
1021
1022 CallInst *C = CreateIntrinsic(ID, {L->getType()},
1023 {L, R, RoundingV, ExceptV}, nullptr, Name);
1025 setFPAttrs(C, FPMathTag, UseFMF);
1026 return C;
1027}
1028
1030 const Twine &Name, MDNode *FPMathTag) {
1031 if (Instruction::isBinaryOp(Opc)) {
1032 assert(Ops.size() == 2 && "Invalid number of operands!");
1033 return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc),
1034 Ops[0], Ops[1], Name, FPMathTag);
1035 }
1036 if (Instruction::isUnaryOp(Opc)) {
1037 assert(Ops.size() == 1 && "Invalid number of operands!");
1038 return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc),
1039 Ops[0], Name, FPMathTag);
1040 }
1041 llvm_unreachable("Unexpected opcode!");
1042}
1043
1045 Intrinsic::ID ID, Value *V, Type *DestTy,
1046 Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag,
1047 std::optional<RoundingMode> Rounding,
1048 std::optional<fp::ExceptionBehavior> Except) {
1049 Value *ExceptV = getConstrainedFPExcept(Except);
1050
1051 FastMathFlags UseFMF = FMF;
1052 if (FMFSource)
1053 UseFMF = FMFSource->getFastMathFlags();
1054
1055 CallInst *C;
1056 bool HasRoundingMD = false;
1057 switch (ID) {
1058 default:
1059 break;
1060#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
1061 case Intrinsic::INTRINSIC: \
1062 HasRoundingMD = ROUND_MODE; \
1063 break;
1064#include "llvm/IR/ConstrainedOps.def"
1065 }
1066 if (HasRoundingMD) {
1067 Value *RoundingV = getConstrainedFPRounding(Rounding);
1068 C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
1069 nullptr, Name);
1070 } else
1071 C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,
1072 Name);
1073
1075
1076 if (isa<FPMathOperator>(C))
1077 setFPAttrs(C, FPMathTag, UseFMF);
1078 return C;
1079}
1080
1081Value *IRBuilderBase::CreateFCmpHelper(
1082 CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name,
1083 MDNode *FPMathTag, bool IsSignaling) {
1084 if (IsFPConstrained) {
1085 auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
1086 : Intrinsic::experimental_constrained_fcmp;
1087 return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name);
1088 }
1089
1090 if (auto *LC = dyn_cast<Constant>(LHS))
1091 if (auto *RC = dyn_cast<Constant>(RHS))
1092 return Insert(Folder.CreateFCmp(P, LC, RC), Name);
1093 return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name);
1094}
1095
1098 const Twine &Name, std::optional<fp::ExceptionBehavior> Except) {
1099 Value *PredicateV = getConstrainedFPPredicate(P);
1100 Value *ExceptV = getConstrainedFPExcept(Except);
1101
1102 CallInst *C = CreateIntrinsic(ID, {L->getType()},
1103 {L, R, PredicateV, ExceptV}, nullptr, Name);
1105 return C;
1106}
1107
1110 std::optional<RoundingMode> Rounding,
1111 std::optional<fp::ExceptionBehavior> Except) {
1113
1114 append_range(UseArgs, Args);
1115 bool HasRoundingMD = false;
1116 switch (Callee->getIntrinsicID()) {
1117 default:
1118 break;
1119#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
1120 case Intrinsic::INTRINSIC: \
1121 HasRoundingMD = ROUND_MODE; \
1122 break;
1123#include "llvm/IR/ConstrainedOps.def"
1124 }
1125 if (HasRoundingMD)
1126 UseArgs.push_back(getConstrainedFPRounding(Rounding));
1127 UseArgs.push_back(getConstrainedFPExcept(Except));
1128
1129 CallInst *C = CreateCall(Callee, UseArgs, Name);
1131 return C;
1132}
1133
1135 const Twine &Name, Instruction *MDFrom) {
1136 if (auto *V = Folder.FoldSelect(C, True, False))
1137 return V;
1138
1139 SelectInst *Sel = SelectInst::Create(C, True, False);
1140 if (MDFrom) {
1141 MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof);
1142 MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable);
1143 Sel = addBranchMetadata(Sel, Prof, Unpred);
1144 }
1145 if (isa<FPMathOperator>(Sel))
1146 setFPAttrs(Sel, nullptr /* MDNode* */, FMF);
1147 return Insert(Sel, Name);
1148}
1149
1151 const Twine &Name) {
1152 assert(LHS->getType() == RHS->getType() &&
1153 "Pointer subtraction operand types must match!");
1154 assert(cast<PointerType>(LHS->getType())
1155 ->isOpaqueOrPointeeTypeMatches(ElemTy) &&
1156 "Pointer type must match element type");
1159 Value *Difference = CreateSub(LHS_int, RHS_int);
1160 return CreateExactSDiv(Difference, ConstantExpr::getSizeOf(ElemTy),
1161 Name);
1162}
1163
1165 assert(isa<PointerType>(Ptr->getType()) &&
1166 "launder.invariant.group only applies to pointers.");
1167 // FIXME: we could potentially avoid casts to/from i8*.
1168 auto *PtrType = Ptr->getType();
1169 auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
1170 if (PtrType != Int8PtrTy)
1171 Ptr = CreateBitCast(Ptr, Int8PtrTy);
1172 Module *M = BB->getParent()->getParent();
1173 Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration(
1174 M, Intrinsic::launder_invariant_group, {Int8PtrTy});
1175
1176 assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy &&
1177 FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
1178 Int8PtrTy &&
1179 "LaunderInvariantGroup should take and return the same type");
1180
1181 CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr});
1182
1183 if (PtrType != Int8PtrTy)
1184 return CreateBitCast(Fn, PtrType);
1185 return Fn;
1186}
1187
1189 assert(isa<PointerType>(Ptr->getType()) &&
1190 "strip.invariant.group only applies to pointers.");
1191
1192 // FIXME: we could potentially avoid casts to/from i8*.
1193 auto *PtrType = Ptr->getType();
1194 auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
1195 if (PtrType != Int8PtrTy)
1196 Ptr = CreateBitCast(Ptr, Int8PtrTy);
1197 Module *M = BB->getParent()->getParent();
1198 Function *FnStripInvariantGroup = Intrinsic::getDeclaration(
1199 M, Intrinsic::strip_invariant_group, {Int8PtrTy});
1200
1201 assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy &&
1202 FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
1203 Int8PtrTy &&
1204 "StripInvariantGroup should take and return the same type");
1205
1206 CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr});
1207
1208 if (PtrType != Int8PtrTy)
1209 return CreateBitCast(Fn, PtrType);
1210 return Fn;
1211}
1212
1214 auto *Ty = cast<VectorType>(V->getType());
1215 if (isa<ScalableVectorType>(Ty)) {
1216 Module *M = BB->getParent()->getParent();
1218 M, Intrinsic::experimental_vector_reverse, Ty);
1219 return Insert(CallInst::Create(F, V), Name);
1220 }
1221 // Keep the original behaviour for fixed vector
1222 SmallVector<int, 8> ShuffleMask;
1223 int NumElts = Ty->getElementCount().getKnownMinValue();
1224 for (int i = 0; i < NumElts; ++i)
1225 ShuffleMask.push_back(NumElts - i - 1);
1226 return CreateShuffleVector(V, ShuffleMask, Name);
1227}
1228
1230 const Twine &Name) {
1231 assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1232 assert(V1->getType() == V2->getType() &&
1233 "Splice expects matching operand types!");
1234
1235 if (auto *VTy = dyn_cast<ScalableVectorType>(V1->getType())) {
1236 Module *M = BB->getParent()->getParent();
1238 M, Intrinsic::experimental_vector_splice, VTy);
1239
1240 Value *Ops[] = {V1, V2, getInt32(Imm)};
1241 return Insert(CallInst::Create(F, Ops), Name);
1242 }
1243
1244 unsigned NumElts = cast<FixedVectorType>(V1->getType())->getNumElements();
1245 assert(((-Imm <= NumElts) || (Imm < NumElts)) &&
1246 "Invalid immediate for vector splice!");
1247
1248 // Keep the original behaviour for fixed vector
1249 unsigned Idx = (NumElts + Imm) % NumElts;
1251 for (unsigned I = 0; I < NumElts; ++I)
1252 Mask.push_back(Idx + I);
1253
1254 return CreateShuffleVector(V1, V2, Mask);
1255}
1256
1258 const Twine &Name) {
1259 auto EC = ElementCount::getFixed(NumElts);
1260 return CreateVectorSplat(EC, V, Name);
1261}
1262
1264 const Twine &Name) {
1265 assert(EC.isNonZero() && "Cannot splat to an empty vector!");
1266
1267 // First insert it into a poison vector so we can shuffle it.
1268 Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));
1269 V = CreateInsertElement(Poison, V, getInt64(0), Name + ".splatinsert");
1270
1271 // Shuffle the value across the desired number of elements.
1273 Zeros.resize(EC.getKnownMinValue());
1274 return CreateShuffleVector(V, Zeros, Name + ".splat");
1275}
1276
1278 const DataLayout &DL, Value *From, IntegerType *ExtractedTy,
1279 uint64_t Offset, const Twine &Name) {
1280 auto *IntTy = cast<IntegerType>(From->getType());
1281 assert(DL.getTypeStoreSize(ExtractedTy) + Offset <=
1282 DL.getTypeStoreSize(IntTy) &&
1283 "Element extends past full value");
1284 uint64_t ShAmt = 8 * Offset;
1285 Value *V = From;
1286 if (DL.isBigEndian())
1287 ShAmt = 8 * (DL.getTypeStoreSize(IntTy) -
1288 DL.getTypeStoreSize(ExtractedTy) - Offset);
1289 if (ShAmt) {
1290 V = CreateLShr(V, ShAmt, Name + ".shift");
1291 }
1292 assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() &&
1293 "Cannot extract to a larger integer!");
1294 if (ExtractedTy != IntTy) {
1295 V = CreateTrunc(V, ExtractedTy, Name + ".trunc");
1296 }
1297 return V;
1298}
1299
1301 Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex,
1302 MDNode *DbgInfo) {
1303 auto *BaseType = Base->getType();
1304 assert(isa<PointerType>(BaseType) &&
1305 "Invalid Base ptr type for preserve.array.access.index.");
1306 assert(cast<PointerType>(BaseType)->isOpaqueOrPointeeTypeMatches(ElTy) &&
1307 "Pointer element type mismatch");
1308
1309 Value *LastIndexV = getInt32(LastIndex);
1311 SmallVector<Value *, 4> IdxList(Dimension, Zero);
1312 IdxList.push_back(LastIndexV);
1313
1314 Type *ResultType =
1316
1317 Module *M = BB->getParent()->getParent();
1318 Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration(
1319 M, Intrinsic::preserve_array_access_index, {ResultType, BaseType});
1320
1321 Value *DimV = getInt32(Dimension);
1322 CallInst *Fn =
1323 CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV});
1324 Fn->addParamAttr(
1325 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
1326 if (DbgInfo)
1327 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1328
1329 return Fn;
1330}
1331
1333 Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1334 assert(isa<PointerType>(Base->getType()) &&
1335 "Invalid Base ptr type for preserve.union.access.index.");
1336 auto *BaseType = Base->getType();
1337
1338 Module *M = BB->getParent()->getParent();
1339 Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration(
1340 M, Intrinsic::preserve_union_access_index, {BaseType, BaseType});
1341
1342 Value *DIIndex = getInt32(FieldIndex);
1343 CallInst *Fn =
1344 CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex});
1345 if (DbgInfo)
1346 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1347
1348 return Fn;
1349}
1350
1352 Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1353 MDNode *DbgInfo) {
1354 auto *BaseType = Base->getType();
1355 assert(isa<PointerType>(BaseType) &&
1356 "Invalid Base ptr type for preserve.struct.access.index.");
1357 assert(cast<PointerType>(BaseType)->isOpaqueOrPointeeTypeMatches(ElTy) &&
1358 "Pointer element type mismatch");
1359
1360 Value *GEPIndex = getInt32(Index);
1362 Type *ResultType =
1363 GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex});
1364
1365 Module *M = BB->getParent()->getParent();
1366 Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration(
1367 M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType});
1368
1369 Value *DIIndex = getInt32(FieldIndex);
1370 CallInst *Fn = CreateCall(FnPreserveStructAccessIndex,
1371 {Base, GEPIndex, DIIndex});
1372 Fn->addParamAttr(
1373 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
1374 if (DbgInfo)
1375 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1376
1377 return Fn;
1378}
1379
1380CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1381 Value *PtrValue,
1382 Value *AlignValue,
1383 Value *OffsetValue) {
1384 SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1385 if (OffsetValue)
1386 Vals.push_back(OffsetValue);
1387 OperandBundleDefT<Value *> AlignOpB("align", Vals);
1388 return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB});
1389}
1390
1392 Value *PtrValue,
1393 unsigned Alignment,
1394 Value *OffsetValue) {
1395 assert(isa<PointerType>(PtrValue->getType()) &&
1396 "trying to create an alignment assumption on a non-pointer?");
1397 assert(Alignment != 0 && "Invalid Alignment");
1398 auto *PtrTy = cast<PointerType>(PtrValue->getType());
1399 Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
1400 Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment);
1401 return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
1402}
1403
1405 Value *PtrValue,
1406 Value *Alignment,
1407 Value *OffsetValue) {
1408 assert(isa<PointerType>(PtrValue->getType()) &&
1409 "trying to create an alignment assumption on a non-pointer?");
1410 return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
1411}
1412
1416void ConstantFolder::anchor() {}
1417void NoFolder::anchor() {}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Callee
ArrayRef< TableEntry > TableRef
assume Assume Builder
SmallVector< MachineOperand, 4 > Cond
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
std::string Name
uint64_t Size
static InvokeInst * CreateGCStatepointInvokeCommon(IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags, ArrayRef< T0 > InvokeArgs, std::optional< ArrayRef< T1 > > TransitionArgs, std::optional< ArrayRef< T2 > > DeoptArgs, ArrayRef< T3 > GCArgs, const Twine &Name)
Definition: IRBuilder.cpp:856
static CallInst * CreateGCStatepointCallCommon(IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee, uint32_t Flags, ArrayRef< T0 > CallArgs, std::optional< ArrayRef< T1 > > TransitionArgs, std::optional< ArrayRef< T2 > > DeoptArgs, ArrayRef< T3 > GCArgs, const Twine &Name)
Definition: IRBuilder.cpp:802
static std::vector< OperandBundleDef > getStatepointBundles(std::optional< ArrayRef< T1 > > TransitionArgs, std::optional< ArrayRef< T2 > > DeoptArgs, ArrayRef< T3 > GCArgs)
Definition: IRBuilder.cpp:779
static std::vector< Value * > getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags, ArrayRef< T0 > CallArgs)
Definition: IRBuilder.cpp:760
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:524
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define P(N)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
@ Flags
Definition: TextStubV5.cpp:93
Value * RHS
Value * LHS
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:158
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:91
static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
Definition: Attributes.cpp:167
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:112
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Definition: BasicBlock.cpp:146
void addRetAttr(Attribute::AttrKind Kind)
Adds the attribute to the return value.
Definition: InstrTypes.h:1532
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Definition: InstrTypes.h:1542
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:711
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2925
static Constant * getSizeOf(Type *Ty)
getSizeOf constant expr - computes the (alloc) size of a type (in address-units, not bits) in a targe...
Definition: Constants.cpp:2349
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:833
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
static Constant * get(ArrayRef< Constant * > V)
Definition: Constants.cpp:1342
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:403
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
A debug info location.
Definition: DebugLoc.h:33
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:291
This instruction compares its operands according to the predicate given to the constructor.
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:20
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
FunctionType * getFunctionType()
Definition: DerivedTypes.h:182
Class to represent function types.
Definition: DerivedTypes.h:103
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:174
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:179
static Type * getGEPReturnType(Type *ElTy, Value *Ptr, ArrayRef< Value * > IdxList)
Returns the pointer type returned by the GEP instruction, which may be a vector of pointers.
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
Value * CreateExactSDiv(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1329
CallInst * CreateElementUnorderedAtomicMemCpy(Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, uint32_t ElementSize, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert an element unordered-atomic memcpy between the specified pointers.
Definition: IRBuilder.cpp:306
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Definition: IRBuilder.h:447
CallInst * CreateMaskedCompressStore(Value *Val, Value *Ptr, Value *Mask=nullptr)
Create a call to Masked Compress Store intrinsic.
Definition: IRBuilder.cpp:742
BasicBlock * BB
Definition: IRBuilder.h:119
CallInst * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
Definition: IRBuilder.cpp:956
Value * CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS, const Twine &Name="")
Return the i64 difference between two pointer values, dividing out the size of the pointed-to objects...
Definition: IRBuilder.cpp:1150
CallInst * CreateMulReduce(Value *Src)
Create a vector int mul reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:449
CallInst * CreateFAddReduce(Value *Acc, Value *Src)
Create a sequential vector fadd reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:429
CallInst * CreateMaskedExpandLoad(Type *Ty, Value *Ptr, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Expand Load intrinsic.
Definition: IRBuilder.cpp:719
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1926
Value * CreateVScale(Constant *Scaling, const Twine &Name="")
Create a call to llvm.vscale, multiplied by Scaling.
Definition: IRBuilder.cpp:97
Value * CreateLaunderInvariantGroup(Value *Ptr)
Create a launder.invariant.group intrinsic call.
Definition: IRBuilder.cpp:1164
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2367
CallInst * CreateThreadLocalAddress(Value *Ptr)
Create a call to llvm.threadlocal.address intrinsic.
Definition: IRBuilder.cpp:545
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Definition: IRBuilder.h:497
Type * getCurrentFunctionReturnType() const
Get the return type of the current function that we're emitting into.
Definition: IRBuilder.cpp:58
CallInst * CreateGCGetPointerBase(Value *DerivedPtr, const Twine &Name="")
Create a call to the experimental.gc.pointer.base intrinsic to get the base pointer for the specified...
Definition: IRBuilder.cpp:938
CallInst * CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee, ArrayRef< Value * > CallArgs, std::optional< ArrayRef< Value * > > DeoptArgs, ArrayRef< Value * > GCArgs, const Twine &Name="")
Create a call to the experimental.gc.statepoint intrinsic to start a new statepoint sequence.
Definition: IRBuilder.cpp:826
GlobalVariable * CreateGlobalString(StringRef Str, const Twine &Name="", unsigned AddressSpace=0, Module *M=nullptr)
Make a new global variable with initializer type i8*.
Definition: IRBuilder.cpp:43
CallInst * CreateLifetimeStart(Value *Ptr, ConstantInt *Size=nullptr)
Create a lifetime.start intrinsic.
Definition: IRBuilder.cpp:485
CallInst * CreateConstrainedFPCmp(Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R, const Twine &Name="", std::optional< fp::ExceptionBehavior > Except=std::nullopt)
Definition: IRBuilder.cpp:1096
CallInst * CreateAndReduce(Value *Src)
Create a vector int AND reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:453
Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")
Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.
Definition: IRBuilder.cpp:1229
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Definition: IRBuilder.cpp:1257
CallInst * CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, Value *Size, bool IsVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Definition: IRBuilder.cpp:268
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Definition: IRBuilder.cpp:973
Value * CreatePreserveStructAccessIndex(Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex, MDNode *DbgInfo)
Definition: IRBuilder.cpp:1351
CallInst * CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue, unsigned Alignment, Value *OffsetValue=nullptr)
Create an assume intrinsic call that represents an alignment assumption on the provided pointer.
Definition: IRBuilder.cpp:1391
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
Definition: IRBuilder.cpp:596
CallInst * CreateConstrainedFPCall(Function *Callee, ArrayRef< Value * > Args, const Twine &Name="", std::optional< RoundingMode > Rounding=std::nullopt, std::optional< fp::ExceptionBehavior > Except=std::nullopt)
Definition: IRBuilder.cpp:1108
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, MaybeAlign Align, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memset to the specified pointer and the specified value.
Definition: IRBuilder.h:586
LLVMContext & Context
Definition: IRBuilder.h:121
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.cpp:1134
CallInst * CreateGCGetPointerOffset(Value *DerivedPtr, const Twine &Name="")
Create a call to the experimental.gc.get.pointer.offset intrinsic to get the offset of the specified ...
Definition: IRBuilder.cpp:947
Value * CreateTypeSize(Type *DstType, TypeSize Size)
Create an expression which evaluates to the number of units in Size at runtime.
Definition: IRBuilder.cpp:113
CallInst * CreateAddReduce(Value *Src)
Create a vector int add reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:445
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1366
IntegerType * getIntPtrTy(const DataLayout &DL, unsigned AddrSpace=0)
Fetch the type of an integer with size at least as big as that of a pointer in the given address spac...
Definition: IRBuilder.h:566
BasicBlock * GetInsertBlock() const
Definition: IRBuilder.h:174
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Definition: IRBuilder.h:517
CallInst * CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, Value *Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Definition: IRBuilder.cpp:231
Value * CreateVectorReverse(Value *V, const Twine &Name="")
Return a vector value that contains the vector V reversed.
Definition: IRBuilder.cpp:1213
CallInst * CreateXorReduce(Value *Src)
Create a vector int XOR reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:461
FastMathFlags FMF
Definition: IRBuilder.h:126
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition: IRBuilder.h:477
Value * CreateUnOp(Instruction::UnaryOps Opc, Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1682
CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:457
CallInst * CreateFPMinReduce(Value *Src)
Create a vector float min reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:481
CallInst * CreateFPMaxReduce(Value *Src)
Create a vector float max reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:477
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:472
Value * CreateExtractInteger(const DataLayout &DL, Value *From, IntegerType *ExtractedTy, uint64_t Offset, const Twine &Name)
Return a value that has been extracted from a larger integer type.
Definition: IRBuilder.cpp:1277
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
Definition: IRBuilder.h:145
CallInst * CreateConstrainedFPBinOp(Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource=nullptr, const Twine &Name="", MDNode *FPMathTag=nullptr, std::optional< RoundingMode > Rounding=std::nullopt, std::optional< fp::ExceptionBehavior > Except=std::nullopt)
Definition: IRBuilder.cpp:1010
DebugLoc getCurrentDebugLocation() const
Get location information used by debugging information.
Definition: IRBuilder.cpp:72
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1273
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2022
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
Definition: IRBuilder.h:560
Value * CreateNAryOp(unsigned Opc, ArrayRef< Value * > Ops, const Twine &Name="", MDNode *FPMathTag=nullptr)
Create either a UnaryOperator or BinaryOperator depending on Opc.
Definition: IRBuilder.cpp:1029
CallInst * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Definition: IRBuilder.cpp:964
CallInst * CreateAssumption(Value *Cond, ArrayRef< OperandBundleDef > OpBundles=std::nullopt)
Create an assume intrinsic call that allows the optimizer to assume that the provided condition will ...
Definition: IRBuilder.cpp:569
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Definition: IRBuilder.h:2389
LLVMContext & getContext() const
Definition: IRBuilder.h:176
Value * CreatePreserveUnionAccessIndex(Value *Base, unsigned FieldIndex, MDNode *DbgInfo)
Definition: IRBuilder.cpp:1332
CallInst * CreateIntMaxReduce(Value *Src, bool IsSigned=false)
Create a vector integer max reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:465
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Definition: IRBuilder.cpp:617
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2012
CallInst * CreateGCResult(Instruction *Statepoint, Type *ResultType, const Twine &Name="")
Create a call to the experimental.gc.result intrinsic to extract the result from a call wrapped in a ...
Definition: IRBuilder.cpp:915
CallInst * CreateLifetimeEnd(Value *Ptr, ConstantInt *Size=nullptr)
Create a lifetime.end intrinsic.
Definition: IRBuilder.cpp:501
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1595
Value * CreateElementCount(Type *DstType, ElementCount EC)
Create an expression which evaluates to the number of elements in EC at runtime.
Definition: IRBuilder.cpp:108
CallInst * CreateConstrainedFPCast(Intrinsic::ID ID, Value *V, Type *DestTy, Instruction *FMFSource=nullptr, const Twine &Name="", MDNode *FPMathTag=nullptr, std::optional< RoundingMode > Rounding=std::nullopt, std::optional< fp::ExceptionBehavior > Except=std::nullopt)
Definition: IRBuilder.cpp:1044
CallInst * CreateElementUnorderedAtomicMemMove(Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, uint32_t ElementSize, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert an element unordered-atomic memmove between the specified pointers.
Definition: IRBuilder.cpp:381
CallInst * CreateIntMinReduce(Value *Src, bool IsSigned=false)
Create a vector integer min reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:471
void setConstrainedFPCallAttr(CallBase *I)
Definition: IRBuilder.h:347
InvokeInst * CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef< Value * > InvokeArgs, std::optional< ArrayRef< Value * > > DeoptArgs, ArrayRef< Value * > GCArgs, const Twine &Name="")
Create an invoke to the experimental.gc.statepoint intrinsic to start a new statepoint sequence.
Definition: IRBuilder.cpp:882
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2307
const IRBuilderFolder & Folder
Definition: IRBuilder.h:122
CallInst * CreateMemSetInline(Value *Dst, MaybeAlign DstAlign, Value *Val, Value *Size, bool IsVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Definition: IRBuilder.cpp:174
CallInst * CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val, uint64_t Size, Align Alignment, uint32_t ElementSize, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert an element unordered-atomic memset of the region of memory starting at the given po...
Definition: IRBuilder.h:611
CallInst * CreateFMulReduce(Value *Acc, Value *Src)
Create a sequential vector fmul reduction intrinsic of the source vector.
Definition: IRBuilder.cpp:437
void SetInstDebugLocation(Instruction *I) const
If this builder has a current debug location, set it on the specified instruction.
Definition: IRBuilder.cpp:79
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Definition: IRBuilder.h:502
CallInst * CreateGCRelocate(Instruction *Statepoint, int BaseOffset, int DerivedOffset, Type *ResultType, const Twine &Name="")
Create a call to the experimental.gc.relocate intrinsics to project the relocated value of one pointe...
Definition: IRBuilder.cpp:926
Value * CreateStepVector(Type *DstType, const Twine &Name="")
Creates a vector of type DstType with the linear sequence <0, 1, ...>
Definition: IRBuilder.cpp:118
CallInst * CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, uint64_t Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Definition: IRBuilder.h:682
Value * CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex, MDNode *DbgInfo)
Definition: IRBuilder.cpp:1300
CallInst * CreateInvariantStart(Value *Ptr, ConstantInt *Size=nullptr)
Create a call to invariant.start intrinsic.
Definition: IRBuilder.cpp:517
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1290
Instruction * CreateNoAliasScopeDeclaration(Value *Scope)
Create a llvm.experimental.noalias.scope.decl intrinsic call.
Definition: IRBuilder.cpp:580
CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
Definition: IRBuilder.cpp:686
Value * CreateStripInvariantGroup(Value *Ptr)
Create a strip.invariant.group intrinsic call.
Definition: IRBuilder.cpp:1188
CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
Definition: IRBuilder.cpp:650
virtual Value * FoldSelect(Value *C, Value *True, Value *False) const =0
virtual Value * CreateFCmp(CmpInst::Predicate P, Constant *LHS, Constant *RHS) const =0
virtual ~IRBuilderFolder()
void copyFastMathFlags(FastMathFlags FMF)
Convenience function for transferring all fast-math flag values to this instruction,...
bool isBinaryOp() const
Definition: Instruction.h:173
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:275
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1521
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
bool isUnaryOp() const
Definition: Instruction.h:172
Class to represent integer types.
Definition: DerivedTypes.h:40
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
Definition: DerivedTypes.h:72
Invoke instruction.
Metadata node.
Definition: Metadata.h:950
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
A container for an operand bundle being viewed as a set of values rather than a set of uses.
Definition: InstrTypes.h:1143
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1743
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", Instruction *InsertBefore=nullptr, Instruction *MDFrom=nullptr)
void reserve(size_type N)
Definition: SmallVector.h:667
void resize(size_type N)
Definition: SmallVector.h:642
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:265
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:348
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1069
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:738
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl< IITDescriptor > &T)
Return the IIT table descriptor for the specified intrinsic into an array of IITDescriptors.
Definition: Function.cpp:1293
@ MatchIntrinsicTypes_Match
Definition: Intrinsics.h:214
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1465
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:406
AddressSpace
Definition: NVPTXBaseInfo.h:21
bool getAlign(const Function &F, unsigned index, unsigned &align)
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:2129
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117