Line data Source code
1 : //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file implements the IRBuilder class, which is used as a convenient way
11 : // to create LLVM instructions with a consistent and simplified interface.
12 : //
13 : //===----------------------------------------------------------------------===//
14 :
15 : #include "llvm/IR/IRBuilder.h"
16 : #include "llvm/ADT/ArrayRef.h"
17 : #include "llvm/ADT/None.h"
18 : #include "llvm/IR/Constant.h"
19 : #include "llvm/IR/Constants.h"
20 : #include "llvm/IR/DerivedTypes.h"
21 : #include "llvm/IR/Function.h"
22 : #include "llvm/IR/GlobalValue.h"
23 : #include "llvm/IR/GlobalVariable.h"
24 : #include "llvm/IR/IntrinsicInst.h"
25 : #include "llvm/IR/Intrinsics.h"
26 : #include "llvm/IR/LLVMContext.h"
27 : #include "llvm/IR/Operator.h"
28 : #include "llvm/IR/Statepoint.h"
29 : #include "llvm/IR/Type.h"
30 : #include "llvm/IR/Value.h"
31 : #include "llvm/Support/Casting.h"
32 : #include "llvm/Support/MathExtras.h"
33 : #include <cassert>
34 : #include <cstdint>
35 : #include <vector>
36 :
37 : using namespace llvm;
38 :
39 : /// CreateGlobalString - Make a new global variable with an initializer that
40 : /// has array of i8 type filled in with the nul terminated string value
41 : /// specified. If Name is specified, it is the name of the global variable
42 : /// created.
43 25359 : GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
44 : const Twine &Name,
45 : unsigned AddressSpace) {
46 25359 : Constant *StrConstant = ConstantDataArray::getString(Context, Str);
47 25359 : Module &M = *BB->getParent()->getParent();
48 25359 : auto *GV = new GlobalVariable(M, StrConstant->getType(), true,
49 : GlobalValue::PrivateLinkage, StrConstant, Name,
50 : nullptr, GlobalVariable::NotThreadLocal,
51 25359 : AddressSpace);
52 : GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
53 25359 : GV->setAlignment(1);
54 25359 : return GV;
55 : }
56 :
57 0 : Type *IRBuilderBase::getCurrentFunctionReturnType() const {
58 : assert(BB && BB->getParent() && "No current function!");
59 0 : return BB->getParent()->getReturnType();
60 : }
61 :
62 410399 : Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
63 410399 : auto *PT = cast<PointerType>(Ptr->getType());
64 410399 : if (PT->getElementType()->isIntegerTy(8))
65 : return Ptr;
66 :
67 : // Otherwise, we need to insert a bitcast.
68 31047 : PT = getInt8PtrTy(PT->getAddressSpace());
69 31047 : BitCastInst *BCI = new BitCastInst(Ptr, PT, "");
70 31047 : BB->getInstList().insert(InsertPt, BCI);
71 31047 : SetInstDebugLocation(BCI);
72 31047 : return BCI;
73 : }
74 :
75 265807 : static CallInst *createCallHelper(Value *Callee, ArrayRef<Value *> Ops,
76 : IRBuilderBase *Builder,
77 : const Twine &Name = "",
78 : Instruction *FMFSource = nullptr) {
79 265807 : CallInst *CI = CallInst::Create(Callee, Ops, Name);
80 265807 : if (FMFSource)
81 530 : CI->copyFastMathFlags(FMFSource);
82 265807 : Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),CI);
83 265807 : Builder->SetInstDebugLocation(CI);
84 265807 : return CI;
85 : }
86 :
87 34 : static InvokeInst *createInvokeHelper(Value *Invokee, BasicBlock *NormalDest,
88 : BasicBlock *UnwindDest,
89 : ArrayRef<Value *> Ops,
90 : IRBuilderBase *Builder,
91 : const Twine &Name = "") {
92 : InvokeInst *II =
93 34 : InvokeInst::Create(Invokee, NormalDest, UnwindDest, Ops, Name);
94 34 : Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),
95 34 : II);
96 34 : Builder->SetInstDebugLocation(II);
97 34 : return II;
98 : }
99 :
100 32830 : CallInst *IRBuilderBase::
101 : CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align,
102 : bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
103 : MDNode *NoAliasTag) {
104 32830 : Ptr = getCastedInt8PtrValue(Ptr);
105 32830 : Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
106 32830 : Type *Tys[] = { Ptr->getType(), Size->getType() };
107 32830 : Module *M = BB->getParent()->getParent();
108 32830 : Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
109 :
110 32830 : CallInst *CI = createCallHelper(TheFn, Ops, this);
111 :
112 32830 : if (Align > 0)
113 32821 : cast<MemSetInst>(CI)->setDestAlignment(Align);
114 :
115 : // Set the TBAA info if present.
116 32830 : if (TBAATag)
117 2 : CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
118 :
119 32830 : if (ScopeTag)
120 2 : CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
121 :
122 32830 : if (NoAliasTag)
123 2 : CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
124 :
125 32830 : return CI;
126 : }
127 :
128 0 : CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
129 : Value *Ptr, Value *Val, Value *Size, unsigned Align, uint32_t ElementSize,
130 : MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
131 : assert(Align >= ElementSize &&
132 : "Pointer alignment must be at least element size.");
133 :
134 0 : Ptr = getCastedInt8PtrValue(Ptr);
135 0 : Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
136 0 : Type *Tys[] = {Ptr->getType(), Size->getType()};
137 0 : Module *M = BB->getParent()->getParent();
138 0 : Value *TheFn = Intrinsic::getDeclaration(
139 : M, Intrinsic::memset_element_unordered_atomic, Tys);
140 :
141 0 : CallInst *CI = createCallHelper(TheFn, Ops, this);
142 :
143 0 : cast<AtomicMemSetInst>(CI)->setDestAlignment(Align);
144 :
145 : // Set the TBAA info if present.
146 0 : if (TBAATag)
147 0 : CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
148 :
149 0 : if (ScopeTag)
150 0 : CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
151 :
152 0 : if (NoAliasTag)
153 0 : CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
154 :
155 0 : return CI;
156 : }
157 :
158 161241 : CallInst *IRBuilderBase::
159 : CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
160 : Value *Size, bool isVolatile, MDNode *TBAATag,
161 : MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
162 : assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) && "Must be 0 or a power of 2");
163 : assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) && "Must be 0 or a power of 2");
164 161241 : Dst = getCastedInt8PtrValue(Dst);
165 161241 : Src = getCastedInt8PtrValue(Src);
166 :
167 161241 : Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
168 161241 : Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
169 161241 : Module *M = BB->getParent()->getParent();
170 161241 : Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
171 :
172 161241 : CallInst *CI = createCallHelper(TheFn, Ops, this);
173 :
174 : auto* MCI = cast<MemCpyInst>(CI);
175 161241 : if (DstAlign > 0)
176 161240 : MCI->setDestAlignment(DstAlign);
177 161241 : if (SrcAlign > 0)
178 161238 : MCI->setSourceAlignment(SrcAlign);
179 :
180 : // Set the TBAA info if present.
181 161241 : if (TBAATag)
182 8 : CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
183 :
184 : // Set the TBAA Struct info if present.
185 161241 : if (TBAAStructTag)
186 1 : CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
187 :
188 161241 : if (ScopeTag)
189 8 : CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
190 :
191 161241 : if (NoAliasTag)
192 8 : CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
193 :
194 161241 : return CI;
195 : }
196 :
197 7 : CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
198 : Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
199 : uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
200 : MDNode *ScopeTag, MDNode *NoAliasTag) {
201 : assert(DstAlign >= ElementSize &&
202 : "Pointer alignment must be at least element size");
203 : assert(SrcAlign >= ElementSize &&
204 : "Pointer alignment must be at least element size");
205 7 : Dst = getCastedInt8PtrValue(Dst);
206 7 : Src = getCastedInt8PtrValue(Src);
207 :
208 7 : Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
209 7 : Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
210 7 : Module *M = BB->getParent()->getParent();
211 7 : Value *TheFn = Intrinsic::getDeclaration(
212 : M, Intrinsic::memcpy_element_unordered_atomic, Tys);
213 :
214 7 : CallInst *CI = createCallHelper(TheFn, Ops, this);
215 :
216 : // Set the alignment of the pointer args.
217 : auto *AMCI = cast<AtomicMemCpyInst>(CI);
218 7 : AMCI->setDestAlignment(DstAlign);
219 7 : AMCI->setSourceAlignment(SrcAlign);
220 :
221 : // Set the TBAA info if present.
222 7 : if (TBAATag)
223 0 : CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
224 :
225 : // Set the TBAA Struct info if present.
226 7 : if (TBAAStructTag)
227 0 : CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
228 :
229 7 : if (ScopeTag)
230 0 : CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
231 :
232 7 : if (NoAliasTag)
233 0 : CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
234 :
235 7 : return CI;
236 : }
237 :
238 1128 : CallInst *IRBuilderBase::
239 : CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
240 : Value *Size, bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
241 : MDNode *NoAliasTag) {
242 : assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) && "Must be 0 or a power of 2");
243 : assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) && "Must be 0 or a power of 2");
244 1128 : Dst = getCastedInt8PtrValue(Dst);
245 1128 : Src = getCastedInt8PtrValue(Src);
246 :
247 1128 : Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
248 1128 : Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
249 1128 : Module *M = BB->getParent()->getParent();
250 1128 : Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
251 :
252 1128 : CallInst *CI = createCallHelper(TheFn, Ops, this);
253 :
254 : auto *MMI = cast<MemMoveInst>(CI);
255 1128 : if (DstAlign > 0)
256 1127 : MMI->setDestAlignment(DstAlign);
257 1128 : if (SrcAlign > 0)
258 1127 : MMI->setSourceAlignment(SrcAlign);
259 :
260 : // Set the TBAA info if present.
261 1128 : if (TBAATag)
262 1 : CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
263 :
264 1128 : if (ScopeTag)
265 1 : CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
266 :
267 1128 : if (NoAliasTag)
268 1 : CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
269 :
270 1128 : return CI;
271 : }
272 :
273 0 : CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
274 : Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
275 : uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
276 : MDNode *ScopeTag, MDNode *NoAliasTag) {
277 : assert(DstAlign >= ElementSize &&
278 : "Pointer alignment must be at least element size");
279 : assert(SrcAlign >= ElementSize &&
280 : "Pointer alignment must be at least element size");
281 0 : Dst = getCastedInt8PtrValue(Dst);
282 0 : Src = getCastedInt8PtrValue(Src);
283 :
284 0 : Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
285 0 : Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
286 0 : Module *M = BB->getParent()->getParent();
287 0 : Value *TheFn = Intrinsic::getDeclaration(
288 : M, Intrinsic::memmove_element_unordered_atomic, Tys);
289 :
290 0 : CallInst *CI = createCallHelper(TheFn, Ops, this);
291 :
292 : // Set the alignment of the pointer args.
293 0 : CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
294 0 : CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
295 :
296 : // Set the TBAA info if present.
297 0 : if (TBAATag)
298 0 : CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
299 :
300 : // Set the TBAA Struct info if present.
301 0 : if (TBAAStructTag)
302 0 : CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
303 :
304 0 : if (ScopeTag)
305 0 : CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
306 :
307 0 : if (NoAliasTag)
308 0 : CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
309 :
310 0 : return CI;
311 : }
312 :
313 20 : static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
314 : Value *Src) {
315 20 : Module *M = Builder->GetInsertBlock()->getParent()->getParent();
316 20 : Value *Ops[] = {Src};
317 40 : Type *Tys[] = { Src->getType()->getVectorElementType(), Src->getType() };
318 20 : auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
319 20 : return createCallHelper(Decl, Ops, Builder);
320 : }
321 :
322 0 : CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
323 0 : Module *M = GetInsertBlock()->getParent()->getParent();
324 0 : Value *Ops[] = {Acc, Src};
325 0 : Type *Tys[] = {Src->getType()->getVectorElementType(), Acc->getType(),
326 0 : Src->getType()};
327 0 : auto Decl = Intrinsic::getDeclaration(
328 : M, Intrinsic::experimental_vector_reduce_fadd, Tys);
329 0 : return createCallHelper(Decl, Ops, this);
330 : }
331 :
332 0 : CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
333 0 : Module *M = GetInsertBlock()->getParent()->getParent();
334 0 : Value *Ops[] = {Acc, Src};
335 0 : Type *Tys[] = {Src->getType()->getVectorElementType(), Acc->getType(),
336 0 : Src->getType()};
337 0 : auto Decl = Intrinsic::getDeclaration(
338 : M, Intrinsic::experimental_vector_reduce_fmul, Tys);
339 0 : return createCallHelper(Decl, Ops, this);
340 : }
341 :
342 20 : CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
343 20 : return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
344 20 : Src);
345 : }
346 :
347 0 : CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
348 0 : return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
349 0 : Src);
350 : }
351 :
352 0 : CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
353 0 : return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
354 0 : Src);
355 : }
356 :
357 0 : CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
358 0 : return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
359 0 : Src);
360 : }
361 :
362 0 : CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
363 0 : return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
364 0 : Src);
365 : }
366 :
367 0 : CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
368 0 : auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
369 : : Intrinsic::experimental_vector_reduce_umax;
370 0 : return getReductionIntrinsic(this, ID, Src);
371 : }
372 :
373 0 : CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
374 0 : auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
375 : : Intrinsic::experimental_vector_reduce_umin;
376 0 : return getReductionIntrinsic(this, ID, Src);
377 : }
378 :
379 0 : CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
380 0 : auto Rdx = getReductionIntrinsic(
381 : this, Intrinsic::experimental_vector_reduce_fmax, Src);
382 0 : if (NoNaN) {
383 : FastMathFlags FMF;
384 : FMF.setNoNaNs();
385 0 : Rdx->setFastMathFlags(FMF);
386 : }
387 0 : return Rdx;
388 : }
389 :
390 0 : CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
391 0 : auto Rdx = getReductionIntrinsic(
392 : this, Intrinsic::experimental_vector_reduce_fmin, Src);
393 0 : if (NoNaN) {
394 : FastMathFlags FMF;
395 : FMF.setNoNaNs();
396 0 : Rdx->setFastMathFlags(FMF);
397 : }
398 0 : return Rdx;
399 : }
400 :
401 23524 : CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
402 : assert(isa<PointerType>(Ptr->getType()) &&
403 : "lifetime.start only applies to pointers.");
404 23524 : Ptr = getCastedInt8PtrValue(Ptr);
405 23524 : if (!Size)
406 2 : Size = getInt64(-1);
407 : else
408 : assert(Size->getType() == getInt64Ty() &&
409 : "lifetime.start requires the size to be an i64");
410 23524 : Value *Ops[] = { Size, Ptr };
411 23524 : Module *M = BB->getParent()->getParent();
412 23524 : Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::lifetime_start,
413 23524 : { Ptr->getType() });
414 23524 : return createCallHelper(TheFn, Ops, this);
415 : }
416 :
417 29293 : CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
418 : assert(isa<PointerType>(Ptr->getType()) &&
419 : "lifetime.end only applies to pointers.");
420 29293 : Ptr = getCastedInt8PtrValue(Ptr);
421 29293 : if (!Size)
422 3 : Size = getInt64(-1);
423 : else
424 : assert(Size->getType() == getInt64Ty() &&
425 : "lifetime.end requires the size to be an i64");
426 29293 : Value *Ops[] = { Size, Ptr };
427 29293 : Module *M = BB->getParent()->getParent();
428 29293 : Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::lifetime_end,
429 29293 : { Ptr->getType() });
430 29293 : return createCallHelper(TheFn, Ops, this);
431 : }
432 :
433 0 : CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
434 :
435 : assert(isa<PointerType>(Ptr->getType()) &&
436 : "invariant.start only applies to pointers.");
437 0 : Ptr = getCastedInt8PtrValue(Ptr);
438 0 : if (!Size)
439 0 : Size = getInt64(-1);
440 : else
441 : assert(Size->getType() == getInt64Ty() &&
442 : "invariant.start requires the size to be an i64");
443 :
444 0 : Value *Ops[] = {Size, Ptr};
445 : // Fill in the single overloaded type: memory object type.
446 0 : Type *ObjectPtr[1] = {Ptr->getType()};
447 0 : Module *M = BB->getParent()->getParent();
448 : Value *TheFn =
449 0 : Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
450 0 : return createCallHelper(TheFn, Ops, this);
451 : }
452 :
453 391 : CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
454 : assert(Cond->getType() == getInt1Ty() &&
455 : "an assumption condition must be of type i1");
456 :
457 391 : Value *Ops[] = { Cond };
458 391 : Module *M = BB->getParent()->getParent();
459 391 : Value *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
460 391 : return createCallHelper(FnAssume, Ops, this);
461 : }
462 :
463 : /// Create a call to a Masked Load intrinsic.
464 : /// \p Ptr - base pointer for the load
465 : /// \p Align - alignment of the source location
466 : /// \p Mask - vector of booleans which indicates what vector lanes should
467 : /// be accessed in memory
468 : /// \p PassThru - pass-through value that is used to fill the masked-off lanes
469 : /// of the result
470 : /// \p Name - name of the result variable
471 295 : CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align,
472 : Value *Mask, Value *PassThru,
473 : const Twine &Name) {
474 295 : auto *PtrTy = cast<PointerType>(Ptr->getType());
475 295 : Type *DataTy = PtrTy->getElementType();
476 : assert(DataTy->isVectorTy() && "Ptr should point to a vector");
477 : assert(Mask && "Mask should not be all-ones (null)");
478 295 : if (!PassThru)
479 0 : PassThru = UndefValue::get(DataTy);
480 295 : Type *OverloadedTypes[] = { DataTy, PtrTy };
481 295 : Value *Ops[] = { Ptr, getInt32(Align), Mask, PassThru};
482 295 : return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
483 295 : OverloadedTypes, Name);
484 : }
485 :
486 : /// Create a call to a Masked Store intrinsic.
487 : /// \p Val - data to be stored,
488 : /// \p Ptr - base pointer for the store
489 : /// \p Align - alignment of the destination location
490 : /// \p Mask - vector of booleans which indicates what vector lanes should
491 : /// be accessed in memory
492 201 : CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
493 : unsigned Align, Value *Mask) {
494 201 : auto *PtrTy = cast<PointerType>(Ptr->getType());
495 201 : Type *DataTy = PtrTy->getElementType();
496 : assert(DataTy->isVectorTy() && "Ptr should point to a vector");
497 : assert(Mask && "Mask should not be all-ones (null)");
498 201 : Type *OverloadedTypes[] = { DataTy, PtrTy };
499 201 : Value *Ops[] = { Val, Ptr, getInt32(Align), Mask };
500 201 : return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
501 : }
502 :
503 : /// Create a call to a Masked intrinsic, with given intrinsic Id,
504 : /// an array of operands - Ops, and an array of overloaded types -
505 : /// OverloadedTypes.
506 576 : CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
507 : ArrayRef<Value *> Ops,
508 : ArrayRef<Type *> OverloadedTypes,
509 : const Twine &Name) {
510 576 : Module *M = BB->getParent()->getParent();
511 576 : Value *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
512 576 : return createCallHelper(TheFn, Ops, this, Name);
513 : }
514 :
515 : /// Create a call to a Masked Gather intrinsic.
516 : /// \p Ptrs - vector of pointers for loading
517 : /// \p Align - alignment for one element
518 : /// \p Mask - vector of booleans which indicates what vector lanes should
519 : /// be accessed in memory
520 : /// \p PassThru - pass-through value that is used to fill the masked-off lanes
521 : /// of the result
522 : /// \p Name - name of the result variable
523 61 : CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align,
524 : Value *Mask, Value *PassThru,
525 : const Twine& Name) {
526 61 : auto PtrsTy = cast<VectorType>(Ptrs->getType());
527 61 : auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
528 : unsigned NumElts = PtrsTy->getVectorNumElements();
529 61 : Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
530 :
531 61 : if (!Mask)
532 29 : Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
533 : NumElts));
534 :
535 61 : if (!PassThru)
536 61 : PassThru = UndefValue::get(DataTy);
537 :
538 61 : Type *OverloadedTypes[] = {DataTy, PtrsTy};
539 61 : Value * Ops[] = {Ptrs, getInt32(Align), Mask, PassThru};
540 :
541 : // We specify only one type when we create this intrinsic. Types of other
542 : // arguments are derived from this type.
543 61 : return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
544 61 : Name);
545 : }
546 :
547 : /// Create a call to a Masked Scatter intrinsic.
548 : /// \p Data - data to be stored,
549 : /// \p Ptrs - the vector of pointers, where the \p Data elements should be
550 : /// stored
551 : /// \p Align - alignment for one element
552 : /// \p Mask - vector of booleans which indicates what vector lanes should
553 : /// be accessed in memory
554 19 : CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
555 : unsigned Align, Value *Mask) {
556 19 : auto PtrsTy = cast<VectorType>(Ptrs->getType());
557 19 : auto DataTy = cast<VectorType>(Data->getType());
558 : unsigned NumElts = PtrsTy->getVectorNumElements();
559 :
560 : #ifndef NDEBUG
561 : auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
562 : assert(NumElts == DataTy->getVectorNumElements() &&
563 : PtrTy->getElementType() == DataTy->getElementType() &&
564 : "Incompatible pointer and data types");
565 : #endif
566 :
567 19 : if (!Mask)
568 7 : Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
569 : NumElts));
570 :
571 19 : Type *OverloadedTypes[] = {DataTy, PtrsTy};
572 19 : Value * Ops[] = {Data, Ptrs, getInt32(Align), Mask};
573 :
574 : // We specify only one type when we create this intrinsic. Types of other
575 : // arguments are derived from this type.
576 19 : return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
577 : }
578 :
579 : template <typename T0, typename T1, typename T2, typename T3>
580 : static std::vector<Value *>
581 312 : getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
582 : Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
583 : ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs,
584 : ArrayRef<T3> GCArgs) {
585 : std::vector<Value *> Args;
586 312 : Args.push_back(B.getInt64(ID));
587 312 : Args.push_back(B.getInt32(NumPatchBytes));
588 312 : Args.push_back(ActualCallee);
589 312 : Args.push_back(B.getInt32(CallArgs.size()));
590 312 : Args.push_back(B.getInt32(Flags));
591 : Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
592 312 : Args.push_back(B.getInt32(TransitionArgs.size()));
593 : Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end());
594 312 : Args.push_back(B.getInt32(DeoptArgs.size()));
595 : Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
596 312 : Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
597 :
598 312 : return Args;
599 : }
600 0 :
601 : template <typename T0, typename T1, typename T2, typename T3>
602 : static CallInst *CreateGCStatepointCallCommon(
603 : IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
604 : Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
605 0 : ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs,
606 0 : const Twine &Name) {
607 0 : // Extract out the type of the callee.
608 0 : auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
609 0 : assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
610 : "actual callee must be a callable value");
611 0 :
612 : Module *M = Builder->GetInsertBlock()->getParent()->getParent();
613 0 : // Fill in the one generic type'd argument (the function is also vararg)
614 : Type *ArgTypes[] = { FuncPtrType };
615 0 : Function *FnStatepoint =
616 : Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
617 0 : ArgTypes);
618 :
619 312 : std::vector<Value *> Args =
620 : getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
621 : CallArgs, TransitionArgs, DeoptArgs, GCArgs);
622 : return createCallHelper(FnStatepoint, Args, Builder, Name);
623 : }
624 312 :
625 312 : CallInst *IRBuilderBase::CreateGCStatepointCall(
626 312 : uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
627 312 : ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
628 312 : ArrayRef<Value *> GCArgs, const Twine &Name) {
629 : return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
630 312 : this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
631 : CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
632 312 : }
633 :
634 312 : CallInst *IRBuilderBase::CreateGCStatepointCall(
635 : uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
636 312 : ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs,
637 : ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
638 0 : return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
639 : this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
640 : DeoptArgs, GCArgs, Name);
641 : }
642 :
643 0 : CallInst *IRBuilderBase::CreateGCStatepointCall(
644 0 : uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
645 0 : ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
646 0 : ArrayRef<Value *> GCArgs, const Twine &Name) {
647 0 : return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
648 : this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
649 0 : CallArgs, None, DeoptArgs, GCArgs, Name);
650 : }
651 0 :
652 : template <typename T0, typename T1, typename T2, typename T3>
653 0 : static InvokeInst *CreateGCStatepointInvokeCommon(
654 : IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
655 0 : Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
656 : uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs,
657 : ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) {
658 : // Extract out the type of the callee.
659 278 : auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
660 : assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
661 : "actual callee must be a callable value");
662 :
663 : Module *M = Builder->GetInsertBlock()->getParent()->getParent();
664 : // Fill in the one generic type'd argument (the function is also vararg)
665 278 : Function *FnStatepoint = Intrinsic::getDeclaration(
666 : M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
667 :
668 : std::vector<Value *> Args =
669 278 : getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
670 : InvokeArgs, TransitionArgs, DeoptArgs, GCArgs);
671 278 : return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, Builder,
672 : Name);
673 278 : }
674 :
675 : InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
676 278 : uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
677 : BasicBlock *NormalDest, BasicBlock *UnwindDest,
678 : ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
679 278 : ArrayRef<Value *> GCArgs, const Twine &Name) {
680 : return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
681 0 : this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
682 : uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
683 : DeoptArgs, GCArgs, Name);
684 : }
685 :
686 : InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
687 0 : uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
688 : BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
689 : ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs,
690 : ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
691 0 : return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
692 : this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
693 0 : InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
694 : }
695 0 :
696 : InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
697 : uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
698 0 : BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
699 : ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
700 : return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
701 0 : this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
702 : uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
703 278 : Name);
704 : }
705 :
706 : CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
707 : Type *ResultType,
708 : const Twine &Name) {
709 278 : Intrinsic::ID ID = Intrinsic::experimental_gc_result;
710 : Module *M = BB->getParent()->getParent();
711 : Type *Types[] = {ResultType};
712 : Value *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
713 278 :
714 : Value *Args[] = {Statepoint};
715 278 : return createCallHelper(FnGCResult, Args, this, Name);
716 : }
717 278 :
718 : CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
719 : int BaseOffset,
720 278 : int DerivedOffset,
721 : Type *ResultType,
722 : const Twine &Name) {
723 278 : Module *M = BB->getParent()->getParent();
724 : Type *Types[] = {ResultType};
725 0 : Value *FnGCRelocate =
726 : Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
727 :
728 : Value *Args[] = {Statepoint,
729 : getInt32(BaseOffset),
730 : getInt32(DerivedOffset)};
731 0 : return createCallHelper(FnGCRelocate, Args, this, Name);
732 : }
733 :
734 : CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
735 0 : Instruction *FMFSource,
736 : const Twine &Name) {
737 0 : Module *M = BB->getModule();
738 : Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
739 0 : return createCallHelper(Fn, {V}, this, Name, FMFSource);
740 : }
741 :
742 0 : CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
743 : Value *RHS,
744 : Instruction *FMFSource,
745 0 : const Twine &Name) {
746 : Module *M = BB->getModule();
747 : Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
748 0 : return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
749 : }
750 :
751 : CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
752 0 : ArrayRef<Type *> Types,
753 : ArrayRef<Value *> Args,
754 0 : Instruction *FMFSource,
755 : const Twine &Name) {
756 : Module *M = BB->getModule();
757 278 : Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
758 : return createCallHelper(Fn, Args, this, Name, FMFSource);
759 : }
|
