Bug Summary

File:llvm/include/llvm/IR/IRBuilder.h
Warning:line 1694, column 30
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name OMPIRBuilder.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/lib/Frontend/OpenMP -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/lib/Frontend/OpenMP -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../x86_64-linux-gnu/include -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/lib/Frontend/OpenMP -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-04-05-202135-9119-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp

/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp

1//===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file implements the OpenMPIRBuilder class, which is used as a
11/// convenient way to create LLVM instructions for OpenMP directives.
12///
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
16
17#include "llvm/ADT/StringRef.h"
18#include "llvm/ADT/Triple.h"
19#include "llvm/IR/CFG.h"
20#include "llvm/IR/DebugInfo.h"
21#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/MDBuilder.h"
23#include "llvm/Support/CommandLine.h"
24#include "llvm/Support/Error.h"
25#include "llvm/Transforms/Utils/BasicBlockUtils.h"
26#include "llvm/Transforms/Utils/CodeExtractor.h"
27
28#include <sstream>
29
30#define DEBUG_TYPE"openmp-ir-builder" "openmp-ir-builder"
31
32using namespace llvm;
33using namespace omp;
34
35static cl::opt<bool>
36 OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
37 cl::desc("Use optimistic attributes describing "
38 "'as-if' properties of runtime calls."),
39 cl::init(false));
40
41void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) {
42 LLVMContext &Ctx = Fn.getContext();
43
44 // Get the function's current attributes.
45 auto Attrs = Fn.getAttributes();
46 auto FnAttrs = Attrs.getFnAttributes();
47 auto RetAttrs = Attrs.getRetAttributes();
48 SmallVector<AttributeSet, 4> ArgAttrs;
49 for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)
50 ArgAttrs.emplace_back(Attrs.getParamAttributes(ArgNo));
51
52#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
53#include "llvm/Frontend/OpenMP/OMPKinds.def"
54
55 // Add attributes to the function declaration.
56 switch (FnID) {
57#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \
58 case Enum: \
59 FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet); \
60 RetAttrs = RetAttrs.addAttributes(Ctx, RetAttrSet); \
61 for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo) \
62 ArgAttrs[ArgNo] = \
63 ArgAttrs[ArgNo].addAttributes(Ctx, ArgAttrSets[ArgNo]); \
64 Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs)); \
65 break;
66#include "llvm/Frontend/OpenMP/OMPKinds.def"
67 default:
68 // Attributes are optional.
69 break;
70 }
71}
72
73FunctionCallee
74OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) {
75 FunctionType *FnTy = nullptr;
76 Function *Fn = nullptr;
77
78 // Try to find the declation in the module first.
79 switch (FnID) {
80#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \
81 case Enum: \
82 FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \
83 IsVarArg); \
84 Fn = M.getFunction(Str); \
85 break;
86#include "llvm/Frontend/OpenMP/OMPKinds.def"
87 }
88
89 if (!Fn) {
90 // Create a new declaration if we need one.
91 switch (FnID) {
92#define OMP_RTL(Enum, Str, ...) \
93 case Enum: \
94 Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \
95 break;
96#include "llvm/Frontend/OpenMP/OMPKinds.def"
97 }
98
99 // Add information if the runtime function takes a callback function
100 if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
101 if (!Fn->hasMetadata(LLVMContext::MD_callback)) {
102 LLVMContext &Ctx = Fn->getContext();
103 MDBuilder MDB(Ctx);
104 // Annotate the callback behavior of the runtime function:
105 // - The callback callee is argument number 2 (microtask).
106 // - The first two arguments of the callback callee are unknown (-1).
107 // - All variadic arguments to the runtime function are passed to the
108 // callback callee.
109 Fn->addMetadata(
110 LLVMContext::MD_callback,
111 *MDNode::get(Ctx, {MDB.createCallbackEncoding(
112 2, {-1, -1}, /* VarArgsArePassed */ true)}));
113 }
114 }
115
116 LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Created OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
117 << " with type " << *Fn->getFunctionType() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Created OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
;
118 addAttributes(FnID, *Fn);
119
120 } else {
121 LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Found OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
122 << " with type " << *Fn->getFunctionType() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Found OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
;
123 }
124
125 assert(Fn && "Failed to create OpenMP runtime function")((Fn && "Failed to create OpenMP runtime function") ?
static_cast<void> (0) : __assert_fail ("Fn && \"Failed to create OpenMP runtime function\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 125, __PRETTY_FUNCTION__))
;
126
127 // Cast the function to the expected type if necessary
128 Constant *C = ConstantExpr::getBitCast(Fn, FnTy->getPointerTo());
129 return {FnTy, C};
130}
131
132Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) {
133 FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID);
134 auto *Fn = dyn_cast<llvm::Function>(RTLFn.getCallee());
135 assert(Fn && "Failed to create OpenMP runtime function pointer")((Fn && "Failed to create OpenMP runtime function pointer"
) ? static_cast<void> (0) : __assert_fail ("Fn && \"Failed to create OpenMP runtime function pointer\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 135, __PRETTY_FUNCTION__))
;
136 return Fn;
137}
138
139void OpenMPIRBuilder::initialize() { initializeTypes(M); }
140
141void OpenMPIRBuilder::finalize(Function *Fn, bool AllowExtractorSinking) {
142 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
143 SmallVector<BasicBlock *, 32> Blocks;
144 SmallVector<OutlineInfo, 16> DeferredOutlines;
145 for (OutlineInfo &OI : OutlineInfos) {
146 // Skip functions that have not finalized yet; may happen with nested
147 // function generation.
148 if (Fn && OI.getFunction() != Fn) {
149 DeferredOutlines.push_back(OI);
150 continue;
151 }
152
153 ParallelRegionBlockSet.clear();
154 Blocks.clear();
155 OI.collectBlocks(ParallelRegionBlockSet, Blocks);
156
157 Function *OuterFn = OI.getFunction();
158 CodeExtractorAnalysisCache CEAC(*OuterFn);
159 CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
160 /* AggregateArgs */ false,
161 /* BlockFrequencyInfo */ nullptr,
162 /* BranchProbabilityInfo */ nullptr,
163 /* AssumptionCache */ nullptr,
164 /* AllowVarArgs */ true,
165 /* AllowAlloca */ true,
166 /* Suffix */ ".omp_par");
167
168 LLVM_DEBUG(dbgs() << "Before outlining: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before outlining: "
<< *OuterFn << "\n"; } } while (false)
;
169 LLVM_DEBUG(dbgs() << "Entry " << OI.EntryBB->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Entry " << OI
.EntryBB->getName() << " Exit: " << OI.ExitBB->
getName() << "\n"; } } while (false)
170 << " Exit: " << OI.ExitBB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Entry " << OI
.EntryBB->getName() << " Exit: " << OI.ExitBB->
getName() << "\n"; } } while (false)
;
171 assert(Extractor.isEligible() &&((Extractor.isEligible() && "Expected OpenMP outlining to be possible!"
) ? static_cast<void> (0) : __assert_fail ("Extractor.isEligible() && \"Expected OpenMP outlining to be possible!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 172, __PRETTY_FUNCTION__))
172 "Expected OpenMP outlining to be possible!")((Extractor.isEligible() && "Expected OpenMP outlining to be possible!"
) ? static_cast<void> (0) : __assert_fail ("Extractor.isEligible() && \"Expected OpenMP outlining to be possible!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 172, __PRETTY_FUNCTION__))
;
173
174 Function *OutlinedFn = Extractor.extractCodeRegion(CEAC);
175
176 LLVM_DEBUG(dbgs() << "After outlining: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After outlining: "
<< *OuterFn << "\n"; } } while (false)
;
177 LLVM_DEBUG(dbgs() << " Outlined function: " << *OutlinedFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << " Outlined function: "
<< *OutlinedFn << "\n"; } } while (false)
;
178 assert(OutlinedFn->getReturnType()->isVoidTy() &&((OutlinedFn->getReturnType()->isVoidTy() && "OpenMP outlined functions should not return a value!"
) ? static_cast<void> (0) : __assert_fail ("OutlinedFn->getReturnType()->isVoidTy() && \"OpenMP outlined functions should not return a value!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 179, __PRETTY_FUNCTION__))
179 "OpenMP outlined functions should not return a value!")((OutlinedFn->getReturnType()->isVoidTy() && "OpenMP outlined functions should not return a value!"
) ? static_cast<void> (0) : __assert_fail ("OutlinedFn->getReturnType()->isVoidTy() && \"OpenMP outlined functions should not return a value!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 179, __PRETTY_FUNCTION__))
;
180
181 // For compability with the clang CG we move the outlined function after the
182 // one with the parallel region.
183 OutlinedFn->removeFromParent();
184 M.getFunctionList().insertAfter(OuterFn->getIterator(), OutlinedFn);
185
186 // Remove the artificial entry introduced by the extractor right away, we
187 // made our own entry block after all.
188 {
189 BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock();
190 assert(ArtificialEntry.getUniqueSuccessor() == OI.EntryBB)((ArtificialEntry.getUniqueSuccessor() == OI.EntryBB) ? static_cast
<void> (0) : __assert_fail ("ArtificialEntry.getUniqueSuccessor() == OI.EntryBB"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 190, __PRETTY_FUNCTION__))
;
191 assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry)((OI.EntryBB->getUniquePredecessor() == &ArtificialEntry
) ? static_cast<void> (0) : __assert_fail ("OI.EntryBB->getUniquePredecessor() == &ArtificialEntry"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 191, __PRETTY_FUNCTION__))
;
192 if (AllowExtractorSinking) {
193 // Move instructions from the to-be-deleted ArtificialEntry to the entry
194 // basic block of the parallel region. CodeExtractor may have sunk
195 // allocas/bitcasts for values that are solely used in the outlined
196 // region and do not escape.
197 assert(!ArtificialEntry.empty() &&((!ArtificialEntry.empty() && "Expected instructions to sink in the outlined region"
) ? static_cast<void> (0) : __assert_fail ("!ArtificialEntry.empty() && \"Expected instructions to sink in the outlined region\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 198, __PRETTY_FUNCTION__))
198 "Expected instructions to sink in the outlined region")((!ArtificialEntry.empty() && "Expected instructions to sink in the outlined region"
) ? static_cast<void> (0) : __assert_fail ("!ArtificialEntry.empty() && \"Expected instructions to sink in the outlined region\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 198, __PRETTY_FUNCTION__))
;
199 for (BasicBlock::iterator It = ArtificialEntry.begin(),
200 End = ArtificialEntry.end();
201 It != End;) {
202 Instruction &I = *It;
203 It++;
204
205 if (I.isTerminator())
206 continue;
207
208 I.moveBefore(*OI.EntryBB, OI.EntryBB->getFirstInsertionPt());
209 }
210 }
211 OI.EntryBB->moveBefore(&ArtificialEntry);
212 ArtificialEntry.eraseFromParent();
213 }
214 assert(&OutlinedFn->getEntryBlock() == OI.EntryBB)((&OutlinedFn->getEntryBlock() == OI.EntryBB) ? static_cast
<void> (0) : __assert_fail ("&OutlinedFn->getEntryBlock() == OI.EntryBB"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 214, __PRETTY_FUNCTION__))
;
215 assert(OutlinedFn && OutlinedFn->getNumUses() == 1)((OutlinedFn && OutlinedFn->getNumUses() == 1) ? static_cast
<void> (0) : __assert_fail ("OutlinedFn && OutlinedFn->getNumUses() == 1"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 215, __PRETTY_FUNCTION__))
;
216
217 // Run a user callback, e.g. to add attributes.
218 if (OI.PostOutlineCB)
219 OI.PostOutlineCB(*OutlinedFn);
220 }
221
222 // Remove work items that have been completed.
223 OutlineInfos = std::move(DeferredOutlines);
224}
225
226OpenMPIRBuilder::~OpenMPIRBuilder() {
227 assert(OutlineInfos.empty() && "There must be no outstanding outlinings")((OutlineInfos.empty() && "There must be no outstanding outlinings"
) ? static_cast<void> (0) : __assert_fail ("OutlineInfos.empty() && \"There must be no outstanding outlinings\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 227, __PRETTY_FUNCTION__))
;
228}
229
230Value *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr,
231 IdentFlag LocFlags,
232 unsigned Reserve2Flags) {
233 // Enable "C-mode".
234 LocFlags |= OMP_IDENT_FLAG_KMPC;
235
236 Value *&Ident =
237 IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];
238 if (!Ident) {
239 Constant *I32Null = ConstantInt::getNullValue(Int32);
240 Constant *IdentData[] = {
241 I32Null, ConstantInt::get(Int32, uint32_t(LocFlags)),
242 ConstantInt::get(Int32, Reserve2Flags), I32Null, SrcLocStr};
243 Constant *Initializer = ConstantStruct::get(
244 cast<StructType>(IdentPtr->getPointerElementType()), IdentData);
245
246 // Look for existing encoding of the location + flags, not needed but
247 // minimizes the difference to the existing solution while we transition.
248 for (GlobalVariable &GV : M.getGlobalList())
249 if (GV.getType() == IdentPtr && GV.hasInitializer())
250 if (GV.getInitializer() == Initializer)
251 return Ident = &GV;
252
253 auto *GV = new GlobalVariable(M, IdentPtr->getPointerElementType(),
254 /* isConstant = */ true,
255 GlobalValue::PrivateLinkage, Initializer);
256 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
257 GV->setAlignment(Align(8));
258 Ident = GV;
259 }
260 return Builder.CreatePointerCast(Ident, IdentPtr);
261}
262
263Type *OpenMPIRBuilder::getLanemaskType() {
264 LLVMContext &Ctx = M.getContext();
265 Triple triple(M.getTargetTriple());
266
267 // This test is adequate until deviceRTL has finer grained lane widths
268 return triple.isAMDGCN() ? Type::getInt64Ty(Ctx) : Type::getInt32Ty(Ctx);
269}
270
271Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr) {
272 Constant *&SrcLocStr = SrcLocStrMap[LocStr];
273 if (!SrcLocStr) {
274 Constant *Initializer =
275 ConstantDataArray::getString(M.getContext(), LocStr);
276
277 // Look for existing encoding of the location, not needed but minimizes the
278 // difference to the existing solution while we transition.
279 for (GlobalVariable &GV : M.getGlobalList())
280 if (GV.isConstant() && GV.hasInitializer() &&
281 GV.getInitializer() == Initializer)
282 return SrcLocStr = ConstantExpr::getPointerCast(&GV, Int8Ptr);
283
284 SrcLocStr = Builder.CreateGlobalStringPtr(LocStr, /* Name */ "",
285 /* AddressSpace */ 0, &M);
286 }
287 return SrcLocStr;
288}
289
290Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName,
291 StringRef FileName,
292 unsigned Line,
293 unsigned Column) {
294 SmallString<128> Buffer;
295 Buffer.push_back(';');
296 Buffer.append(FileName);
297 Buffer.push_back(';');
298 Buffer.append(FunctionName);
299 Buffer.push_back(';');
300 Buffer.append(std::to_string(Line));
301 Buffer.push_back(';');
302 Buffer.append(std::to_string(Column));
303 Buffer.push_back(';');
304 Buffer.push_back(';');
305 return getOrCreateSrcLocStr(Buffer.str());
306}
307
308Constant *OpenMPIRBuilder::getOrCreateDefaultSrcLocStr() {
309 return getOrCreateSrcLocStr(";unknown;unknown;0;0;;");
310}
311
312Constant *
313OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc) {
314 DILocation *DIL = Loc.DL.get();
315 if (!DIL)
316 return getOrCreateDefaultSrcLocStr();
317 StringRef FileName = M.getName();
318 if (DIFile *DIF = DIL->getFile())
319 if (Optional<StringRef> Source = DIF->getSource())
320 FileName = *Source;
321 StringRef Function = DIL->getScope()->getSubprogram()->getName();
322 Function =
323 !Function.empty() ? Function : Loc.IP.getBlock()->getParent()->getName();
324 return getOrCreateSrcLocStr(Function, FileName, DIL->getLine(),
325 DIL->getColumn());
326}
327
328Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {
329 return Builder.CreateCall(
330 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num), Ident,
331 "omp_global_thread_num");
332}
333
334OpenMPIRBuilder::InsertPointTy
335OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive DK,
336 bool ForceSimpleCall, bool CheckCancelFlag) {
337 if (!updateToLocation(Loc))
338 return Loc.IP;
339 return emitBarrierImpl(Loc, DK, ForceSimpleCall, CheckCancelFlag);
340}
341
342OpenMPIRBuilder::InsertPointTy
343OpenMPIRBuilder::emitBarrierImpl(const LocationDescription &Loc, Directive Kind,
344 bool ForceSimpleCall, bool CheckCancelFlag) {
345 // Build call __kmpc_cancel_barrier(loc, thread_id) or
346 // __kmpc_barrier(loc, thread_id);
347
348 IdentFlag BarrierLocFlags;
349 switch (Kind) {
350 case OMPD_for:
351 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
352 break;
353 case OMPD_sections:
354 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
355 break;
356 case OMPD_single:
357 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
358 break;
359 case OMPD_barrier:
360 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
361 break;
362 default:
363 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
364 break;
365 }
366
367 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
368 Value *Args[] = {getOrCreateIdent(SrcLocStr, BarrierLocFlags),
369 getOrCreateThreadID(getOrCreateIdent(SrcLocStr))};
370
371 // If we are in a cancellable parallel region, barriers are cancellation
372 // points.
373 // TODO: Check why we would force simple calls or to ignore the cancel flag.
374 bool UseCancelBarrier =
375 !ForceSimpleCall && isLastFinalizationInfoCancellable(OMPD_parallel);
376
377 Value *Result =
378 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(
379 UseCancelBarrier ? OMPRTL___kmpc_cancel_barrier
380 : OMPRTL___kmpc_barrier),
381 Args);
382
383 if (UseCancelBarrier && CheckCancelFlag)
384 emitCancelationCheckImpl(Result, OMPD_parallel);
385
386 return Builder.saveIP();
387}
388
389OpenMPIRBuilder::InsertPointTy
390OpenMPIRBuilder::createCancel(const LocationDescription &Loc,
391 Value *IfCondition,
392 omp::Directive CanceledDirective) {
393 if (!updateToLocation(Loc))
394 return Loc.IP;
395
396 // LLVM utilities like blocks with terminators.
397 auto *UI = Builder.CreateUnreachable();
398
399 Instruction *ThenTI = UI, *ElseTI = nullptr;
400 if (IfCondition)
401 SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI);
402 Builder.SetInsertPoint(ThenTI);
403
404 Value *CancelKind = nullptr;
405 switch (CanceledDirective) {
406#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
407 case DirectiveEnum: \
408 CancelKind = Builder.getInt32(Value); \
409 break;
410#include "llvm/Frontend/OpenMP/OMPKinds.def"
411 default:
412 llvm_unreachable("Unknown cancel kind!")::llvm::llvm_unreachable_internal("Unknown cancel kind!", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 412)
;
413 }
414
415 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
416 Value *Ident = getOrCreateIdent(SrcLocStr);
417 Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
418 Value *Result = Builder.CreateCall(
419 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_cancel), Args);
420
421 // The actual cancel logic is shared with others, e.g., cancel_barriers.
422 emitCancelationCheckImpl(Result, CanceledDirective);
423
424 // Update the insertion point and remove the terminator we introduced.
425 Builder.SetInsertPoint(UI->getParent());
426 UI->eraseFromParent();
427
428 return Builder.saveIP();
429}
430
431void OpenMPIRBuilder::emitCancelationCheckImpl(
432 Value *CancelFlag, omp::Directive CanceledDirective) {
433 assert(isLastFinalizationInfoCancellable(CanceledDirective) &&((isLastFinalizationInfoCancellable(CanceledDirective) &&
"Unexpected cancellation!") ? static_cast<void> (0) : __assert_fail
("isLastFinalizationInfoCancellable(CanceledDirective) && \"Unexpected cancellation!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 434, __PRETTY_FUNCTION__))
434 "Unexpected cancellation!")((isLastFinalizationInfoCancellable(CanceledDirective) &&
"Unexpected cancellation!") ? static_cast<void> (0) : __assert_fail
("isLastFinalizationInfoCancellable(CanceledDirective) && \"Unexpected cancellation!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 434, __PRETTY_FUNCTION__))
;
435
436 // For a cancel barrier we create two new blocks.
437 BasicBlock *BB = Builder.GetInsertBlock();
438 BasicBlock *NonCancellationBlock;
439 if (Builder.GetInsertPoint() == BB->end()) {
440 // TODO: This branch will not be needed once we moved to the
441 // OpenMPIRBuilder codegen completely.
442 NonCancellationBlock = BasicBlock::Create(
443 BB->getContext(), BB->getName() + ".cont", BB->getParent());
444 } else {
445 NonCancellationBlock = SplitBlock(BB, &*Builder.GetInsertPoint());
446 BB->getTerminator()->eraseFromParent();
447 Builder.SetInsertPoint(BB);
448 }
449 BasicBlock *CancellationBlock = BasicBlock::Create(
450 BB->getContext(), BB->getName() + ".cncl", BB->getParent());
451
452 // Jump to them based on the return value.
453 Value *Cmp = Builder.CreateIsNull(CancelFlag);
454 Builder.CreateCondBr(Cmp, NonCancellationBlock, CancellationBlock,
455 /* TODO weight */ nullptr, nullptr);
456
457 // From the cancellation block we finalize all variables and go to the
458 // post finalization block that is known to the FiniCB callback.
459 Builder.SetInsertPoint(CancellationBlock);
460 auto &FI = FinalizationStack.back();
461 FI.FiniCB(Builder.saveIP());
462
463 // The continuation block is where code generation continues.
464 Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin());
465}
466
467IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
468 const LocationDescription &Loc, InsertPointTy OuterAllocaIP,
469 BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
470 FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads,
471 omp::ProcBindKind ProcBind, bool IsCancellable) {
472 if (!updateToLocation(Loc))
1
Taking false branch
473 return Loc.IP;
474
475 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
476 Value *Ident = getOrCreateIdent(SrcLocStr);
477 Value *ThreadID = getOrCreateThreadID(Ident);
478
479 if (NumThreads) {
2
Assuming 'NumThreads' is null
3
Taking false branch
480 // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads)
481 Value *Args[] = {
482 Ident, ThreadID,
483 Builder.CreateIntCast(NumThreads, Int32, /*isSigned*/ false)};
484 Builder.CreateCall(
485 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_num_threads), Args);
486 }
487
488 if (ProcBind != OMP_PROC_BIND_default) {
4
Assuming 'ProcBind' is equal to 'OMP_PROC_BIND_default'
5
Taking false branch
489 // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind)
490 Value *Args[] = {
491 Ident, ThreadID,
492 ConstantInt::get(Int32, unsigned(ProcBind), /*isSigned=*/true)};
493 Builder.CreateCall(
494 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_proc_bind), Args);
495 }
496
497 BasicBlock *InsertBB = Builder.GetInsertBlock();
498 Function *OuterFn = InsertBB->getParent();
499
500 // Save the outer alloca block because the insertion iterator may get
501 // invalidated and we still need this later.
502 BasicBlock *OuterAllocaBlock = OuterAllocaIP.getBlock();
503
504 // Vector to remember instructions we used only during the modeling but which
505 // we want to delete at the end.
506 SmallVector<Instruction *, 4> ToBeDeleted;
507
508 // Change the location to the outer alloca insertion point to create and
509 // initialize the allocas we pass into the parallel region.
510 Builder.restoreIP(OuterAllocaIP);
511 AllocaInst *TIDAddr = Builder.CreateAlloca(Int32, nullptr, "tid.addr");
512 AllocaInst *ZeroAddr = Builder.CreateAlloca(Int32, nullptr, "zero.addr");
513
514 // If there is an if condition we actually use the TIDAddr and ZeroAddr in the
515 // program, otherwise we only need them for modeling purposes to get the
516 // associated arguments in the outlined function. In the former case,
517 // initialize the allocas properly, in the latter case, delete them later.
518 if (IfCondition) {
6
Assuming 'IfCondition' is null
7
Taking false branch
519 Builder.CreateStore(Constant::getNullValue(Int32), TIDAddr);
520 Builder.CreateStore(Constant::getNullValue(Int32), ZeroAddr);
521 } else {
522 ToBeDeleted.push_back(TIDAddr);
523 ToBeDeleted.push_back(ZeroAddr);
524 }
525
526 // Create an artificial insertion point that will also ensure the blocks we
527 // are about to split are not degenerated.
528 auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);
529
530 Instruction *ThenTI = UI, *ElseTI = nullptr;
531 if (IfCondition
7.1
'IfCondition' is null
7.1
'IfCondition' is null
7.1
'IfCondition' is null
)
8
Taking false branch
532 SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI);
533
534 BasicBlock *ThenBB = ThenTI->getParent();
535 BasicBlock *PRegEntryBB = ThenBB->splitBasicBlock(ThenTI, "omp.par.entry");
536 BasicBlock *PRegBodyBB =
537 PRegEntryBB->splitBasicBlock(ThenTI, "omp.par.region");
538 BasicBlock *PRegPreFiniBB =
539 PRegBodyBB->splitBasicBlock(ThenTI, "omp.par.pre_finalize");
540 BasicBlock *PRegExitBB =
541 PRegPreFiniBB->splitBasicBlock(ThenTI, "omp.par.exit");
542
543 auto FiniCBWrapper = [&](InsertPointTy IP) {
544 // Hide "open-ended" blocks from the given FiniCB by setting the right jump
545 // target to the region exit block.
546 if (IP.getBlock()->end() == IP.getPoint()) {
547 IRBuilder<>::InsertPointGuard IPG(Builder);
548 Builder.restoreIP(IP);
549 Instruction *I = Builder.CreateBr(PRegExitBB);
550 IP = InsertPointTy(I->getParent(), I->getIterator());
551 }
552 assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&((IP.getBlock()->getTerminator()->getNumSuccessors() ==
1 && IP.getBlock()->getTerminator()->getSuccessor
(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 554, __PRETTY_FUNCTION__))
553 IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&((IP.getBlock()->getTerminator()->getNumSuccessors() ==
1 && IP.getBlock()->getTerminator()->getSuccessor
(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 554, __PRETTY_FUNCTION__))
554 "Unexpected insertion point for finalization call!")((IP.getBlock()->getTerminator()->getNumSuccessors() ==
1 && IP.getBlock()->getTerminator()->getSuccessor
(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 554, __PRETTY_FUNCTION__))
;
555 return FiniCB(IP);
556 };
557
558 FinalizationStack.push_back({FiniCBWrapper, OMPD_parallel, IsCancellable});
559
560 // Generate the privatization allocas in the block that will become the entry
561 // of the outlined function.
562 Builder.SetInsertPoint(PRegEntryBB->getTerminator());
563 InsertPointTy InnerAllocaIP = Builder.saveIP();
564
565 AllocaInst *PrivTIDAddr =
566 Builder.CreateAlloca(Int32, nullptr, "tid.addr.local");
567 Instruction *PrivTID = Builder.CreateLoad(Int32, PrivTIDAddr, "tid");
568
569 // Add some fake uses for OpenMP provided arguments.
570 ToBeDeleted.push_back(Builder.CreateLoad(Int32, TIDAddr, "tid.addr.use"));
571 Instruction *ZeroAddrUse = Builder.CreateLoad(Int32, ZeroAddr,
572 "zero.addr.use");
573 ToBeDeleted.push_back(ZeroAddrUse);
574
575 // ThenBB
576 // |
577 // V
578 // PRegionEntryBB <- Privatization allocas are placed here.
579 // |
580 // V
581 // PRegionBodyBB <- BodeGen is invoked here.
582 // |
583 // V
584 // PRegPreFiniBB <- The block we will start finalization from.
585 // |
586 // V
587 // PRegionExitBB <- A common exit to simplify block collection.
588 //
589
590 LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before body codegen: "
<< *OuterFn << "\n"; } } while (false)
;
9
Assuming 'DebugFlag' is false
10
Loop condition is false. Exiting loop
591
592 // Let the caller create the body.
593 assert(BodyGenCB && "Expected body generation callback!")((BodyGenCB && "Expected body generation callback!") ?
static_cast<void> (0) : __assert_fail ("BodyGenCB && \"Expected body generation callback!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 593, __PRETTY_FUNCTION__))
;
11
Assuming the condition is true
12
'?' condition is true
594 InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());
595 BodyGenCB(InnerAllocaIP, CodeGenIP, *PRegPreFiniBB);
596
597 LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After body codegen: "
<< *OuterFn << "\n"; } } while (false)
;
13
Assuming 'DebugFlag' is false
14
Loop condition is false. Exiting loop
598
599 FunctionCallee RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call);
600 if (auto *F
15.1
'F' is null
15.1
'F' is null
15.1
'F' is null
= dyn_cast<llvm::Function>(RTLFn.getCallee())) {
15
Assuming the object is not a 'Function'
16
Taking false branch
601 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
602 llvm::LLVMContext &Ctx = F->getContext();
603 MDBuilder MDB(Ctx);
604 // Annotate the callback behavior of the __kmpc_fork_call:
605 // - The callback callee is argument number 2 (microtask).
606 // - The first two arguments of the callback callee are unknown (-1).
607 // - All variadic arguments to the __kmpc_fork_call are passed to the
608 // callback callee.
609 F->addMetadata(
610 llvm::LLVMContext::MD_callback,
611 *llvm::MDNode::get(
612 Ctx, {MDB.createCallbackEncoding(2, {-1, -1},
613 /* VarArgsArePassed */ true)}));
614 }
615 }
616
617 OutlineInfo OI;
618 OI.PostOutlineCB = [=](Function &OutlinedFn) {
619 // Add some known attributes.
620 OutlinedFn.addParamAttr(0, Attribute::NoAlias);
621 OutlinedFn.addParamAttr(1, Attribute::NoAlias);
622 OutlinedFn.addFnAttr(Attribute::NoUnwind);
623 OutlinedFn.addFnAttr(Attribute::NoRecurse);
624
625 assert(OutlinedFn.arg_size() >= 2 &&((OutlinedFn.arg_size() >= 2 && "Expected at least tid and bounded tid as arguments"
) ? static_cast<void> (0) : __assert_fail ("OutlinedFn.arg_size() >= 2 && \"Expected at least tid and bounded tid as arguments\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 626, __PRETTY_FUNCTION__))
626 "Expected at least tid and bounded tid as arguments")((OutlinedFn.arg_size() >= 2 && "Expected at least tid and bounded tid as arguments"
) ? static_cast<void> (0) : __assert_fail ("OutlinedFn.arg_size() >= 2 && \"Expected at least tid and bounded tid as arguments\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 626, __PRETTY_FUNCTION__))
;
627 unsigned NumCapturedVars =
628 OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
629
630 CallInst *CI = cast<CallInst>(OutlinedFn.user_back());
631 CI->getParent()->setName("omp_parallel");
632 Builder.SetInsertPoint(CI);
633
634 // Build call __kmpc_fork_call(Ident, n, microtask, var1, .., varn);
635 Value *ForkCallArgs[] = {
636 Ident, Builder.getInt32(NumCapturedVars),
637 Builder.CreateBitCast(&OutlinedFn, ParallelTaskPtr)};
638
639 SmallVector<Value *, 16> RealArgs;
640 RealArgs.append(std::begin(ForkCallArgs), std::end(ForkCallArgs));
641 RealArgs.append(CI->arg_begin() + /* tid & bound tid */ 2, CI->arg_end());
642
643 Builder.CreateCall(RTLFn, RealArgs);
644
645 LLVM_DEBUG(dbgs() << "With fork_call placed: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With fork_call placed: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
646 << *Builder.GetInsertBlock()->getParent() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With fork_call placed: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
;
647
648 InsertPointTy ExitIP(PRegExitBB, PRegExitBB->end());
649
650 // Initialize the local TID stack location with the argument value.
651 Builder.SetInsertPoint(PrivTID);
652 Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
653 Builder.CreateStore(Builder.CreateLoad(Int32, OutlinedAI), PrivTIDAddr);
654
655 // If no "if" clause was present we do not need the call created during
656 // outlining, otherwise we reuse it in the serialized parallel region.
657 if (!ElseTI) {
658 CI->eraseFromParent();
659 } else {
660
661 // If an "if" clause was present we are now generating the serialized
662 // version into the "else" branch.
663 Builder.SetInsertPoint(ElseTI);
664
665 // Build calls __kmpc_serialized_parallel(&Ident, GTid);
666 Value *SerializedParallelCallArgs[] = {Ident, ThreadID};
667 Builder.CreateCall(
668 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_serialized_parallel),
669 SerializedParallelCallArgs);
670
671 // OutlinedFn(&GTid, &zero, CapturedStruct);
672 CI->removeFromParent();
673 Builder.Insert(CI);
674
675 // __kmpc_end_serialized_parallel(&Ident, GTid);
676 Value *EndArgs[] = {Ident, ThreadID};
677 Builder.CreateCall(
678 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_serialized_parallel),
679 EndArgs);
680
681 LLVM_DEBUG(dbgs() << "With serialized parallel region: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With serialized parallel region: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
682 << *Builder.GetInsertBlock()->getParent() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With serialized parallel region: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
;
683 }
684
685 for (Instruction *I : ToBeDeleted)
686 I->eraseFromParent();
687 };
688
689 // Adjust the finalization stack, verify the adjustment, and call the
690 // finalize function a last time to finalize values between the pre-fini
691 // block and the exit block if we left the parallel "the normal way".
692 auto FiniInfo = FinalizationStack.pop_back_val();
693 (void)FiniInfo;
694 assert(FiniInfo.DK == OMPD_parallel &&((FiniInfo.DK == OMPD_parallel && "Unexpected finalization stack state!"
) ? static_cast<void> (0) : __assert_fail ("FiniInfo.DK == OMPD_parallel && \"Unexpected finalization stack state!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 695, __PRETTY_FUNCTION__))
17
Assuming 'OMPD_parallel' is equal to field 'DK'
18
'?' condition is true
695 "Unexpected finalization stack state!")((FiniInfo.DK == OMPD_parallel && "Unexpected finalization stack state!"
) ? static_cast<void> (0) : __assert_fail ("FiniInfo.DK == OMPD_parallel && \"Unexpected finalization stack state!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 695, __PRETTY_FUNCTION__))
;
696
697 Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();
698
699 InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator());
700 FiniCB(PreFiniIP);
701
702 OI.EntryBB = PRegEntryBB;
703 OI.ExitBB = PRegExitBB;
704
705 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
706 SmallVector<BasicBlock *, 32> Blocks;
707 OI.collectBlocks(ParallelRegionBlockSet, Blocks);
708
709 // Ensure a single exit node for the outlined region by creating one.
710 // We might have multiple incoming edges to the exit now due to finalizations,
711 // e.g., cancel calls that cause the control flow to leave the region.
712 BasicBlock *PRegOutlinedExitBB = PRegExitBB;
713 PRegExitBB = SplitBlock(PRegExitBB, &*PRegExitBB->getFirstInsertionPt());
714 PRegOutlinedExitBB->setName("omp.par.outlined.exit");
715 Blocks.push_back(PRegOutlinedExitBB);
716
717 CodeExtractorAnalysisCache CEAC(*OuterFn);
718 CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
719 /* AggregateArgs */ false,
720 /* BlockFrequencyInfo */ nullptr,
721 /* BranchProbabilityInfo */ nullptr,
722 /* AssumptionCache */ nullptr,
723 /* AllowVarArgs */ true,
724 /* AllowAlloca */ true,
725 /* Suffix */ ".omp_par");
726
727 // Find inputs to, outputs from the code region.
728 BasicBlock *CommonExit = nullptr;
729 SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
730 Extractor.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
731 Extractor.findInputsOutputs(Inputs, Outputs, SinkingCands);
732
733 LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before privatization: "
<< *OuterFn << "\n"; } } while (false)
;
19
Assuming 'DebugFlag' is false
20
Loop condition is false. Exiting loop
734
735 FunctionCallee TIDRTLFn =
736 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num);
737
738 auto PrivHelper = [&](Value &V) {
739 if (&V == TIDAddr || &V == ZeroAddr)
24
Assuming the condition is false
25
Assuming the condition is false
26
Taking false branch
740 return;
741
742 SetVector<Use *> Uses;
743 for (Use &U : V.uses())
744 if (auto *UserI = dyn_cast<Instruction>(U.getUser()))
745 if (ParallelRegionBlockSet.count(UserI->getParent()))
746 Uses.insert(&U);
747
748 // __kmpc_fork_call expects extra arguments as pointers. If the input
749 // already has a pointer type, everything is fine. Otherwise, store the
750 // value onto stack and load it back inside the to-be-outlined region. This
751 // will ensure only the pointer will be passed to the function.
752 // FIXME: if there are more than 15 trailing arguments, they must be
753 // additionally packed in a struct.
754 Value *Inner = &V;
755 if (!V.getType()->isPointerTy()) {
27
Taking true branch
756 IRBuilder<>::InsertPointGuard Guard(Builder);
757 LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { llvm::dbgs() << "Forwarding input as pointer: "
<< V << "\n"; } } while (false)
;
28
Assuming 'DebugFlag' is false
29
Loop condition is false. Exiting loop
758
759 Builder.restoreIP(OuterAllocaIP);
760 Value *Ptr =
761 Builder.CreateAlloca(V.getType(), nullptr, V.getName() + ".reloaded");
762
763 // Store to stack at end of the block that currently branches to the entry
764 // block of the to-be-outlined region.
765 Builder.SetInsertPoint(InsertBB,
766 InsertBB->getTerminator()->getIterator());
767 Builder.CreateStore(&V, Ptr);
768
769 // Load back next to allocations in the to-be-outlined region.
770 Builder.restoreIP(InnerAllocaIP);
30
Calling 'IRBuilderBase::restoreIP'
35
Returning from 'IRBuilderBase::restoreIP'
771 Inner = Builder.CreateLoad(V.getType(), Ptr);
36
Calling 'IRBuilderBase::CreateLoad'
772 }
773
774 Value *ReplacementValue = nullptr;
775 CallInst *CI = dyn_cast<CallInst>(&V);
776 if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {
777 ReplacementValue = PrivTID;
778 } else {
779 Builder.restoreIP(
780 PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue));
781 assert(ReplacementValue &&((ReplacementValue && "Expected copy/create callback to set replacement value!"
) ? static_cast<void> (0) : __assert_fail ("ReplacementValue && \"Expected copy/create callback to set replacement value!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 782, __PRETTY_FUNCTION__))
782 "Expected copy/create callback to set replacement value!")((ReplacementValue && "Expected copy/create callback to set replacement value!"
) ? static_cast<void> (0) : __assert_fail ("ReplacementValue && \"Expected copy/create callback to set replacement value!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 782, __PRETTY_FUNCTION__))
;
783 if (ReplacementValue == &V)
784 return;
785 }
786
787 for (Use *UPtr : Uses)
788 UPtr->set(ReplacementValue);
789 };
790
791 // Reset the inner alloca insertion as it will be used for loading the values
792 // wrapped into pointers before passing them into the to-be-outlined region.
793 // Configure it to insert immediately after the fake use of zero address so
794 // that they are available in the generated body and so that the
795 // OpenMP-related values (thread ID and zero address pointers) remain leading
796 // in the argument list.
797 InnerAllocaIP = IRBuilder<>::InsertPoint(
798 ZeroAddrUse->getParent(), ZeroAddrUse->getNextNode()->getIterator());
799
800 // Reset the outer alloca insertion point to the entry of the relevant block
801 // in case it was invalidated.
802 OuterAllocaIP = IRBuilder<>::InsertPoint(
803 OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt());
804
805 for (Value *Input : Inputs) {
806 LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Captured input: " <<
*Input << "\n"; } } while (false)
;
21
Assuming 'DebugFlag' is false
22
Loop condition is false. Exiting loop
807 PrivHelper(*Input);
23
Calling 'operator()'
808 }
809 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
810 for (Value *Output : Outputs)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
811 LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
812 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
;
813 assert(Outputs.empty() &&((Outputs.empty() && "OpenMP outlining should not produce live-out values!"
) ? static_cast<void> (0) : __assert_fail ("Outputs.empty() && \"OpenMP outlining should not produce live-out values!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 814, __PRETTY_FUNCTION__))
814 "OpenMP outlining should not produce live-out values!")((Outputs.empty() && "OpenMP outlining should not produce live-out values!"
) ? static_cast<void> (0) : __assert_fail ("Outputs.empty() && \"OpenMP outlining should not produce live-out values!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 814, __PRETTY_FUNCTION__))
;
815
816 LLVM_DEBUG(dbgs() << "After privatization: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After privatization: "
<< *OuterFn << "\n"; } } while (false)
;
817 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
818 for (auto *BB : Blocks)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
819 dbgs() << " PBR: " << BB->getName() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
820 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
;
821
822 // Register the outlined info.
823 addOutlineInfo(std::move(OI));
824
825 InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());
826 UI->eraseFromParent();
827
828 return AfterIP;
829}
830
831void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) {
832 // Build call void __kmpc_flush(ident_t *loc)
833 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
834 Value *Args[] = {getOrCreateIdent(SrcLocStr)};
835
836 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_flush), Args);
837}
838
839void OpenMPIRBuilder::createFlush(const LocationDescription &Loc) {
840 if (!updateToLocation(Loc))
841 return;
842 emitFlush(Loc);
843}
844
845void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) {
846 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
847 // global_tid);
848 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
849 Value *Ident = getOrCreateIdent(SrcLocStr);
850 Value *Args[] = {Ident, getOrCreateThreadID(Ident)};
851
852 // Ignore return result until untied tasks are supported.
853 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskwait),
854 Args);
855}
856
857void OpenMPIRBuilder::createTaskwait(const LocationDescription &Loc) {
858 if (!updateToLocation(Loc))
859 return;
860 emitTaskwaitImpl(Loc);
861}
862
863void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) {
864 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
865 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
866 Value *Ident = getOrCreateIdent(SrcLocStr);
867 Constant *I32Null = ConstantInt::getNullValue(Int32);
868 Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};
869
870 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskyield),
871 Args);
872}
873
874void OpenMPIRBuilder::createTaskyield(const LocationDescription &Loc) {
875 if (!updateToLocation(Loc))
876 return;
877 emitTaskyieldImpl(Loc);
878}
879
880OpenMPIRBuilder::InsertPointTy
881OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
882 BodyGenCallbackTy BodyGenCB,
883 FinalizeCallbackTy FiniCB) {
884
885 if (!updateToLocation(Loc))
886 return Loc.IP;
887
888 Directive OMPD = Directive::OMPD_master;
889 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
890 Value *Ident = getOrCreateIdent(SrcLocStr);
891 Value *ThreadId = getOrCreateThreadID(Ident);
892 Value *Args[] = {Ident, ThreadId};
893
894 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_master);
895 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
896
897 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_master);
898 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
899
900 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
901 /*Conditional*/ true, /*hasFinalize*/ true);
902}
903
904CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
905 DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
906 BasicBlock *PostInsertBefore, const Twine &Name) {
907 Module *M = F->getParent();
908 LLVMContext &Ctx = M->getContext();
909 Type *IndVarTy = TripCount->getType();
910
911 // Create the basic block structure.
912 BasicBlock *Preheader =
913 BasicBlock::Create(Ctx, "omp_" + Name + ".preheader", F, PreInsertBefore);
914 BasicBlock *Header =
915 BasicBlock::Create(Ctx, "omp_" + Name + ".header", F, PreInsertBefore);
916 BasicBlock *Cond =
917 BasicBlock::Create(Ctx, "omp_" + Name + ".cond", F, PreInsertBefore);
918 BasicBlock *Body =
919 BasicBlock::Create(Ctx, "omp_" + Name + ".body", F, PreInsertBefore);
920 BasicBlock *Latch =
921 BasicBlock::Create(Ctx, "omp_" + Name + ".inc", F, PostInsertBefore);
922 BasicBlock *Exit =
923 BasicBlock::Create(Ctx, "omp_" + Name + ".exit", F, PostInsertBefore);
924 BasicBlock *After =
925 BasicBlock::Create(Ctx, "omp_" + Name + ".after", F, PostInsertBefore);
926
927 // Use specified DebugLoc for new instructions.
928 Builder.SetCurrentDebugLocation(DL);
929
930 Builder.SetInsertPoint(Preheader);
931 Builder.CreateBr(Header);
932
933 Builder.SetInsertPoint(Header);
934 PHINode *IndVarPHI = Builder.CreatePHI(IndVarTy, 2, "omp_" + Name + ".iv");
935 IndVarPHI->addIncoming(ConstantInt::get(IndVarTy, 0), Preheader);
936 Builder.CreateBr(Cond);
937
938 Builder.SetInsertPoint(Cond);
939 Value *Cmp =
940 Builder.CreateICmpULT(IndVarPHI, TripCount, "omp_" + Name + ".cmp");
941 Builder.CreateCondBr(Cmp, Body, Exit);
942
943 Builder.SetInsertPoint(Body);
944 Builder.CreateBr(Latch);
945
946 Builder.SetInsertPoint(Latch);
947 Value *Next = Builder.CreateAdd(IndVarPHI, ConstantInt::get(IndVarTy, 1),
948 "omp_" + Name + ".next", /*HasNUW=*/true);
949 Builder.CreateBr(Header);
950 IndVarPHI->addIncoming(Next, Latch);
951
952 Builder.SetInsertPoint(Exit);
953 Builder.CreateBr(After);
954
955 // Remember and return the canonical control flow.
956 LoopInfos.emplace_front();
957 CanonicalLoopInfo *CL = &LoopInfos.front();
958
959 CL->Preheader = Preheader;
960 CL->Header = Header;
961 CL->Cond = Cond;
962 CL->Body = Body;
963 CL->Latch = Latch;
964 CL->Exit = Exit;
965 CL->After = After;
966
967 CL->IsValid = true;
968
969#ifndef NDEBUG
970 CL->assertOK();
971#endif
972 return CL;
973}
974
975CanonicalLoopInfo *
976OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
977 LoopBodyGenCallbackTy BodyGenCB,
978 Value *TripCount, const Twine &Name) {
979 BasicBlock *BB = Loc.IP.getBlock();
980 BasicBlock *NextBB = BB->getNextNode();
981
982 CanonicalLoopInfo *CL = createLoopSkeleton(Loc.DL, TripCount, BB->getParent(),
983 NextBB, NextBB, Name);
984 BasicBlock *After = CL->getAfter();
985
986 // If location is not set, don't connect the loop.
987 if (updateToLocation(Loc)) {
988 // Split the loop at the insertion point: Branch to the preheader and move
989 // every following instruction to after the loop (the After BB). Also, the
990 // new successor is the loop's after block.
991 Builder.CreateBr(CL->Preheader);
992 After->getInstList().splice(After->begin(), BB->getInstList(),
993 Builder.GetInsertPoint(), BB->end());
994 After->replaceSuccessorsPhiUsesWith(BB, After);
995 }
996
997 // Emit the body content. We do it after connecting the loop to the CFG to
998 // avoid that the callback encounters degenerate BBs.
999 BodyGenCB(CL->getBodyIP(), CL->getIndVar());
1000
1001#ifndef NDEBUG
1002 CL->assertOK();
1003#endif
1004 return CL;
1005}
1006
1007CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop(
1008 const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
1009 Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
1010 InsertPointTy ComputeIP, const Twine &Name) {
1011
1012 // Consider the following difficulties (assuming 8-bit signed integers):
1013 // * Adding \p Step to the loop counter which passes \p Stop may overflow:
1014 // DO I = 1, 100, 50
1015 /// * A \p Step of INT_MIN cannot not be normalized to a positive direction:
1016 // DO I = 100, 0, -128
1017
1018 // Start, Stop and Step must be of the same integer type.
1019 auto *IndVarTy = cast<IntegerType>(Start->getType());
1020 assert(IndVarTy == Stop->getType() && "Stop type mismatch")((IndVarTy == Stop->getType() && "Stop type mismatch"
) ? static_cast<void> (0) : __assert_fail ("IndVarTy == Stop->getType() && \"Stop type mismatch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1020, __PRETTY_FUNCTION__))
;
1021 assert(IndVarTy == Step->getType() && "Step type mismatch")((IndVarTy == Step->getType() && "Step type mismatch"
) ? static_cast<void> (0) : __assert_fail ("IndVarTy == Step->getType() && \"Step type mismatch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1021, __PRETTY_FUNCTION__))
;
1022
1023 LocationDescription ComputeLoc =
1024 ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
1025 updateToLocation(ComputeLoc);
1026
1027 ConstantInt *Zero = ConstantInt::get(IndVarTy, 0);
1028 ConstantInt *One = ConstantInt::get(IndVarTy, 1);
1029
1030 // Like Step, but always positive.
1031 Value *Incr = Step;
1032
1033 // Distance between Start and Stop; always positive.
1034 Value *Span;
1035
1036 // Condition whether there are no iterations are executed at all, e.g. because
1037 // UB < LB.
1038 Value *ZeroCmp;
1039
1040 if (IsSigned) {
1041 // Ensure that increment is positive. If not, negate and invert LB and UB.
1042 Value *IsNeg = Builder.CreateICmpSLT(Step, Zero);
1043 Incr = Builder.CreateSelect(IsNeg, Builder.CreateNeg(Step), Step);
1044 Value *LB = Builder.CreateSelect(IsNeg, Stop, Start);
1045 Value *UB = Builder.CreateSelect(IsNeg, Start, Stop);
1046 Span = Builder.CreateSub(UB, LB, "", false, true);
1047 ZeroCmp = Builder.CreateICmp(
1048 InclusiveStop ? CmpInst::ICMP_SLT : CmpInst::ICMP_SLE, UB, LB);
1049 } else {
1050 Span = Builder.CreateSub(Stop, Start, "", true);
1051 ZeroCmp = Builder.CreateICmp(
1052 InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Stop, Start);
1053 }
1054
1055 Value *CountIfLooping;
1056 if (InclusiveStop) {
1057 CountIfLooping = Builder.CreateAdd(Builder.CreateUDiv(Span, Incr), One);
1058 } else {
1059 // Avoid incrementing past stop since it could overflow.
1060 Value *CountIfTwo = Builder.CreateAdd(
1061 Builder.CreateUDiv(Builder.CreateSub(Span, One), Incr), One);
1062 Value *OneCmp = Builder.CreateICmp(
1063 InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Span, Incr);
1064 CountIfLooping = Builder.CreateSelect(OneCmp, One, CountIfTwo);
1065 }
1066 Value *TripCount = Builder.CreateSelect(ZeroCmp, Zero, CountIfLooping,
1067 "omp_" + Name + ".tripcount");
1068
1069 auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
1070 Builder.restoreIP(CodeGenIP);
1071 Value *Span = Builder.CreateMul(IV, Step);
1072 Value *IndVar = Builder.CreateAdd(Span, Start);
1073 BodyGenCB(Builder.saveIP(), IndVar);
1074 };
1075 LocationDescription LoopLoc = ComputeIP.isSet() ? Loc.IP : Builder.saveIP();
1076 return createCanonicalLoop(LoopLoc, BodyGen, TripCount, Name);
1077}
1078
1079// Returns an LLVM function to call for initializing loop bounds using OpenMP
1080// static scheduling depending on `type`. Only i32 and i64 are supported by the
1081// runtime. Always interpret integers as unsigned similarly to
1082// CanonicalLoopInfo.
1083static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,
1084 OpenMPIRBuilder &OMPBuilder) {
1085 unsigned Bitwidth = Ty->getIntegerBitWidth();
1086 if (Bitwidth == 32)
1087 return OMPBuilder.getOrCreateRuntimeFunction(
1088 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
1089 if (Bitwidth == 64)
1090 return OMPBuilder.getOrCreateRuntimeFunction(
1091 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
1092 llvm_unreachable("unknown OpenMP loop iterator bitwidth")::llvm::llvm_unreachable_internal("unknown OpenMP loop iterator bitwidth"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1092)
;
1093}
1094
1095// Sets the number of loop iterations to the given value. This value must be
1096// valid in the condition block (i.e., defined in the preheader) and is
1097// interpreted as an unsigned integer.
1098void setCanonicalLoopTripCount(CanonicalLoopInfo *CLI, Value *TripCount) {
1099 Instruction *CmpI = &CLI->getCond()->front();
1100 assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount")((isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount"
) ? static_cast<void> (0) : __assert_fail ("isa<CmpInst>(CmpI) && \"First inst must compare IV with TripCount\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1100, __PRETTY_FUNCTION__))
;
1101 CmpI->setOperand(1, TripCount);
1102 CLI->assertOK();
1103}
1104
1105CanonicalLoopInfo *OpenMPIRBuilder::createStaticWorkshareLoop(
1106 const LocationDescription &Loc, CanonicalLoopInfo *CLI,
1107 InsertPointTy AllocaIP, bool NeedsBarrier, Value *Chunk) {
1108 // Set up the source location value for OpenMP runtime.
1109 if (!updateToLocation(Loc))
1110 return nullptr;
1111
1112 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1113 Value *SrcLoc = getOrCreateIdent(SrcLocStr);
1114
1115 // Declare useful OpenMP runtime functions.
1116 Value *IV = CLI->getIndVar();
1117 Type *IVTy = IV->getType();
1118 FunctionCallee StaticInit = getKmpcForStaticInitForType(IVTy, M, *this);
1119 FunctionCallee StaticFini =
1120 getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_for_static_fini);
1121
1122 // Allocate space for computed loop bounds as expected by the "init" function.
1123 Builder.restoreIP(AllocaIP);
1124 Type *I32Type = Type::getInt32Ty(M.getContext());
1125 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
1126 Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");
1127 Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");
1128 Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");
1129
1130 // At the end of the preheader, prepare for calling the "init" function by
1131 // storing the current loop bounds into the allocated space. A canonical loop
1132 // always iterates from 0 to trip-count with step 1. Note that "init" expects
1133 // and produces an inclusive upper bound.
1134 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
1135 Constant *Zero = ConstantInt::get(IVTy, 0);
1136 Constant *One = ConstantInt::get(IVTy, 1);
1137 Builder.CreateStore(Zero, PLowerBound);
1138 Value *UpperBound = Builder.CreateSub(CLI->getTripCount(), One);
1139 Builder.CreateStore(UpperBound, PUpperBound);
1140 Builder.CreateStore(One, PStride);
1141
1142 if (!Chunk)
1143 Chunk = One;
1144
1145 Value *ThreadNum = getOrCreateThreadID(SrcLoc);
1146
1147 // TODO: extract scheduling type and map it to OMP constant. This is curently
1148 // happening in kmp.h and its ilk and needs to be moved to OpenMP.td first.
1149 constexpr int StaticSchedType = 34;
1150 Constant *SchedulingType = ConstantInt::get(I32Type, StaticSchedType);
1151
1152 // Call the "init" function and update the trip count of the loop with the
1153 // value it produced.
1154 Builder.CreateCall(StaticInit,
1155 {SrcLoc, ThreadNum, SchedulingType, PLastIter, PLowerBound,
1156 PUpperBound, PStride, One, Chunk});
1157 Value *LowerBound = Builder.CreateLoad(IVTy, PLowerBound);
1158 Value *InclusiveUpperBound = Builder.CreateLoad(IVTy, PUpperBound);
1159 Value *TripCountMinusOne = Builder.CreateSub(InclusiveUpperBound, LowerBound);
1160 Value *TripCount = Builder.CreateAdd(TripCountMinusOne, One);
1161 setCanonicalLoopTripCount(CLI, TripCount);
1162
1163 // Update all uses of the induction variable except the one in the condition
1164 // block that compares it with the actual upper bound, and the increment in
1165 // the latch block.
1166 // TODO: this can eventually move to CanonicalLoopInfo or to a new
1167 // CanonicalLoopInfoUpdater interface.
1168 Builder.SetInsertPoint(CLI->getBody(), CLI->getBody()->getFirstInsertionPt());
1169 Value *UpdatedIV = Builder.CreateAdd(IV, LowerBound);
1170 IV->replaceUsesWithIf(UpdatedIV, [&](Use &U) {
1171 auto *Instr = dyn_cast<Instruction>(U.getUser());
1172 return !Instr ||
1173 (Instr->getParent() != CLI->getCond() &&
1174 Instr->getParent() != CLI->getLatch() && Instr != UpdatedIV);
1175 });
1176
1177 // In the "exit" block, call the "fini" function.
1178 Builder.SetInsertPoint(CLI->getExit(),
1179 CLI->getExit()->getTerminator()->getIterator());
1180 Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});
1181
1182 // Add the barrier if requested.
1183 if (NeedsBarrier)
1184 createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
1185 omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
1186 /* CheckCancelFlag */ false);
1187
1188 CLI->assertOK();
1189 return CLI;
1190}
1191
1192CanonicalLoopInfo *OpenMPIRBuilder::createWorkshareLoop(
1193 const LocationDescription &Loc, CanonicalLoopInfo *CLI,
1194 InsertPointTy AllocaIP, bool NeedsBarrier) {
1195 // Currently only supports static schedules.
1196 return createStaticWorkshareLoop(Loc, CLI, AllocaIP, NeedsBarrier);
1197}
1198
1199/// Make \p Source branch to \p Target.
1200///
1201/// Handles two situations:
1202/// * \p Source already has an unconditional branch.
1203/// * \p Source is a degenerate block (no terminator because the BB is
1204/// the current head of the IR construction).
1205static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL) {
1206 if (Instruction *Term = Source->getTerminator()) {
1207 auto *Br = cast<BranchInst>(Term);
1208 assert(!Br->isConditional() &&((!Br->isConditional() && "BB's terminator must be an unconditional branch (or degenerate)"
) ? static_cast<void> (0) : __assert_fail ("!Br->isConditional() && \"BB's terminator must be an unconditional branch (or degenerate)\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1209, __PRETTY_FUNCTION__))
1209 "BB's terminator must be an unconditional branch (or degenerate)")((!Br->isConditional() && "BB's terminator must be an unconditional branch (or degenerate)"
) ? static_cast<void> (0) : __assert_fail ("!Br->isConditional() && \"BB's terminator must be an unconditional branch (or degenerate)\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1209, __PRETTY_FUNCTION__))
;
1210 BasicBlock *Succ = Br->getSuccessor(0);
1211 Succ->removePredecessor(Source, /*KeepOneInputPHIs=*/true);
1212 Br->setSuccessor(0, Target);
1213 return;
1214 }
1215
1216 auto *NewBr = BranchInst::Create(Target, Source);
1217 NewBr->setDebugLoc(DL);
1218}
1219
1220/// Redirect all edges that branch to \p OldTarget to \p NewTarget. That is,
1221/// after this \p OldTarget will be orphaned.
1222static void redirectAllPredecessorsTo(BasicBlock *OldTarget,
1223 BasicBlock *NewTarget, DebugLoc DL) {
1224 for (BasicBlock *Pred : make_early_inc_range(predecessors(OldTarget)))
1225 redirectTo(Pred, NewTarget, DL);
1226}
1227
1228/// Determine which blocks in \p BBs are reachable from outside and remove the
1229/// ones that are not reachable from the function.
1230static void removeUnusedBlocksFromParent(ArrayRef<BasicBlock *> BBs) {
1231 SmallPtrSet<BasicBlock *, 6> BBsToErase{BBs.begin(), BBs.end()};
1232 auto HasRemainingUses = [&BBsToErase](BasicBlock *BB) {
1233 for (Use &U : BB->uses()) {
1234 auto *UseInst = dyn_cast<Instruction>(U.getUser());
1235 if (!UseInst)
1236 continue;
1237 if (BBsToErase.count(UseInst->getParent()))
1238 continue;
1239 return true;
1240 }
1241 return false;
1242 };
1243
1244 while (true) {
1245 bool Changed = false;
1246 for (BasicBlock *BB : make_early_inc_range(BBsToErase)) {
1247 if (HasRemainingUses(BB)) {
1248 BBsToErase.erase(BB);
1249 Changed = true;
1250 }
1251 }
1252 if (!Changed)
1253 break;
1254 }
1255
1256 SmallVector<BasicBlock *, 7> BBVec(BBsToErase.begin(), BBsToErase.end());
1257 DeleteDeadBlocks(BBVec);
1258}
1259
1260CanonicalLoopInfo *
1261OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
1262 InsertPointTy ComputeIP) {
1263 assert(Loops.size() >= 1 && "At least one loop required")((Loops.size() >= 1 && "At least one loop required"
) ? static_cast<void> (0) : __assert_fail ("Loops.size() >= 1 && \"At least one loop required\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1263, __PRETTY_FUNCTION__))
;
1264 size_t NumLoops = Loops.size();
1265
1266 // Nothing to do if there is already just one loop.
1267 if (NumLoops == 1)
1268 return Loops.front();
1269
1270 CanonicalLoopInfo *Outermost = Loops.front();
1271 CanonicalLoopInfo *Innermost = Loops.back();
1272 BasicBlock *OrigPreheader = Outermost->getPreheader();
1273 BasicBlock *OrigAfter = Outermost->getAfter();
1274 Function *F = OrigPreheader->getParent();
1275
1276 // Setup the IRBuilder for inserting the trip count computation.
1277 Builder.SetCurrentDebugLocation(DL);
1278 if (ComputeIP.isSet())
1279 Builder.restoreIP(ComputeIP);
1280 else
1281 Builder.restoreIP(Outermost->getPreheaderIP());
1282
1283 // Derive the collapsed' loop trip count.
1284 // TODO: Find common/largest indvar type.
1285 Value *CollapsedTripCount = nullptr;
1286 for (CanonicalLoopInfo *L : Loops) {
1287 Value *OrigTripCount = L->getTripCount();
1288 if (!CollapsedTripCount) {
1289 CollapsedTripCount = OrigTripCount;
1290 continue;
1291 }
1292
1293 // TODO: Enable UndefinedSanitizer to diagnose an overflow here.
1294 CollapsedTripCount = Builder.CreateMul(CollapsedTripCount, OrigTripCount,
1295 {}, /*HasNUW=*/true);
1296 }
1297
1298 // Create the collapsed loop control flow.
1299 CanonicalLoopInfo *Result =
1300 createLoopSkeleton(DL, CollapsedTripCount, F,
1301 OrigPreheader->getNextNode(), OrigAfter, "collapsed");
1302
1303 // Build the collapsed loop body code.
1304 // Start with deriving the input loop induction variables from the collapsed
1305 // one, using a divmod scheme. To preserve the original loops' order, the
1306 // innermost loop use the least significant bits.
1307 Builder.restoreIP(Result->getBodyIP());
1308
1309 Value *Leftover = Result->getIndVar();
1310 SmallVector<Value *> NewIndVars;
1311 NewIndVars.set_size(NumLoops);
1312 for (int i = NumLoops - 1; i >= 1; --i) {
1313 Value *OrigTripCount = Loops[i]->getTripCount();
1314
1315 Value *NewIndVar = Builder.CreateURem(Leftover, OrigTripCount);
1316 NewIndVars[i] = NewIndVar;
1317
1318 Leftover = Builder.CreateUDiv(Leftover, OrigTripCount);
1319 }
1320 // Outermost loop gets all the remaining bits.
1321 NewIndVars[0] = Leftover;
1322
1323 // Construct the loop body control flow.
1324 // We progressively construct the branch structure following in direction of
1325 // the control flow, from the leading in-between code, the loop nest body, the
1326 // trailing in-between code, and rejoining the collapsed loop's latch.
1327 // ContinueBlock and ContinuePred keep track of the source(s) of next edge. If
1328 // the ContinueBlock is set, continue with that block. If ContinuePred, use
1329 // its predecessors as sources.
1330 BasicBlock *ContinueBlock = Result->getBody();
1331 BasicBlock *ContinuePred = nullptr;
1332 auto ContinueWith = [&ContinueBlock, &ContinuePred, DL](BasicBlock *Dest,
1333 BasicBlock *NextSrc) {
1334 if (ContinueBlock)
1335 redirectTo(ContinueBlock, Dest, DL);
1336 else
1337 redirectAllPredecessorsTo(ContinuePred, Dest, DL);
1338
1339 ContinueBlock = nullptr;
1340 ContinuePred = NextSrc;
1341 };
1342
1343 // The code before the nested loop of each level.
1344 // Because we are sinking it into the nest, it will be executed more often
1345 // that the original loop. More sophisticated schemes could keep track of what
1346 // the in-between code is and instantiate it only once per thread.
1347 for (size_t i = 0; i < NumLoops - 1; ++i)
1348 ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader());
1349
1350 // Connect the loop nest body.
1351 ContinueWith(Innermost->getBody(), Innermost->getLatch());
1352
1353 // The code after the nested loop at each level.
1354 for (size_t i = NumLoops - 1; i > 0; --i)
1355 ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch());
1356
1357 // Connect the finished loop to the collapsed loop latch.
1358 ContinueWith(Result->getLatch(), nullptr);
1359
1360 // Replace the input loops with the new collapsed loop.
1361 redirectTo(Outermost->getPreheader(), Result->getPreheader(), DL);
1362 redirectTo(Result->getAfter(), Outermost->getAfter(), DL);
1363
1364 // Replace the input loop indvars with the derived ones.
1365 for (size_t i = 0; i < NumLoops; ++i)
1366 Loops[i]->getIndVar()->replaceAllUsesWith(NewIndVars[i]);
1367
1368 // Remove unused parts of the input loops.
1369 SmallVector<BasicBlock *, 12> OldControlBBs;
1370 OldControlBBs.reserve(6 * Loops.size());
1371 for (CanonicalLoopInfo *Loop : Loops)
1372 Loop->collectControlBlocks(OldControlBBs);
1373 removeUnusedBlocksFromParent(OldControlBBs);
1374
1375#ifndef NDEBUG
1376 Result->assertOK();
1377#endif
1378 return Result;
1379}
1380
1381std::vector<CanonicalLoopInfo *>
1382OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
1383 ArrayRef<Value *> TileSizes) {
1384 assert(TileSizes.size() == Loops.size() &&((TileSizes.size() == Loops.size() && "Must pass as many tile sizes as there are loops"
) ? static_cast<void> (0) : __assert_fail ("TileSizes.size() == Loops.size() && \"Must pass as many tile sizes as there are loops\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1385, __PRETTY_FUNCTION__))
1385 "Must pass as many tile sizes as there are loops")((TileSizes.size() == Loops.size() && "Must pass as many tile sizes as there are loops"
) ? static_cast<void> (0) : __assert_fail ("TileSizes.size() == Loops.size() && \"Must pass as many tile sizes as there are loops\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1385, __PRETTY_FUNCTION__))
;
1386 int NumLoops = Loops.size();
1387 assert(NumLoops >= 1 && "At least one loop to tile required")((NumLoops >= 1 && "At least one loop to tile required"
) ? static_cast<void> (0) : __assert_fail ("NumLoops >= 1 && \"At least one loop to tile required\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1387, __PRETTY_FUNCTION__))
;
1388
1389 CanonicalLoopInfo *OutermostLoop = Loops.front();
1390 CanonicalLoopInfo *InnermostLoop = Loops.back();
1391 Function *F = OutermostLoop->getBody()->getParent();
1392 BasicBlock *InnerEnter = InnermostLoop->getBody();
1393 BasicBlock *InnerLatch = InnermostLoop->getLatch();
1394
1395 // Collect original trip counts and induction variable to be accessible by
1396 // index. Also, the structure of the original loops is not preserved during
1397 // the construction of the tiled loops, so do it before we scavenge the BBs of
1398 // any original CanonicalLoopInfo.
1399 SmallVector<Value *, 4> OrigTripCounts, OrigIndVars;
1400 for (CanonicalLoopInfo *L : Loops) {
1401 OrigTripCounts.push_back(L->getTripCount());
1402 OrigIndVars.push_back(L->getIndVar());
1403 }
1404
1405 // Collect the code between loop headers. These may contain SSA definitions
1406 // that are used in the loop nest body. To be usable with in the innermost
1407 // body, these BasicBlocks will be sunk into the loop nest body. That is,
1408 // these instructions may be executed more often than before the tiling.
1409 // TODO: It would be sufficient to only sink them into body of the
1410 // corresponding tile loop.
1411 SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> InbetweenCode;
1412 for (int i = 0; i < NumLoops - 1; ++i) {
1413 CanonicalLoopInfo *Surrounding = Loops[i];
1414 CanonicalLoopInfo *Nested = Loops[i + 1];
1415
1416 BasicBlock *EnterBB = Surrounding->getBody();
1417 BasicBlock *ExitBB = Nested->getHeader();
1418 InbetweenCode.emplace_back(EnterBB, ExitBB);
1419 }
1420
1421 // Compute the trip counts of the floor loops.
1422 Builder.SetCurrentDebugLocation(DL);
1423 Builder.restoreIP(OutermostLoop->getPreheaderIP());
1424 SmallVector<Value *, 4> FloorCount, FloorRems;
1425 for (int i = 0; i < NumLoops; ++i) {
1426 Value *TileSize = TileSizes[i];
1427 Value *OrigTripCount = OrigTripCounts[i];
1428 Type *IVType = OrigTripCount->getType();
1429
1430 Value *FloorTripCount = Builder.CreateUDiv(OrigTripCount, TileSize);
1431 Value *FloorTripRem = Builder.CreateURem(OrigTripCount, TileSize);
1432
1433 // 0 if tripcount divides the tilesize, 1 otherwise.
1434 // 1 means we need an additional iteration for a partial tile.
1435 //
1436 // Unfortunately we cannot just use the roundup-formula
1437 // (tripcount + tilesize - 1)/tilesize
1438 // because the summation might overflow. We do not want introduce undefined
1439 // behavior when the untiled loop nest did not.
1440 Value *FloorTripOverflow =
1441 Builder.CreateICmpNE(FloorTripRem, ConstantInt::get(IVType, 0));
1442
1443 FloorTripOverflow = Builder.CreateZExt(FloorTripOverflow, IVType);
1444 FloorTripCount =
1445 Builder.CreateAdd(FloorTripCount, FloorTripOverflow,
1446 "omp_floor" + Twine(i) + ".tripcount", true);
1447
1448 // Remember some values for later use.
1449 FloorCount.push_back(FloorTripCount);
1450 FloorRems.push_back(FloorTripRem);
1451 }
1452
1453 // Generate the new loop nest, from the outermost to the innermost.
1454 std::vector<CanonicalLoopInfo *> Result;
1455 Result.reserve(NumLoops * 2);
1456
1457 // The basic block of the surrounding loop that enters the nest generated
1458 // loop.
1459 BasicBlock *Enter = OutermostLoop->getPreheader();
1460
1461 // The basic block of the surrounding loop where the inner code should
1462 // continue.
1463 BasicBlock *Continue = OutermostLoop->getAfter();
1464
1465 // Where the next loop basic block should be inserted.
1466 BasicBlock *OutroInsertBefore = InnermostLoop->getExit();
1467
1468 auto EmbeddNewLoop =
1469 [this, DL, F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](
1470 Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * {
1471 CanonicalLoopInfo *EmbeddedLoop = createLoopSkeleton(
1472 DL, TripCount, F, InnerEnter, OutroInsertBefore, Name);
1473 redirectTo(Enter, EmbeddedLoop->getPreheader(), DL);
1474 redirectTo(EmbeddedLoop->getAfter(), Continue, DL);
1475
1476 // Setup the position where the next embedded loop connects to this loop.
1477 Enter = EmbeddedLoop->getBody();
1478 Continue = EmbeddedLoop->getLatch();
1479 OutroInsertBefore = EmbeddedLoop->getLatch();
1480 return EmbeddedLoop;
1481 };
1482
1483 auto EmbeddNewLoops = [&Result, &EmbeddNewLoop](ArrayRef<Value *> TripCounts,
1484 const Twine &NameBase) {
1485 for (auto P : enumerate(TripCounts)) {
1486 CanonicalLoopInfo *EmbeddedLoop =
1487 EmbeddNewLoop(P.value(), NameBase + Twine(P.index()));
1488 Result.push_back(EmbeddedLoop);
1489 }
1490 };
1491
1492 EmbeddNewLoops(FloorCount, "floor");
1493
1494 // Within the innermost floor loop, emit the code that computes the tile
1495 // sizes.
1496 Builder.SetInsertPoint(Enter->getTerminator());
1497 SmallVector<Value *, 4> TileCounts;
1498 for (int i = 0; i < NumLoops; ++i) {
1499 CanonicalLoopInfo *FloorLoop = Result[i];
1500 Value *TileSize = TileSizes[i];
1501
1502 Value *FloorIsEpilogue =
1503 Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCount[i]);
1504 Value *TileTripCount =
1505 Builder.CreateSelect(FloorIsEpilogue, FloorRems[i], TileSize);
1506
1507 TileCounts.push_back(TileTripCount);
1508 }
1509
1510 // Create the tile loops.
1511 EmbeddNewLoops(TileCounts, "tile");
1512
1513 // Insert the inbetween code into the body.
1514 BasicBlock *BodyEnter = Enter;
1515 BasicBlock *BodyEntered = nullptr;
1516 for (std::pair<BasicBlock *, BasicBlock *> P : InbetweenCode) {
1517 BasicBlock *EnterBB = P.first;
1518 BasicBlock *ExitBB = P.second;
1519
1520 if (BodyEnter)
1521 redirectTo(BodyEnter, EnterBB, DL);
1522 else
1523 redirectAllPredecessorsTo(BodyEntered, EnterBB, DL);
1524
1525 BodyEnter = nullptr;
1526 BodyEntered = ExitBB;
1527 }
1528
1529 // Append the original loop nest body into the generated loop nest body.
1530 if (BodyEnter)
1531 redirectTo(BodyEnter, InnerEnter, DL);
1532 else
1533 redirectAllPredecessorsTo(BodyEntered, InnerEnter, DL);
1534 redirectAllPredecessorsTo(InnerLatch, Continue, DL);
1535
1536 // Replace the original induction variable with an induction variable computed
1537 // from the tile and floor induction variables.
1538 Builder.restoreIP(Result.back()->getBodyIP());
1539 for (int i = 0; i < NumLoops; ++i) {
1540 CanonicalLoopInfo *FloorLoop = Result[i];
1541 CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
1542 Value *OrigIndVar = OrigIndVars[i];
1543 Value *Size = TileSizes[i];
1544
1545 Value *Scale =
1546 Builder.CreateMul(Size, FloorLoop->getIndVar(), {}, /*HasNUW=*/true);
1547 Value *Shift =
1548 Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true);
1549 OrigIndVar->replaceAllUsesWith(Shift);
1550 }
1551
1552 // Remove unused parts of the original loops.
1553 SmallVector<BasicBlock *, 12> OldControlBBs;
1554 OldControlBBs.reserve(6 * Loops.size());
1555 for (CanonicalLoopInfo *Loop : Loops)
1556 Loop->collectControlBlocks(OldControlBBs);
1557 removeUnusedBlocksFromParent(OldControlBBs);
1558
1559#ifndef NDEBUG
1560 for (CanonicalLoopInfo *GenL : Result)
1561 GenL->assertOK();
1562#endif
1563 return Result;
1564}
1565
1566OpenMPIRBuilder::InsertPointTy
1567OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc,
1568 llvm::Value *BufSize, llvm::Value *CpyBuf,
1569 llvm::Value *CpyFn, llvm::Value *DidIt) {
1570 if (!updateToLocation(Loc))
1571 return Loc.IP;
1572
1573 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1574 Value *Ident = getOrCreateIdent(SrcLocStr);
1575 Value *ThreadId = getOrCreateThreadID(Ident);
1576
1577 llvm::Value *DidItLD = Builder.CreateLoad(Builder.getInt32Ty(), DidIt);
1578
1579 Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};
1580
1581 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_copyprivate);
1582 Builder.CreateCall(Fn, Args);
1583
1584 return Builder.saveIP();
1585}
1586
1587OpenMPIRBuilder::InsertPointTy
1588OpenMPIRBuilder::createSingle(const LocationDescription &Loc,
1589 BodyGenCallbackTy BodyGenCB,
1590 FinalizeCallbackTy FiniCB, llvm::Value *DidIt) {
1591
1592 if (!updateToLocation(Loc))
1593 return Loc.IP;
1594
1595 // If needed (i.e. not null), initialize `DidIt` with 0
1596 if (DidIt) {
1597 Builder.CreateStore(Builder.getInt32(0), DidIt);
1598 }
1599
1600 Directive OMPD = Directive::OMPD_single;
1601 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1602 Value *Ident = getOrCreateIdent(SrcLocStr);
1603 Value *ThreadId = getOrCreateThreadID(Ident);
1604 Value *Args[] = {Ident, ThreadId};
1605
1606 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_single);
1607 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
1608
1609 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_single);
1610 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
1611
1612 // generates the following:
1613 // if (__kmpc_single()) {
1614 // .... single region ...
1615 // __kmpc_end_single
1616 // }
1617
1618 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
1619 /*Conditional*/ true, /*hasFinalize*/ true);
1620}
1621
1622OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCritical(
1623 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
1624 FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {
1625
1626 if (!updateToLocation(Loc))
1627 return Loc.IP;
1628
1629 Directive OMPD = Directive::OMPD_critical;
1630 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1631 Value *Ident = getOrCreateIdent(SrcLocStr);
1632 Value *ThreadId = getOrCreateThreadID(Ident);
1633 Value *LockVar = getOMPCriticalRegionLock(CriticalName);
1634 Value *Args[] = {Ident, ThreadId, LockVar};
1635
1636 SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args), std::end(Args));
1637 Function *RTFn = nullptr;
1638 if (HintInst) {
1639 // Add Hint to entry Args and create call
1640 EnterArgs.push_back(HintInst);
1641 RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical_with_hint);
1642 } else {
1643 RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical);
1644 }
1645 Instruction *EntryCall = Builder.CreateCall(RTFn, EnterArgs);
1646
1647 Function *ExitRTLFn =
1648 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_critical);
1649 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
1650
1651 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
1652 /*Conditional*/ false, /*hasFinalize*/ true);
1653}
1654
1655OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::EmitOMPInlinedRegion(
1656 Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,
1657 BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,
1658 bool HasFinalize) {
1659
1660 if (HasFinalize)
1661 FinalizationStack.push_back({FiniCB, OMPD, /*IsCancellable*/ false});
1662
1663 // Create inlined region's entry and body blocks, in preparation
1664 // for conditional creation
1665 BasicBlock *EntryBB = Builder.GetInsertBlock();
1666 Instruction *SplitPos = EntryBB->getTerminator();
1667 if (!isa_and_nonnull<BranchInst>(SplitPos))
1668 SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);
1669 BasicBlock *ExitBB = EntryBB->splitBasicBlock(SplitPos, "omp_region.end");
1670 BasicBlock *FiniBB =
1671 EntryBB->splitBasicBlock(EntryBB->getTerminator(), "omp_region.finalize");
1672
1673 Builder.SetInsertPoint(EntryBB->getTerminator());
1674 emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
1675
1676 // generate body
1677 BodyGenCB(/* AllocaIP */ InsertPointTy(),
1678 /* CodeGenIP */ Builder.saveIP(), *FiniBB);
1679
1680 // If we didn't emit a branch to FiniBB during body generation, it means
1681 // FiniBB is unreachable (e.g. while(1);). stop generating all the
1682 // unreachable blocks, and remove anything we are not going to use.
1683 auto SkipEmittingRegion = FiniBB->hasNPredecessors(0);
1684 if (SkipEmittingRegion) {
1685 FiniBB->eraseFromParent();
1686 ExitCall->eraseFromParent();
1687 // Discard finalization if we have it.
1688 if (HasFinalize) {
1689 assert(!FinalizationStack.empty() &&((!FinalizationStack.empty() && "Unexpected finalization stack state!"
) ? static_cast<void> (0) : __assert_fail ("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1690, __PRETTY_FUNCTION__))
1690 "Unexpected finalization stack state!")((!FinalizationStack.empty() && "Unexpected finalization stack state!"
) ? static_cast<void> (0) : __assert_fail ("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1690, __PRETTY_FUNCTION__))
;
1691 FinalizationStack.pop_back();
1692 }
1693 } else {
1694 // emit exit call and do any needed finalization.
1695 auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt());
1696 assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&((FiniBB->getTerminator()->getNumSuccessors() == 1 &&
FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
"Unexpected control flow graph state!!") ? static_cast<void
> (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1698, __PRETTY_FUNCTION__))
1697 FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&((FiniBB->getTerminator()->getNumSuccessors() == 1 &&
FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
"Unexpected control flow graph state!!") ? static_cast<void
> (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1698, __PRETTY_FUNCTION__))
1698 "Unexpected control flow graph state!!")((FiniBB->getTerminator()->getNumSuccessors() == 1 &&
FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
"Unexpected control flow graph state!!") ? static_cast<void
> (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1698, __PRETTY_FUNCTION__))
;
1699 emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
1700 assert(FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB &&((FiniBB->getUniquePredecessor()->getUniqueSuccessor() ==
FiniBB && "Unexpected Control Flow State!") ? static_cast
<void> (0) : __assert_fail ("FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB && \"Unexpected Control Flow State!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1701, __PRETTY_FUNCTION__))
1701 "Unexpected Control Flow State!")((FiniBB->getUniquePredecessor()->getUniqueSuccessor() ==
FiniBB && "Unexpected Control Flow State!") ? static_cast
<void> (0) : __assert_fail ("FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB && \"Unexpected Control Flow State!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1701, __PRETTY_FUNCTION__))
;
1702 MergeBlockIntoPredecessor(FiniBB);
1703 }
1704
1705 // If we are skipping the region of a non conditional, remove the exit
1706 // block, and clear the builder's insertion point.
1707 assert(SplitPos->getParent() == ExitBB &&((SplitPos->getParent() == ExitBB && "Unexpected Insertion point location!"
) ? static_cast<void> (0) : __assert_fail ("SplitPos->getParent() == ExitBB && \"Unexpected Insertion point location!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1708, __PRETTY_FUNCTION__))
1708 "Unexpected Insertion point location!")((SplitPos->getParent() == ExitBB && "Unexpected Insertion point location!"
) ? static_cast<void> (0) : __assert_fail ("SplitPos->getParent() == ExitBB && \"Unexpected Insertion point location!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1708, __PRETTY_FUNCTION__))
;
1709 if (!Conditional && SkipEmittingRegion) {
1710 ExitBB->eraseFromParent();
1711 Builder.ClearInsertionPoint();
1712 } else {
1713 auto merged = MergeBlockIntoPredecessor(ExitBB);
1714 BasicBlock *ExitPredBB = SplitPos->getParent();
1715 auto InsertBB = merged ? ExitPredBB : ExitBB;
1716 if (!isa_and_nonnull<BranchInst>(SplitPos))
1717 SplitPos->eraseFromParent();
1718 Builder.SetInsertPoint(InsertBB);
1719 }
1720
1721 return Builder.saveIP();
1722}
1723
1724OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(
1725 Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) {
1726
1727 // if nothing to do, Return current insertion point.
1728 if (!Conditional)
1729 return Builder.saveIP();
1730
1731 BasicBlock *EntryBB = Builder.GetInsertBlock();
1732 Value *CallBool = Builder.CreateIsNotNull(EntryCall);
1733 auto *ThenBB = BasicBlock::Create(M.getContext(), "omp_region.body");
1734 auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);
1735
1736 // Emit thenBB and set the Builder's insertion point there for
1737 // body generation next. Place the block after the current block.
1738 Function *CurFn = EntryBB->getParent();
1739 CurFn->getBasicBlockList().insertAfter(EntryBB->getIterator(), ThenBB);
1740
1741 // Move Entry branch to end of ThenBB, and replace with conditional
1742 // branch (If-stmt)
1743 Instruction *EntryBBTI = EntryBB->getTerminator();
1744 Builder.CreateCondBr(CallBool, ThenBB, ExitBB);
1745 EntryBBTI->removeFromParent();
1746 Builder.SetInsertPoint(UI);
1747 Builder.Insert(EntryBBTI);
1748 UI->eraseFromParent();
1749 Builder.SetInsertPoint(ThenBB->getTerminator());
1750
1751 // return an insertion point to ExitBB.
1752 return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt());
1753}
1754
1755OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit(
1756 omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,
1757 bool HasFinalize) {
1758
1759 Builder.restoreIP(FinIP);
1760
1761 // If there is finalization to do, emit it before the exit call
1762 if (HasFinalize) {
1763 assert(!FinalizationStack.empty() &&((!FinalizationStack.empty() && "Unexpected finalization stack state!"
) ? static_cast<void> (0) : __assert_fail ("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1764, __PRETTY_FUNCTION__))
1764 "Unexpected finalization stack state!")((!FinalizationStack.empty() && "Unexpected finalization stack state!"
) ? static_cast<void> (0) : __assert_fail ("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1764, __PRETTY_FUNCTION__))
;
1765
1766 FinalizationInfo Fi = FinalizationStack.pop_back_val();
1767 assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!")((Fi.DK == OMPD && "Unexpected Directive for Finalization call!"
) ? static_cast<void> (0) : __assert_fail ("Fi.DK == OMPD && \"Unexpected Directive for Finalization call!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1767, __PRETTY_FUNCTION__))
;
1768
1769 Fi.FiniCB(FinIP);
1770
1771 BasicBlock *FiniBB = FinIP.getBlock();
1772 Instruction *FiniBBTI = FiniBB->getTerminator();
1773
1774 // set Builder IP for call creation
1775 Builder.SetInsertPoint(FiniBBTI);
1776 }
1777
1778 // place the Exitcall as last instruction before Finalization block terminator
1779 ExitCall->removeFromParent();
1780 Builder.Insert(ExitCall);
1781
1782 return IRBuilder<>::InsertPoint(ExitCall->getParent(),
1783 ExitCall->getIterator());
1784}
1785
1786OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCopyinClauseBlocks(
1787 InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,
1788 llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {
1789 if (!IP.isSet())
1790 return IP;
1791
1792 IRBuilder<>::InsertPointGuard IPG(Builder);
1793
1794 // creates the following CFG structure
1795 // OMP_Entry : (MasterAddr != PrivateAddr)?
1796 // F T
1797 // | \
1798 // | copin.not.master
1799 // | /
1800 // v /
1801 // copyin.not.master.end
1802 // |
1803 // v
1804 // OMP.Entry.Next
1805
1806 BasicBlock *OMP_Entry = IP.getBlock();
1807 Function *CurFn = OMP_Entry->getParent();
1808 BasicBlock *CopyBegin =
1809 BasicBlock::Create(M.getContext(), "copyin.not.master", CurFn);
1810 BasicBlock *CopyEnd = nullptr;
1811
1812 // If entry block is terminated, split to preserve the branch to following
1813 // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.
1814 if (isa_and_nonnull<BranchInst>(OMP_Entry->getTerminator())) {
1815 CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(),
1816 "copyin.not.master.end");
1817 OMP_Entry->getTerminator()->eraseFromParent();
1818 } else {
1819 CopyEnd =
1820 BasicBlock::Create(M.getContext(), "copyin.not.master.end", CurFn);
1821 }
1822
1823 Builder.SetInsertPoint(OMP_Entry);
1824 Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy);
1825 Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy);
1826 Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr);
1827 Builder.CreateCondBr(cmp, CopyBegin, CopyEnd);
1828
1829 Builder.SetInsertPoint(CopyBegin);
1830 if (BranchtoEnd)
1831 Builder.SetInsertPoint(Builder.CreateBr(CopyEnd));
1832
1833 return Builder.saveIP();
1834}
1835
1836CallInst *OpenMPIRBuilder::createOMPAlloc(const LocationDescription &Loc,
1837 Value *Size, Value *Allocator,
1838 std::string Name) {
1839 IRBuilder<>::InsertPointGuard IPG(Builder);
1840 Builder.restoreIP(Loc.IP);
1841
1842 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1843 Value *Ident = getOrCreateIdent(SrcLocStr);
1844 Value *ThreadId = getOrCreateThreadID(Ident);
1845 Value *Args[] = {ThreadId, Size, Allocator};
1846
1847 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_alloc);
1848
1849 return Builder.CreateCall(Fn, Args, Name);
1850}
1851
1852CallInst *OpenMPIRBuilder::createOMPFree(const LocationDescription &Loc,
1853 Value *Addr, Value *Allocator,
1854 std::string Name) {
1855 IRBuilder<>::InsertPointGuard IPG(Builder);
1856 Builder.restoreIP(Loc.IP);
1857
1858 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1859 Value *Ident = getOrCreateIdent(SrcLocStr);
1860 Value *ThreadId = getOrCreateThreadID(Ident);
1861 Value *Args[] = {ThreadId, Addr, Allocator};
1862 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_free);
1863 return Builder.CreateCall(Fn, Args, Name);
1864}
1865
1866CallInst *OpenMPIRBuilder::createCachedThreadPrivate(
1867 const LocationDescription &Loc, llvm::Value *Pointer,
1868 llvm::ConstantInt *Size, const llvm::Twine &Name) {
1869 IRBuilder<>::InsertPointGuard IPG(Builder);
1870 Builder.restoreIP(Loc.IP);
1871
1872 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1873 Value *Ident = getOrCreateIdent(SrcLocStr);
1874 Value *ThreadId = getOrCreateThreadID(Ident);
1875 Constant *ThreadPrivateCache =
1876 getOrCreateOMPInternalVariable(Int8PtrPtr, Name);
1877 llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};
1878
1879 Function *Fn =
1880 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_threadprivate_cached);
1881
1882 return Builder.CreateCall(Fn, Args);
1883}
1884
1885std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
1886 StringRef FirstSeparator,
1887 StringRef Separator) {
1888 SmallString<128> Buffer;
1889 llvm::raw_svector_ostream OS(Buffer);
1890 StringRef Sep = FirstSeparator;
1891 for (StringRef Part : Parts) {
1892 OS << Sep << Part;
1893 Sep = Separator;
1894 }
1895 return OS.str().str();
1896}
1897
1898Constant *OpenMPIRBuilder::getOrCreateOMPInternalVariable(
1899 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
1900 // TODO: Replace the twine arg with stringref to get rid of the conversion
1901 // logic. However This is taken from current implementation in clang as is.
1902 // Since this method is used in many places exclusively for OMP internal use
1903 // we will keep it as is for temporarily until we move all users to the
1904 // builder and then, if possible, fix it everywhere in one go.
1905 SmallString<256> Buffer;
1906 llvm::raw_svector_ostream Out(Buffer);
1907 Out << Name;
1908 StringRef RuntimeName = Out.str();
1909 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
1910 if (Elem.second) {
1911 assert(Elem.second->getType()->getPointerElementType() == Ty &&((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1912, __PRETTY_FUNCTION__))
1912 "OMP internal variable has different type than requested")((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1912, __PRETTY_FUNCTION__))
;
1913 } else {
1914 // TODO: investigate the appropriate linkage type used for the global
1915 // variable for possibly changing that to internal or private, or maybe
1916 // create different versions of the function for different OMP internal
1917 // variables.
1918 Elem.second = new llvm::GlobalVariable(
1919 M, Ty, /*IsConstant*/ false, llvm::GlobalValue::CommonLinkage,
1920 llvm::Constant::getNullValue(Ty), Elem.first(),
1921 /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
1922 AddressSpace);
1923 }
1924
1925 return Elem.second;
1926}
1927
1928Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {
1929 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
1930 std::string Name = getNameWithSeparators({Prefix, "var"}, ".", ".");
1931 return getOrCreateOMPInternalVariable(KmpCriticalNameTy, Name);
1932}
1933
1934// Create all simple and struct types exposed by the runtime and remember
1935// the llvm::PointerTypes of them for easy access later.
1936void OpenMPIRBuilder::initializeTypes(Module &M) {
1937 LLVMContext &Ctx = M.getContext();
1938 StructType *T;
1939#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
1940#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \
1941 VarName##Ty = ArrayType::get(ElemTy, ArraySize); \
1942 VarName##PtrTy = PointerType::getUnqual(VarName##Ty);
1943#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \
1944 VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \
1945 VarName##Ptr = PointerType::getUnqual(VarName);
1946#define OMP_STRUCT_TYPE(VarName, StructName, ...) \
1947 T = StructType::getTypeByName(Ctx, StructName); \
1948 if (!T) \
1949 T = StructType::create(Ctx, {__VA_ARGS__}, StructName); \
1950 VarName = T; \
1951 VarName##Ptr = PointerType::getUnqual(T);
1952#include "llvm/Frontend/OpenMP/OMPKinds.def"
1953}
1954
1955void OpenMPIRBuilder::OutlineInfo::collectBlocks(
1956 SmallPtrSetImpl<BasicBlock *> &BlockSet,
1957 SmallVectorImpl<BasicBlock *> &BlockVector) {
1958 SmallVector<BasicBlock *, 32> Worklist;
1959 BlockSet.insert(EntryBB);
1960 BlockSet.insert(ExitBB);
1961
1962 Worklist.push_back(EntryBB);
1963 while (!Worklist.empty()) {
1964 BasicBlock *BB = Worklist.pop_back_val();
1965 BlockVector.push_back(BB);
1966 for (BasicBlock *SuccBB : successors(BB))
1967 if (BlockSet.insert(SuccBB).second)
1968 Worklist.push_back(SuccBB);
1969 }
1970}
1971
1972void CanonicalLoopInfo::collectControlBlocks(
1973 SmallVectorImpl<BasicBlock *> &BBs) {
1974 // We only count those BBs as control block for which we do not need to
1975 // reverse the CFG, i.e. not the loop body which can contain arbitrary control
1976 // flow. For consistency, this also means we do not add the Body block, which
1977 // is just the entry to the body code.
1978 BBs.reserve(BBs.size() + 6);
1979 BBs.append({Preheader, Header, Cond, Latch, Exit, After});
1980}
1981
1982void CanonicalLoopInfo::assertOK() const {
1983#ifndef NDEBUG
1984 if (!IsValid)
1985 return;
1986
1987 // Verify standard control-flow we use for OpenMP loops.
1988 assert(Preheader)((Preheader) ? static_cast<void> (0) : __assert_fail ("Preheader"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1988, __PRETTY_FUNCTION__))
;
1989 assert(isa<BranchInst>(Preheader->getTerminator()) &&((isa<BranchInst>(Preheader->getTerminator()) &&
"Preheader must terminate with unconditional branch") ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(Preheader->getTerminator()) && \"Preheader must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1990, __PRETTY_FUNCTION__))
1990 "Preheader must terminate with unconditional branch")((isa<BranchInst>(Preheader->getTerminator()) &&
"Preheader must terminate with unconditional branch") ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(Preheader->getTerminator()) && \"Preheader must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1990, __PRETTY_FUNCTION__))
;
1991 assert(Preheader->getSingleSuccessor() == Header &&((Preheader->getSingleSuccessor() == Header && "Preheader must jump to header"
) ? static_cast<void> (0) : __assert_fail ("Preheader->getSingleSuccessor() == Header && \"Preheader must jump to header\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1992, __PRETTY_FUNCTION__))
1992 "Preheader must jump to header")((Preheader->getSingleSuccessor() == Header && "Preheader must jump to header"
) ? static_cast<void> (0) : __assert_fail ("Preheader->getSingleSuccessor() == Header && \"Preheader must jump to header\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1992, __PRETTY_FUNCTION__))
;
1993
1994 assert(Header)((Header) ? static_cast<void> (0) : __assert_fail ("Header"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1994, __PRETTY_FUNCTION__))
;
1995 assert(isa<BranchInst>(Header->getTerminator()) &&((isa<BranchInst>(Header->getTerminator()) &&
"Header must terminate with unconditional branch") ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(Header->getTerminator()) && \"Header must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1996, __PRETTY_FUNCTION__))
1996 "Header must terminate with unconditional branch")((isa<BranchInst>(Header->getTerminator()) &&
"Header must terminate with unconditional branch") ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(Header->getTerminator()) && \"Header must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1996, __PRETTY_FUNCTION__))
;
1997 assert(Header->getSingleSuccessor() == Cond &&((Header->getSingleSuccessor() == Cond && "Header must jump to exiting block"
) ? static_cast<void> (0) : __assert_fail ("Header->getSingleSuccessor() == Cond && \"Header must jump to exiting block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1998, __PRETTY_FUNCTION__))
1998 "Header must jump to exiting block")((Header->getSingleSuccessor() == Cond && "Header must jump to exiting block"
) ? static_cast<void> (0) : __assert_fail ("Header->getSingleSuccessor() == Cond && \"Header must jump to exiting block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1998, __PRETTY_FUNCTION__))
;
1999
2000 assert(Cond)((Cond) ? static_cast<void> (0) : __assert_fail ("Cond"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2000, __PRETTY_FUNCTION__))
;
2001 assert(Cond->getSinglePredecessor() == Header &&((Cond->getSinglePredecessor() == Header && "Exiting block only reachable from header"
) ? static_cast<void> (0) : __assert_fail ("Cond->getSinglePredecessor() == Header && \"Exiting block only reachable from header\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2002, __PRETTY_FUNCTION__))
2002 "Exiting block only reachable from header")((Cond->getSinglePredecessor() == Header && "Exiting block only reachable from header"
) ? static_cast<void> (0) : __assert_fail ("Cond->getSinglePredecessor() == Header && \"Exiting block only reachable from header\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2002, __PRETTY_FUNCTION__))
;
2003
2004 assert(isa<BranchInst>(Cond->getTerminator()) &&((isa<BranchInst>(Cond->getTerminator()) && "Exiting block must terminate with conditional branch"
) ? static_cast<void> (0) : __assert_fail ("isa<BranchInst>(Cond->getTerminator()) && \"Exiting block must terminate with conditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2005, __PRETTY_FUNCTION__))
2005 "Exiting block must terminate with conditional branch")((isa<BranchInst>(Cond->getTerminator()) && "Exiting block must terminate with conditional branch"
) ? static_cast<void> (0) : __assert_fail ("isa<BranchInst>(Cond->getTerminator()) && \"Exiting block must terminate with conditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2005, __PRETTY_FUNCTION__))
;
2006 assert(size(successors(Cond)) == 2 &&((size(successors(Cond)) == 2 && "Exiting block must have two successors"
) ? static_cast<void> (0) : __assert_fail ("size(successors(Cond)) == 2 && \"Exiting block must have two successors\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2007, __PRETTY_FUNCTION__))
2007 "Exiting block must have two successors")((size(successors(Cond)) == 2 && "Exiting block must have two successors"
) ? static_cast<void> (0) : __assert_fail ("size(successors(Cond)) == 2 && \"Exiting block must have two successors\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2007, __PRETTY_FUNCTION__))
;
2008 assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body &&((cast<BranchInst>(Cond->getTerminator())->getSuccessor
(0) == Body && "Exiting block's first successor jump to the body"
) ? static_cast<void> (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body && \"Exiting block's first successor jump to the body\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2009, __PRETTY_FUNCTION__))
2009 "Exiting block's first successor jump to the body")((cast<BranchInst>(Cond->getTerminator())->getSuccessor
(0) == Body && "Exiting block's first successor jump to the body"
) ? static_cast<void> (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body && \"Exiting block's first successor jump to the body\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2009, __PRETTY_FUNCTION__))
;
2010 assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit &&((cast<BranchInst>(Cond->getTerminator())->getSuccessor
(1) == Exit && "Exiting block's second successor must exit the loop"
) ? static_cast<void> (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit && \"Exiting block's second successor must exit the loop\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2011, __PRETTY_FUNCTION__))
2011 "Exiting block's second successor must exit the loop")((cast<BranchInst>(Cond->getTerminator())->getSuccessor
(1) == Exit && "Exiting block's second successor must exit the loop"
) ? static_cast<void> (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit && \"Exiting block's second successor must exit the loop\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2011, __PRETTY_FUNCTION__))
;
2012
2013 assert(Body)((Body) ? static_cast<void> (0) : __assert_fail ("Body"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2013, __PRETTY_FUNCTION__))
;
2014 assert(Body->getSinglePredecessor() == Cond &&((Body->getSinglePredecessor() == Cond && "Body only reachable from exiting block"
) ? static_cast<void> (0) : __assert_fail ("Body->getSinglePredecessor() == Cond && \"Body only reachable from exiting block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2015, __PRETTY_FUNCTION__))
2015 "Body only reachable from exiting block")((Body->getSinglePredecessor() == Cond && "Body only reachable from exiting block"
) ? static_cast<void> (0) : __assert_fail ("Body->getSinglePredecessor() == Cond && \"Body only reachable from exiting block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2015, __PRETTY_FUNCTION__))
;
2016 assert(!isa<PHINode>(Body->front()))((!isa<PHINode>(Body->front())) ? static_cast<void
> (0) : __assert_fail ("!isa<PHINode>(Body->front())"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2016, __PRETTY_FUNCTION__))
;
2017
2018 assert(Latch)((Latch) ? static_cast<void> (0) : __assert_fail ("Latch"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2018, __PRETTY_FUNCTION__))
;
2019 assert(isa<BranchInst>(Latch->getTerminator()) &&((isa<BranchInst>(Latch->getTerminator()) &&
"Latch must terminate with unconditional branch") ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(Latch->getTerminator()) && \"Latch must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2020, __PRETTY_FUNCTION__))
2020 "Latch must terminate with unconditional branch")((isa<BranchInst>(Latch->getTerminator()) &&
"Latch must terminate with unconditional branch") ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(Latch->getTerminator()) && \"Latch must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2020, __PRETTY_FUNCTION__))
;
2021 assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header")((Latch->getSingleSuccessor() == Header && "Latch must jump to header"
) ? static_cast<void> (0) : __assert_fail ("Latch->getSingleSuccessor() == Header && \"Latch must jump to header\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2021, __PRETTY_FUNCTION__))
;
2022 // TODO: To support simple redirecting of the end of the body code that has
2023 // multiple; introduce another auxiliary basic block like preheader and after.
2024 assert(Latch->getSinglePredecessor() != nullptr)((Latch->getSinglePredecessor() != nullptr) ? static_cast<
void> (0) : __assert_fail ("Latch->getSinglePredecessor() != nullptr"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2024, __PRETTY_FUNCTION__))
;
2025 assert(!isa<PHINode>(Latch->front()))((!isa<PHINode>(Latch->front())) ? static_cast<void
> (0) : __assert_fail ("!isa<PHINode>(Latch->front())"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2025, __PRETTY_FUNCTION__))
;
2026
2027 assert(Exit)((Exit) ? static_cast<void> (0) : __assert_fail ("Exit"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2027, __PRETTY_FUNCTION__))
;
2028 assert(isa<BranchInst>(Exit->getTerminator()) &&((isa<BranchInst>(Exit->getTerminator()) && "Exit block must terminate with unconditional branch"
) ? static_cast<void> (0) : __assert_fail ("isa<BranchInst>(Exit->getTerminator()) && \"Exit block must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2029, __PRETTY_FUNCTION__))
2029 "Exit block must terminate with unconditional branch")((isa<BranchInst>(Exit->getTerminator()) && "Exit block must terminate with unconditional branch"
) ? static_cast<void> (0) : __assert_fail ("isa<BranchInst>(Exit->getTerminator()) && \"Exit block must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2029, __PRETTY_FUNCTION__))
;
2030 assert(Exit->getSingleSuccessor() == After &&((Exit->getSingleSuccessor() == After && "Exit block must jump to after block"
) ? static_cast<void> (0) : __assert_fail ("Exit->getSingleSuccessor() == After && \"Exit block must jump to after block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2031, __PRETTY_FUNCTION__))
2031 "Exit block must jump to after block")((Exit->getSingleSuccessor() == After && "Exit block must jump to after block"
) ? static_cast<void> (0) : __assert_fail ("Exit->getSingleSuccessor() == After && \"Exit block must jump to after block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2031, __PRETTY_FUNCTION__))
;
2032
2033 assert(After)((After) ? static_cast<void> (0) : __assert_fail ("After"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2033, __PRETTY_FUNCTION__))
;
2034 assert(After->getSinglePredecessor() == Exit &&((After->getSinglePredecessor() == Exit && "After block only reachable from exit block"
) ? static_cast<void> (0) : __assert_fail ("After->getSinglePredecessor() == Exit && \"After block only reachable from exit block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2035, __PRETTY_FUNCTION__))
2035 "After block only reachable from exit block")((After->getSinglePredecessor() == Exit && "After block only reachable from exit block"
) ? static_cast<void> (0) : __assert_fail ("After->getSinglePredecessor() == Exit && \"After block only reachable from exit block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2035, __PRETTY_FUNCTION__))
;
2036 assert(After->empty() || !isa<PHINode>(After->front()))((After->empty() || !isa<PHINode>(After->front())
) ? static_cast<void> (0) : __assert_fail ("After->empty() || !isa<PHINode>(After->front())"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2036, __PRETTY_FUNCTION__))
;
2037
2038 Instruction *IndVar = getIndVar();
2039 assert(IndVar && "Canonical induction variable not found?")((IndVar && "Canonical induction variable not found?"
) ? static_cast<void> (0) : __assert_fail ("IndVar && \"Canonical induction variable not found?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2039, __PRETTY_FUNCTION__))
;
2040 assert(isa<IntegerType>(IndVar->getType()) &&((isa<IntegerType>(IndVar->getType()) && "Induction variable must be an integer"
) ? static_cast<void> (0) : __assert_fail ("isa<IntegerType>(IndVar->getType()) && \"Induction variable must be an integer\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2041, __PRETTY_FUNCTION__))
2041 "Induction variable must be an integer")((isa<IntegerType>(IndVar->getType()) && "Induction variable must be an integer"
) ? static_cast<void> (0) : __assert_fail ("isa<IntegerType>(IndVar->getType()) && \"Induction variable must be an integer\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2041, __PRETTY_FUNCTION__))
;
2042 assert(cast<PHINode>(IndVar)->getParent() == Header &&((cast<PHINode>(IndVar)->getParent() == Header &&
"Induction variable must be a PHI in the loop header") ? static_cast
<void> (0) : __assert_fail ("cast<PHINode>(IndVar)->getParent() == Header && \"Induction variable must be a PHI in the loop header\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2043, __PRETTY_FUNCTION__))
2043 "Induction variable must be a PHI in the loop header")((cast<PHINode>(IndVar)->getParent() == Header &&
"Induction variable must be a PHI in the loop header") ? static_cast
<void> (0) : __assert_fail ("cast<PHINode>(IndVar)->getParent() == Header && \"Induction variable must be a PHI in the loop header\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2043, __PRETTY_FUNCTION__))
;
2044 assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader)((cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader
) ? static_cast<void> (0) : __assert_fail ("cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2044, __PRETTY_FUNCTION__))
;
2045 assert(((cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue
(0))->isZero()) ? static_cast<void> (0) : __assert_fail
("cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero()"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2046, __PRETTY_FUNCTION__))
2046 cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero())((cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue
(0))->isZero()) ? static_cast<void> (0) : __assert_fail
("cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero()"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2046, __PRETTY_FUNCTION__))
;
2047 assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch)((cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch
) ? static_cast<void> (0) : __assert_fail ("cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2047, __PRETTY_FUNCTION__))
;
2048
2049 auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
2050 assert(cast<Instruction>(NextIndVar)->getParent() == Latch)((cast<Instruction>(NextIndVar)->getParent() == Latch
) ? static_cast<void> (0) : __assert_fail ("cast<Instruction>(NextIndVar)->getParent() == Latch"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2050, __PRETTY_FUNCTION__))
;
2051 assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add)((cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator
::Add) ? static_cast<void> (0) : __assert_fail ("cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2051, __PRETTY_FUNCTION__))
;
2052 assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar)((cast<BinaryOperator>(NextIndVar)->getOperand(0) ==
IndVar) ? static_cast<void> (0) : __assert_fail ("cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2052, __PRETTY_FUNCTION__))
;
2053 assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))((cast<ConstantInt>(cast<BinaryOperator>(NextIndVar
)->getOperand(1)) ->isOne()) ? static_cast<void> (
0) : __assert_fail ("cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1)) ->isOne()"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2054, __PRETTY_FUNCTION__))
2054 ->isOne())((cast<ConstantInt>(cast<BinaryOperator>(NextIndVar
)->getOperand(1)) ->isOne()) ? static_cast<void> (
0) : __assert_fail ("cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1)) ->isOne()"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2054, __PRETTY_FUNCTION__))
;
2055
2056 Value *TripCount = getTripCount();
2057 assert(TripCount && "Loop trip count not found?")((TripCount && "Loop trip count not found?") ? static_cast
<void> (0) : __assert_fail ("TripCount && \"Loop trip count not found?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2057, __PRETTY_FUNCTION__))
;
2058 assert(IndVar->getType() == TripCount->getType() &&((IndVar->getType() == TripCount->getType() && "Trip count and induction variable must have the same type"
) ? static_cast<void> (0) : __assert_fail ("IndVar->getType() == TripCount->getType() && \"Trip count and induction variable must have the same type\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2059, __PRETTY_FUNCTION__))
2059 "Trip count and induction variable must have the same type")((IndVar->getType() == TripCount->getType() && "Trip count and induction variable must have the same type"
) ? static_cast<void> (0) : __assert_fail ("IndVar->getType() == TripCount->getType() && \"Trip count and induction variable must have the same type\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2059, __PRETTY_FUNCTION__))
;
2060
2061 auto *CmpI = cast<CmpInst>(&Cond->front());
2062 assert(CmpI->getPredicate() == CmpInst::ICMP_ULT &&((CmpI->getPredicate() == CmpInst::ICMP_ULT && "Exit condition must be a signed less-than comparison"
) ? static_cast<void> (0) : __assert_fail ("CmpI->getPredicate() == CmpInst::ICMP_ULT && \"Exit condition must be a signed less-than comparison\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2063, __PRETTY_FUNCTION__))
2063 "Exit condition must be a signed less-than comparison")((CmpI->getPredicate() == CmpInst::ICMP_ULT && "Exit condition must be a signed less-than comparison"
) ? static_cast<void> (0) : __assert_fail ("CmpI->getPredicate() == CmpInst::ICMP_ULT && \"Exit condition must be a signed less-than comparison\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2063, __PRETTY_FUNCTION__))
;
2064 assert(CmpI->getOperand(0) == IndVar &&((CmpI->getOperand(0) == IndVar && "Exit condition must compare the induction variable"
) ? static_cast<void> (0) : __assert_fail ("CmpI->getOperand(0) == IndVar && \"Exit condition must compare the induction variable\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2065, __PRETTY_FUNCTION__))
2065 "Exit condition must compare the induction variable")((CmpI->getOperand(0) == IndVar && "Exit condition must compare the induction variable"
) ? static_cast<void> (0) : __assert_fail ("CmpI->getOperand(0) == IndVar && \"Exit condition must compare the induction variable\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2065, __PRETTY_FUNCTION__))
;
2066 assert(CmpI->getOperand(1) == TripCount &&((CmpI->getOperand(1) == TripCount && "Exit condition must compare with the trip count"
) ? static_cast<void> (0) : __assert_fail ("CmpI->getOperand(1) == TripCount && \"Exit condition must compare with the trip count\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2067, __PRETTY_FUNCTION__))
2067 "Exit condition must compare with the trip count")((CmpI->getOperand(1) == TripCount && "Exit condition must compare with the trip count"
) ? static_cast<void> (0) : __assert_fail ("CmpI->getOperand(1) == TripCount && \"Exit condition must compare with the trip count\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2067, __PRETTY_FUNCTION__))
;
2068#endif
2069}

/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h

1//===- llvm/IRBuilder.h - Builder for LLVM Instructions ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_IR_IRBUILDER_H
15#define LLVM_IR_IRBUILDER_H
16
17#include "llvm-c/Types.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/None.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/IR/BasicBlock.h"
24#include "llvm/IR/Constant.h"
25#include "llvm/IR/ConstantFolder.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/IR/DebugInfoMetadata.h"
29#include "llvm/IR/DebugLoc.h"
30#include "llvm/IR/DerivedTypes.h"
31#include "llvm/IR/Function.h"
32#include "llvm/IR/GlobalVariable.h"
33#include "llvm/IR/InstrTypes.h"
34#include "llvm/IR/Instruction.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/IR/IntrinsicInst.h"
37#include "llvm/IR/LLVMContext.h"
38#include "llvm/IR/Module.h"
39#include "llvm/IR/Operator.h"
40#include "llvm/IR/Type.h"
41#include "llvm/IR/Value.h"
42#include "llvm/IR/ValueHandle.h"
43#include "llvm/Support/AtomicOrdering.h"
44#include "llvm/Support/CBindingWrapping.h"
45#include "llvm/Support/Casting.h"
46#include <cassert>
47#include <cstddef>
48#include <cstdint>
49#include <functional>
50#include <utility>
51
52namespace llvm {
53
54class APInt;
55class MDNode;
56class Use;
57
58/// This provides the default implementation of the IRBuilder
59/// 'InsertHelper' method that is called whenever an instruction is created by
60/// IRBuilder and needs to be inserted.
61///
62/// By default, this inserts the instruction at the insertion point.
63class IRBuilderDefaultInserter {
64public:
65 virtual ~IRBuilderDefaultInserter();
66
67 virtual void InsertHelper(Instruction *I, const Twine &Name,
68 BasicBlock *BB,
69 BasicBlock::iterator InsertPt) const {
70 if (BB) BB->getInstList().insert(InsertPt, I);
71 I->setName(Name);
72 }
73};
74
75/// Provides an 'InsertHelper' that calls a user-provided callback after
76/// performing the default insertion.
77class IRBuilderCallbackInserter : public IRBuilderDefaultInserter {
78 std::function<void(Instruction *)> Callback;
79
80public:
81 virtual ~IRBuilderCallbackInserter();
82
83 IRBuilderCallbackInserter(std::function<void(Instruction *)> Callback)
84 : Callback(std::move(Callback)) {}
85
86 void InsertHelper(Instruction *I, const Twine &Name,
87 BasicBlock *BB,
88 BasicBlock::iterator InsertPt) const override {
89 IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
90 Callback(I);
91 }
92};
93
94/// Common base class shared among various IRBuilders.
95class IRBuilderBase {
96 /// Pairs of (metadata kind, MDNode *) that should be added to all newly
97 /// created instructions, like !dbg metadata.
98 SmallVector<std::pair<unsigned, MDNode *>, 2> MetadataToCopy;
99
100 /// Add or update the an entry (Kind, MD) to MetadataToCopy, if \p MD is not
101 /// null. If \p MD is null, remove the entry with \p Kind.
102 void AddOrRemoveMetadataToCopy(unsigned Kind, MDNode *MD) {
103 if (!MD) {
104 erase_if(MetadataToCopy, [Kind](const std::pair<unsigned, MDNode *> &KV) {
105 return KV.first == Kind;
106 });
107 return;
108 }
109
110 for (auto &KV : MetadataToCopy)
111 if (KV.first == Kind) {
112 KV.second = MD;
113 return;
114 }
115
116 MetadataToCopy.emplace_back(Kind, MD);
117 }
118
119protected:
120 BasicBlock *BB;
121 BasicBlock::iterator InsertPt;
122 LLVMContext &Context;
123 const IRBuilderFolder &Folder;
124 const IRBuilderDefaultInserter &Inserter;
125
126 MDNode *DefaultFPMathTag;
127 FastMathFlags FMF;
128
129 bool IsFPConstrained;
130 fp::ExceptionBehavior DefaultConstrainedExcept;
131 RoundingMode DefaultConstrainedRounding;
132
133 ArrayRef<OperandBundleDef> DefaultOperandBundles;
134
135public:
136 IRBuilderBase(LLVMContext &context, const IRBuilderFolder &Folder,
137 const IRBuilderDefaultInserter &Inserter,
138 MDNode *FPMathTag, ArrayRef<OperandBundleDef> OpBundles)
139 : Context(context), Folder(Folder), Inserter(Inserter),
140 DefaultFPMathTag(FPMathTag), IsFPConstrained(false),
141 DefaultConstrainedExcept(fp::ebStrict),
142 DefaultConstrainedRounding(RoundingMode::Dynamic),
143 DefaultOperandBundles(OpBundles) {
144 ClearInsertionPoint();
145 }
146
147 /// Insert and return the specified instruction.
148 template<typename InstTy>
149 InstTy *Insert(InstTy *I, const Twine &Name = "") const {
150 Inserter.InsertHelper(I, Name, BB, InsertPt);
151 AddMetadataToInst(I);
152 return I;
153 }
154
155 /// No-op overload to handle constants.
156 Constant *Insert(Constant *C, const Twine& = "") const {
157 return C;
158 }
159
160 Value *Insert(Value *V, const Twine &Name = "") const {
161 if (Instruction *I = dyn_cast<Instruction>(V))
162 return Insert(I, Name);
163 assert(isa<Constant>(V))((isa<Constant>(V)) ? static_cast<void> (0) : __assert_fail
("isa<Constant>(V)", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 163, __PRETTY_FUNCTION__))
;
164 return V;
165 }
166
167 //===--------------------------------------------------------------------===//
168 // Builder configuration methods
169 //===--------------------------------------------------------------------===//
170
171 /// Clear the insertion point: created instructions will not be
172 /// inserted into a block.
173 void ClearInsertionPoint() {
174 BB = nullptr;
33
Null pointer value stored to field 'BB'
175 InsertPt = BasicBlock::iterator();
176 }
177
178 BasicBlock *GetInsertBlock() const { return BB; }
179 BasicBlock::iterator GetInsertPoint() const { return InsertPt; }
180 LLVMContext &getContext() const { return Context; }
181
182 /// This specifies that created instructions should be appended to the
183 /// end of the specified block.
184 void SetInsertPoint(BasicBlock *TheBB) {
185 BB = TheBB;
186 InsertPt = BB->end();
187 }
188
189 /// This specifies that created instructions should be inserted before
190 /// the specified instruction.
191 void SetInsertPoint(Instruction *I) {
192 BB = I->getParent();
193 InsertPt = I->getIterator();
194 assert(InsertPt != BB->end() && "Can't read debug loc from end()")((InsertPt != BB->end() && "Can't read debug loc from end()"
) ? static_cast<void> (0) : __assert_fail ("InsertPt != BB->end() && \"Can't read debug loc from end()\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 194, __PRETTY_FUNCTION__))
;
195 SetCurrentDebugLocation(I->getDebugLoc());
196 }
197
198 /// This specifies that created instructions should be inserted at the
199 /// specified point.
200 void SetInsertPoint(BasicBlock *TheBB, BasicBlock::iterator IP) {
201 BB = TheBB;
202 InsertPt = IP;
203 if (IP != TheBB->end())
204 SetCurrentDebugLocation(IP->getDebugLoc());
205 }
206
207 /// Set location information used by debugging information.
208 void SetCurrentDebugLocation(DebugLoc L) {
209 AddOrRemoveMetadataToCopy(LLVMContext::MD_dbg, L.getAsMDNode());
210 }
211
212 /// Collect metadata with IDs \p MetadataKinds from \p Src which should be
213 /// added to all created instructions. Entries present in MedataDataToCopy but
214 /// not on \p Src will be dropped from MetadataToCopy.
215 void CollectMetadataToCopy(Instruction *Src,
216 ArrayRef<unsigned> MetadataKinds) {
217 for (unsigned K : MetadataKinds)
218 AddOrRemoveMetadataToCopy(K, Src->getMetadata(K));
219 }
220
221 /// Get location information used by debugging information.
222 DebugLoc getCurrentDebugLocation() const {
223 for (auto &KV : MetadataToCopy)
224 if (KV.first == LLVMContext::MD_dbg)
225 return {cast<DILocation>(KV.second)};
226
227 return {};
228 }
229
230 /// If this builder has a current debug location, set it on the
231 /// specified instruction.
232 void SetInstDebugLocation(Instruction *I) const {
233 for (const auto &KV : MetadataToCopy)
234 if (KV.first == LLVMContext::MD_dbg) {
235 I->setDebugLoc(DebugLoc(KV.second));
236 return;
237 }
238 }
239
240 /// Add all entries in MetadataToCopy to \p I.
241 void AddMetadataToInst(Instruction *I) const {
242 for (auto &KV : MetadataToCopy)
243 I->setMetadata(KV.first, KV.second);
244 }
245
246 /// Get the return type of the current function that we're emitting
247 /// into.
248 Type *getCurrentFunctionReturnType() const;
249
250 /// InsertPoint - A saved insertion point.
251 class InsertPoint {
252 BasicBlock *Block = nullptr;
253 BasicBlock::iterator Point;
254
255 public:
256 /// Creates a new insertion point which doesn't point to anything.
257 InsertPoint() = default;
258
259 /// Creates a new insertion point at the given location.
260 InsertPoint(BasicBlock *InsertBlock, BasicBlock::iterator InsertPoint)
261 : Block(InsertBlock), Point(InsertPoint) {}
262
263 /// Returns true if this insert point is set.
264 bool isSet() const { return (Block != nullptr); }
265
266 BasicBlock *getBlock() const { return Block; }
267 BasicBlock::iterator getPoint() const { return Point; }
268 };
269
270 /// Returns the current insert point.
271 InsertPoint saveIP() const {
272 return InsertPoint(GetInsertBlock(), GetInsertPoint());
273 }
274
275 /// Returns the current insert point, clearing it in the process.
276 InsertPoint saveAndClearIP() {
277 InsertPoint IP(GetInsertBlock(), GetInsertPoint());
278 ClearInsertionPoint();
279 return IP;
280 }
281
282 /// Sets the current insert point to a previously-saved location.
283 void restoreIP(InsertPoint IP) {
284 if (IP.isSet())
31
Taking false branch
285 SetInsertPoint(IP.getBlock(), IP.getPoint());
286 else
287 ClearInsertionPoint();
32
Calling 'IRBuilderBase::ClearInsertionPoint'
34
Returning from 'IRBuilderBase::ClearInsertionPoint'
288 }
289
290 /// Get the floating point math metadata being used.
291 MDNode *getDefaultFPMathTag() const { return DefaultFPMathTag; }
292
293 /// Get the flags to be applied to created floating point ops
294 FastMathFlags getFastMathFlags() const { return FMF; }
295
296 FastMathFlags &getFastMathFlags() { return FMF; }
297
298 /// Clear the fast-math flags.
299 void clearFastMathFlags() { FMF.clear(); }
300
301 /// Set the floating point math metadata to be used.
302 void setDefaultFPMathTag(MDNode *FPMathTag) { DefaultFPMathTag = FPMathTag; }
303
304 /// Set the fast-math flags to be used with generated fp-math operators
305 void setFastMathFlags(FastMathFlags NewFMF) { FMF = NewFMF; }
306
307 /// Enable/Disable use of constrained floating point math. When
308 /// enabled the CreateF<op>() calls instead create constrained
309 /// floating point intrinsic calls. Fast math flags are unaffected
310 /// by this setting.
311 void setIsFPConstrained(bool IsCon) { IsFPConstrained = IsCon; }
312
313 /// Query for the use of constrained floating point math
314 bool getIsFPConstrained() { return IsFPConstrained; }
315
316 /// Set the exception handling to be used with constrained floating point
317 void setDefaultConstrainedExcept(fp::ExceptionBehavior NewExcept) {
318#ifndef NDEBUG
319 Optional<StringRef> ExceptStr = ExceptionBehaviorToStr(NewExcept);
320 assert(ExceptStr.hasValue() && "Garbage strict exception behavior!")((ExceptStr.hasValue() && "Garbage strict exception behavior!"
) ? static_cast<void> (0) : __assert_fail ("ExceptStr.hasValue() && \"Garbage strict exception behavior!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 320, __PRETTY_FUNCTION__))
;
321#endif
322 DefaultConstrainedExcept = NewExcept;
323 }
324
325 /// Set the rounding mode handling to be used with constrained floating point
326 void setDefaultConstrainedRounding(RoundingMode NewRounding) {
327#ifndef NDEBUG
328 Optional<StringRef> RoundingStr = RoundingModeToStr(NewRounding);
329 assert(RoundingStr.hasValue() && "Garbage strict rounding mode!")((RoundingStr.hasValue() && "Garbage strict rounding mode!"
) ? static_cast<void> (0) : __assert_fail ("RoundingStr.hasValue() && \"Garbage strict rounding mode!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 329, __PRETTY_FUNCTION__))
;
330#endif
331 DefaultConstrainedRounding = NewRounding;
332 }
333
334 /// Get the exception handling used with constrained floating point
335 fp::ExceptionBehavior getDefaultConstrainedExcept() {
336 return DefaultConstrainedExcept;
337 }
338
339 /// Get the rounding mode handling used with constrained floating point
340 RoundingMode getDefaultConstrainedRounding() {
341 return DefaultConstrainedRounding;
342 }
343
344 void setConstrainedFPFunctionAttr() {
345 assert(BB && "Must have a basic block to set any function attributes!")((BB && "Must have a basic block to set any function attributes!"
) ? static_cast<void> (0) : __assert_fail ("BB && \"Must have a basic block to set any function attributes!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 345, __PRETTY_FUNCTION__))
;
346
347 Function *F = BB->getParent();
348 if (!F->hasFnAttribute(Attribute::StrictFP)) {
349 F->addFnAttr(Attribute::StrictFP);
350 }
351 }
352
353 void setConstrainedFPCallAttr(CallBase *I) {
354 I->addAttribute(AttributeList::FunctionIndex, Attribute::StrictFP);
355 }
356
357 void setDefaultOperandBundles(ArrayRef<OperandBundleDef> OpBundles) {
358 DefaultOperandBundles = OpBundles;
359 }
360
361 //===--------------------------------------------------------------------===//
362 // RAII helpers.
363 //===--------------------------------------------------------------------===//
364
365 // RAII object that stores the current insertion point and restores it
366 // when the object is destroyed. This includes the debug location.
367 class InsertPointGuard {
368 IRBuilderBase &Builder;
369 AssertingVH<BasicBlock> Block;
370 BasicBlock::iterator Point;
371 DebugLoc DbgLoc;
372
373 public:
374 InsertPointGuard(IRBuilderBase &B)
375 : Builder(B), Block(B.GetInsertBlock()), Point(B.GetInsertPoint()),
376 DbgLoc(B.getCurrentDebugLocation()) {}
377
378 InsertPointGuard(const InsertPointGuard &) = delete;
379 InsertPointGuard &operator=(const InsertPointGuard &) = delete;
380
381 ~InsertPointGuard() {
382 Builder.restoreIP(InsertPoint(Block, Point));
383 Builder.SetCurrentDebugLocation(DbgLoc);
384 }
385 };
386
387 // RAII object that stores the current fast math settings and restores
388 // them when the object is destroyed.
389 class FastMathFlagGuard {
390 IRBuilderBase &Builder;
391 FastMathFlags FMF;
392 MDNode *FPMathTag;
393 bool IsFPConstrained;
394 fp::ExceptionBehavior DefaultConstrainedExcept;
395 RoundingMode DefaultConstrainedRounding;
396
397 public:
398 FastMathFlagGuard(IRBuilderBase &B)
399 : Builder(B), FMF(B.FMF), FPMathTag(B.DefaultFPMathTag),
400 IsFPConstrained(B.IsFPConstrained),
401 DefaultConstrainedExcept(B.DefaultConstrainedExcept),
402 DefaultConstrainedRounding(B.DefaultConstrainedRounding) {}
403
404 FastMathFlagGuard(const FastMathFlagGuard &) = delete;
405 FastMathFlagGuard &operator=(const FastMathFlagGuard &) = delete;
406
407 ~FastMathFlagGuard() {
408 Builder.FMF = FMF;
409 Builder.DefaultFPMathTag = FPMathTag;
410 Builder.IsFPConstrained = IsFPConstrained;
411 Builder.DefaultConstrainedExcept = DefaultConstrainedExcept;
412 Builder.DefaultConstrainedRounding = DefaultConstrainedRounding;
413 }
414 };
415
416 // RAII object that stores the current default operand bundles and restores
417 // them when the object is destroyed.
418 class OperandBundlesGuard {
419 IRBuilderBase &Builder;
420 ArrayRef<OperandBundleDef> DefaultOperandBundles;
421
422 public:
423 OperandBundlesGuard(IRBuilderBase &B)
424 : Builder(B), DefaultOperandBundles(B.DefaultOperandBundles) {}
425
426 OperandBundlesGuard(const OperandBundlesGuard &) = delete;
427 OperandBundlesGuard &operator=(const OperandBundlesGuard &) = delete;
428
429 ~OperandBundlesGuard() {
430 Builder.DefaultOperandBundles = DefaultOperandBundles;
431 }
432 };
433
434
435 //===--------------------------------------------------------------------===//
436 // Miscellaneous creation methods.
437 //===--------------------------------------------------------------------===//
438
439 /// Make a new global variable with initializer type i8*
440 ///
441 /// Make a new global variable with an initializer that has array of i8 type
442 /// filled in with the null terminated string value specified. The new global
443 /// variable will be marked mergable with any others of the same contents. If
444 /// Name is specified, it is the name of the global variable created.
445 ///
446 /// If no module is given via \p M, it is take from the insertion point basic
447 /// block.
448 GlobalVariable *CreateGlobalString(StringRef Str, const Twine &Name = "",
449 unsigned AddressSpace = 0,
450 Module *M = nullptr);
451
452 /// Get a constant value representing either true or false.
453 ConstantInt *getInt1(bool V) {
454 return ConstantInt::get(getInt1Ty(), V);
455 }
456
457 /// Get the constant value for i1 true.
458 ConstantInt *getTrue() {
459 return ConstantInt::getTrue(Context);
460 }
461
462 /// Get the constant value for i1 false.
463 ConstantInt *getFalse() {
464 return ConstantInt::getFalse(Context);
465 }
466
467 /// Get a constant 8-bit value.
468 ConstantInt *getInt8(uint8_t C) {
469 return ConstantInt::get(getInt8Ty(), C);
470 }
471
472 /// Get a constant 16-bit value.
473 ConstantInt *getInt16(uint16_t C) {
474 return ConstantInt::get(getInt16Ty(), C);
475 }
476
477 /// Get a constant 32-bit value.
478 ConstantInt *getInt32(uint32_t C) {
479 return ConstantInt::get(getInt32Ty(), C);
480 }
481
482 /// Get a constant 64-bit value.
483 ConstantInt *getInt64(uint64_t C) {
484 return ConstantInt::get(getInt64Ty(), C);
485 }
486
487 /// Get a constant N-bit value, zero extended or truncated from
488 /// a 64-bit value.
489 ConstantInt *getIntN(unsigned N, uint64_t C) {
490 return ConstantInt::get(getIntNTy(N), C);
491 }
492
493 /// Get a constant integer value.
494 ConstantInt *getInt(const APInt &AI) {
495 return ConstantInt::get(Context, AI);
496 }
497
498 //===--------------------------------------------------------------------===//
499 // Type creation methods
500 //===--------------------------------------------------------------------===//
501
502 /// Fetch the type representing a single bit
503 IntegerType *getInt1Ty() {
504 return Type::getInt1Ty(Context);
505 }
506
507 /// Fetch the type representing an 8-bit integer.
508 IntegerType *getInt8Ty() {
509 return Type::getInt8Ty(Context);
510 }
511
512 /// Fetch the type representing a 16-bit integer.
513 IntegerType *getInt16Ty() {
514 return Type::getInt16Ty(Context);
515 }
516
517 /// Fetch the type representing a 32-bit integer.
518 IntegerType *getInt32Ty() {
519 return Type::getInt32Ty(Context);
520 }
521
522 /// Fetch the type representing a 64-bit integer.
523 IntegerType *getInt64Ty() {
524 return Type::getInt64Ty(Context);
525 }
526
527 /// Fetch the type representing a 128-bit integer.
528 IntegerType *getInt128Ty() { return Type::getInt128Ty(Context); }
529
530 /// Fetch the type representing an N-bit integer.
531 IntegerType *getIntNTy(unsigned N) {
532 return Type::getIntNTy(Context, N);
533 }
534
535 /// Fetch the type representing a 16-bit floating point value.
536 Type *getHalfTy() {
537 return Type::getHalfTy(Context);
538 }
539
540 /// Fetch the type representing a 16-bit brain floating point value.
541 Type *getBFloatTy() {
542 return Type::getBFloatTy(Context);
543 }
544
545 /// Fetch the type representing a 32-bit floating point value.
546 Type *getFloatTy() {
547 return Type::getFloatTy(Context);
548 }
549
550 /// Fetch the type representing a 64-bit floating point value.
551 Type *getDoubleTy() {
552 return Type::getDoubleTy(Context);
553 }
554
555 /// Fetch the type representing void.
556 Type *getVoidTy() {
557 return Type::getVoidTy(Context);
558 }
559
560 /// Fetch the type representing a pointer to an 8-bit integer value.
561 PointerType *getInt8PtrTy(unsigned AddrSpace = 0) {
562 return Type::getInt8PtrTy(Context, AddrSpace);
563 }
564
565 /// Fetch the type representing a pointer to an integer value.
566 IntegerType *getIntPtrTy(const DataLayout &DL, unsigned AddrSpace = 0) {
567 return DL.getIntPtrType(Context, AddrSpace);
568 }
569
570 //===--------------------------------------------------------------------===//
571 // Intrinsic creation methods
572 //===--------------------------------------------------------------------===//
573
574 /// Create and insert a memset to the specified pointer and the
575 /// specified value.
576 ///
577 /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
578 /// specified, it will be added to the instruction. Likewise with alias.scope
579 /// and noalias tags.
580 CallInst *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size,
581 MaybeAlign Align, bool isVolatile = false,
582 MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr,
583 MDNode *NoAliasTag = nullptr) {
584 return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile,
585 TBAATag, ScopeTag, NoAliasTag);
586 }
587
588 CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size, MaybeAlign Align,
589 bool isVolatile = false, MDNode *TBAATag = nullptr,
590 MDNode *ScopeTag = nullptr,
591 MDNode *NoAliasTag = nullptr);
592
593 /// Create and insert an element unordered-atomic memset of the region of
594 /// memory starting at the given pointer to the given value.
595 ///
596 /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
597 /// specified, it will be added to the instruction. Likewise with alias.scope
598 /// and noalias tags.
599 CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
600 uint64_t Size, Align Alignment,
601 uint32_t ElementSize,
602 MDNode *TBAATag = nullptr,
603 MDNode *ScopeTag = nullptr,
604 MDNode *NoAliasTag = nullptr) {
605 return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size),
606 Align(Alignment), ElementSize,
607 TBAATag, ScopeTag, NoAliasTag);
608 }
609
610 CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
611 Value *Size, Align Alignment,
612 uint32_t ElementSize,
613 MDNode *TBAATag = nullptr,
614 MDNode *ScopeTag = nullptr,
615 MDNode *NoAliasTag = nullptr);
616
617 /// Create and insert a memcpy between the specified pointers.
618 ///
619 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
620 /// specified, it will be added to the instruction. Likewise with alias.scope
621 /// and noalias tags.
622 CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
623 MaybeAlign SrcAlign, uint64_t Size,
624 bool isVolatile = false, MDNode *TBAATag = nullptr,
625 MDNode *TBAAStructTag = nullptr,
626 MDNode *ScopeTag = nullptr,
627 MDNode *NoAliasTag = nullptr) {
628 return CreateMemCpy(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
629 isVolatile, TBAATag, TBAAStructTag, ScopeTag,
630 NoAliasTag);
631 }
632
633 CallInst *CreateMemTransferInst(
634 Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
635 MaybeAlign SrcAlign, Value *Size, bool isVolatile = false,
636 MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
637 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr);
638
639 CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
640 MaybeAlign SrcAlign, Value *Size,
641 bool isVolatile = false, MDNode *TBAATag = nullptr,
642 MDNode *TBAAStructTag = nullptr,
643 MDNode *ScopeTag = nullptr,
644 MDNode *NoAliasTag = nullptr) {
645 return CreateMemTransferInst(Intrinsic::memcpy, Dst, DstAlign, Src,
646 SrcAlign, Size, isVolatile, TBAATag,
647 TBAAStructTag, ScopeTag, NoAliasTag);
648 }
649
650 CallInst *CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign, Value *Src,
651 MaybeAlign SrcAlign, Value *Size);
652
653 /// Create and insert an element unordered-atomic memcpy between the
654 /// specified pointers.
655 ///
656 /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers, respectively.
657 ///
658 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
659 /// specified, it will be added to the instruction. Likewise with alias.scope
660 /// and noalias tags.
661 CallInst *CreateElementUnorderedAtomicMemCpy(
662 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
663 uint32_t ElementSize, MDNode *TBAATag = nullptr,
664 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
665 MDNode *NoAliasTag = nullptr);
666
667 CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
668 MaybeAlign SrcAlign, uint64_t Size,
669 bool isVolatile = false, MDNode *TBAATag = nullptr,
670 MDNode *ScopeTag = nullptr,
671 MDNode *NoAliasTag = nullptr) {
672 return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
673 isVolatile, TBAATag, ScopeTag, NoAliasTag);
674 }
675
676 CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
677 MaybeAlign SrcAlign, Value *Size,
678 bool isVolatile = false, MDNode *TBAATag = nullptr,
679 MDNode *ScopeTag = nullptr,
680 MDNode *NoAliasTag = nullptr);
681
682 /// \brief Create and insert an element unordered-atomic memmove between the
683 /// specified pointers.
684 ///
685 /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers,
686 /// respectively.
687 ///
688 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
689 /// specified, it will be added to the instruction. Likewise with alias.scope
690 /// and noalias tags.
691 CallInst *CreateElementUnorderedAtomicMemMove(
692 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
693 uint32_t ElementSize, MDNode *TBAATag = nullptr,
694 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
695 MDNode *NoAliasTag = nullptr);
696
697 /// Create a vector fadd reduction intrinsic of the source vector.
698 /// The first parameter is a scalar accumulator value for ordered reductions.
699 CallInst *CreateFAddReduce(Value *Acc, Value *Src);
700
701 /// Create a vector fmul reduction intrinsic of the source vector.
702 /// The first parameter is a scalar accumulator value for ordered reductions.
703 CallInst *CreateFMulReduce(Value *Acc, Value *Src);
704
705 /// Create a vector int add reduction intrinsic of the source vector.
706 CallInst *CreateAddReduce(Value *Src);
707
708 /// Create a vector int mul reduction intrinsic of the source vector.
709 CallInst *CreateMulReduce(Value *Src);
710
711 /// Create a vector int AND reduction intrinsic of the source vector.
712 CallInst *CreateAndReduce(Value *Src);
713
714 /// Create a vector int OR reduction intrinsic of the source vector.
715 CallInst *CreateOrReduce(Value *Src);
716
717 /// Create a vector int XOR reduction intrinsic of the source vector.
718 CallInst *CreateXorReduce(Value *Src);
719
720 /// Create a vector integer max reduction intrinsic of the source
721 /// vector.
722 CallInst *CreateIntMaxReduce(Value *Src, bool IsSigned = false);
723
724 /// Create a vector integer min reduction intrinsic of the source
725 /// vector.
726 CallInst *CreateIntMinReduce(Value *Src, bool IsSigned = false);
727
728 /// Create a vector float max reduction intrinsic of the source
729 /// vector.
730 CallInst *CreateFPMaxReduce(Value *Src);
731
732 /// Create a vector float min reduction intrinsic of the source
733 /// vector.
734 CallInst *CreateFPMinReduce(Value *Src);
735
736 /// Create a lifetime.start intrinsic.
737 ///
738 /// If the pointer isn't i8* it will be converted.
739 CallInst *CreateLifetimeStart(Value *Ptr, ConstantInt *Size = nullptr);
740
741 /// Create a lifetime.end intrinsic.
742 ///
743 /// If the pointer isn't i8* it will be converted.
744 CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr);
745
746 /// Create a call to invariant.start intrinsic.
747 ///
748 /// If the pointer isn't i8* it will be converted.
749 CallInst *CreateInvariantStart(Value *Ptr, ConstantInt *Size = nullptr);
750
751 /// Create a call to Masked Load intrinsic
752 CallInst *CreateMaskedLoad(Value *Ptr, Align Alignment, Value *Mask,
753 Value *PassThru = nullptr, const Twine &Name = "");
754
755 /// Create a call to Masked Store intrinsic
756 CallInst *CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
757 Value *Mask);
758
759 /// Create a call to Masked Gather intrinsic
760 CallInst *CreateMaskedGather(Value *Ptrs, Align Alignment,
761 Value *Mask = nullptr, Value *PassThru = nullptr,
762 const Twine &Name = "");
763
764 /// Create a call to Masked Scatter intrinsic
765 CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment,
766 Value *Mask = nullptr);
767
768 /// Create an assume intrinsic call that allows the optimizer to
769 /// assume that the provided condition will be true.
770 ///
771 /// The optional argument \p OpBundles specifies operand bundles that are
772 /// added to the call instruction.
773 CallInst *CreateAssumption(Value *Cond,
774 ArrayRef<OperandBundleDef> OpBundles = llvm::None);
775
776 /// Create a llvm.experimental.noalias.scope.decl intrinsic call.
777 Instruction *CreateNoAliasScopeDeclaration(Value *Scope);
778 Instruction *CreateNoAliasScopeDeclaration(MDNode *ScopeTag) {
779 return CreateNoAliasScopeDeclaration(
780 MetadataAsValue::get(Context, ScopeTag));
781 }
782
783 /// Create a call to the experimental.gc.statepoint intrinsic to
784 /// start a new statepoint sequence.
785 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
786 Value *ActualCallee,
787 ArrayRef<Value *> CallArgs,
788 Optional<ArrayRef<Value *>> DeoptArgs,
789 ArrayRef<Value *> GCArgs,
790 const Twine &Name = "");
791
792 /// Create a call to the experimental.gc.statepoint intrinsic to
793 /// start a new statepoint sequence.
794 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
795 Value *ActualCallee, uint32_t Flags,
796 ArrayRef<Value *> CallArgs,
797 Optional<ArrayRef<Use>> TransitionArgs,
798 Optional<ArrayRef<Use>> DeoptArgs,
799 ArrayRef<Value *> GCArgs,
800 const Twine &Name = "");
801
802 /// Conveninence function for the common case when CallArgs are filled
803 /// in using makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be
804 /// .get()'ed to get the Value pointer.
805 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
806 Value *ActualCallee, ArrayRef<Use> CallArgs,
807 Optional<ArrayRef<Value *>> DeoptArgs,
808 ArrayRef<Value *> GCArgs,
809 const Twine &Name = "");
810
811 /// Create an invoke to the experimental.gc.statepoint intrinsic to
812 /// start a new statepoint sequence.
813 InvokeInst *
814 CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
815 Value *ActualInvokee, BasicBlock *NormalDest,
816 BasicBlock *UnwindDest, ArrayRef<Value *> InvokeArgs,
817 Optional<ArrayRef<Value *>> DeoptArgs,
818 ArrayRef<Value *> GCArgs, const Twine &Name = "");
819
820 /// Create an invoke to the experimental.gc.statepoint intrinsic to
821 /// start a new statepoint sequence.
822 InvokeInst *CreateGCStatepointInvoke(
823 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
824 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
825 ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs,
826 Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
827 const Twine &Name = "");
828
829 // Convenience function for the common case when CallArgs are filled in using
830 // makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be .get()'ed to
831 // get the Value *.
832 InvokeInst *
833 CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
834 Value *ActualInvokee, BasicBlock *NormalDest,
835 BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
836 Optional<ArrayRef<Value *>> DeoptArgs,
837 ArrayRef<Value *> GCArgs, const Twine &Name = "");
838
839 /// Create a call to the experimental.gc.result intrinsic to extract
840 /// the result from a call wrapped in a statepoint.
841 CallInst *CreateGCResult(Instruction *Statepoint,
842 Type *ResultType,
843 const Twine &Name = "");
844
845 /// Create a call to the experimental.gc.relocate intrinsics to
846 /// project the relocated value of one pointer from the statepoint.
847 CallInst *CreateGCRelocate(Instruction *Statepoint,
848 int BaseOffset,
849 int DerivedOffset,
850 Type *ResultType,
851 const Twine &Name = "");
852
853 /// Create a call to llvm.vscale, multiplied by \p Scaling. The type of VScale
854 /// will be the same type as that of \p Scaling.
855 Value *CreateVScale(Constant *Scaling, const Twine &Name = "");
856
857 /// Creates a vector of type \p DstType with the linear sequence <0, 1, ...>
858 Value *CreateStepVector(Type *DstType, const Twine &Name = "");
859
860 /// Create a call to intrinsic \p ID with 1 operand which is mangled on its
861 /// type.
862 CallInst *CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
863 Instruction *FMFSource = nullptr,
864 const Twine &Name = "");
865
866 /// Create a call to intrinsic \p ID with 2 operands which is mangled on the
867 /// first type.
868 CallInst *CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS,
869 Instruction *FMFSource = nullptr,
870 const Twine &Name = "");
871
872 /// Create a call to intrinsic \p ID with \p args, mangled using \p Types. If
873 /// \p FMFSource is provided, copy fast-math-flags from that instruction to
874 /// the intrinsic.
875 CallInst *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Type *> Types,
876 ArrayRef<Value *> Args,
877 Instruction *FMFSource = nullptr,
878 const Twine &Name = "");
879
880 /// Create call to the minnum intrinsic.
881 CallInst *CreateMinNum(Value *LHS, Value *RHS, const Twine &Name = "") {
882 return CreateBinaryIntrinsic(Intrinsic::minnum, LHS, RHS, nullptr, Name);
883 }
884
885 /// Create call to the maxnum intrinsic.
886 CallInst *CreateMaxNum(Value *LHS, Value *RHS, const Twine &Name = "") {
887 return CreateBinaryIntrinsic(Intrinsic::maxnum, LHS, RHS, nullptr, Name);
888 }
889
890 /// Create call to the minimum intrinsic.
891 CallInst *CreateMinimum(Value *LHS, Value *RHS, const Twine &Name = "") {
892 return CreateBinaryIntrinsic(Intrinsic::minimum, LHS, RHS, nullptr, Name);
893 }
894
895 /// Create call to the maximum intrinsic.
896 CallInst *CreateMaximum(Value *LHS, Value *RHS, const Twine &Name = "") {
897 return CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS, nullptr, Name);
898 }
899
900 /// Create a call to the experimental.vector.extract intrinsic.
901 CallInst *CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx,
902 const Twine &Name = "") {
903 return CreateIntrinsic(Intrinsic::experimental_vector_extract,
904 {DstType, SrcVec->getType()}, {SrcVec, Idx}, nullptr,
905 Name);
906 }
907
908 /// Create a call to the experimental.vector.insert intrinsic.
909 CallInst *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
910 Value *Idx, const Twine &Name = "") {
911 return CreateIntrinsic(Intrinsic::experimental_vector_insert,
912 {DstType, SubVec->getType()}, {SrcVec, SubVec, Idx},
913 nullptr, Name);
914 }
915
916private:
917 /// Create a call to a masked intrinsic with given Id.
918 CallInst *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops,
919 ArrayRef<Type *> OverloadedTypes,
920 const Twine &Name = "");
921
922 Value *getCastedInt8PtrValue(Value *Ptr);
923
924 //===--------------------------------------------------------------------===//
925 // Instruction creation methods: Terminators
926 //===--------------------------------------------------------------------===//
927
928private:
929 /// Helper to add branch weight and unpredictable metadata onto an
930 /// instruction.
931 /// \returns The annotated instruction.
932 template <typename InstTy>
933 InstTy *addBranchMetadata(InstTy *I, MDNode *Weights, MDNode *Unpredictable) {
934 if (Weights)
935 I->setMetadata(LLVMContext::MD_prof, Weights);
936 if (Unpredictable)
937 I->setMetadata(LLVMContext::MD_unpredictable, Unpredictable);
938 return I;
939 }
940
941public:
942 /// Create a 'ret void' instruction.
943 ReturnInst *CreateRetVoid() {
944 return Insert(ReturnInst::Create(Context));
945 }
946
947 /// Create a 'ret <val>' instruction.
948 ReturnInst *CreateRet(Value *V) {
949 return Insert(ReturnInst::Create(Context, V));
950 }
951
952 /// Create a sequence of N insertvalue instructions,
953 /// with one Value from the retVals array each, that build a aggregate
954 /// return value one value at a time, and a ret instruction to return
955 /// the resulting aggregate value.
956 ///
957 /// This is a convenience function for code that uses aggregate return values
958 /// as a vehicle for having multiple return values.
959 ReturnInst *CreateAggregateRet(Value *const *retVals, unsigned N) {
960 Value *V = UndefValue::get(getCurrentFunctionReturnType());
961 for (unsigned i = 0; i != N; ++i)
962 V = CreateInsertValue(V, retVals[i], i, "mrv");
963 return Insert(ReturnInst::Create(Context, V));
964 }
965
966 /// Create an unconditional 'br label X' instruction.
967 BranchInst *CreateBr(BasicBlock *Dest) {
968 return Insert(BranchInst::Create(Dest));
969 }
970
971 /// Create a conditional 'br Cond, TrueDest, FalseDest'
972 /// instruction.
973 BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False,
974 MDNode *BranchWeights = nullptr,
975 MDNode *Unpredictable = nullptr) {
976 return Insert(addBranchMetadata(BranchInst::Create(True, False, Cond),
977 BranchWeights, Unpredictable));
978 }
979
980 /// Create a conditional 'br Cond, TrueDest, FalseDest'
981 /// instruction. Copy branch meta data if available.
982 BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False,
983 Instruction *MDSrc) {
984 BranchInst *Br = BranchInst::Create(True, False, Cond);
985 if (MDSrc) {
986 unsigned WL[4] = {LLVMContext::MD_prof, LLVMContext::MD_unpredictable,
987 LLVMContext::MD_make_implicit, LLVMContext::MD_dbg};
988 Br->copyMetadata(*MDSrc, makeArrayRef(&WL[0], 4));
989 }
990 return Insert(Br);
991 }
992
993 /// Create a switch instruction with the specified value, default dest,
994 /// and with a hint for the number of cases that will be added (for efficient
995 /// allocation).
996 SwitchInst *CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases = 10,
997 MDNode *BranchWeights = nullptr,
998 MDNode *Unpredictable = nullptr) {
999 return Insert(addBranchMetadata(SwitchInst::Create(V, Dest, NumCases),
1000 BranchWeights, Unpredictable));
1001 }
1002
1003 /// Create an indirect branch instruction with the specified address
1004 /// operand, with an optional hint for the number of destinations that will be
1005 /// added (for efficient allocation).
1006 IndirectBrInst *CreateIndirectBr(Value *Addr, unsigned NumDests = 10) {
1007 return Insert(IndirectBrInst::Create(Addr, NumDests));
1008 }
1009
1010 /// Create an invoke instruction.
1011 InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee,
1012 BasicBlock *NormalDest, BasicBlock *UnwindDest,
1013 ArrayRef<Value *> Args,
1014 ArrayRef<OperandBundleDef> OpBundles,
1015 const Twine &Name = "") {
1016 InvokeInst *II =
1017 InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args, OpBundles);
1018 if (IsFPConstrained)
1019 setConstrainedFPCallAttr(II);
1020 return Insert(II, Name);
1021 }
1022 InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee,
1023 BasicBlock *NormalDest, BasicBlock *UnwindDest,
1024 ArrayRef<Value *> Args = None,
1025 const Twine &Name = "") {
1026 InvokeInst *II =
1027 InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args);
1028 if (IsFPConstrained)
1029 setConstrainedFPCallAttr(II);
1030 return Insert(II, Name);
1031 }
1032
1033 InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest,
1034 BasicBlock *UnwindDest, ArrayRef<Value *> Args,
1035 ArrayRef<OperandBundleDef> OpBundles,
1036 const Twine &Name = "") {
1037 return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(),
1038 NormalDest, UnwindDest, Args, OpBundles, Name);
1039 }
1040
1041 InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest,
1042 BasicBlock *UnwindDest,
1043 ArrayRef<Value *> Args = None,
1044 const Twine &Name = "") {
1045 return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(),
1046 NormalDest, UnwindDest, Args, Name);
1047 }
1048
1049 /// \brief Create a callbr instruction.
1050 CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee,
1051 BasicBlock *DefaultDest,
1052 ArrayRef<BasicBlock *> IndirectDests,
1053 ArrayRef<Value *> Args = None,
1054 const Twine &Name = "") {
1055 return Insert(CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests,
1056 Args), Name);
1057 }
1058 CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee,
1059 BasicBlock *DefaultDest,
1060 ArrayRef<BasicBlock *> IndirectDests,
1061 ArrayRef<Value *> Args,
1062 ArrayRef<OperandBundleDef> OpBundles,
1063 const Twine &Name = "") {
1064 return Insert(
1065 CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args,
1066 OpBundles), Name);
1067 }
1068
1069 CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest,
1070 ArrayRef<BasicBlock *> IndirectDests,
1071 ArrayRef<Value *> Args = None,
1072 const Twine &Name = "") {
1073 return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(),
1074 DefaultDest, IndirectDests, Args, Name);
1075 }
1076 CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest,
1077 ArrayRef<BasicBlock *> IndirectDests,
1078 ArrayRef<Value *> Args,
1079 ArrayRef<OperandBundleDef> OpBundles,
1080 const Twine &Name = "") {
1081 return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(),
1082 DefaultDest, IndirectDests, Args, Name);
1083 }
1084
1085 ResumeInst *CreateResume(Value *Exn) {
1086 return Insert(ResumeInst::Create(Exn));
1087 }
1088
1089 CleanupReturnInst *CreateCleanupRet(CleanupPadInst *CleanupPad,
1090 BasicBlock *UnwindBB = nullptr) {
1091 return Insert(CleanupReturnInst::Create(CleanupPad, UnwindBB));
1092 }
1093
1094 CatchSwitchInst *CreateCatchSwitch(Value *ParentPad, BasicBlock *UnwindBB,
1095 unsigned NumHandlers,
1096 const Twine &Name = "") {
1097 return Insert(CatchSwitchInst::Create(ParentPad, UnwindBB, NumHandlers),
1098 Name);
1099 }
1100
1101 CatchPadInst *CreateCatchPad(Value *ParentPad, ArrayRef<Value *> Args,
1102 const Twine &Name = "") {
1103 return Insert(CatchPadInst::Create(ParentPad, Args), Name);
1104 }
1105
1106 CleanupPadInst *CreateCleanupPad(Value *ParentPad,
1107 ArrayRef<Value *> Args = None,
1108 const Twine &Name = "") {
1109 return Insert(CleanupPadInst::Create(ParentPad, Args), Name);
1110 }
1111
1112 CatchReturnInst *CreateCatchRet(CatchPadInst *CatchPad, BasicBlock *BB) {
1113 return Insert(CatchReturnInst::Create(CatchPad, BB));
1114 }
1115
1116 UnreachableInst *CreateUnreachable() {
1117 return Insert(new UnreachableInst(Context));
1118 }
1119
1120 //===--------------------------------------------------------------------===//
1121 // Instruction creation methods: Binary Operators
1122 //===--------------------------------------------------------------------===//
1123private:
1124 BinaryOperator *CreateInsertNUWNSWBinOp(BinaryOperator::BinaryOps Opc,
1125 Value *LHS, Value *RHS,
1126 const Twine &Name,
1127 bool HasNUW, bool HasNSW) {
1128 BinaryOperator *BO = Insert(BinaryOperator::Create(Opc, LHS, RHS), Name);
1129 if (HasNUW) BO->setHasNoUnsignedWrap();
1130 if (HasNSW) BO->setHasNoSignedWrap();
1131 return BO;
1132 }
1133
1134 Instruction *setFPAttrs(Instruction *I, MDNode *FPMD,
1135 FastMathFlags FMF) const {
1136 if (!FPMD)
1137 FPMD = DefaultFPMathTag;
1138 if (FPMD)
1139 I->setMetadata(LLVMContext::MD_fpmath, FPMD);
1140 I->setFastMathFlags(FMF);
1141 return I;
1142 }
1143
1144 Value *foldConstant(Instruction::BinaryOps Opc, Value *L,
1145 Value *R, const Twine &Name) const {
1146 auto *LC = dyn_cast<Constant>(L);
1147 auto *RC = dyn_cast<Constant>(R);
1148 return (LC && RC) ? Insert(Folder.CreateBinOp(Opc, LC, RC), Name) : nullptr;
1149 }
1150
1151 Value *getConstrainedFPRounding(Optional<RoundingMode> Rounding) {
1152 RoundingMode UseRounding = DefaultConstrainedRounding;
1153
1154 if (Rounding.hasValue())
1155 UseRounding = Rounding.getValue();
1156
1157 Optional<StringRef> RoundingStr = RoundingModeToStr(UseRounding);
1158 assert(RoundingStr.hasValue() && "Garbage strict rounding mode!")((RoundingStr.hasValue() && "Garbage strict rounding mode!"
) ? static_cast<void> (0) : __assert_fail ("RoundingStr.hasValue() && \"Garbage strict rounding mode!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1158, __PRETTY_FUNCTION__))
;
1159 auto *RoundingMDS = MDString::get(Context, RoundingStr.getValue());
1160
1161 return MetadataAsValue::get(Context, RoundingMDS);
1162 }
1163
1164 Value *getConstrainedFPExcept(Optional<fp::ExceptionBehavior> Except) {
1165 fp::ExceptionBehavior UseExcept = DefaultConstrainedExcept;
1166
1167 if (Except.hasValue())
1168 UseExcept = Except.getValue();
1169
1170 Optional<StringRef> ExceptStr = ExceptionBehaviorToStr(UseExcept);
1171 assert(ExceptStr.hasValue() && "Garbage strict exception behavior!")((ExceptStr.hasValue() && "Garbage strict exception behavior!"
) ? static_cast<void> (0) : __assert_fail ("ExceptStr.hasValue() && \"Garbage strict exception behavior!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1171, __PRETTY_FUNCTION__))
;
1172 auto *ExceptMDS = MDString::get(Context, ExceptStr.getValue());
1173
1174 return MetadataAsValue::get(Context, ExceptMDS);
1175 }
1176
1177 Value *getConstrainedFPPredicate(CmpInst::Predicate Predicate) {
1178 assert(CmpInst::isFPPredicate(Predicate) &&((CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst
::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE &&
"Invalid constrained FP comparison predicate!") ? static_cast
<void> (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1181, __PRETTY_FUNCTION__))
1179 Predicate != CmpInst::FCMP_FALSE &&((CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst
::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE &&
"Invalid constrained FP comparison predicate!") ? static_cast
<void> (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1181, __PRETTY_FUNCTION__))
1180 Predicate != CmpInst::FCMP_TRUE &&((CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst
::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE &&
"Invalid constrained FP comparison predicate!") ? static_cast
<void> (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1181, __PRETTY_FUNCTION__))
1181 "Invalid constrained FP comparison predicate!")((CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst
::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE &&
"Invalid constrained FP comparison predicate!") ? static_cast
<void> (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1181, __PRETTY_FUNCTION__))
;
1182
1183 StringRef PredicateStr = CmpInst::getPredicateName(Predicate);
1184 auto *PredicateMDS = MDString::get(Context, PredicateStr);
1185
1186 return MetadataAsValue::get(Context, PredicateMDS);
1187 }
1188
1189public:
1190 Value *CreateAdd(Value *LHS, Value *RHS, const Twine &Name = "",
1191 bool HasNUW = false, bool HasNSW = false) {
1192 if (auto *LC = dyn_cast<Constant>(LHS))
1193 if (auto *RC = dyn_cast<Constant>(RHS))
1194 return Insert(Folder.CreateAdd(LC, RC, HasNUW, HasNSW), Name);
1195 return CreateInsertNUWNSWBinOp(Instruction::Add, LHS, RHS, Name,
1196 HasNUW, HasNSW);
1197 }
1198
1199 Value *CreateNSWAdd(Value *LHS, Value *RHS, const Twine &Name = "") {
1200 return CreateAdd(LHS, RHS, Name, false, true);
1201 }
1202
1203 Value *CreateNUWAdd(Value *LHS, Value *RHS, const Twine &Name = "") {
1204 return CreateAdd(LHS, RHS, Name, true, false);
1205 }
1206
1207 Value *CreateSub(Value *LHS, Value *RHS, const Twine &Name = "",
1208 bool HasNUW = false, bool HasNSW = false) {
1209 if (auto *LC = dyn_cast<Constant>(LHS))
1210 if (auto *RC = dyn_cast<Constant>(RHS))
1211 return Insert(Folder.CreateSub(LC, RC, HasNUW, HasNSW), Name);
1212 return CreateInsertNUWNSWBinOp(Instruction::Sub, LHS, RHS, Name,
1213 HasNUW, HasNSW);
1214 }
1215
1216 Value *CreateNSWSub(Value *LHS, Value *RHS, const Twine &Name = "") {
1217 return CreateSub(LHS, RHS, Name, false, true);
1218 }
1219
1220 Value *CreateNUWSub(Value *LHS, Value *RHS, const Twine &Name = "") {
1221 return CreateSub(LHS, RHS, Name, true, false);
1222 }
1223
1224 Value *CreateMul(Value *LHS, Value *RHS, const Twine &Name = "",
1225 bool HasNUW = false, bool HasNSW = false) {
1226 if (auto *LC = dyn_cast<Constant>(LHS))
1227 if (auto *RC = dyn_cast<Constant>(RHS))
1228 return Insert(Folder.CreateMul(LC, RC, HasNUW, HasNSW), Name);
1229 return CreateInsertNUWNSWBinOp(Instruction::Mul, LHS, RHS, Name,
1230 HasNUW, HasNSW);
1231 }
1232
1233 Value *CreateNSWMul(Value *LHS, Value *RHS, const Twine &Name = "") {
1234 return CreateMul(LHS, RHS, Name, false, true);
1235 }
1236
1237 Value *CreateNUWMul(Value *LHS, Value *RHS, const Twine &Name = "") {
1238 return CreateMul(LHS, RHS, Name, true, false);
1239 }
1240
1241 Value *CreateUDiv(Value *LHS, Value *RHS, const Twine &Name = "",
1242 bool isExact = false) {
1243 if (auto *LC = dyn_cast<Constant>(LHS))
1244 if (auto *RC = dyn_cast<Constant>(RHS))
1245 return Insert(Folder.CreateUDiv(LC, RC, isExact), Name);
1246 if (!isExact)
1247 return Insert(BinaryOperator::CreateUDiv(LHS, RHS), Name);
1248 return Insert(BinaryOperator::CreateExactUDiv(LHS, RHS), Name);
1249 }
1250
1251 Value *CreateExactUDiv(Value *LHS, Value *RHS, const Twine &Name = "") {
1252 return CreateUDiv(LHS, RHS, Name, true);
1253 }
1254
1255 Value *CreateSDiv(Value *LHS, Value *RHS, const Twine &Name = "",
1256 bool isExact = false) {
1257 if (auto *LC = dyn_cast<Constant>(LHS))
1258 if (auto *RC = dyn_cast<Constant>(RHS))
1259 return Insert(Folder.CreateSDiv(LC, RC, isExact), Name);
1260 if (!isExact)
1261 return Insert(BinaryOperator::CreateSDiv(LHS, RHS), Name);
1262 return Insert(BinaryOperator::CreateExactSDiv(LHS, RHS), Name);
1263 }
1264
1265 Value *CreateExactSDiv(Value *LHS, Value *RHS, const Twine &Name = "") {
1266 return CreateSDiv(LHS, RHS, Name, true);
1267 }
1268
1269 Value *CreateURem(Value *LHS, Value *RHS, const Twine &Name = "") {
1270 if (Value *V = foldConstant(Instruction::URem, LHS, RHS, Name)) return V;
1271 return Insert(BinaryOperator::CreateURem(LHS, RHS), Name);
1272 }
1273
1274 Value *CreateSRem(Value *LHS, Value *RHS, const Twine &Name = "") {
1275 if (Value *V = foldConstant(Instruction::SRem, LHS, RHS, Name)) return V;
1276 return Insert(BinaryOperator::CreateSRem(LHS, RHS), Name);
1277 }
1278
1279 Value *CreateShl(Value *LHS, Value *RHS, const Twine &Name = "",
1280 bool HasNUW = false, bool HasNSW = false) {
1281 if (auto *LC = dyn_cast<Constant>(LHS))
1282 if (auto *RC = dyn_cast<Constant>(RHS))
1283 return Insert(Folder.CreateShl(LC, RC, HasNUW, HasNSW), Name);
1284 return CreateInsertNUWNSWBinOp(Instruction::Shl, LHS, RHS, Name,
1285 HasNUW, HasNSW);
1286 }
1287
1288 Value *CreateShl(Value *LHS, const APInt &RHS, const Twine &Name = "",
1289 bool HasNUW = false, bool HasNSW = false) {
1290 return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name,
1291 HasNUW, HasNSW);
1292 }
1293
1294 Value *CreateShl(Value *LHS, uint64_t RHS, const Twine &Name = "",
1295 bool HasNUW = false, bool HasNSW = false) {
1296 return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name,
1297 HasNUW, HasNSW);
1298 }
1299
1300 Value *CreateLShr(Value *LHS, Value *RHS, const Twine &Name = "",
1301 bool isExact = false) {
1302 if (auto *LC = dyn_cast<Constant>(LHS))
1303 if (auto *RC = dyn_cast<Constant>(RHS))
1304 return Insert(Folder.CreateLShr(LC, RC, isExact), Name);
1305 if (!isExact)
1306 return Insert(BinaryOperator::CreateLShr(LHS, RHS), Name);
1307 return Insert(BinaryOperator::CreateExactLShr(LHS, RHS), Name);
1308 }
1309
1310 Value *CreateLShr(Value *LHS, const APInt &RHS, const Twine &Name = "",
1311 bool isExact = false) {
1312 return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1313 }
1314
1315 Value *CreateLShr(Value *LHS, uint64_t RHS, const Twine &Name = "",
1316 bool isExact = false) {
1317 return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1318 }
1319
1320 Value *CreateAShr(Value *LHS, Value *RHS, const Twine &Name = "",
1321 bool isExact = false) {
1322 if (auto *LC = dyn_cast<Constant>(LHS))
1323 if (auto *RC = dyn_cast<Constant>(RHS))
1324 return Insert(Folder.CreateAShr(LC, RC, isExact), Name);
1325 if (!isExact)
1326 return Insert(BinaryOperator::CreateAShr(LHS, RHS), Name);
1327 return Insert(BinaryOperator::CreateExactAShr(LHS, RHS), Name);
1328 }
1329
1330 Value *CreateAShr(Value *LHS, const APInt &RHS, const Twine &Name = "",
1331 bool isExact = false) {
1332 return CreateAShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1333 }
1334
1335 Value *CreateAShr(Value *LHS, uint64_t RHS, const Twine &Name = "",
1336 bool isExact = false) {
1337 return CreateAShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1338 }
1339
1340 Value *CreateAnd(Value *LHS, Value *RHS, const Twine &Name = "") {
1341 if (auto *RC = dyn_cast<Constant>(RHS)) {
1342 if (isa<ConstantInt>(RC) && cast<ConstantInt>(RC)->isMinusOne())
1343 return LHS; // LHS & -1 -> LHS
1344 if (auto *LC = dyn_cast<Constant>(LHS))
1345 return Insert(Folder.CreateAnd(LC, RC), Name);
1346 }
1347 return Insert(BinaryOperator::CreateAnd(LHS, RHS), Name);
1348 }
1349
1350 Value *CreateAnd(Value *LHS, const APInt &RHS, const Twine &Name = "") {
1351 return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1352 }
1353
1354 Value *CreateAnd(Value *LHS, uint64_t RHS, const Twine &Name = "") {
1355 return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1356 }
1357
1358 Value *CreateAnd(ArrayRef<Value*> Ops) {
1359 assert(!Ops.empty())((!Ops.empty()) ? static_cast<void> (0) : __assert_fail
("!Ops.empty()", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1359, __PRETTY_FUNCTION__))
;
1360 Value *Accum = Ops[0];
1361 for (unsigned i = 1; i < Ops.size(); i++)
1362 Accum = CreateAnd(Accum, Ops[i]);
1363 return Accum;
1364 }
1365
1366 Value *CreateOr(Value *LHS, Value *RHS, const Twine &Name = "") {
1367 if (auto *RC = dyn_cast<Constant>(RHS)) {
1368 if (RC->isNullValue())
1369 return LHS; // LHS | 0 -> LHS
1370 if (auto *LC = dyn_cast<Constant>(LHS))
1371 return Insert(Folder.CreateOr(LC, RC), Name);
1372 }
1373 return Insert(BinaryOperator::CreateOr(LHS, RHS), Name);
1374 }
1375
1376 Value *CreateOr(Value *LHS, const APInt &RHS, const Twine &Name = "") {
1377 return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1378 }
1379
1380 Value *CreateOr(Value *LHS, uint64_t RHS, const Twine &Name = "") {
1381 return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1382 }
1383
1384 Value *CreateOr(ArrayRef<Value*> Ops) {
1385 assert(!Ops.empty())((!Ops.empty()) ? static_cast<void> (0) : __assert_fail
("!Ops.empty()", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1385, __PRETTY_FUNCTION__))
;
1386 Value *Accum = Ops[0];
1387 for (unsigned i = 1; i < Ops.size(); i++)
1388 Accum = CreateOr(Accum, Ops[i]);
1389 return Accum;
1390 }
1391
1392 Value *CreateXor(Value *LHS, Value *RHS, const Twine &Name = "") {
1393 if (Value *V = foldConstant(Instruction::Xor, LHS, RHS, Name)) return V;
1394 return Insert(BinaryOperator::CreateXor(LHS, RHS), Name);
1395 }
1396
1397 Value *CreateXor(Value *LHS, const APInt &RHS, const Twine &Name = "") {
1398 return CreateXor(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1399 }
1400
1401 Value *CreateXor(Value *LHS, uint64_t RHS, const Twine &Name = "") {
1402 return CreateXor(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1403 }
1404
1405 Value *CreateFAdd(Value *L, Value *R, const Twine &Name = "",
1406 MDNode *FPMD = nullptr) {
1407 if (IsFPConstrained)
1408 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd,
1409 L, R, nullptr, Name, FPMD);
1410
1411 if (Value *V = foldConstant(Instruction::FAdd, L, R, Name)) return V;
1412 Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), FPMD, FMF);
1413 return Insert(I, Name);
1414 }
1415
1416 /// Copy fast-math-flags from an instruction rather than using the builder's
1417 /// default FMF.
1418 Value *CreateFAddFMF(Value *L, Value *R, Instruction *FMFSource,
1419 const Twine &Name = "") {
1420 if (IsFPConstrained)
1421 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd,
1422 L, R, FMFSource, Name);
1423
1424 if (Value *V = foldConstant(Instruction::FAdd, L, R, Name)) return V;
1425 Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), nullptr,
1426 FMFSource->getFastMathFlags());
1427 return Insert(I, Name);
1428 }
1429
1430 Value *CreateFSub(Value *L, Value *R, const Twine &Name = "",
1431 MDNode *FPMD = nullptr) {
1432 if (IsFPConstrained)
1433 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub,
1434 L, R, nullptr, Name, FPMD);
1435
1436 if (Value *V = foldConstant(Instruction::FSub, L, R, Name)) return V;
1437 Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), FPMD, FMF);
1438 return Insert(I, Name);
1439 }
1440
1441 /// Copy fast-math-flags from an instruction rather than using the builder's
1442 /// default FMF.
1443 Value *CreateFSubFMF(Value *L, Value *R, Instruction *FMFSource,
1444 const Twine &Name = "") {
1445 if (IsFPConstrained)
1446 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub,
1447 L, R, FMFSource, Name);
1448
1449 if (Value *V = foldConstant(Instruction::FSub, L, R, Name)) return V;
1450 Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), nullptr,
1451 FMFSource->getFastMathFlags());
1452 return Insert(I, Name);
1453 }
1454
1455 Value *CreateFMul(Value *L, Value *R, const Twine &Name = "",
1456 MDNode *FPMD = nullptr) {
1457 if (IsFPConstrained)
1458 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul,
1459 L, R, nullptr, Name, FPMD);
1460
1461 if (Value *V = foldConstant(Instruction::FMul, L, R, Name)) return V;
1462 Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), FPMD, FMF);
1463 return Insert(I, Name);
1464 }
1465
1466 /// Copy fast-math-flags from an instruction rather than using the builder's
1467 /// default FMF.
1468 Value *CreateFMulFMF(Value *L, Value *R, Instruction *FMFSource,
1469 const Twine &Name = "") {
1470 if (IsFPConstrained)
1471 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul,
1472 L, R, FMFSource, Name);
1473
1474 if (Value *V = foldConstant(Instruction::FMul, L, R, Name)) return V;
1475 Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), nullptr,
1476 FMFSource->getFastMathFlags());
1477 return Insert(I, Name);
1478 }
1479
1480 Value *CreateFDiv(Value *L, Value *R, const Twine &Name = "",
1481 MDNode *FPMD = nullptr) {
1482 if (IsFPConstrained)
1483 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv,
1484 L, R, nullptr, Name, FPMD);
1485
1486 if (Value *V = foldConstant(Instruction::FDiv, L, R, Name)) return V;
1487 Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), FPMD, FMF);
1488 return Insert(I, Name);
1489 }
1490
1491 /// Copy fast-math-flags from an instruction rather than using the builder's
1492 /// default FMF.
1493 Value *CreateFDivFMF(Value *L, Value *R, Instruction *FMFSource,
1494 const Twine &Name = "") {
1495 if (IsFPConstrained)
1496 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv,
1497 L, R, FMFSource, Name);
1498
1499 if (Value *V = foldConstant(Instruction::FDiv, L, R, Name)) return V;
1500 Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), nullptr,
1501 FMFSource->getFastMathFlags());
1502 return Insert(I, Name);
1503 }
1504
1505 Value *CreateFRem(Value *L, Value *R, const Twine &Name = "",
1506 MDNode *FPMD = nullptr) {
1507 if (IsFPConstrained)
1508 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem,
1509 L, R, nullptr, Name, FPMD);
1510
1511 if (Value *V = foldConstant(Instruction::FRem, L, R, Name)) return V;
1512 Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), FPMD, FMF);
1513 return Insert(I, Name);
1514 }
1515
1516 /// Copy fast-math-flags from an instruction rather than using the builder's
1517 /// default FMF.
1518 Value *CreateFRemFMF(Value *L, Value *R, Instruction *FMFSource,
1519 const Twine &Name = "") {
1520 if (IsFPConstrained)
1521 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem,
1522 L, R, FMFSource, Name);
1523
1524 if (Value *V = foldConstant(Instruction::FRem, L, R, Name)) return V;
1525 Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), nullptr,
1526 FMFSource->getFastMathFlags());
1527 return Insert(I, Name);
1528 }
1529
1530 Value *CreateBinOp(Instruction::BinaryOps Opc,
1531 Value *LHS, Value *RHS, const Twine &Name = "",
1532 MDNode *FPMathTag = nullptr) {
1533 if (Value *V = foldConstant(Opc, LHS, RHS, Name)) return V;
1534 Instruction *BinOp = BinaryOperator::Create(Opc, LHS, RHS);
1535 if (isa<FPMathOperator>(BinOp))
1536 setFPAttrs(BinOp, FPMathTag, FMF);
1537 return Insert(BinOp, Name);
1538 }
1539
1540 Value *CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name = "") {
1541 assert(Cond2->getType()->isIntOrIntVectorTy(1))((Cond2->getType()->isIntOrIntVectorTy(1)) ? static_cast
<void> (0) : __assert_fail ("Cond2->getType()->isIntOrIntVectorTy(1)"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1541, __PRETTY_FUNCTION__))
;
1542 return CreateSelect(Cond1, Cond2,
1543 ConstantInt::getNullValue(Cond2->getType()), Name);
1544 }
1545
1546 Value *CreateLogicalOr(Value *Cond1, Value *Cond2, const Twine &Name = "") {
1547 assert(Cond2->getType()->isIntOrIntVectorTy(1))((Cond2->getType()->isIntOrIntVectorTy(1)) ? static_cast
<void> (0) : __assert_fail ("Cond2->getType()->isIntOrIntVectorTy(1)"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1547, __PRETTY_FUNCTION__))
;
1548 return CreateSelect(Cond1, ConstantInt::getAllOnesValue(Cond2->getType()),
1549 Cond2, Name);
1550 }
1551
1552 CallInst *CreateConstrainedFPBinOp(
1553 Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource = nullptr,
1554 const Twine &Name = "", MDNode *FPMathTag = nullptr,
1555 Optional<RoundingMode> Rounding = None,
1556 Optional<fp::ExceptionBehavior> Except = None);
1557
1558 Value *CreateNeg(Value *V, const Twine &Name = "",
1559 bool HasNUW = false, bool HasNSW = false) {
1560 if (auto *VC = dyn_cast<Constant>(V))
1561 return Insert(Folder.CreateNeg(VC, HasNUW, HasNSW), Name);
1562 BinaryOperator *BO = Insert(BinaryOperator::CreateNeg(V), Name);
1563 if (HasNUW) BO->setHasNoUnsignedWrap();
1564 if (HasNSW) BO->setHasNoSignedWrap();
1565 return BO;
1566 }
1567
1568 Value *CreateNSWNeg(Value *V, const Twine &Name = "") {
1569 return CreateNeg(V, Name, false, true);
1570 }
1571
1572 Value *CreateNUWNeg(Value *V, const Twine &Name = "") {
1573 return CreateNeg(V, Name, true, false);
1574 }
1575
1576 Value *CreateFNeg(Value *V, const Twine &Name = "",
1577 MDNode *FPMathTag = nullptr) {
1578 if (auto *VC = dyn_cast<Constant>(V))
1579 return Insert(Folder.CreateFNeg(VC), Name);
1580 return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), FPMathTag, FMF),
1581 Name);
1582 }
1583
1584 /// Copy fast-math-flags from an instruction rather than using the builder's
1585 /// default FMF.
1586 Value *CreateFNegFMF(Value *V, Instruction *FMFSource,
1587 const Twine &Name = "") {
1588 if (auto *VC = dyn_cast<Constant>(V))
1589 return Insert(Folder.CreateFNeg(VC), Name);
1590 return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), nullptr,
1591 FMFSource->getFastMathFlags()),
1592 Name);
1593 }
1594
1595 Value *CreateNot(Value *V, const Twine &Name = "") {
1596 if (auto *VC = dyn_cast<Constant>(V))
1597 return Insert(Folder.CreateNot(VC), Name);
1598 return Insert(BinaryOperator::CreateNot(V), Name);
1599 }
1600
1601 Value *CreateUnOp(Instruction::UnaryOps Opc,
1602 Value *V, const Twine &Name = "",
1603 MDNode *FPMathTag = nullptr) {
1604 if (auto *VC = dyn_cast<Constant>(V))
1605 return Insert(Folder.CreateUnOp(Opc, VC), Name);
1606 Instruction *UnOp = UnaryOperator::Create(Opc, V);
1607 if (isa<FPMathOperator>(UnOp))
1608 setFPAttrs(UnOp, FPMathTag, FMF);
1609 return Insert(UnOp, Name);
1610 }
1611
1612 /// Create either a UnaryOperator or BinaryOperator depending on \p Opc.
1613 /// Correct number of operands must be passed accordingly.
1614 Value *CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
1615 const Twine &Name = "", MDNode *FPMathTag = nullptr);
1616
1617 //===--------------------------------------------------------------------===//
1618 // Instruction creation methods: Memory Instructions
1619 //===--------------------------------------------------------------------===//
1620
1621 AllocaInst *CreateAlloca(Type *Ty, unsigned AddrSpace,
1622 Value *ArraySize = nullptr, const Twine &Name = "") {
1623 const DataLayout &DL = BB->getModule()->getDataLayout();
1624 Align AllocaAlign = DL.getPrefTypeAlign(Ty);
1625 return Insert(new AllocaInst(Ty, AddrSpace, ArraySize, AllocaAlign), Name);
1626 }
1627
1628 AllocaInst *CreateAlloca(Type *Ty, Value *ArraySize = nullptr,
1629 const Twine &Name = "") {
1630 const DataLayout &DL = BB->getModule()->getDataLayout();
1631 Align AllocaAlign = DL.getPrefTypeAlign(Ty);
1632 unsigned AddrSpace = DL.getAllocaAddrSpace();
1633 return Insert(new AllocaInst(Ty, AddrSpace, ArraySize, AllocaAlign), Name);
1634 }
1635
1636 /// Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of
1637 /// converting the string to 'bool' for the isVolatile parameter.
1638 LoadInst *CreateLoad(Type *Ty, Value *Ptr, const char *Name) {
1639 return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), Name);
1640 }
1641
1642 LoadInst *CreateLoad(Type *Ty, Value *Ptr, const Twine &Name = "") {
1643 return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), Name);
37
Calling 'IRBuilderBase::CreateAlignedLoad'
1644 }
1645
1646 LoadInst *CreateLoad(Type *Ty, Value *Ptr, bool isVolatile,
1647 const Twine &Name = "") {
1648 return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), isVolatile, Name);
1649 }
1650
1651 // Deprecated [opaque pointer types]
1652 LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateLoad(Value *Ptr,[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
char *Name)
1653 const char *Name),[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
char *Name)
1654 "Use the version that explicitly specifies the "[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
char *Name)
1655 "loaded type instead")[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
char *Name)
{
1656 return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, Name);
1657 }
1658
1659 // Deprecated [opaque pointer types]
1660 LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateLoad(Value *Ptr,[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
Twine &Name = "")
1661 const Twine &Name = ""),[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
Twine &Name = "")
1662 "Use the version that explicitly specifies the "[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
Twine &Name = "")
1663 "loaded type instead")[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
Twine &Name = "")
{
1664 return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, Name);
1665 }
1666
1667 // Deprecated [opaque pointer types]
1668 LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateLoad(Value *Ptr,[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, bool
isVolatile, const Twine &Name = "")
1669 bool isVolatile,[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, bool
isVolatile, const Twine &Name = "")
1670 const Twine &Name = ""),[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, bool
isVolatile, const Twine &Name = "")
1671 "Use the version that explicitly specifies the "[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, bool
isVolatile, const Twine &Name = "")
1672 "loaded type instead")[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, bool
isVolatile, const Twine &Name = "")
{
1673 return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, isVolatile,
1674 Name);
1675 }
1676
1677 StoreInst *CreateStore(Value *Val, Value *Ptr, bool isVolatile = false) {
1678 return CreateAlignedStore(Val, Ptr, MaybeAlign(), isVolatile);
1679 }
1680
1681 LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
1682 const char *Name) {
1683 return CreateAlignedLoad(Ty, Ptr, Align, /*isVolatile*/false, Name);
1684 }
1685
1686 LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
1687 const Twine &Name = "") {
1688 return CreateAlignedLoad(Ty, Ptr, Align, /*isVolatile*/false, Name);
38
Calling 'IRBuilderBase::CreateAlignedLoad'
1689 }
1690
1691 LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
1692 bool isVolatile, const Twine &Name = "") {
1693 if (!Align) {
39
Calling 'Optional::operator bool'
47
Returning from 'Optional::operator bool'
48
Taking true branch
1694 const DataLayout &DL = BB->getModule()->getDataLayout();
49
Called C++ object pointer is null
1695 Align = DL.getABITypeAlign(Ty);
1696 }
1697 return Insert(new LoadInst(Ty, Ptr, Twine(), isVolatile, *Align), Name);
1698 }
1699
1700 // Deprecated [opaque pointer types]
1701 LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateAlignedLoad(Value *Ptr,[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const char *Name)
1702 MaybeAlign Align,[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const char *Name)
1703 const char *Name),[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const char *Name)
1704 "Use the version that explicitly specifies the "[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const char *Name)
1705 "loaded type instead")[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const char *Name)
{
1706 return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr,
1707 Align, Name);
1708 }
1709 // Deprecated [opaque pointer types]
1710 LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateAlignedLoad(Value *Ptr,[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const Twine &Name = "")
1711 MaybeAlign Align,[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const Twine &Name = "")
1712 const Twine &Name = ""),[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const Twine &Name = "")
1713 "Use the version that explicitly specifies the "[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const Twine &Name = "")
1714 "loaded type instead")[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const Twine &Name = "")
{
1715 return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr,
1716 Align, Name);
1717 }
1718 // Deprecated [opaque pointer types]
1719 LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateAlignedLoad(Value *Ptr,[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, bool isVolatile, const Twine &Name
= "")
1720 MaybeAlign Align,[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, bool isVolatile, const Twine &Name
= "")
1721 bool isVolatile,[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, bool isVolatile, const Twine &Name
= "")
1722 const Twine &Name = ""),[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, bool isVolatile, const Twine &Name
= "")
1723 "Use the version that explicitly specifies the "[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, bool isVolatile, const Twine &Name
= "")
1724 "loaded type instead")[[deprecated("Use the version that explicitly specifies the "
"loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, bool isVolatile, const Twine &Name
= "")
{
1725 return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr,
1726 Align, isVolatile, Name);
1727 }
1728
1729 StoreInst *CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align,
1730 bool isVolatile = false) {
1731 if (!Align) {
1732 const DataLayout &DL = BB->getModule()->getDataLayout();
1733 Align = DL.getABITypeAlign(Val->getType());
1734 }
1735 return Insert(new StoreInst(Val, Ptr, isVolatile, *Align));
1736 }
1737 FenceInst *CreateFence(AtomicOrdering Ordering,
1738 SyncScope::ID SSID = SyncScope::System,
1739 const Twine &Name = "") {
1740 return Insert(new FenceInst(Context, Ordering, SSID), Name);
1741 }
1742
1743 AtomicCmpXchgInst *
1744 CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align,
1745 AtomicOrdering SuccessOrdering,
1746 AtomicOrdering FailureOrdering,
1747 SyncScope::ID SSID = SyncScope::System) {
1748 if (!Align) {
1749 const DataLayout &DL = BB->getModule()->getDataLayout();
1750 Align = llvm::Align(DL.getTypeStoreSize(New->getType()));
1751 }
1752
1753 return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, *Align, SuccessOrdering,
1754 FailureOrdering, SSID));
1755 }
1756
1757 AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr,
1758 Value *Val, MaybeAlign Align,
1759 AtomicOrdering Ordering,
1760 SyncScope::ID SSID = SyncScope::System) {
1761 if (!Align) {
1762 const DataLayout &DL = BB->getModule()->getDataLayout();
1763 Align = llvm::Align(DL.getTypeStoreSize(Val->getType()));
1764 }
1765
1766 return Insert(new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID));
1767 }
1768
1769 Value *CreateGEP(Value *Ptr, ArrayRef<Value *> IdxList,
1770 const Twine &Name = "") {
1771 return CreateGEP(nullptr, Ptr, IdxList, Name);
1772 }
1773
1774 Value *CreateGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList,
1775 const Twine &Name = "") {
1776 if (auto *PC = dyn_cast<Constant>(Ptr)) {
1777 // Every index must be constant.
1778 size_t i, e;
1779 for (i = 0, e = IdxList.size(); i != e; ++i)
1780 if (!isa<Constant>(IdxList[i]))
1781 break;
1782 if (i == e)
1783 return Insert(Folder.CreateGetElementPtr(Ty, PC, IdxList), Name);
1784 }
1785 return Insert(GetElementPtrInst::Create(Ty, Ptr, IdxList), Name);
1786 }
1787
1788 Value *CreateInBoundsGEP(Value *Ptr, ArrayRef<Value *> IdxList,
1789 const Twine &Name = "") {
1790 return CreateInBoundsGEP(nullptr, Ptr, IdxList, Name);
1791 }
1792
1793 Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList,
1794 const Twine &Name = "") {
1795 if (auto *PC = dyn_cast<Constant>(Ptr)) {
1796 // Every index must be constant.
1797 size_t i, e;
1798 for (i = 0, e = IdxList.size(); i != e; ++i)
1799 if (!isa<Constant>(IdxList[i]))
1800 break;
1801 if (i == e)
1802 return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, IdxList),
1803 Name);
1804 }
1805 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, IdxList), Name);
1806 }
1807
1808 Value *CreateGEP(Value *Ptr, Value *Idx, const Twine &Name = "") {
1809 return CreateGEP(nullptr, Ptr, Idx, Name);
1810 }
1811
1812 Value *CreateGEP(Type *Ty, Value *Ptr, Value *Idx, const Twine &Name = "") {
1813 if (auto *PC = dyn_cast<Constant>(Ptr))
1814 if (auto *IC = dyn_cast<Constant>(Idx))
1815 return Insert(Folder.CreateGetElementPtr(Ty, PC, IC), Name);
1816 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name);
1817 }
1818
1819 Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, Value *Idx,
1820 const Twine &Name = "") {
1821 if (auto *PC = dyn_cast<Constant>(Ptr))
1822 if (auto *IC = dyn_cast<Constant>(Idx))
1823 return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, IC), Name);
1824 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name);
1825 }
1826
1827 Value *CreateConstGEP1_32(Value *Ptr, unsigned Idx0, const Twine &Name = "") {
1828 return CreateConstGEP1_32(nullptr, Ptr, Idx0, Name);
1829 }
1830
1831 Value *CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0,
1832 const Twine &Name = "") {
1833 Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0);
1834
1835 if (auto *PC = dyn_cast<Constant>(Ptr))
1836 return Insert(Folder.CreateGetElementPtr(Ty, PC, Idx), Name);
1837
1838 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name);
1839 }
1840
1841 Value *CreateConstInBoundsGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0,
1842 const Twine &Name = "") {
1843 Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0);
1844
1845 if (auto *PC = dyn_cast<Constant>(Ptr))
1846 return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idx), Name);
1847
1848 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name);
1849 }
1850
1851 Value *CreateConstGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, unsigned Idx1,
1852 const Twine &Name = "") {
1853 Value *Idxs[] = {
1854 ConstantInt::get(Type::getInt32Ty(Context), Idx0),
1855 ConstantInt::get(Type::getInt32Ty(Context), Idx1)
1856 };
1857
1858 if (auto *PC = dyn_cast<Constant>(Ptr))
1859 return Insert(Folder.CreateGetElementPtr(Ty, PC, Idxs), Name);
1860
1861 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name);
1862 }
1863
1864 Value *CreateConstInBoundsGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0,
1865 unsigned Idx1, const Twine &Name = "") {
1866 Value *Idxs[] = {
1867 ConstantInt::get(Type::getInt32Ty(Context), Idx0),
1868 ConstantInt::get(Type::getInt32Ty(Context), Idx1)
1869 };
1870
1871 if (auto *PC = dyn_cast<Constant>(Ptr))
1872 return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idxs), Name);
1873
1874 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name);
1875 }
1876
1877 Value *CreateConstGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0,
1878 const Twine &Name = "") {
1879 Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0);
1880
1881 if (auto *PC = dyn_cast<Constant>(Ptr))
1882 return Insert(Folder.CreateGetElementPtr(Ty, PC, Idx), Name);
1883
1884 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name);
1885 }
1886
1887 Value *CreateConstGEP1_64(Value *Ptr, uint64_t Idx0, const Twine &Name = "") {
1888 return CreateConstGEP1_64(nullptr, Ptr, Idx0, Name);
1889 }
1890
1891 Value *CreateConstInBoundsGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0,
1892 const Twine &Name = "") {
1893 Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0);
1894
1895 if (auto *PC = dyn_cast<Constant>(Ptr))
1896 return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idx), Name);
1897
1898 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name);
1899 }
1900
1901 Value *CreateConstInBoundsGEP1_64(Value *Ptr, uint64_t Idx0,
1902 const Twine &Name = "") {
1903 return CreateConstInBoundsGEP1_64(nullptr, Ptr, Idx0, Name);
1904 }
1905
1906 Value *CreateConstGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1,
1907 const Twine &Name = "") {
1908 Value *Idxs[] = {
1909 ConstantInt::get(Type::getInt64Ty(Context), Idx0),
1910 ConstantInt::get(Type::getInt64Ty(Context), Idx1)
1911 };
1912
1913 if (auto *PC = dyn_cast<Constant>(Ptr))
1914 return Insert(Folder.CreateGetElementPtr(Ty, PC, Idxs), Name);
1915
1916 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name);
1917 }
1918
1919 Value *CreateConstGEP2_64(Value *Ptr, uint64_t Idx0, uint64_t Idx1,
1920 const Twine &Name = "") {
1921 return CreateConstGEP2_64(nullptr, Ptr, Idx0, Idx1, Name);
1922 }
1923
1924 Value *CreateConstInBoundsGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0,
1925 uint64_t Idx1, const Twine &Name = "") {
1926 Value *Idxs[] = {
1927 ConstantInt::get(Type::getInt64Ty(Context), Idx0),
1928 ConstantInt::get(Type::getInt64Ty(Context), Idx1)
1929 };
1930
1931 if (auto *PC = dyn_cast<Constant>(Ptr))
1932 return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idxs), Name);
1933
1934 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name);
1935 }
1936
1937 Value *CreateConstInBoundsGEP2_64(Value *Ptr, uint64_t Idx0, uint64_t Idx1,
1938 const Twine &Name = "") {
1939 return CreateConstInBoundsGEP2_64(nullptr, Ptr, Idx0, Idx1, Name);
1940 }
1941
1942 Value *CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx,
1943 const Twine &Name = "") {
1944 return CreateConstInBoundsGEP2_32(Ty, Ptr, 0, Idx, Name);
1945 }
1946
1947 Value *CreateStructGEP(Value *Ptr, unsigned Idx, const Twine &Name = "") {
1948 return CreateConstInBoundsGEP2_32(nullptr, Ptr, 0, Idx, Name);
1949 }
1950
1951 /// Same as CreateGlobalString, but return a pointer with "i8*" type
1952 /// instead of a pointer to array of i8.
1953 ///
1954 /// If no module is given via \p M, it is take from the insertion point basic
1955 /// block.
1956 Constant *CreateGlobalStringPtr(StringRef Str, const Twine &Name = "",
1957 unsigned AddressSpace = 0,
1958 Module *M = nullptr) {
1959 GlobalVariable *GV = CreateGlobalString(Str, Name, AddressSpace, M);
1960 Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1961 Constant *Indices[] = {Zero, Zero};
1962 return ConstantExpr::getInBoundsGetElementPtr(GV->getValueType(), GV,
1963 Indices);
1964 }
1965
1966 //===--------------------------------------------------------------------===//
1967 // Instruction creation methods: Cast/Conversion Operators
1968 //===--------------------------------------------------------------------===//
1969
1970 Value *CreateTrunc(Value *V, Type *DestTy, const Twine &Name = "") {
1971 return CreateCast(Instruction::Trunc, V, DestTy, Name);
1972 }
1973
1974 Value *CreateZExt(Value *V, Type *DestTy, const Twine &Name = "") {
1975 return CreateCast(Instruction::ZExt, V, DestTy, Name);
1976 }
1977
1978 Value *CreateSExt(Value *V, Type *DestTy, const Twine &Name = "") {
1979 return CreateCast(Instruction::SExt, V, DestTy, Name);
1980 }
1981
1982 /// Create a ZExt or Trunc from the integer value V to DestTy. Return
1983 /// the value untouched if the type of V is already DestTy.
1984 Value *CreateZExtOrTrunc(Value *V, Type *DestTy,
1985 const Twine &Name = "") {
1986 assert(V->getType()->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy->
isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1988, __PRETTY_FUNCTION__))
1987 DestTy->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy->
isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1988, __PRETTY_FUNCTION__))
1988 "Can only zero extend/truncate integers!")((V->getType()->isIntOrIntVectorTy() && DestTy->
isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1988, __PRETTY_FUNCTION__))
;
1989 Type *VTy = V->getType();
1990 if (VTy->getScalarSizeInBits() < DestTy->getScalarSizeInBits())
1991 return CreateZExt(V, DestTy, Name);
1992 if (VTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits())
1993 return CreateTrunc(V, DestTy, Name);
1994 return V;
1995 }
1996
1997 /// Create a SExt or Trunc from the integer value V to DestTy. Return
1998 /// the value untouched if the type of V is already DestTy.
1999 Value *CreateSExtOrTrunc(Value *V, Type *DestTy,
2000 const Twine &Name = "") {
2001 assert(V->getType()->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy->
isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 2003, __PRETTY_FUNCTION__))
2002 DestTy->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy->
isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 2003, __PRETTY_FUNCTION__))
2003 "Can only sign extend/truncate integers!")((V->getType()->isIntOrIntVectorTy() && DestTy->
isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 2003, __PRETTY_FUNCTION__))
;
2004 Type *VTy = V->getType();
2005 if (VTy->getScalarSizeInBits() < DestTy->getScalarSizeInBits())
2006 return CreateSExt(V, DestTy, Name);
2007 if (VTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits())
2008 return CreateTrunc(V, DestTy, Name);
2009 return V;
2010 }
2011
2012 Value *CreateFPToUI(Value *V, Type *DestTy, const Twine &Name = "") {
2013 if (IsFPConstrained)
2014 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptoui,
2015 V, DestTy, nullptr, Name);
2016 return CreateCast(Instruction::FPToUI, V, DestTy, Name);
2017 }
2018
2019 Value *CreateFPToSI(Value *V, Type *DestTy, const Twine &Name = "") {
2020 if (IsFPConstrained)
2021 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptosi,
2022 V, DestTy, nullptr, Name);
2023 return CreateCast(Instruction::FPToSI, V, DestTy, Name);
2024 }
2025
2026 Value *CreateUIToFP(Value *V, Type *DestTy, const Twine &Name = ""){
2027 if (IsFPConstrained)
2028 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_uitofp,
2029 V, DestTy, nullptr, Name);
2030 return CreateCast(Instruction::UIToFP, V, DestTy, Name);
2031 }
2032
2033 Value *CreateSIToFP(Value *V, Type *DestTy, const Twine &Name = ""){
2034 if (IsFPConstrained)
2035 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_sitofp,
2036 V, DestTy, nullptr, Name);
2037 return CreateCast(Instruction::SIToFP, V, DestTy, Name);
2038 }
2039
2040 Value *CreateFPTrunc(Value *V, Type *DestTy,
2041 const Twine &Name = "") {
2042 if (IsFPConstrained)
2043 return CreateConstrainedFPCast(
2044 Intrinsic::experimental_constrained_fptrunc, V, DestTy, nullptr,
2045 Name);
2046 return CreateCast(Instruction::FPTrunc, V, DestTy, Name);
2047 }
2048
2049 Value *CreateFPExt(Value *V, Type *DestTy, const Twine &Name = "") {
2050 if (IsFPConstrained)
2051 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fpext,
2052 V, DestTy, nullptr, Name);
2053 return CreateCast(Instruction::FPExt, V, DestTy, Name);
2054 }
2055
2056 Value *CreatePtrToInt(Value *V, Type *DestTy,
2057 const Twine &Name = "") {
2058 return CreateCast(Instruction::PtrToInt, V, DestTy, Name);
2059 }
2060
2061 Value *CreateIntToPtr(Value *V, Type *DestTy,
2062 const Twine &Name = "") {
2063 return CreateCast(Instruction::IntToPtr, V, DestTy, Name);
2064 }
2065
2066 Value *CreateBitCast(Value *V, Type *DestTy,
2067 const Twine &Name = "") {
2068 return CreateCast(Instruction::BitCast, V, DestTy, Name);
2069 }
2070
2071 Value *CreateAddrSpaceCast(Value *V, Type *DestTy,
2072 const Twine &Name = "") {
2073 return CreateCast(Instruction::AddrSpaceCast, V, DestTy, Name);
2074 }
2075
2076 Value *CreateZExtOrBitCast(Value *V, Type *DestTy,
2077 const Twine &Name = "") {
2078 if (V->getType() == DestTy)
2079 return V;
2080 if (auto *VC = dyn_cast<Constant>(V))
2081 return Insert(Folder.CreateZExtOrBitCast(VC, DestTy), Name);
2082 return Insert(CastInst::CreateZExtOrBitCast(V, DestTy), Name);
2083 }
2084
2085 Value *CreateSExtOrBitCast(Value *V, Type *DestTy,
2086 const Twine &Name = "") {
2087 if (V->getType() == DestTy)
2088 return V;
2089 if (auto *VC = dyn_cast<Constant>(V))
2090 return Insert(Folder.CreateSExtOrBitCast(VC, DestTy), Name);
2091 return Insert(CastInst::CreateSExtOrBitCast(V, DestTy), Name);
2092 }
2093
2094 Value *CreateTruncOrBitCast(Value *V, Type *DestTy,
2095 const Twine &Name = "") {
2096 if (V->getType() == DestTy)
2097 return V;
2098 if (auto *VC = dyn_cast<Constant>(V))
2099 return Insert(Folder.CreateTruncOrBitCast(VC, DestTy), Name);
2100 return Insert(CastInst::CreateTruncOrBitCast(V, DestTy), Name);
2101 }
2102
2103 Value *CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy,
2104 const Twine &Name = "") {
2105 if (V->getType() == DestTy)
2106 return V;
2107 if (auto *VC = dyn_cast<Constant>(V))
2108 return Insert(Folder.CreateCast(Op, VC, DestTy), Name);
2109 return Insert(CastInst::Create(Op, V, DestTy), Name);
2110 }
2111
2112 Value *CreatePointerCast(Value *V, Type *DestTy,
2113 const Twine &Name = "") {
2114 if (V->getType() == DestTy)
2115 return V;
2116 if (auto *VC = dyn_cast<Constant>(V))
2117 return Insert(Folder.CreatePointerCast(VC, DestTy), Name);
2118 return Insert(CastInst::CreatePointerCast(V, DestTy), Name);
2119 }
2120
2121 Value *CreatePointerBitCastOrAddrSpaceCast(Value *V, Type *DestTy,
2122 const Twine &Name = "") {
2123 if (V->getType() == DestTy)
2124 return V;
2125
2126 if (auto *VC = dyn_cast<Constant>(V)) {
2127 return Insert(Folder.CreatePointerBitCastOrAddrSpaceCast(VC, DestTy),
2128 Name);
2129 }
2130
2131 return Insert(CastInst::CreatePointerBitCastOrAddrSpaceCast(V, DestTy),
2132 Name);
2133 }
2134
2135 Value *CreateIntCast(Value *V, Type *DestTy, bool isSigned,
2136 const Twine &Name = "") {
2137 if (V->getType() == DestTy)
2138 return V;
2139 if (auto *VC = dyn_cast<Constant>(V))
2140 return Insert(Folder.CreateIntCast(VC, DestTy, isSigned), Name);
2141 return Insert(CastInst::CreateIntegerCast(V, DestTy, isSigned), Name);
2142 }
2143
2144 Value *CreateBitOrPointerCast(Value *V, Type *DestTy,
2145 const Twine &Name = "") {
2146 if (V->getType() == DestTy)
2147 return V;
2148 if (V->getType()->isPtrOrPtrVectorTy() && DestTy->isIntOrIntVectorTy())
2149 return CreatePtrToInt(V, DestTy, Name);
2150 if (V->getType()->isIntOrIntVectorTy() && DestTy->isPtrOrPtrVectorTy())
2151 return CreateIntToPtr(V, DestTy, Name);
2152
2153 return CreateBitCast(V, DestTy, Name);
2154 }
2155
2156 Value *CreateFPCast(Value *V, Type *DestTy, const Twine &Name = "") {
2157 if (V->getType() == DestTy)
2158 return V;
2159 if (auto *VC = dyn_cast<Constant>(V))
2160 return Insert(Folder.CreateFPCast(VC, DestTy), Name);
2161 return Insert(CastInst::CreateFPCast(V, DestTy), Name);
2162 }
2163
2164 CallInst *CreateConstrainedFPCast(
2165 Intrinsic::ID ID, Value *V, Type *DestTy,
2166 Instruction *FMFSource = nullptr, const Twine &Name = "",
2167 MDNode *FPMathTag = nullptr,
2168 Optional<RoundingMode> Rounding = None,
2169 Optional<fp::ExceptionBehavior> Except = None);
2170
2171 // Provided to resolve 'CreateIntCast(Ptr, Ptr, "...")', giving a
2172 // compile time error, instead of converting the string to bool for the
2173 // isSigned parameter.
2174 Value *CreateIntCast(Value *, Type *, const char *) = delete;
2175
2176 //===--------------------------------------------------------------------===//
2177 // Instruction creation methods: Compare Instructions
2178 //===--------------------------------------------------------------------===//
2179
2180 Value *CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name = "") {
2181 return CreateICmp(ICmpInst::ICMP_EQ, LHS, RHS, Name);
2182 }
2183
2184 Value *CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name = "") {
2185 return CreateICmp(ICmpInst::ICMP_NE, LHS, RHS, Name);
2186 }
2187
2188 Value *CreateICmpUGT(Value *LHS, Value *RHS, const Twine &Name = "") {
2189 return CreateICmp(ICmpInst::ICMP_UGT, LHS, RHS, Name);
2190 }
2191
2192 Value *CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name = "") {
2193 return CreateICmp(ICmpInst::ICMP_UGE, LHS, RHS, Name);
2194 }
2195
2196 Value *CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name = "") {
2197 return CreateICmp(ICmpInst::ICMP_ULT, LHS, RHS, Name);
2198 }
2199
2200 Value *CreateICmpULE(Value *LHS, Value *RHS, const Twine &Name = "") {
2201 return CreateICmp(ICmpInst::ICMP_ULE, LHS, RHS, Name);
2202 }
2203
2204 Value *CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name = "") {
2205 return CreateICmp(ICmpInst::ICMP_SGT, LHS, RHS, Name);
2206 }
2207
2208 Value *CreateICmpSGE(Value *LHS, Value *RHS, const Twine &Name = "") {
2209 return CreateICmp(ICmpInst::ICMP_SGE, LHS, RHS, Name);
2210 }
2211
2212 Value *CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name = "") {
2213 return CreateICmp(ICmpInst::ICMP_SLT, LHS, RHS, Name);
2214 }
2215
2216 Value *CreateICmpSLE(Value *LHS, Value *RHS, const Twine &Name = "") {
2217 return CreateICmp(ICmpInst::ICMP_SLE, LHS, RHS, Name);
2218 }
2219
2220 Value *CreateFCmpOEQ(Value *LHS, Value *RHS, const Twine &Name = "",
2221 MDNode *FPMathTag = nullptr) {
2222 return CreateFCmp(FCmpInst::FCMP_OEQ, LHS, RHS, Name, FPMathTag);
2223 }
2224
2225 Value *CreateFCmpOGT(Value *LHS, Value *RHS, const Twine &Name = "",
2226 MDNode *FPMathTag = nullptr) {
2227 return CreateFCmp(FCmpInst::FCMP_OGT, LHS, RHS, Name, FPMathTag);
2228 }
2229
2230 Value *CreateFCmpOGE(Value *LHS, Value *RHS, const Twine &Name = "",
2231 MDNode *FPMathTag = nullptr) {
2232 return CreateFCmp(FCmpInst::FCMP_OGE, LHS, RHS, Name, FPMathTag);
2233 }
2234
2235 Value *CreateFCmpOLT(Value *LHS, Value *RHS, const Twine &Name = "",
2236 MDNode *FPMathTag = nullptr) {
2237 return CreateFCmp(FCmpInst::FCMP_OLT, LHS, RHS, Name, FPMathTag);
2238 }
2239
2240 Value *CreateFCmpOLE(Value *LHS, Value *RHS, const Twine &Name = "",
2241 MDNode *FPMathTag = nullptr) {
2242 return CreateFCmp(FCmpInst::FCMP_OLE, LHS, RHS, Name, FPMathTag);
2243 }
2244
2245 Value *CreateFCmpONE(Value *LHS, Value *RHS, const Twine &Name = "",
2246 MDNode *FPMathTag = nullptr) {
2247 return CreateFCmp(FCmpInst::FCMP_ONE, LHS, RHS, Name, FPMathTag);
2248 }
2249
2250 Value *CreateFCmpORD(Value *LHS, Value *RHS, const Twine &Name = "",
2251 MDNode *FPMathTag = nullptr) {
2252 return CreateFCmp(FCmpInst::FCMP_ORD, LHS, RHS, Name, FPMathTag);
2253 }
2254
2255 Value *CreateFCmpUNO(Value *LHS, Value *RHS, const Twine &Name = "",
2256 MDNode *FPMathTag = nullptr) {
2257 return CreateFCmp(FCmpInst::FCMP_UNO, LHS, RHS, Name, FPMathTag);
2258 }
2259
2260 Value *CreateFCmpUEQ(Value *LHS, Value *RHS, const Twine &Name = "",
2261 MDNode *FPMathTag = nullptr) {
2262 return CreateFCmp(FCmpInst::FCMP_UEQ, LHS, RHS, Name, FPMathTag);
2263 }
2264
2265 Value *CreateFCmpUGT(Value *LHS, Value *RHS, const Twine &Name = "",
2266 MDNode *FPMathTag = nullptr) {
2267 return CreateFCmp(FCmpInst::FCMP_UGT, LHS, RHS, Name, FPMathTag);
2268 }
2269
2270 Value *CreateFCmpUGE(Value *LHS, Value *RHS, const Twine &Name = "",
2271 MDNode *FPMathTag = nullptr) {
2272 return CreateFCmp(FCmpInst::FCMP_UGE, LHS, RHS, Name, FPMathTag);
2273 }
2274
2275 Value *CreateFCmpULT(Value *LHS, Value *RHS, const Twine &Name = "",
2276 MDNode *FPMathTag = nullptr) {
2277 return CreateFCmp(FCmpInst::FCMP_ULT, LHS, RHS, Name, FPMathTag);
2278 }
2279
2280 Value *CreateFCmpULE(Value *LHS, Value *RHS, const Twine &Name = "",
2281 MDNode *FPMathTag = nullptr) {
2282 return CreateFCmp(FCmpInst::FCMP_ULE, LHS, RHS, Name, FPMathTag);
2283 }
2284
2285 Value *CreateFCmpUNE(Value *LHS, Value *RHS, const Twine &Name = "",
2286 MDNode *FPMathTag = nullptr) {
2287 return CreateFCmp(FCmpInst::FCMP_UNE, LHS, RHS, Name, FPMathTag);
2288 }
2289
2290 Value *CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS,
2291 const Twine &Name = "") {
2292 if (auto *LC = dyn_cast<Constant>(LHS))
2293 if (auto *RC = dyn_cast<Constant>(RHS))
2294 return Insert(Folder.CreateICmp(P, LC, RC), Name);
2295 return Insert(new ICmpInst(P, LHS, RHS), Name);
2296 }
2297
2298 // Create a quiet floating-point comparison (i.e. one that raises an FP
2299 // exception only in the case where an input is a signaling NaN).
2300 // Note that this differs from CreateFCmpS only if IsFPConstrained is true.
2301 Value *CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS,
2302 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2303 return CreateFCmpHelper(P, LHS, RHS, Name, FPMathTag, false);
2304 }
2305
2306 Value *CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS,
2307 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2308 return CmpInst::isFPPredicate(Pred)
2309 ? CreateFCmp(Pred, LHS, RHS, Name, FPMathTag)
2310 : CreateICmp(Pred, LHS, RHS, Name);
2311 }
2312
2313 // Create a signaling floating-point comparison (i.e. one that raises an FP
2314 // exception whenever an input is any NaN, signaling or quiet).
2315 // Note that this differs from CreateFCmp only if IsFPConstrained is true.
2316 Value *CreateFCmpS(CmpInst::Predicate P, Value *LHS, Value *RHS,
2317 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2318 return CreateFCmpHelper(P, LHS, RHS, Name, FPMathTag, true);
2319 }
2320
2321private:
2322 // Helper routine to create either a signaling or a quiet FP comparison.
2323 Value *CreateFCmpHelper(CmpInst::Predicate P, Value *LHS, Value *RHS,
2324 const Twine &Name, MDNode *FPMathTag,
2325 bool IsSignaling);
2326
2327public:
2328 CallInst *CreateConstrainedFPCmp(
2329 Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
2330 const Twine &Name = "", Optional<fp::ExceptionBehavior> Except = None);
2331
2332 //===--------------------------------------------------------------------===//
2333 // Instruction creation methods: Other Instructions
2334 //===--------------------------------------------------------------------===//
2335
2336 PHINode *CreatePHI(Type *Ty, unsigned NumReservedValues,
2337 const Twine &Name = "") {
2338 PHINode *Phi = PHINode::Create(Ty, NumReservedValues);
2339 if (isa<FPMathOperator>(Phi))
2340 setFPAttrs(Phi, nullptr /* MDNode* */, FMF);
2341 return Insert(Phi, Name);
2342 }
2343
2344 CallInst *CreateCall(FunctionType *FTy, Value *Callee,
2345 ArrayRef<Value *> Args = None, const Twine &Name = "",
2346 MDNode *FPMathTag = nullptr) {
2347 CallInst *CI = CallInst::Create(FTy, Callee, Args, DefaultOperandBundles);
2348 if (IsFPConstrained)
2349 setConstrainedFPCallAttr(CI);
2350 if (isa<FPMathOperator>(CI))
2351 setFPAttrs(CI, FPMathTag, FMF);
2352 return Insert(CI, Name);
2353 }
2354
2355 CallInst *CreateCall(FunctionType *FTy, Value *Callee, ArrayRef<Value *> Args,
2356 ArrayRef<OperandBundleDef> OpBundles,
2357 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2358 CallInst *CI = CallInst::Create(FTy, Callee, Args, OpBundles);
2359 if (IsFPConstrained)
2360 setConstrainedFPCallAttr(CI);
2361 if (isa<FPMathOperator>(CI))
2362 setFPAttrs(CI, FPMathTag, FMF);
2363 return Insert(CI, Name);
2364 }
2365
2366 CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args = None,
2367 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2368 return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args, Name,
2369 FPMathTag);
2370 }
2371
2372 CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args,
2373 ArrayRef<OperandBundleDef> OpBundles,
2374 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2375 return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args,
2376 OpBundles, Name, FPMathTag);
2377 }
2378
2379 CallInst *CreateConstrainedFPCall(
2380 Function *Callee, ArrayRef<Value *> Args, const Twine &Name = "",
2381 Optional<RoundingMode> Rounding = None,
2382 Optional<fp::ExceptionBehavior> Except = None);
2383
2384 Value *CreateSelect(Value *C, Value *True, Value *False,
2385 const Twine &Name = "", Instruction *MDFrom = nullptr);
2386
2387 VAArgInst *CreateVAArg(Value *List, Type *Ty, const Twine &Name = "") {
2388 return Insert(new VAArgInst(List, Ty), Name);
2389 }
2390
2391 Value *CreateExtractElement(Value *Vec, Value *Idx,
2392 const Twine &Name = "") {
2393 if (auto *VC = dyn_cast<Constant>(Vec))
2394 if (auto *IC = dyn_cast<Constant>(Idx))
2395 return Insert(Folder.CreateExtractElement(VC, IC), Name);
2396 return Insert(ExtractElementInst::Create(Vec, Idx), Name);
2397 }
2398
2399 Value *CreateExtractElement(Value *Vec, uint64_t Idx,
2400 const Twine &Name = "") {
2401 return CreateExtractElement(Vec, getInt64(Idx), Name);
2402 }
2403
2404 Value *CreateInsertElement(Value *Vec, Value *NewElt, Value *Idx,
2405 const Twine &Name = "") {
2406 if (auto *VC = dyn_cast<Constant>(Vec))
2407 if (auto *NC = dyn_cast<Constant>(NewElt))
2408 if (auto *IC = dyn_cast<Constant>(Idx))
2409 return Insert(Folder.CreateInsertElement(VC, NC, IC), Name);
2410 return Insert(InsertElementInst::Create(Vec, NewElt, Idx), Name);
2411 }
2412
2413 Value *CreateInsertElement(Value *Vec, Value *NewElt, uint64_t Idx,
2414 const Twine &Name = "") {
2415 return CreateInsertElement(Vec, NewElt, getInt64(Idx), Name);
2416 }
2417
2418 Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask,
2419 const Twine &Name = "") {
2420 SmallVector<int, 16> IntMask;
2421 ShuffleVectorInst::getShuffleMask(cast<Constant>(Mask), IntMask);
2422 return CreateShuffleVector(V1, V2, IntMask, Name);
2423 }
2424
2425 LLVM_ATTRIBUTE_DEPRECATED(Value *CreateShuffleVector(Value *V1, Value *V2,[[deprecated("Pass indices as 'int' instead")]] Value *CreateShuffleVector
(Value *V1, Value *V2, ArrayRef<uint32_t> Mask, const Twine
&Name = "")
2426 ArrayRef<uint32_t> Mask,[[deprecated("Pass indices as 'int' instead")]] Value *CreateShuffleVector
(Value *V1, Value *V2, ArrayRef<uint32_t> Mask, const Twine
&Name = "")
2427 const Twine &Name = ""),[[deprecated("Pass indices as 'int' instead")]] Value *CreateShuffleVector
(Value *V1, Value *V2, ArrayRef<uint32_t> Mask, const Twine
&Name = "")
2428 "Pass indices as 'int' instead")[[deprecated("Pass indices as 'int' instead")]] Value *CreateShuffleVector
(Value *V1, Value *V2, ArrayRef<uint32_t> Mask, const Twine
&Name = "")
{
2429 SmallVector<int, 16> IntMask;
2430 IntMask.assign(Mask.begin(), Mask.end());
2431 return CreateShuffleVector(V1, V2, IntMask, Name);
2432 }
2433
2434 /// See class ShuffleVectorInst for a description of the mask representation.
2435 Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<int> Mask,
2436 const Twine &Name = "") {
2437 if (auto *V1C = dyn_cast<Constant>(V1))
2438 if (auto *V2C = dyn_cast<Constant>(V2))
2439 return Insert(Folder.CreateShuffleVector(V1C, V2C, Mask), Name);
2440 return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
2441 }
2442
2443 /// Create a unary shuffle. The second vector operand of the IR instruction
2444 /// is poison.
2445 Value *CreateShuffleVector(Value *V, ArrayRef<int> Mask,
2446 const Twine &Name = "") {
2447 return CreateShuffleVector(V, PoisonValue::get(V->getType()), Mask, Name);
2448 }
2449
2450 Value *CreateExtractValue(Value *Agg,
2451 ArrayRef<unsigned> Idxs,
2452 const Twine &Name = "") {
2453 if (auto *AggC = dyn_cast<Constant>(Agg))
2454 return Insert(Folder.CreateExtractValue(AggC, Idxs), Name);
2455 return Insert(ExtractValueInst::Create(Agg, Idxs), Name);
2456 }
2457
2458 Value *CreateInsertValue(Value *Agg, Value *Val,
2459 ArrayRef<unsigned> Idxs,
2460 const Twine &Name = "") {
2461 if (auto *AggC = dyn_cast<Constant>(Agg))
2462 if (auto *ValC = dyn_cast<Constant>(Val))
2463 return Insert(Folder.CreateInsertValue(AggC, ValC, Idxs), Name);
2464 return Insert(InsertValueInst::Create(Agg, Val, Idxs), Name);
2465 }
2466
2467 LandingPadInst *CreateLandingPad(Type *Ty, unsigned NumClauses,
2468 const Twine &Name = "") {
2469 return Insert(LandingPadInst::Create(Ty, NumClauses), Name);
2470 }
2471
2472 Value *CreateFreeze(Value *V, const Twine &Name = "") {
2473 return Insert(new FreezeInst(V), Name);
2474 }
2475
2476 //===--------------------------------------------------------------------===//
2477 // Utility creation methods
2478 //===--------------------------------------------------------------------===//
2479
2480 /// Return an i1 value testing if \p Arg is null.
2481 Value *CreateIsNull(Value *Arg, const Twine &Name = "") {
2482 return CreateICmpEQ(Arg, Constant::getNullValue(Arg->getType()),
2483 Name);
2484 }
2485
2486 /// Return an i1 value testing if \p Arg is not null.
2487 Value *CreateIsNotNull(Value *Arg, const Twine &Name = "") {
2488 return CreateICmpNE(Arg, Constant::getNullValue(Arg->getType()),
2489 Name);
2490 }
2491
2492 /// Return the i64 difference between two pointer values, dividing out
2493 /// the size of the pointed-to objects.
2494 ///
2495 /// This is intended to implement C-style pointer subtraction. As such, the
2496 /// pointers must be appropriately aligned for their element types and
2497 /// pointing into the same object.
2498 Value *CreatePtrDiff(Value *LHS, Value *RHS, const Twine &Name = "");
2499
2500 /// Create a launder.invariant.group intrinsic call. If Ptr type is
2501 /// different from pointer to i8, it's casted to pointer to i8 in the same
2502 /// address space before call and casted back to Ptr type after call.
2503 Value *CreateLaunderInvariantGroup(Value *Ptr);
2504
2505 /// \brief Create a strip.invariant.group intrinsic call. If Ptr type is
2506 /// different from pointer to i8, it's casted to pointer to i8 in the same
2507 /// address space before call and casted back to Ptr type after call.
2508 Value *CreateStripInvariantGroup(Value *Ptr);
2509
2510 /// Return a vector value that contains the vector V reversed
2511 Value *CreateVectorReverse(Value *V, const Twine &Name = "");
2512
2513 /// Return a vector value that contains \arg V broadcasted to \p
2514 /// NumElts elements.
2515 Value *CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name = "");
2516
2517 /// Return a vector value that contains \arg V broadcasted to \p
2518 /// EC elements.
2519 Value *CreateVectorSplat(ElementCount EC, Value *V, const Twine &Name = "");
2520
2521 /// Return a value that has been extracted from a larger integer type.
2522 Value *CreateExtractInteger(const DataLayout &DL, Value *From,
2523 IntegerType *ExtractedTy, uint64_t Offset,
2524 const Twine &Name);
2525
2526 Value *CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,
2527 unsigned Dimension, unsigned LastIndex,
2528 MDNode *DbgInfo);
2529
2530 Value *CreatePreserveUnionAccessIndex(Value *Base, unsigned FieldIndex,
2531 MDNode *DbgInfo);
2532
2533 Value *CreatePreserveStructAccessIndex(Type *ElTy, Value *Base,
2534 unsigned Index, unsigned FieldIndex,
2535 MDNode *DbgInfo);
2536
2537private:
2538 /// Helper function that creates an assume intrinsic call that
2539 /// represents an alignment assumption on the provided pointer \p PtrValue
2540 /// with offset \p OffsetValue and alignment value \p AlignValue.
2541 CallInst *CreateAlignmentAssumptionHelper(const DataLayout &DL,
2542 Value *PtrValue, Value *AlignValue,
2543 Value *OffsetValue);
2544
2545public:
2546 /// Create an assume intrinsic call that represents an alignment
2547 /// assumption on the provided pointer.
2548 ///
2549 /// An optional offset can be provided, and if it is provided, the offset
2550 /// must be subtracted from the provided pointer to get the pointer with the
2551 /// specified alignment.
2552 CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue,
2553 unsigned Alignment,
2554 Value *OffsetValue = nullptr);
2555
2556 /// Create an assume intrinsic call that represents an alignment
2557 /// assumption on the provided pointer.
2558 ///
2559 /// An optional offset can be provided, and if it is provided, the offset
2560 /// must be subtracted from the provided pointer to get the pointer with the
2561 /// specified alignment.
2562 ///
2563 /// This overload handles the condition where the Alignment is dependent
2564 /// on an existing value rather than a static value.
2565 CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue,
2566 Value *Alignment,
2567 Value *OffsetValue = nullptr);
2568};
2569
2570/// This provides a uniform API for creating instructions and inserting
2571/// them into a basic block: either at the end of a BasicBlock, or at a specific
2572/// iterator location in a block.
2573///
2574/// Note that the builder does not expose the full generality of LLVM
2575/// instructions. For access to extra instruction properties, use the mutators
2576/// (e.g. setVolatile) on the instructions after they have been
2577/// created. Convenience state exists to specify fast-math flags and fp-math
2578/// tags.
2579///
2580/// The first template argument specifies a class to use for creating constants.
2581/// This defaults to creating minimally folded constants. The second template
2582/// argument allows clients to specify custom insertion hooks that are called on
2583/// every newly created insertion.
2584template <typename FolderTy = ConstantFolder,
2585 typename InserterTy = IRBuilderDefaultInserter>
2586class IRBuilder : public IRBuilderBase {
2587private:
2588 FolderTy Folder;
2589 InserterTy Inserter;
2590
2591public:
2592 IRBuilder(LLVMContext &C, FolderTy Folder, InserterTy Inserter = InserterTy(),
2593 MDNode *FPMathTag = nullptr,
2594 ArrayRef<OperandBundleDef> OpBundles = None)
2595 : IRBuilderBase(C, this->Folder, this->Inserter, FPMathTag, OpBundles),
2596 Folder(Folder), Inserter(Inserter) {}
2597
2598 explicit IRBuilder(LLVMContext &C, MDNode *FPMathTag = nullptr,
2599 ArrayRef<OperandBundleDef> OpBundles = None)
2600 : IRBuilderBase(C, this->Folder, this->Inserter, FPMathTag, OpBundles) {}
2601
2602 explicit IRBuilder(BasicBlock *TheBB, FolderTy Folder,
2603 MDNode *FPMathTag = nullptr,
2604 ArrayRef<OperandBundleDef> OpBundles = None)
2605 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2606 FPMathTag, OpBundles), Folder(Folder) {
2607 SetInsertPoint(TheBB);
2608 }
2609
2610 explicit IRBuilder(BasicBlock *TheBB, MDNode *FPMathTag = nullptr,
2611 ArrayRef<OperandBundleDef> OpBundles = None)
2612 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2613 FPMathTag, OpBundles) {
2614 SetInsertPoint(TheBB);
2615 }
2616
2617 explicit IRBuilder(Instruction *IP, MDNode *FPMathTag = nullptr,
2618 ArrayRef<OperandBundleDef> OpBundles = None)
2619 : IRBuilderBase(IP->getContext(), this->Folder, this->Inserter,
2620 FPMathTag, OpBundles) {
2621 SetInsertPoint(IP);
2622 }
2623
2624 IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP, FolderTy Folder,
2625 MDNode *FPMathTag = nullptr,
2626 ArrayRef<OperandBundleDef> OpBundles = None)
2627 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2628 FPMathTag, OpBundles), Folder(Folder) {
2629 SetInsertPoint(TheBB, IP);
2630 }
2631
2632 IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP,
2633 MDNode *FPMathTag = nullptr,
2634 ArrayRef<OperandBundleDef> OpBundles = None)
2635 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2636 FPMathTag, OpBundles) {
2637 SetInsertPoint(TheBB, IP);
2638 }
2639
2640 /// Avoid copying the full IRBuilder. Prefer using InsertPointGuard
2641 /// or FastMathFlagGuard instead.
2642 IRBuilder(const IRBuilder &) = delete;
2643
2644 InserterTy &getInserter() { return Inserter; }
2645};
2646
2647// Create wrappers for C Binding types (see CBindingWrapping.h).
2648DEFINE_SIMPLE_CONVERSION_FUNCTIONS(IRBuilder<>, LLVMBuilderRef)inline IRBuilder<> *unwrap(LLVMBuilderRef P) { return reinterpret_cast
<IRBuilder<>*>(P); } inline LLVMBuilderRef wrap(const
IRBuilder<> *P) { return reinterpret_cast<LLVMBuilderRef
>(const_cast<IRBuilder<>*>(P)); }
2649
2650} // end namespace llvm
2651
2652#endif // LLVM_IR_IRBUILDER_H

/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/ADT/Optional.h

1//===- Optional.h - Simple variant for passing optional values --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides Optional, a template class modeled in the spirit of
10// OCaml's 'opt' variant. The idea is to strongly type whether or not
11// a value can be optional.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_ADT_OPTIONAL_H
16#define LLVM_ADT_OPTIONAL_H
17
18#include "llvm/ADT/Hashing.h"
19#include "llvm/ADT/None.h"
20#include "llvm/Support/Compiler.h"
21#include "llvm/Support/type_traits.h"
22#include <cassert>
23#include <memory>
24#include <new>
25#include <utility>
26
27namespace llvm {
28
29class raw_ostream;
30
31namespace optional_detail {
32
33struct in_place_t {};
34
35/// Storage for any type.
36//
37// The specialization condition intentionally uses
38// llvm::is_trivially_copy_constructible instead of
39// std::is_trivially_copy_constructible. GCC versions prior to 7.4 may
40// instantiate the copy constructor of `T` when
41// std::is_trivially_copy_constructible is instantiated. This causes
42// compilation to fail if we query the trivially copy constructible property of
43// a class which is not copy constructible.
44//
45// The current implementation of OptionalStorage insists that in order to use
46// the trivial specialization, the value_type must be trivially copy
47// constructible and trivially copy assignable due to =default implementations
48// of the copy/move constructor/assignment. It does not follow that this is
49// necessarily the case std::is_trivially_copyable is true (hence the expanded
50// specialization condition).
51//
52// The move constructible / assignable conditions emulate the remaining behavior
53// of std::is_trivially_copyable.
54template <typename T, bool = (llvm::is_trivially_copy_constructible<T>::value &&
55 std::is_trivially_copy_assignable<T>::value &&
56 (std::is_trivially_move_constructible<T>::value ||
57 !std::is_move_constructible<T>::value) &&
58 (std::is_trivially_move_assignable<T>::value ||
59 !std::is_move_assignable<T>::value))>
60class OptionalStorage {
61 union {
62 char empty;
63 T value;
64 };
65 bool hasVal;
66
67public:
68 ~OptionalStorage() { reset(); }
69
70 constexpr OptionalStorage() noexcept : empty(), hasVal(false) {}
71
72 constexpr OptionalStorage(OptionalStorage const &other) : OptionalStorage() {
73 if (other.hasValue()) {
74 emplace(other.value);
75 }
76 }
77 constexpr OptionalStorage(OptionalStorage &&other) : OptionalStorage() {
78 if (other.hasValue()) {
79 emplace(std::move(other.value));
80 }
81 }
82
83 template <class... Args>
84 constexpr explicit OptionalStorage(in_place_t, Args &&... args)
85 : value(std::forward<Args>(args)...), hasVal(true) {}
86
87 void reset() noexcept {
88 if (hasVal) {
89 value.~T();
90 hasVal = false;
91 }
92 }
93
94 constexpr bool hasValue() const noexcept { return hasVal; }
95
96 T &getValue() LLVM_LVALUE_FUNCTION& noexcept {
97 assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/ADT/Optional.h"
, 97, __PRETTY_FUNCTION__))
;
98 return value;
99 }
100 constexpr T const &getValue() const LLVM_LVALUE_FUNCTION& noexcept {
101 assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/ADT/Optional.h"
, 101, __PRETTY_FUNCTION__))
;
102 return value;
103 }
104#if LLVM_HAS_RVALUE_REFERENCE_THIS1
105 T &&getValue() && noexcept {
106 assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/ADT/Optional.h"
, 106, __PRETTY_FUNCTION__))
;
107 return std::move(value);
108 }
109#endif
110
111 template <class... Args> void emplace(Args &&... args) {
112 reset();
113 ::new ((void *)std::addressof(value)) T(std::forward<Args>(args)...);
114 hasVal = true;
115 }
116
117 OptionalStorage &operator=(T const &y) {
118 if (hasValue()) {
119 value = y;
120 } else {
121 ::new ((void *)std::addressof(value)) T(y);
122 hasVal = true;
123 }
124 return *this;
125 }
126 OptionalStorage &operator=(T &&y) {
127 if (hasValue()) {
128 value = std::move(y);
129 } else {
130 ::new ((void *)std::addressof(value)) T(std::move(y));
131 hasVal = true;
132 }
133 return *this;
134 }
135
136 OptionalStorage &operator=(OptionalStorage const &other) {
137 if (other.hasValue()) {
138 if (hasValue()) {
139 value = other.value;
140 } else {
141 ::new ((void *)std::addressof(value)) T(other.value);
142 hasVal = true;
143 }
144 } else {
145 reset();
146 }
147 return *this;
148 }
149
150 OptionalStorage &operator=(OptionalStorage &&other) {
151 if (other.hasValue()) {
152 if (hasValue()) {
153 value = std::move(other.value);
154 } else {
155 ::new ((void *)std::addressof(value)) T(std::move(other.value));
156 hasVal = true;
157 }
158 } else {
159 reset();
160 }
161 return *this;
162 }
163};
164
165template <typename T> class OptionalStorage<T, true> {
166 union {
167 char empty;
168 T value;
169 };
170 bool hasVal = false;
171
172public:
173 ~OptionalStorage() = default;
174
175 constexpr OptionalStorage() noexcept : empty{} {}
176
177 constexpr OptionalStorage(OptionalStorage const &other) = default;
178 constexpr OptionalStorage(OptionalStorage &&other) = default;
179
180 OptionalStorage &operator=(OptionalStorage const &other) = default;
181 OptionalStorage &operator=(OptionalStorage &&other) = default;
182
183 template <class... Args>
184 constexpr explicit OptionalStorage(in_place_t, Args &&... args)
185 : value(std::forward<Args>(args)...), hasVal(true) {}
186
187 void reset() noexcept {
188 if (hasVal) {
189 value.~T();
190 hasVal = false;
191 }
192 }
193
194 constexpr bool hasValue() const noexcept { return hasVal; }
42
Returning zero, which participates in a condition later
195
196 T &getValue() LLVM_LVALUE_FUNCTION& noexcept {
197 assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/ADT/Optional.h"
, 197, __PRETTY_FUNCTION__))
;
198 return value;
199 }
200 constexpr T const &getValue() const LLVM_LVALUE_FUNCTION& noexcept {
201 assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/ADT/Optional.h"
, 201, __PRETTY_FUNCTION__))
;
202 return value;
203 }
204#if LLVM_HAS_RVALUE_REFERENCE_THIS1
205 T &&getValue() && noexcept {
206 assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/ADT/Optional.h"
, 206, __PRETTY_FUNCTION__))
;
207 return std::move(value);
208 }
209#endif
210
211 template <class... Args> void emplace(Args &&... args) {
212 reset();
213 ::new ((void *)std::addressof(value)) T(std::forward<Args>(args)...);
214 hasVal = true;
215 }
216
217 OptionalStorage &operator=(T const &y) {
218 if (hasValue()) {
219 value = y;
220 } else {
221 ::new ((void *)std::addressof(value)) T(y);
222 hasVal = true;
223 }
224 return *this;
225 }
226 OptionalStorage &operator=(T &&y) {
227 if (hasValue()) {
228 value = std::move(y);
229 } else {
230 ::new ((void *)std::addressof(value)) T(std::move(y));
231 hasVal = true;
232 }
233 return *this;
234 }
235};
236
237} // namespace optional_detail
238
239template <typename T> class Optional {
240 optional_detail::OptionalStorage<T> Storage;
241
242public:
243 using value_type = T;
244
245 constexpr Optional() {}
246 constexpr Optional(NoneType) {}
247
248 constexpr Optional(const T &y) : Storage(optional_detail::in_place_t{}, y) {}
249 constexpr Optional(const Optional &O) = default;
250
251 constexpr Optional(T &&y)
252 : Storage(optional_detail::in_place_t{}, std::move(y)) {}
253 constexpr Optional(Optional &&O) = default;
254
255 Optional &operator=(T &&y) {
256 Storage = std::move(y);
257 return *this;
258 }
259 Optional &operator=(Optional &&O) = default;
260
261 /// Create a new object by constructing it in place with the given arguments.
262 template <typename... ArgTypes> void emplace(ArgTypes &&... Args) {
263 Storage.emplace(std::forward<ArgTypes>(Args)...);
264 }
265
266 static constexpr Optional create(const T *y) {
267 return y ? Optional(*y) : Optional();
268 }
269
270 Optional &operator=(const T &y) {
271 Storage = y;
272 return *this;
273 }
274 Optional &operator=(const Optional &O) = default;
275
276 void reset() { Storage.reset(); }
277
278 constexpr const T *getPointer() const { return &Storage.getValue(); }
279 T *getPointer() { return &Storage.getValue(); }
280 constexpr const T &getValue() const LLVM_LVALUE_FUNCTION& {
281 return Storage.getValue();
282 }
283 T &getValue() LLVM_LVALUE_FUNCTION& { return Storage.getValue(); }
284
285 constexpr explicit operator bool() const { return hasValue(); }
40
Calling 'Optional::hasValue'
45
Returning from 'Optional::hasValue'
46
Returning zero, which participates in a condition later
286 constexpr bool hasValue() const { return Storage.hasValue(); }
41
Calling 'OptionalStorage::hasValue'
43
Returning from 'OptionalStorage::hasValue'
44
Returning zero, which participates in a condition later
287 constexpr const T *operator->() const { return getPointer(); }
288 T *operator->() { return getPointer(); }
289 constexpr const T &operator*() const LLVM_LVALUE_FUNCTION& {
290 return getValue();
291 }
292 T &operator*() LLVM_LVALUE_FUNCTION& { return getValue(); }
293
294 template <typename U>
295 constexpr T getValueOr(U &&value) const LLVM_LVALUE_FUNCTION& {
296 return hasValue() ? getValue() : std::forward<U>(value);
297 }
298
299 /// Apply a function to the value if present; otherwise return None.
300 template <class Function>
301 auto map(const Function &F) const LLVM_LVALUE_FUNCTION&
302 -> Optional<decltype(F(getValue()))> {
303 if (*this) return F(getValue());
304 return None;
305 }
306
307#if LLVM_HAS_RVALUE_REFERENCE_THIS1
308 T &&getValue() && { return std::move(Storage.getValue()); }
309 T &&operator*() && { return std::move(Storage.getValue()); }
310
311 template <typename U>
312 T getValueOr(U &&value) && {
313 return hasValue() ? std::move(getValue()) : std::forward<U>(value);
314 }
315
316 /// Apply a function to the value if present; otherwise return None.
317 template <class Function>
318 auto map(const Function &F) &&
319 -> Optional<decltype(F(std::move(*this).getValue()))> {
320 if (*this) return F(std::move(*this).getValue());
321 return None;
322 }
323#endif
324};
325
326template <class T> llvm::hash_code hash_value(const Optional<T> &O) {
327 return O ? hash_combine(true, *O) : hash_value(false);
328}
329
330template <typename T, typename U>
331constexpr bool operator==(const Optional<T> &X, const Optional<U> &Y) {
332 if (X && Y)
333 return *X == *Y;
334 return X.hasValue() == Y.hasValue();
335}
336
337template <typename T, typename U>
338constexpr bool operator!=(const Optional<T> &X, const Optional<U> &Y) {
339 return !(X == Y);
340}
341
342template <typename T, typename U>
343constexpr bool operator<(const Optional<T> &X, const Optional<U> &Y) {
344 if (X && Y)
345 return *X < *Y;
346 return X.hasValue() < Y.hasValue();
347}
348
349template <typename T, typename U>
350constexpr bool operator<=(const Optional<T> &X, const Optional<U> &Y) {
351 return !(Y < X);
352}
353
354template <typename T, typename U>
355constexpr bool operator>(const Optional<T> &X, const Optional<U> &Y) {
356 return Y < X;
357}
358
359template <typename T, typename U>
360constexpr bool operator>=(const Optional<T> &X, const Optional<U> &Y) {
361 return !(X < Y);
362}
363
364template <typename T>
365constexpr bool operator==(const Optional<T> &X, NoneType) {
366 return !X;
367}
368
369template <typename T>
370constexpr bool operator==(NoneType, const Optional<T> &X) {
371 return X == None;
372}
373
374template <typename T>
375constexpr bool operator!=(const Optional<T> &X, NoneType) {
376 return !(X == None);
377}
378
379template <typename T>
380constexpr bool operator!=(NoneType, const Optional<T> &X) {
381 return X != None;
382}
383
384template <typename T> constexpr bool operator<(const Optional<T> &, NoneType) {
385 return false;
386}
387
388template <typename T> constexpr bool operator<(NoneType, const Optional<T> &X) {
389 return X.hasValue();
390}
391
392template <typename T>
393constexpr bool operator<=(const Optional<T> &X, NoneType) {
394 return !(None < X);
395}
396
397template <typename T>
398constexpr bool operator<=(NoneType, const Optional<T> &X) {
399 return !(X < None);
400}
401
402template <typename T> constexpr bool operator>(const Optional<T> &X, NoneType) {
403 return None < X;
404}
405
406template <typename T> constexpr bool operator>(NoneType, const Optional<T> &X) {
407 return X < None;
408}
409
410template <typename T>
411constexpr bool operator>=(const Optional<T> &X, NoneType) {
412 return None <= X;
413}
414
415template <typename T>
416constexpr bool operator>=(NoneType, const Optional<T> &X) {
417 return X <= None;
418}
419
420template <typename T>
421constexpr bool operator==(const Optional<T> &X, const T &Y) {
422 return X && *X == Y;
423}
424
425template <typename T>
426constexpr bool operator==(const T &X, const Optional<T> &Y) {
427 return Y && X == *Y;
428}
429
430template <typename T>
431constexpr bool operator!=(const Optional<T> &X, const T &Y) {
432 return !(X == Y);
433}
434
435template <typename T>
436constexpr bool operator!=(const T &X, const Optional<T> &Y) {
437 return !(X == Y);
438}
439
440template <typename T>
441constexpr bool operator<(const Optional<T> &X, const T &Y) {
442 return !X || *X < Y;
443}
444
445template <typename T>
446constexpr bool operator<(const T &X, const Optional<T> &Y) {
447 return Y && X < *Y;
448}
449
450template <typename T>
451constexpr bool operator<=(const Optional<T> &X, const T &Y) {
452 return !(Y < X);
453}
454
455template <typename T>
456constexpr bool operator<=(const T &X, const Optional<T> &Y) {
457 return !(Y < X);
458}
459
460template <typename T>
461constexpr bool operator>(const Optional<T> &X, const T &Y) {
462 return Y < X;
463}
464
465template <typename T>
466constexpr bool operator>(const T &X, const Optional<T> &Y) {
467 return Y < X;
468}
469
470template <typename T>
471constexpr bool operator>=(const Optional<T> &X, const T &Y) {
472 return !(X < Y);
473}
474
475template <typename T>
476constexpr bool operator>=(const T &X, const Optional<T> &Y) {
477 return !(X < Y);
478}
479
480raw_ostream &operator<<(raw_ostream &OS, NoneType);
481
482template <typename T, typename = decltype(std::declval<raw_ostream &>()
483 << std::declval<const T &>())>
484raw_ostream &operator<<(raw_ostream &OS, const Optional<T> &O) {
485 if (O)
486 OS << *O;
487 else
488 OS << None;
489 return OS;
490}
491
492} // end namespace llvm
493
494#endif // LLVM_ADT_OPTIONAL_H