Bug Summary

File:llvm/include/llvm/IR/IRBuilder.h
Warning:line 192, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name OMPIRBuilder.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Frontend/OpenMP -I /build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/lib/Frontend/OpenMP -I include -I /build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm=build-llvm -fmacro-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm=build-llvm -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm=build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-01-16-232930-107970-1 -x c++ /build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp

/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp

1//===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file implements the OpenMPIRBuilder class, which is used as a
11/// convenient way to create LLVM instructions for OpenMP directives.
12///
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
16#include "llvm/ADT/StringRef.h"
17#include "llvm/ADT/Triple.h"
18#include "llvm/Analysis/AssumptionCache.h"
19#include "llvm/Analysis/CodeMetrics.h"
20#include "llvm/Analysis/OptimizationRemarkEmitter.h"
21#include "llvm/Analysis/ScalarEvolution.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/IR/CFG.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/GlobalVariable.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/MDBuilder.h"
29#include "llvm/IR/PassManager.h"
30#include "llvm/IR/Value.h"
31#include "llvm/MC/TargetRegistry.h"
32#include "llvm/Support/CommandLine.h"
33#include "llvm/Support/Error.h"
34#include "llvm/Target/TargetMachine.h"
35#include "llvm/Target/TargetOptions.h"
36#include "llvm/Transforms/Utils/BasicBlockUtils.h"
37#include "llvm/Transforms/Utils/CodeExtractor.h"
38#include "llvm/Transforms/Utils/LoopPeel.h"
39#include "llvm/Transforms/Utils/ModuleUtils.h"
40#include "llvm/Transforms/Utils/UnrollLoop.h"
41
42#include <cstdint>
43#include <sstream>
44
45#define DEBUG_TYPE"openmp-ir-builder" "openmp-ir-builder"
46
47using namespace llvm;
48using namespace omp;
49
50static cl::opt<bool>
51 OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
52 cl::desc("Use optimistic attributes describing "
53 "'as-if' properties of runtime calls."),
54 cl::init(false));
55
56static cl::opt<double> UnrollThresholdFactor(
57 "openmp-ir-builder-unroll-threshold-factor", cl::Hidden,
58 cl::desc("Factor for the unroll threshold to account for code "
59 "simplifications still taking place"),
60 cl::init(1.5));
61
62void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) {
63 LLVMContext &Ctx = Fn.getContext();
64
65 // Get the function's current attributes.
66 auto Attrs = Fn.getAttributes();
67 auto FnAttrs = Attrs.getFnAttrs();
68 auto RetAttrs = Attrs.getRetAttrs();
69 SmallVector<AttributeSet, 4> ArgAttrs;
70 for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)
71 ArgAttrs.emplace_back(Attrs.getParamAttrs(ArgNo));
72
73#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
74#include "llvm/Frontend/OpenMP/OMPKinds.def"
75
76 // Add attributes to the function declaration.
77 switch (FnID) {
78#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \
79 case Enum: \
80 FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet); \
81 RetAttrs = RetAttrs.addAttributes(Ctx, RetAttrSet); \
82 for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo) \
83 ArgAttrs[ArgNo] = \
84 ArgAttrs[ArgNo].addAttributes(Ctx, ArgAttrSets[ArgNo]); \
85 Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs)); \
86 break;
87#include "llvm/Frontend/OpenMP/OMPKinds.def"
88 default:
89 // Attributes are optional.
90 break;
91 }
92}
93
94FunctionCallee
95OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) {
96 FunctionType *FnTy = nullptr;
97 Function *Fn = nullptr;
98
99 // Try to find the declation in the module first.
100 switch (FnID) {
101#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \
102 case Enum: \
103 FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \
104 IsVarArg); \
105 Fn = M.getFunction(Str); \
106 break;
107#include "llvm/Frontend/OpenMP/OMPKinds.def"
108 }
109
110 if (!Fn) {
111 // Create a new declaration if we need one.
112 switch (FnID) {
113#define OMP_RTL(Enum, Str, ...) \
114 case Enum: \
115 Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \
116 break;
117#include "llvm/Frontend/OpenMP/OMPKinds.def"
118 }
119
120 // Add information if the runtime function takes a callback function
121 if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
122 if (!Fn->hasMetadata(LLVMContext::MD_callback)) {
123 LLVMContext &Ctx = Fn->getContext();
124 MDBuilder MDB(Ctx);
125 // Annotate the callback behavior of the runtime function:
126 // - The callback callee is argument number 2 (microtask).
127 // - The first two arguments of the callback callee are unknown (-1).
128 // - All variadic arguments to the runtime function are passed to the
129 // callback callee.
130 Fn->addMetadata(
131 LLVMContext::MD_callback,
132 *MDNode::get(Ctx, {MDB.createCallbackEncoding(
133 2, {-1, -1}, /* VarArgsArePassed */ true)}));
134 }
135 }
136
137 LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Created OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
138 << " with type " << *Fn->getFunctionType() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Created OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
;
139 addAttributes(FnID, *Fn);
140
141 } else {
142 LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Found OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
143 << " with type " << *Fn->getFunctionType() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Found OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
;
144 }
145
146 assert(Fn && "Failed to create OpenMP runtime function")(static_cast <bool> (Fn && "Failed to create OpenMP runtime function"
) ? void (0) : __assert_fail ("Fn && \"Failed to create OpenMP runtime function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 146, __extension__
__PRETTY_FUNCTION__))
;
147
148 // Cast the function to the expected type if necessary
149 Constant *C = ConstantExpr::getBitCast(Fn, FnTy->getPointerTo());
150 return {FnTy, C};
151}
152
153Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) {
154 FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID);
155 auto *Fn = dyn_cast<llvm::Function>(RTLFn.getCallee());
156 assert(Fn && "Failed to create OpenMP runtime function pointer")(static_cast <bool> (Fn && "Failed to create OpenMP runtime function pointer"
) ? void (0) : __assert_fail ("Fn && \"Failed to create OpenMP runtime function pointer\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 156, __extension__
__PRETTY_FUNCTION__))
;
157 return Fn;
158}
159
160void OpenMPIRBuilder::initialize() { initializeTypes(M); }
161
162void OpenMPIRBuilder::finalize(Function *Fn, bool AllowExtractorSinking) {
163 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
164 SmallVector<BasicBlock *, 32> Blocks;
165 SmallVector<OutlineInfo, 16> DeferredOutlines;
166 for (OutlineInfo &OI : OutlineInfos) {
167 // Skip functions that have not finalized yet; may happen with nested
168 // function generation.
169 if (Fn && OI.getFunction() != Fn) {
170 DeferredOutlines.push_back(OI);
171 continue;
172 }
173
174 ParallelRegionBlockSet.clear();
175 Blocks.clear();
176 OI.collectBlocks(ParallelRegionBlockSet, Blocks);
177
178 Function *OuterFn = OI.getFunction();
179 CodeExtractorAnalysisCache CEAC(*OuterFn);
180 CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
181 /* AggregateArgs */ false,
182 /* BlockFrequencyInfo */ nullptr,
183 /* BranchProbabilityInfo */ nullptr,
184 /* AssumptionCache */ nullptr,
185 /* AllowVarArgs */ true,
186 /* AllowAlloca */ true,
187 /* Suffix */ ".omp_par");
188
189 LLVM_DEBUG(dbgs() << "Before outlining: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before outlining: "
<< *OuterFn << "\n"; } } while (false)
;
190 LLVM_DEBUG(dbgs() << "Entry " << OI.EntryBB->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Entry " << OI
.EntryBB->getName() << " Exit: " << OI.ExitBB->
getName() << "\n"; } } while (false)
191 << " Exit: " << OI.ExitBB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Entry " << OI
.EntryBB->getName() << " Exit: " << OI.ExitBB->
getName() << "\n"; } } while (false)
;
192 assert(Extractor.isEligible() &&(static_cast <bool> (Extractor.isEligible() && "Expected OpenMP outlining to be possible!"
) ? void (0) : __assert_fail ("Extractor.isEligible() && \"Expected OpenMP outlining to be possible!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 193, __extension__
__PRETTY_FUNCTION__))
193 "Expected OpenMP outlining to be possible!")(static_cast <bool> (Extractor.isEligible() && "Expected OpenMP outlining to be possible!"
) ? void (0) : __assert_fail ("Extractor.isEligible() && \"Expected OpenMP outlining to be possible!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 193, __extension__
__PRETTY_FUNCTION__))
;
194
195 Function *OutlinedFn = Extractor.extractCodeRegion(CEAC);
196
197 LLVM_DEBUG(dbgs() << "After outlining: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After outlining: "
<< *OuterFn << "\n"; } } while (false)
;
198 LLVM_DEBUG(dbgs() << " Outlined function: " << *OutlinedFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << " Outlined function: "
<< *OutlinedFn << "\n"; } } while (false)
;
199 assert(OutlinedFn->getReturnType()->isVoidTy() &&(static_cast <bool> (OutlinedFn->getReturnType()->
isVoidTy() && "OpenMP outlined functions should not return a value!"
) ? void (0) : __assert_fail ("OutlinedFn->getReturnType()->isVoidTy() && \"OpenMP outlined functions should not return a value!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 200, __extension__
__PRETTY_FUNCTION__))
200 "OpenMP outlined functions should not return a value!")(static_cast <bool> (OutlinedFn->getReturnType()->
isVoidTy() && "OpenMP outlined functions should not return a value!"
) ? void (0) : __assert_fail ("OutlinedFn->getReturnType()->isVoidTy() && \"OpenMP outlined functions should not return a value!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 200, __extension__
__PRETTY_FUNCTION__))
;
201
202 // For compability with the clang CG we move the outlined function after the
203 // one with the parallel region.
204 OutlinedFn->removeFromParent();
205 M.getFunctionList().insertAfter(OuterFn->getIterator(), OutlinedFn);
206
207 // Remove the artificial entry introduced by the extractor right away, we
208 // made our own entry block after all.
209 {
210 BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock();
211 assert(ArtificialEntry.getUniqueSuccessor() == OI.EntryBB)(static_cast <bool> (ArtificialEntry.getUniqueSuccessor
() == OI.EntryBB) ? void (0) : __assert_fail ("ArtificialEntry.getUniqueSuccessor() == OI.EntryBB"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 211, __extension__
__PRETTY_FUNCTION__))
;
212 assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry)(static_cast <bool> (OI.EntryBB->getUniquePredecessor
() == &ArtificialEntry) ? void (0) : __assert_fail ("OI.EntryBB->getUniquePredecessor() == &ArtificialEntry"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 212, __extension__
__PRETTY_FUNCTION__))
;
213 if (AllowExtractorSinking) {
214 // Move instructions from the to-be-deleted ArtificialEntry to the entry
215 // basic block of the parallel region. CodeExtractor may have sunk
216 // allocas/bitcasts for values that are solely used in the outlined
217 // region and do not escape.
218 assert(!ArtificialEntry.empty() &&(static_cast <bool> (!ArtificialEntry.empty() &&
"Expected instructions to sink in the outlined region") ? void
(0) : __assert_fail ("!ArtificialEntry.empty() && \"Expected instructions to sink in the outlined region\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 219, __extension__
__PRETTY_FUNCTION__))
219 "Expected instructions to sink in the outlined region")(static_cast <bool> (!ArtificialEntry.empty() &&
"Expected instructions to sink in the outlined region") ? void
(0) : __assert_fail ("!ArtificialEntry.empty() && \"Expected instructions to sink in the outlined region\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 219, __extension__
__PRETTY_FUNCTION__))
;
220 for (BasicBlock::iterator It = ArtificialEntry.begin(),
221 End = ArtificialEntry.end();
222 It != End;) {
223 Instruction &I = *It;
224 It++;
225
226 if (I.isTerminator())
227 continue;
228
229 I.moveBefore(*OI.EntryBB, OI.EntryBB->getFirstInsertionPt());
230 }
231 }
232 OI.EntryBB->moveBefore(&ArtificialEntry);
233 ArtificialEntry.eraseFromParent();
234 }
235 assert(&OutlinedFn->getEntryBlock() == OI.EntryBB)(static_cast <bool> (&OutlinedFn->getEntryBlock(
) == OI.EntryBB) ? void (0) : __assert_fail ("&OutlinedFn->getEntryBlock() == OI.EntryBB"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 235, __extension__
__PRETTY_FUNCTION__))
;
236 assert(OutlinedFn && OutlinedFn->getNumUses() == 1)(static_cast <bool> (OutlinedFn && OutlinedFn->
getNumUses() == 1) ? void (0) : __assert_fail ("OutlinedFn && OutlinedFn->getNumUses() == 1"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 236, __extension__
__PRETTY_FUNCTION__))
;
237
238 // Run a user callback, e.g. to add attributes.
239 if (OI.PostOutlineCB)
240 OI.PostOutlineCB(*OutlinedFn);
241 }
242
243 // Remove work items that have been completed.
244 OutlineInfos = std::move(DeferredOutlines);
245}
246
247OpenMPIRBuilder::~OpenMPIRBuilder() {
248 assert(OutlineInfos.empty() && "There must be no outstanding outlinings")(static_cast <bool> (OutlineInfos.empty() && "There must be no outstanding outlinings"
) ? void (0) : __assert_fail ("OutlineInfos.empty() && \"There must be no outstanding outlinings\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 248, __extension__
__PRETTY_FUNCTION__))
;
249}
250
251GlobalValue *OpenMPIRBuilder::createGlobalFlag(unsigned Value, StringRef Name) {
252 IntegerType *I32Ty = Type::getInt32Ty(M.getContext());
253 auto *GV =
254 new GlobalVariable(M, I32Ty,
255 /* isConstant = */ true, GlobalValue::WeakODRLinkage,
256 ConstantInt::get(I32Ty, Value), Name);
257
258 return GV;
259}
260
261Constant *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr,
262 uint32_t SrcLocStrSize,
263 IdentFlag LocFlags,
264 unsigned Reserve2Flags) {
265 // Enable "C-mode".
266 LocFlags |= OMP_IDENT_FLAG_KMPC;
267
268 Constant *&Ident =
269 IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];
270 if (!Ident) {
271 Constant *I32Null = ConstantInt::getNullValue(Int32);
272 Constant *IdentData[] = {I32Null,
273 ConstantInt::get(Int32, uint32_t(LocFlags)),
274 ConstantInt::get(Int32, Reserve2Flags),
275 ConstantInt::get(Int32, SrcLocStrSize), SrcLocStr};
276 Constant *Initializer =
277 ConstantStruct::get(OpenMPIRBuilder::Ident, IdentData);
278
279 // Look for existing encoding of the location + flags, not needed but
280 // minimizes the difference to the existing solution while we transition.
281 for (GlobalVariable &GV : M.getGlobalList())
282 if (GV.getValueType() == OpenMPIRBuilder::Ident && GV.hasInitializer())
283 if (GV.getInitializer() == Initializer)
284 Ident = &GV;
285
286 if (!Ident) {
287 auto *GV = new GlobalVariable(
288 M, OpenMPIRBuilder::Ident,
289 /* isConstant = */ true, GlobalValue::PrivateLinkage, Initializer, "",
290 nullptr, GlobalValue::NotThreadLocal,
291 M.getDataLayout().getDefaultGlobalsAddressSpace());
292 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
293 GV->setAlignment(Align(8));
294 Ident = GV;
295 }
296 }
297
298 return ConstantExpr::getPointerBitCastOrAddrSpaceCast(Ident, IdentPtr);
299}
300
301Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr,
302 uint32_t &SrcLocStrSize) {
303 SrcLocStrSize = LocStr.size();
304 Constant *&SrcLocStr = SrcLocStrMap[LocStr];
305 if (!SrcLocStr) {
306 Constant *Initializer =
307 ConstantDataArray::getString(M.getContext(), LocStr);
308
309 // Look for existing encoding of the location, not needed but minimizes the
310 // difference to the existing solution while we transition.
311 for (GlobalVariable &GV : M.getGlobalList())
312 if (GV.isConstant() && GV.hasInitializer() &&
313 GV.getInitializer() == Initializer)
314 return SrcLocStr = ConstantExpr::getPointerCast(&GV, Int8Ptr);
315
316 SrcLocStr = Builder.CreateGlobalStringPtr(LocStr, /* Name */ "",
317 /* AddressSpace */ 0, &M);
318 }
319 return SrcLocStr;
320}
321
322Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName,
323 StringRef FileName,
324 unsigned Line, unsigned Column,
325 uint32_t &SrcLocStrSize) {
326 SmallString<128> Buffer;
327 Buffer.push_back(';');
328 Buffer.append(FileName);
329 Buffer.push_back(';');
330 Buffer.append(FunctionName);
331 Buffer.push_back(';');
332 Buffer.append(std::to_string(Line));
333 Buffer.push_back(';');
334 Buffer.append(std::to_string(Column));
335 Buffer.push_back(';');
336 Buffer.push_back(';');
337 return getOrCreateSrcLocStr(Buffer.str(), SrcLocStrSize);
338}
339
340Constant *
341OpenMPIRBuilder::getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize) {
342 StringRef UnknownLoc = ";unknown;unknown;0;0;;";
343 return getOrCreateSrcLocStr(UnknownLoc, SrcLocStrSize);
344}
345
346Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(DebugLoc DL,
347 uint32_t &SrcLocStrSize,
348 Function *F) {
349 DILocation *DIL = DL.get();
350 if (!DIL)
351 return getOrCreateDefaultSrcLocStr(SrcLocStrSize);
352 StringRef FileName = M.getName();
353 if (DIFile *DIF = DIL->getFile())
354 if (Optional<StringRef> Source = DIF->getSource())
355 FileName = *Source;
356 StringRef Function = DIL->getScope()->getSubprogram()->getName();
357 if (Function.empty() && F)
358 Function = F->getName();
359 return getOrCreateSrcLocStr(Function, FileName, DIL->getLine(),
360 DIL->getColumn(), SrcLocStrSize);
361}
362
363Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc,
364 uint32_t &SrcLocStrSize) {
365 return getOrCreateSrcLocStr(Loc.DL, SrcLocStrSize,
366 Loc.IP.getBlock()->getParent());
367}
368
369Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {
370 return Builder.CreateCall(
371 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num), Ident,
372 "omp_global_thread_num");
373}
374
375OpenMPIRBuilder::InsertPointTy
376OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive DK,
377 bool ForceSimpleCall, bool CheckCancelFlag) {
378 if (!updateToLocation(Loc))
379 return Loc.IP;
380 return emitBarrierImpl(Loc, DK, ForceSimpleCall, CheckCancelFlag);
381}
382
383OpenMPIRBuilder::InsertPointTy
384OpenMPIRBuilder::emitBarrierImpl(const LocationDescription &Loc, Directive Kind,
385 bool ForceSimpleCall, bool CheckCancelFlag) {
386 // Build call __kmpc_cancel_barrier(loc, thread_id) or
387 // __kmpc_barrier(loc, thread_id);
388
389 IdentFlag BarrierLocFlags;
390 switch (Kind) {
391 case OMPD_for:
392 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
393 break;
394 case OMPD_sections:
395 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
396 break;
397 case OMPD_single:
398 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
399 break;
400 case OMPD_barrier:
401 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
402 break;
403 default:
404 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
405 break;
406 }
407
408 uint32_t SrcLocStrSize;
409 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
410 Value *Args[] = {
411 getOrCreateIdent(SrcLocStr, SrcLocStrSize, BarrierLocFlags),
412 getOrCreateThreadID(getOrCreateIdent(SrcLocStr, SrcLocStrSize))};
413
414 // If we are in a cancellable parallel region, barriers are cancellation
415 // points.
416 // TODO: Check why we would force simple calls or to ignore the cancel flag.
417 bool UseCancelBarrier =
418 !ForceSimpleCall && isLastFinalizationInfoCancellable(OMPD_parallel);
419
420 Value *Result =
421 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(
422 UseCancelBarrier ? OMPRTL___kmpc_cancel_barrier
423 : OMPRTL___kmpc_barrier),
424 Args);
425
426 if (UseCancelBarrier && CheckCancelFlag)
427 emitCancelationCheckImpl(Result, OMPD_parallel);
428
429 return Builder.saveIP();
430}
431
432OpenMPIRBuilder::InsertPointTy
433OpenMPIRBuilder::createCancel(const LocationDescription &Loc,
434 Value *IfCondition,
435 omp::Directive CanceledDirective) {
436 if (!updateToLocation(Loc))
437 return Loc.IP;
438
439 // LLVM utilities like blocks with terminators.
440 auto *UI = Builder.CreateUnreachable();
441
442 Instruction *ThenTI = UI, *ElseTI = nullptr;
443 if (IfCondition)
444 SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI);
445 Builder.SetInsertPoint(ThenTI);
446
447 Value *CancelKind = nullptr;
448 switch (CanceledDirective) {
449#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
450 case DirectiveEnum: \
451 CancelKind = Builder.getInt32(Value); \
452 break;
453#include "llvm/Frontend/OpenMP/OMPKinds.def"
454 default:
455 llvm_unreachable("Unknown cancel kind!")::llvm::llvm_unreachable_internal("Unknown cancel kind!", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 455)
;
456 }
457
458 uint32_t SrcLocStrSize;
459 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
460 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
461 Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
462 Value *Result = Builder.CreateCall(
463 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_cancel), Args);
464 auto ExitCB = [this, CanceledDirective, Loc](InsertPointTy IP) {
465 if (CanceledDirective == OMPD_parallel) {
466 IRBuilder<>::InsertPointGuard IPG(Builder);
467 Builder.restoreIP(IP);
468 createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
469 omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
470 /* CheckCancelFlag */ false);
471 }
472 };
473
474 // The actual cancel logic is shared with others, e.g., cancel_barriers.
475 emitCancelationCheckImpl(Result, CanceledDirective, ExitCB);
476
477 // Update the insertion point and remove the terminator we introduced.
478 Builder.SetInsertPoint(UI->getParent());
479 UI->eraseFromParent();
480
481 return Builder.saveIP();
482}
483
484void OpenMPIRBuilder::emitCancelationCheckImpl(Value *CancelFlag,
485 omp::Directive CanceledDirective,
486 FinalizeCallbackTy ExitCB) {
487 assert(isLastFinalizationInfoCancellable(CanceledDirective) &&(static_cast <bool> (isLastFinalizationInfoCancellable(
CanceledDirective) && "Unexpected cancellation!") ? void
(0) : __assert_fail ("isLastFinalizationInfoCancellable(CanceledDirective) && \"Unexpected cancellation!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 488, __extension__
__PRETTY_FUNCTION__))
488 "Unexpected cancellation!")(static_cast <bool> (isLastFinalizationInfoCancellable(
CanceledDirective) && "Unexpected cancellation!") ? void
(0) : __assert_fail ("isLastFinalizationInfoCancellable(CanceledDirective) && \"Unexpected cancellation!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 488, __extension__
__PRETTY_FUNCTION__))
;
489
490 // For a cancel barrier we create two new blocks.
491 BasicBlock *BB = Builder.GetInsertBlock();
492 BasicBlock *NonCancellationBlock;
493 if (Builder.GetInsertPoint() == BB->end()) {
494 // TODO: This branch will not be needed once we moved to the
495 // OpenMPIRBuilder codegen completely.
496 NonCancellationBlock = BasicBlock::Create(
497 BB->getContext(), BB->getName() + ".cont", BB->getParent());
498 } else {
499 NonCancellationBlock = SplitBlock(BB, &*Builder.GetInsertPoint());
500 BB->getTerminator()->eraseFromParent();
501 Builder.SetInsertPoint(BB);
502 }
503 BasicBlock *CancellationBlock = BasicBlock::Create(
504 BB->getContext(), BB->getName() + ".cncl", BB->getParent());
505
506 // Jump to them based on the return value.
507 Value *Cmp = Builder.CreateIsNull(CancelFlag);
508 Builder.CreateCondBr(Cmp, NonCancellationBlock, CancellationBlock,
509 /* TODO weight */ nullptr, nullptr);
510
511 // From the cancellation block we finalize all variables and go to the
512 // post finalization block that is known to the FiniCB callback.
513 Builder.SetInsertPoint(CancellationBlock);
514 if (ExitCB)
515 ExitCB(Builder.saveIP());
516 auto &FI = FinalizationStack.back();
517 FI.FiniCB(Builder.saveIP());
518
519 // The continuation block is where code generation continues.
520 Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin());
521}
522
523IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
524 const LocationDescription &Loc, InsertPointTy OuterAllocaIP,
525 BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
526 FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads,
527 omp::ProcBindKind ProcBind, bool IsCancellable) {
528 if (!updateToLocation(Loc))
529 return Loc.IP;
530
531 uint32_t SrcLocStrSize;
532 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
533 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
534 Value *ThreadID = getOrCreateThreadID(Ident);
535
536 if (NumThreads) {
537 // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads)
538 Value *Args[] = {
539 Ident, ThreadID,
540 Builder.CreateIntCast(NumThreads, Int32, /*isSigned*/ false)};
541 Builder.CreateCall(
542 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_num_threads), Args);
543 }
544
545 if (ProcBind != OMP_PROC_BIND_default) {
546 // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind)
547 Value *Args[] = {
548 Ident, ThreadID,
549 ConstantInt::get(Int32, unsigned(ProcBind), /*isSigned=*/true)};
550 Builder.CreateCall(
551 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_proc_bind), Args);
552 }
553
554 BasicBlock *InsertBB = Builder.GetInsertBlock();
555 Function *OuterFn = InsertBB->getParent();
556
557 // Save the outer alloca block because the insertion iterator may get
558 // invalidated and we still need this later.
559 BasicBlock *OuterAllocaBlock = OuterAllocaIP.getBlock();
560
561 // Vector to remember instructions we used only during the modeling but which
562 // we want to delete at the end.
563 SmallVector<Instruction *, 4> ToBeDeleted;
564
565 // Change the location to the outer alloca insertion point to create and
566 // initialize the allocas we pass into the parallel region.
567 Builder.restoreIP(OuterAllocaIP);
568 AllocaInst *TIDAddr = Builder.CreateAlloca(Int32, nullptr, "tid.addr");
569 AllocaInst *ZeroAddr = Builder.CreateAlloca(Int32, nullptr, "zero.addr");
570
571 // If there is an if condition we actually use the TIDAddr and ZeroAddr in the
572 // program, otherwise we only need them for modeling purposes to get the
573 // associated arguments in the outlined function. In the former case,
574 // initialize the allocas properly, in the latter case, delete them later.
575 if (IfCondition) {
576 Builder.CreateStore(Constant::getNullValue(Int32), TIDAddr);
577 Builder.CreateStore(Constant::getNullValue(Int32), ZeroAddr);
578 } else {
579 ToBeDeleted.push_back(TIDAddr);
580 ToBeDeleted.push_back(ZeroAddr);
581 }
582
583 // Create an artificial insertion point that will also ensure the blocks we
584 // are about to split are not degenerated.
585 auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);
586
587 Instruction *ThenTI = UI, *ElseTI = nullptr;
588 if (IfCondition)
589 SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI);
590
591 BasicBlock *ThenBB = ThenTI->getParent();
592 BasicBlock *PRegEntryBB = ThenBB->splitBasicBlock(ThenTI, "omp.par.entry");
593 BasicBlock *PRegBodyBB =
594 PRegEntryBB->splitBasicBlock(ThenTI, "omp.par.region");
595 BasicBlock *PRegPreFiniBB =
596 PRegBodyBB->splitBasicBlock(ThenTI, "omp.par.pre_finalize");
597 BasicBlock *PRegExitBB =
598 PRegPreFiniBB->splitBasicBlock(ThenTI, "omp.par.exit");
599
600 auto FiniCBWrapper = [&](InsertPointTy IP) {
601 // Hide "open-ended" blocks from the given FiniCB by setting the right jump
602 // target to the region exit block.
603 if (IP.getBlock()->end() == IP.getPoint()) {
604 IRBuilder<>::InsertPointGuard IPG(Builder);
605 Builder.restoreIP(IP);
606 Instruction *I = Builder.CreateBr(PRegExitBB);
607 IP = InsertPointTy(I->getParent(), I->getIterator());
608 }
609 assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&(static_cast <bool> (IP.getBlock()->getTerminator()->
getNumSuccessors() == 1 && IP.getBlock()->getTerminator
()->getSuccessor(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? void (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 611, __extension__
__PRETTY_FUNCTION__))
610 IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&(static_cast <bool> (IP.getBlock()->getTerminator()->
getNumSuccessors() == 1 && IP.getBlock()->getTerminator
()->getSuccessor(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? void (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 611, __extension__
__PRETTY_FUNCTION__))
611 "Unexpected insertion point for finalization call!")(static_cast <bool> (IP.getBlock()->getTerminator()->
getNumSuccessors() == 1 && IP.getBlock()->getTerminator
()->getSuccessor(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? void (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 611, __extension__
__PRETTY_FUNCTION__))
;
612 return FiniCB(IP);
613 };
614
615 FinalizationStack.push_back({FiniCBWrapper, OMPD_parallel, IsCancellable});
616
617 // Generate the privatization allocas in the block that will become the entry
618 // of the outlined function.
619 Builder.SetInsertPoint(PRegEntryBB->getTerminator());
620 InsertPointTy InnerAllocaIP = Builder.saveIP();
621
622 AllocaInst *PrivTIDAddr =
623 Builder.CreateAlloca(Int32, nullptr, "tid.addr.local");
624 Instruction *PrivTID = Builder.CreateLoad(Int32, PrivTIDAddr, "tid");
625
626 // Add some fake uses for OpenMP provided arguments.
627 ToBeDeleted.push_back(Builder.CreateLoad(Int32, TIDAddr, "tid.addr.use"));
628 Instruction *ZeroAddrUse =
629 Builder.CreateLoad(Int32, ZeroAddr, "zero.addr.use");
630 ToBeDeleted.push_back(ZeroAddrUse);
631
632 // ThenBB
633 // |
634 // V
635 // PRegionEntryBB <- Privatization allocas are placed here.
636 // |
637 // V
638 // PRegionBodyBB <- BodeGen is invoked here.
639 // |
640 // V
641 // PRegPreFiniBB <- The block we will start finalization from.
642 // |
643 // V
644 // PRegionExitBB <- A common exit to simplify block collection.
645 //
646
647 LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before body codegen: "
<< *OuterFn << "\n"; } } while (false)
;
648
649 // Let the caller create the body.
650 assert(BodyGenCB && "Expected body generation callback!")(static_cast <bool> (BodyGenCB && "Expected body generation callback!"
) ? void (0) : __assert_fail ("BodyGenCB && \"Expected body generation callback!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 650, __extension__
__PRETTY_FUNCTION__))
;
651 InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());
652 BodyGenCB(InnerAllocaIP, CodeGenIP, *PRegPreFiniBB);
653
654 LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After body codegen: "
<< *OuterFn << "\n"; } } while (false)
;
655
656 FunctionCallee RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call);
657 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
658 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
659 llvm::LLVMContext &Ctx = F->getContext();
660 MDBuilder MDB(Ctx);
661 // Annotate the callback behavior of the __kmpc_fork_call:
662 // - The callback callee is argument number 2 (microtask).
663 // - The first two arguments of the callback callee are unknown (-1).
664 // - All variadic arguments to the __kmpc_fork_call are passed to the
665 // callback callee.
666 F->addMetadata(
667 llvm::LLVMContext::MD_callback,
668 *llvm::MDNode::get(
669 Ctx, {MDB.createCallbackEncoding(2, {-1, -1},
670 /* VarArgsArePassed */ true)}));
671 }
672 }
673
674 OutlineInfo OI;
675 OI.PostOutlineCB = [=](Function &OutlinedFn) {
676 // Add some known attributes.
677 OutlinedFn.addParamAttr(0, Attribute::NoAlias);
678 OutlinedFn.addParamAttr(1, Attribute::NoAlias);
679 OutlinedFn.addFnAttr(Attribute::NoUnwind);
680 OutlinedFn.addFnAttr(Attribute::NoRecurse);
681
682 assert(OutlinedFn.arg_size() >= 2 &&(static_cast <bool> (OutlinedFn.arg_size() >= 2 &&
"Expected at least tid and bounded tid as arguments") ? void
(0) : __assert_fail ("OutlinedFn.arg_size() >= 2 && \"Expected at least tid and bounded tid as arguments\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 683, __extension__
__PRETTY_FUNCTION__))
683 "Expected at least tid and bounded tid as arguments")(static_cast <bool> (OutlinedFn.arg_size() >= 2 &&
"Expected at least tid and bounded tid as arguments") ? void
(0) : __assert_fail ("OutlinedFn.arg_size() >= 2 && \"Expected at least tid and bounded tid as arguments\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 683, __extension__
__PRETTY_FUNCTION__))
;
684 unsigned NumCapturedVars =
685 OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
686
687 CallInst *CI = cast<CallInst>(OutlinedFn.user_back());
688 CI->getParent()->setName("omp_parallel");
689 Builder.SetInsertPoint(CI);
690
691 // Build call __kmpc_fork_call(Ident, n, microtask, var1, .., varn);
692 Value *ForkCallArgs[] = {
693 Ident, Builder.getInt32(NumCapturedVars),
694 Builder.CreateBitCast(&OutlinedFn, ParallelTaskPtr)};
695
696 SmallVector<Value *, 16> RealArgs;
697 RealArgs.append(std::begin(ForkCallArgs), std::end(ForkCallArgs));
698 RealArgs.append(CI->arg_begin() + /* tid & bound tid */ 2, CI->arg_end());
699
700 Builder.CreateCall(RTLFn, RealArgs);
701
702 LLVM_DEBUG(dbgs() << "With fork_call placed: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With fork_call placed: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
703 << *Builder.GetInsertBlock()->getParent() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With fork_call placed: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
;
704
705 InsertPointTy ExitIP(PRegExitBB, PRegExitBB->end());
706
707 // Initialize the local TID stack location with the argument value.
708 Builder.SetInsertPoint(PrivTID);
709 Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
710 Builder.CreateStore(Builder.CreateLoad(Int32, OutlinedAI), PrivTIDAddr);
711
712 // If no "if" clause was present we do not need the call created during
713 // outlining, otherwise we reuse it in the serialized parallel region.
714 if (!ElseTI) {
715 CI->eraseFromParent();
716 } else {
717
718 // If an "if" clause was present we are now generating the serialized
719 // version into the "else" branch.
720 Builder.SetInsertPoint(ElseTI);
721
722 // Build calls __kmpc_serialized_parallel(&Ident, GTid);
723 Value *SerializedParallelCallArgs[] = {Ident, ThreadID};
724 Builder.CreateCall(
725 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_serialized_parallel),
726 SerializedParallelCallArgs);
727
728 // OutlinedFn(&GTid, &zero, CapturedStruct);
729 CI->removeFromParent();
730 Builder.Insert(CI);
731
732 // __kmpc_end_serialized_parallel(&Ident, GTid);
733 Value *EndArgs[] = {Ident, ThreadID};
734 Builder.CreateCall(
735 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_serialized_parallel),
736 EndArgs);
737
738 LLVM_DEBUG(dbgs() << "With serialized parallel region: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With serialized parallel region: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
739 << *Builder.GetInsertBlock()->getParent() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With serialized parallel region: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
;
740 }
741
742 for (Instruction *I : ToBeDeleted)
743 I->eraseFromParent();
744 };
745
746 // Adjust the finalization stack, verify the adjustment, and call the
747 // finalize function a last time to finalize values between the pre-fini
748 // block and the exit block if we left the parallel "the normal way".
749 auto FiniInfo = FinalizationStack.pop_back_val();
750 (void)FiniInfo;
751 assert(FiniInfo.DK == OMPD_parallel &&(static_cast <bool> (FiniInfo.DK == OMPD_parallel &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("FiniInfo.DK == OMPD_parallel && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 752, __extension__
__PRETTY_FUNCTION__))
752 "Unexpected finalization stack state!")(static_cast <bool> (FiniInfo.DK == OMPD_parallel &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("FiniInfo.DK == OMPD_parallel && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 752, __extension__
__PRETTY_FUNCTION__))
;
753
754 Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();
755
756 InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator());
757 FiniCB(PreFiniIP);
758
759 OI.EntryBB = PRegEntryBB;
760 OI.ExitBB = PRegExitBB;
761
762 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
763 SmallVector<BasicBlock *, 32> Blocks;
764 OI.collectBlocks(ParallelRegionBlockSet, Blocks);
765
766 // Ensure a single exit node for the outlined region by creating one.
767 // We might have multiple incoming edges to the exit now due to finalizations,
768 // e.g., cancel calls that cause the control flow to leave the region.
769 BasicBlock *PRegOutlinedExitBB = PRegExitBB;
770 PRegExitBB = SplitBlock(PRegExitBB, &*PRegExitBB->getFirstInsertionPt());
771 PRegOutlinedExitBB->setName("omp.par.outlined.exit");
772 Blocks.push_back(PRegOutlinedExitBB);
773
774 CodeExtractorAnalysisCache CEAC(*OuterFn);
775 CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
776 /* AggregateArgs */ false,
777 /* BlockFrequencyInfo */ nullptr,
778 /* BranchProbabilityInfo */ nullptr,
779 /* AssumptionCache */ nullptr,
780 /* AllowVarArgs */ true,
781 /* AllowAlloca */ true,
782 /* Suffix */ ".omp_par");
783
784 // Find inputs to, outputs from the code region.
785 BasicBlock *CommonExit = nullptr;
786 SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
787 Extractor.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
788 Extractor.findInputsOutputs(Inputs, Outputs, SinkingCands);
789
790 LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before privatization: "
<< *OuterFn << "\n"; } } while (false)
;
791
792 FunctionCallee TIDRTLFn =
793 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num);
794
795 auto PrivHelper = [&](Value &V) {
796 if (&V == TIDAddr || &V == ZeroAddr)
797 return;
798
799 SetVector<Use *> Uses;
800 for (Use &U : V.uses())
801 if (auto *UserI = dyn_cast<Instruction>(U.getUser()))
802 if (ParallelRegionBlockSet.count(UserI->getParent()))
803 Uses.insert(&U);
804
805 // __kmpc_fork_call expects extra arguments as pointers. If the input
806 // already has a pointer type, everything is fine. Otherwise, store the
807 // value onto stack and load it back inside the to-be-outlined region. This
808 // will ensure only the pointer will be passed to the function.
809 // FIXME: if there are more than 15 trailing arguments, they must be
810 // additionally packed in a struct.
811 Value *Inner = &V;
812 if (!V.getType()->isPointerTy()) {
813 IRBuilder<>::InsertPointGuard Guard(Builder);
814 LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { llvm::dbgs() << "Forwarding input as pointer: "
<< V << "\n"; } } while (false)
;
815
816 Builder.restoreIP(OuterAllocaIP);
817 Value *Ptr =
818 Builder.CreateAlloca(V.getType(), nullptr, V.getName() + ".reloaded");
819
820 // Store to stack at end of the block that currently branches to the entry
821 // block of the to-be-outlined region.
822 Builder.SetInsertPoint(InsertBB,
823 InsertBB->getTerminator()->getIterator());
824 Builder.CreateStore(&V, Ptr);
825
826 // Load back next to allocations in the to-be-outlined region.
827 Builder.restoreIP(InnerAllocaIP);
828 Inner = Builder.CreateLoad(V.getType(), Ptr);
829 }
830
831 Value *ReplacementValue = nullptr;
832 CallInst *CI = dyn_cast<CallInst>(&V);
833 if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {
834 ReplacementValue = PrivTID;
835 } else {
836 Builder.restoreIP(
837 PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue));
838 assert(ReplacementValue &&(static_cast <bool> (ReplacementValue && "Expected copy/create callback to set replacement value!"
) ? void (0) : __assert_fail ("ReplacementValue && \"Expected copy/create callback to set replacement value!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 839, __extension__
__PRETTY_FUNCTION__))
839 "Expected copy/create callback to set replacement value!")(static_cast <bool> (ReplacementValue && "Expected copy/create callback to set replacement value!"
) ? void (0) : __assert_fail ("ReplacementValue && \"Expected copy/create callback to set replacement value!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 839, __extension__
__PRETTY_FUNCTION__))
;
840 if (ReplacementValue == &V)
841 return;
842 }
843
844 for (Use *UPtr : Uses)
845 UPtr->set(ReplacementValue);
846 };
847
848 // Reset the inner alloca insertion as it will be used for loading the values
849 // wrapped into pointers before passing them into the to-be-outlined region.
850 // Configure it to insert immediately after the fake use of zero address so
851 // that they are available in the generated body and so that the
852 // OpenMP-related values (thread ID and zero address pointers) remain leading
853 // in the argument list.
854 InnerAllocaIP = IRBuilder<>::InsertPoint(
855 ZeroAddrUse->getParent(), ZeroAddrUse->getNextNode()->getIterator());
856
857 // Reset the outer alloca insertion point to the entry of the relevant block
858 // in case it was invalidated.
859 OuterAllocaIP = IRBuilder<>::InsertPoint(
860 OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt());
861
862 for (Value *Input : Inputs) {
863 LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Captured input: " <<
*Input << "\n"; } } while (false)
;
864 PrivHelper(*Input);
865 }
866 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
867 for (Value *Output : Outputs)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
868 LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
869 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
;
870 assert(Outputs.empty() &&(static_cast <bool> (Outputs.empty() && "OpenMP outlining should not produce live-out values!"
) ? void (0) : __assert_fail ("Outputs.empty() && \"OpenMP outlining should not produce live-out values!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 871, __extension__
__PRETTY_FUNCTION__))
871 "OpenMP outlining should not produce live-out values!")(static_cast <bool> (Outputs.empty() && "OpenMP outlining should not produce live-out values!"
) ? void (0) : __assert_fail ("Outputs.empty() && \"OpenMP outlining should not produce live-out values!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 871, __extension__
__PRETTY_FUNCTION__))
;
872
873 LLVM_DEBUG(dbgs() << "After privatization: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After privatization: "
<< *OuterFn << "\n"; } } while (false)
;
874 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
875 for (auto *BB : Blocks)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
876 dbgs() << " PBR: " << BB->getName() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
877 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
;
878
879 // Register the outlined info.
880 addOutlineInfo(std::move(OI));
881
882 InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());
883 UI->eraseFromParent();
884
885 return AfterIP;
886}
887
888void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) {
889 // Build call void __kmpc_flush(ident_t *loc)
890 uint32_t SrcLocStrSize;
891 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
892 Value *Args[] = {getOrCreateIdent(SrcLocStr, SrcLocStrSize)};
893
894 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_flush), Args);
895}
896
897void OpenMPIRBuilder::createFlush(const LocationDescription &Loc) {
898 if (!updateToLocation(Loc))
899 return;
900 emitFlush(Loc);
901}
902
903void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) {
904 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
905 // global_tid);
906 uint32_t SrcLocStrSize;
907 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
908 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
909 Value *Args[] = {Ident, getOrCreateThreadID(Ident)};
910
911 // Ignore return result until untied tasks are supported.
912 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskwait),
913 Args);
914}
915
916void OpenMPIRBuilder::createTaskwait(const LocationDescription &Loc) {
917 if (!updateToLocation(Loc))
918 return;
919 emitTaskwaitImpl(Loc);
920}
921
922void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) {
923 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
924 uint32_t SrcLocStrSize;
925 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
926 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
927 Constant *I32Null = ConstantInt::getNullValue(Int32);
928 Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};
929
930 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskyield),
931 Args);
932}
933
934void OpenMPIRBuilder::createTaskyield(const LocationDescription &Loc) {
935 if (!updateToLocation(Loc))
936 return;
937 emitTaskyieldImpl(Loc);
938}
939
940OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections(
941 const LocationDescription &Loc, InsertPointTy AllocaIP,
942 ArrayRef<StorableBodyGenCallbackTy> SectionCBs, PrivatizeCallbackTy PrivCB,
943 FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait) {
944 if (!updateToLocation(Loc))
945 return Loc.IP;
946
947 auto FiniCBWrapper = [&](InsertPointTy IP) {
948 if (IP.getBlock()->end() != IP.getPoint())
949 return FiniCB(IP);
950 // This must be done otherwise any nested constructs using FinalizeOMPRegion
951 // will fail because that function requires the Finalization Basic Block to
952 // have a terminator, which is already removed by EmitOMPRegionBody.
953 // IP is currently at cancelation block.
954 // We need to backtrack to the condition block to fetch
955 // the exit block and create a branch from cancelation
956 // to exit block.
957 IRBuilder<>::InsertPointGuard IPG(Builder);
958 Builder.restoreIP(IP);
959 auto *CaseBB = IP.getBlock()->getSinglePredecessor();
960 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
961 auto *ExitBB = CondBB->getTerminator()->getSuccessor(1);
962 Instruction *I = Builder.CreateBr(ExitBB);
963 IP = InsertPointTy(I->getParent(), I->getIterator());
964 return FiniCB(IP);
965 };
966
967 FinalizationStack.push_back({FiniCBWrapper, OMPD_sections, IsCancellable});
968
969 // Each section is emitted as a switch case
970 // Each finalization callback is handled from clang.EmitOMPSectionDirective()
971 // -> OMP.createSection() which generates the IR for each section
972 // Iterate through all sections and emit a switch construct:
973 // switch (IV) {
974 // case 0:
975 // <SectionStmt[0]>;
976 // break;
977 // ...
978 // case <NumSection> - 1:
979 // <SectionStmt[<NumSection> - 1]>;
980 // break;
981 // }
982 // ...
983 // section_loop.after:
984 // <FiniCB>;
985 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) {
986 auto *CurFn = CodeGenIP.getBlock()->getParent();
987 auto *ForIncBB = CodeGenIP.getBlock()->getSingleSuccessor();
988 auto *ForExitBB = CodeGenIP.getBlock()
989 ->getSinglePredecessor()
990 ->getTerminator()
991 ->getSuccessor(1);
992 SwitchInst *SwitchStmt = Builder.CreateSwitch(IndVar, ForIncBB);
993 Builder.restoreIP(CodeGenIP);
994 unsigned CaseNumber = 0;
995 for (auto SectionCB : SectionCBs) {
996 auto *CaseBB = BasicBlock::Create(M.getContext(),
997 "omp_section_loop.body.case", CurFn);
998 SwitchStmt->addCase(Builder.getInt32(CaseNumber), CaseBB);
999 Builder.SetInsertPoint(CaseBB);
1000 SectionCB(InsertPointTy(), Builder.saveIP(), *ForExitBB);
1001 CaseNumber++;
1002 }
1003 // remove the existing terminator from body BB since there can be no
1004 // terminators after switch/case
1005 CodeGenIP.getBlock()->getTerminator()->eraseFromParent();
1006 };
1007 // Loop body ends here
1008 // LowerBound, UpperBound, and STride for createCanonicalLoop
1009 Type *I32Ty = Type::getInt32Ty(M.getContext());
1010 Value *LB = ConstantInt::get(I32Ty, 0);
1011 Value *UB = ConstantInt::get(I32Ty, SectionCBs.size());
1012 Value *ST = ConstantInt::get(I32Ty, 1);
1013 llvm::CanonicalLoopInfo *LoopInfo = createCanonicalLoop(
1014 Loc, LoopBodyGenCB, LB, UB, ST, true, false, AllocaIP, "section_loop");
1015 Builder.SetInsertPoint(AllocaIP.getBlock()->getTerminator());
1016 AllocaIP = Builder.saveIP();
1017 InsertPointTy AfterIP =
1018 applyStaticWorkshareLoop(Loc.DL, LoopInfo, AllocaIP, !IsNowait);
1019 BasicBlock *LoopAfterBB = AfterIP.getBlock();
1020 Instruction *SplitPos = LoopAfterBB->getTerminator();
1021 if (!isa_and_nonnull<BranchInst>(SplitPos))
1022 SplitPos = new UnreachableInst(Builder.getContext(), LoopAfterBB);
1023 // ExitBB after LoopAfterBB because LoopAfterBB is used for FinalizationCB,
1024 // which requires a BB with branch
1025 BasicBlock *ExitBB =
1026 LoopAfterBB->splitBasicBlock(SplitPos, "omp_sections.end");
1027 SplitPos->eraseFromParent();
1028
1029 // Apply the finalization callback in LoopAfterBB
1030 auto FiniInfo = FinalizationStack.pop_back_val();
1031 assert(FiniInfo.DK == OMPD_sections &&(static_cast <bool> (FiniInfo.DK == OMPD_sections &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("FiniInfo.DK == OMPD_sections && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1032, __extension__
__PRETTY_FUNCTION__))
1032 "Unexpected finalization stack state!")(static_cast <bool> (FiniInfo.DK == OMPD_sections &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("FiniInfo.DK == OMPD_sections && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1032, __extension__
__PRETTY_FUNCTION__))
;
1033 Builder.SetInsertPoint(LoopAfterBB->getTerminator());
1034 FiniInfo.FiniCB(Builder.saveIP());
1035 Builder.SetInsertPoint(ExitBB);
1036
1037 return Builder.saveIP();
1038}
1039
1040OpenMPIRBuilder::InsertPointTy
1041OpenMPIRBuilder::createSection(const LocationDescription &Loc,
1042 BodyGenCallbackTy BodyGenCB,
1043 FinalizeCallbackTy FiniCB) {
1044 if (!updateToLocation(Loc))
1045 return Loc.IP;
1046
1047 auto FiniCBWrapper = [&](InsertPointTy IP) {
1048 if (IP.getBlock()->end() != IP.getPoint())
1049 return FiniCB(IP);
1050 // This must be done otherwise any nested constructs using FinalizeOMPRegion
1051 // will fail because that function requires the Finalization Basic Block to
1052 // have a terminator, which is already removed by EmitOMPRegionBody.
1053 // IP is currently at cancelation block.
1054 // We need to backtrack to the condition block to fetch
1055 // the exit block and create a branch from cancelation
1056 // to exit block.
1057 IRBuilder<>::InsertPointGuard IPG(Builder);
1058 Builder.restoreIP(IP);
1059 auto *CaseBB = Loc.IP.getBlock();
1060 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
1061 auto *ExitBB = CondBB->getTerminator()->getSuccessor(1);
1062 Instruction *I = Builder.CreateBr(ExitBB);
1063 IP = InsertPointTy(I->getParent(), I->getIterator());
1064 return FiniCB(IP);
1065 };
1066
1067 Directive OMPD = Directive::OMPD_sections;
1068 // Since we are using Finalization Callback here, HasFinalize
1069 // and IsCancellable have to be true
1070 return EmitOMPInlinedRegion(OMPD, nullptr, nullptr, BodyGenCB, FiniCBWrapper,
1071 /*Conditional*/ false, /*hasFinalize*/ true,
1072 /*IsCancellable*/ true);
1073}
1074
1075/// Create a function with a unique name and a "void (i8*, i8*)" signature in
1076/// the given module and return it.
1077Function *getFreshReductionFunc(Module &M) {
1078 Type *VoidTy = Type::getVoidTy(M.getContext());
1079 Type *Int8PtrTy = Type::getInt8PtrTy(M.getContext());
1080 auto *FuncTy =
1081 FunctionType::get(VoidTy, {Int8PtrTy, Int8PtrTy}, /* IsVarArg */ false);
1082 return Function::Create(FuncTy, GlobalVariable::InternalLinkage,
1083 M.getDataLayout().getDefaultGlobalsAddressSpace(),
1084 ".omp.reduction.func", &M);
1085}
1086
1087OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
1088 const LocationDescription &Loc, InsertPointTy AllocaIP,
1089 ArrayRef<ReductionInfo> ReductionInfos, bool IsNoWait) {
1090 for (const ReductionInfo &RI : ReductionInfos) {
1091 (void)RI;
1092 assert(RI.Variable && "expected non-null variable")(static_cast <bool> (RI.Variable && "expected non-null variable"
) ? void (0) : __assert_fail ("RI.Variable && \"expected non-null variable\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1092, __extension__
__PRETTY_FUNCTION__))
;
1093 assert(RI.PrivateVariable && "expected non-null private variable")(static_cast <bool> (RI.PrivateVariable && "expected non-null private variable"
) ? void (0) : __assert_fail ("RI.PrivateVariable && \"expected non-null private variable\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1093, __extension__
__PRETTY_FUNCTION__))
;
1094 assert(RI.ReductionGen && "expected non-null reduction generator callback")(static_cast <bool> (RI.ReductionGen && "expected non-null reduction generator callback"
) ? void (0) : __assert_fail ("RI.ReductionGen && \"expected non-null reduction generator callback\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1094, __extension__
__PRETTY_FUNCTION__))
;
1095 assert(RI.Variable->getType() == RI.PrivateVariable->getType() &&(static_cast <bool> (RI.Variable->getType() == RI.PrivateVariable
->getType() && "expected variables and their private equivalents to have the same "
"type") ? void (0) : __assert_fail ("RI.Variable->getType() == RI.PrivateVariable->getType() && \"expected variables and their private equivalents to have the same \" \"type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1097, __extension__
__PRETTY_FUNCTION__))
1096 "expected variables and their private equivalents to have the same "(static_cast <bool> (RI.Variable->getType() == RI.PrivateVariable
->getType() && "expected variables and their private equivalents to have the same "
"type") ? void (0) : __assert_fail ("RI.Variable->getType() == RI.PrivateVariable->getType() && \"expected variables and their private equivalents to have the same \" \"type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1097, __extension__
__PRETTY_FUNCTION__))
1097 "type")(static_cast <bool> (RI.Variable->getType() == RI.PrivateVariable
->getType() && "expected variables and their private equivalents to have the same "
"type") ? void (0) : __assert_fail ("RI.Variable->getType() == RI.PrivateVariable->getType() && \"expected variables and their private equivalents to have the same \" \"type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1097, __extension__
__PRETTY_FUNCTION__))
;
1098 assert(RI.Variable->getType()->isPointerTy() &&(static_cast <bool> (RI.Variable->getType()->isPointerTy
() && "expected variables to be pointers") ? void (0)
: __assert_fail ("RI.Variable->getType()->isPointerTy() && \"expected variables to be pointers\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1099, __extension__
__PRETTY_FUNCTION__))
1099 "expected variables to be pointers")(static_cast <bool> (RI.Variable->getType()->isPointerTy
() && "expected variables to be pointers") ? void (0)
: __assert_fail ("RI.Variable->getType()->isPointerTy() && \"expected variables to be pointers\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1099, __extension__
__PRETTY_FUNCTION__))
;
1100 }
1101
1102 if (!updateToLocation(Loc))
1103 return InsertPointTy();
1104
1105 BasicBlock *InsertBlock = Loc.IP.getBlock();
1106 BasicBlock *ContinuationBlock =
1107 InsertBlock->splitBasicBlock(Loc.IP.getPoint(), "reduce.finalize");
1108 InsertBlock->getTerminator()->eraseFromParent();
1109
1110 // Create and populate array of type-erased pointers to private reduction
1111 // values.
1112 unsigned NumReductions = ReductionInfos.size();
1113 Type *RedArrayTy = ArrayType::get(Builder.getInt8PtrTy(), NumReductions);
1114 Builder.restoreIP(AllocaIP);
1115 Value *RedArray = Builder.CreateAlloca(RedArrayTy, nullptr, "red.array");
1116
1117 Builder.SetInsertPoint(InsertBlock, InsertBlock->end());
1118
1119 for (auto En : enumerate(ReductionInfos)) {
1120 unsigned Index = En.index();
1121 const ReductionInfo &RI = En.value();
1122 Value *RedArrayElemPtr = Builder.CreateConstInBoundsGEP2_64(
1123 RedArrayTy, RedArray, 0, Index, "red.array.elem." + Twine(Index));
1124 Value *Casted =
1125 Builder.CreateBitCast(RI.PrivateVariable, Builder.getInt8PtrTy(),
1126 "private.red.var." + Twine(Index) + ".casted");
1127 Builder.CreateStore(Casted, RedArrayElemPtr);
1128 }
1129
1130 // Emit a call to the runtime function that orchestrates the reduction.
1131 // Declare the reduction function in the process.
1132 Function *Func = Builder.GetInsertBlock()->getParent();
1133 Module *Module = Func->getParent();
1134 Value *RedArrayPtr =
1135 Builder.CreateBitCast(RedArray, Builder.getInt8PtrTy(), "red.array.ptr");
1136 uint32_t SrcLocStrSize;
1137 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1138 bool CanGenerateAtomic =
1139 llvm::all_of(ReductionInfos, [](const ReductionInfo &RI) {
1140 return RI.AtomicReductionGen;
1141 });
1142 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize,
1143 CanGenerateAtomic
1144 ? IdentFlag::OMP_IDENT_FLAG_ATOMIC_REDUCE
1145 : IdentFlag(0));
1146 Value *ThreadId = getOrCreateThreadID(Ident);
1147 Constant *NumVariables = Builder.getInt32(NumReductions);
1148 const DataLayout &DL = Module->getDataLayout();
1149 unsigned RedArrayByteSize = DL.getTypeStoreSize(RedArrayTy);
1150 Constant *RedArraySize = Builder.getInt64(RedArrayByteSize);
1151 Function *ReductionFunc = getFreshReductionFunc(*Module);
1152 Value *Lock = getOMPCriticalRegionLock(".reduction");
1153 Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(
1154 IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
1155 : RuntimeFunction::OMPRTL___kmpc_reduce);
1156 CallInst *ReduceCall =
1157 Builder.CreateCall(ReduceFunc,
1158 {Ident, ThreadId, NumVariables, RedArraySize,
1159 RedArrayPtr, ReductionFunc, Lock},
1160 "reduce");
1161
1162 // Create final reduction entry blocks for the atomic and non-atomic case.
1163 // Emit IR that dispatches control flow to one of the blocks based on the
1164 // reduction supporting the atomic mode.
1165 BasicBlock *NonAtomicRedBlock =
1166 BasicBlock::Create(Module->getContext(), "reduce.switch.nonatomic", Func);
1167 BasicBlock *AtomicRedBlock =
1168 BasicBlock::Create(Module->getContext(), "reduce.switch.atomic", Func);
1169 SwitchInst *Switch =
1170 Builder.CreateSwitch(ReduceCall, ContinuationBlock, /* NumCases */ 2);
1171 Switch->addCase(Builder.getInt32(1), NonAtomicRedBlock);
1172 Switch->addCase(Builder.getInt32(2), AtomicRedBlock);
1173
1174 // Populate the non-atomic reduction using the elementwise reduction function.
1175 // This loads the elements from the global and private variables and reduces
1176 // them before storing back the result to the global variable.
1177 Builder.SetInsertPoint(NonAtomicRedBlock);
1178 for (auto En : enumerate(ReductionInfos)) {
1179 const ReductionInfo &RI = En.value();
1180 Type *ValueType = RI.ElementType;
1181 Value *RedValue = Builder.CreateLoad(ValueType, RI.Variable,
1182 "red.value." + Twine(En.index()));
1183 Value *PrivateRedValue =
1184 Builder.CreateLoad(ValueType, RI.PrivateVariable,
1185 "red.private.value." + Twine(En.index()));
1186 Value *Reduced;
1187 Builder.restoreIP(
1188 RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced));
1189 if (!Builder.GetInsertBlock())
1190 return InsertPointTy();
1191 Builder.CreateStore(Reduced, RI.Variable);
1192 }
1193 Function *EndReduceFunc = getOrCreateRuntimeFunctionPtr(
1194 IsNoWait ? RuntimeFunction::OMPRTL___kmpc_end_reduce_nowait
1195 : RuntimeFunction::OMPRTL___kmpc_end_reduce);
1196 Builder.CreateCall(EndReduceFunc, {Ident, ThreadId, Lock});
1197 Builder.CreateBr(ContinuationBlock);
1198
1199 // Populate the atomic reduction using the atomic elementwise reduction
1200 // function. There are no loads/stores here because they will be happening
1201 // inside the atomic elementwise reduction.
1202 Builder.SetInsertPoint(AtomicRedBlock);
1203 if (CanGenerateAtomic) {
1204 for (const ReductionInfo &RI : ReductionInfos) {
1205 Builder.restoreIP(RI.AtomicReductionGen(Builder.saveIP(), RI.ElementType,
1206 RI.Variable, RI.PrivateVariable));
1207 if (!Builder.GetInsertBlock())
1208 return InsertPointTy();
1209 }
1210 Builder.CreateBr(ContinuationBlock);
1211 } else {
1212 Builder.CreateUnreachable();
1213 }
1214
1215 // Populate the outlined reduction function using the elementwise reduction
1216 // function. Partial values are extracted from the type-erased array of
1217 // pointers to private variables.
1218 BasicBlock *ReductionFuncBlock =
1219 BasicBlock::Create(Module->getContext(), "", ReductionFunc);
1220 Builder.SetInsertPoint(ReductionFuncBlock);
1221 Value *LHSArrayPtr = Builder.CreateBitCast(ReductionFunc->getArg(0),
1222 RedArrayTy->getPointerTo());
1223 Value *RHSArrayPtr = Builder.CreateBitCast(ReductionFunc->getArg(1),
1224 RedArrayTy->getPointerTo());
1225 for (auto En : enumerate(ReductionInfos)) {
1226 const ReductionInfo &RI = En.value();
1227 Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
1228 RedArrayTy, LHSArrayPtr, 0, En.index());
1229 Value *LHSI8Ptr = Builder.CreateLoad(Builder.getInt8PtrTy(), LHSI8PtrPtr);
1230 Value *LHSPtr = Builder.CreateBitCast(LHSI8Ptr, RI.Variable->getType());
1231 Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
1232 Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
1233 RedArrayTy, RHSArrayPtr, 0, En.index());
1234 Value *RHSI8Ptr = Builder.CreateLoad(Builder.getInt8PtrTy(), RHSI8PtrPtr);
1235 Value *RHSPtr =
1236 Builder.CreateBitCast(RHSI8Ptr, RI.PrivateVariable->getType());
1237 Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
1238 Value *Reduced;
1239 Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced));
1240 if (!Builder.GetInsertBlock())
1241 return InsertPointTy();
1242 Builder.CreateStore(Reduced, LHSPtr);
1243 }
1244 Builder.CreateRetVoid();
1245
1246 Builder.SetInsertPoint(ContinuationBlock);
1247 return Builder.saveIP();
1248}
1249
1250OpenMPIRBuilder::InsertPointTy
1251OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
1252 BodyGenCallbackTy BodyGenCB,
1253 FinalizeCallbackTy FiniCB) {
1254
1255 if (!updateToLocation(Loc))
1256 return Loc.IP;
1257
1258 Directive OMPD = Directive::OMPD_master;
1259 uint32_t SrcLocStrSize;
1260 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1261 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1262 Value *ThreadId = getOrCreateThreadID(Ident);
1263 Value *Args[] = {Ident, ThreadId};
1264
1265 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_master);
1266 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
1267
1268 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_master);
1269 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
1270
1271 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
1272 /*Conditional*/ true, /*hasFinalize*/ true);
1273}
1274
1275OpenMPIRBuilder::InsertPointTy
1276OpenMPIRBuilder::createMasked(const LocationDescription &Loc,
1277 BodyGenCallbackTy BodyGenCB,
1278 FinalizeCallbackTy FiniCB, Value *Filter) {
1279 if (!updateToLocation(Loc))
1280 return Loc.IP;
1281
1282 Directive OMPD = Directive::OMPD_masked;
1283 uint32_t SrcLocStrSize;
1284 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1285 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1286 Value *ThreadId = getOrCreateThreadID(Ident);
1287 Value *Args[] = {Ident, ThreadId, Filter};
1288 Value *ArgsEnd[] = {Ident, ThreadId};
1289
1290 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_masked);
1291 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
1292
1293 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_masked);
1294 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, ArgsEnd);
1295
1296 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
1297 /*Conditional*/ true, /*hasFinalize*/ true);
1298}
1299
1300CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
1301 DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
1302 BasicBlock *PostInsertBefore, const Twine &Name) {
1303 Module *M = F->getParent();
1304 LLVMContext &Ctx = M->getContext();
1305 Type *IndVarTy = TripCount->getType();
1306
1307 // Create the basic block structure.
1308 BasicBlock *Preheader =
1309 BasicBlock::Create(Ctx, "omp_" + Name + ".preheader", F, PreInsertBefore);
1310 BasicBlock *Header =
1311 BasicBlock::Create(Ctx, "omp_" + Name + ".header", F, PreInsertBefore);
1312 BasicBlock *Cond =
1313 BasicBlock::Create(Ctx, "omp_" + Name + ".cond", F, PreInsertBefore);
1314 BasicBlock *Body =
1315 BasicBlock::Create(Ctx, "omp_" + Name + ".body", F, PreInsertBefore);
1316 BasicBlock *Latch =
1317 BasicBlock::Create(Ctx, "omp_" + Name + ".inc", F, PostInsertBefore);
1318 BasicBlock *Exit =
1319 BasicBlock::Create(Ctx, "omp_" + Name + ".exit", F, PostInsertBefore);
1320 BasicBlock *After =
1321 BasicBlock::Create(Ctx, "omp_" + Name + ".after", F, PostInsertBefore);
1322
1323 // Use specified DebugLoc for new instructions.
1324 Builder.SetCurrentDebugLocation(DL);
1325
1326 Builder.SetInsertPoint(Preheader);
1327 Builder.CreateBr(Header);
1328
1329 Builder.SetInsertPoint(Header);
1330 PHINode *IndVarPHI = Builder.CreatePHI(IndVarTy, 2, "omp_" + Name + ".iv");
1331 IndVarPHI->addIncoming(ConstantInt::get(IndVarTy, 0), Preheader);
1332 Builder.CreateBr(Cond);
1333
1334 Builder.SetInsertPoint(Cond);
1335 Value *Cmp =
1336 Builder.CreateICmpULT(IndVarPHI, TripCount, "omp_" + Name + ".cmp");
1337 Builder.CreateCondBr(Cmp, Body, Exit);
1338
1339 Builder.SetInsertPoint(Body);
1340 Builder.CreateBr(Latch);
1341
1342 Builder.SetInsertPoint(Latch);
1343 Value *Next = Builder.CreateAdd(IndVarPHI, ConstantInt::get(IndVarTy, 1),
1344 "omp_" + Name + ".next", /*HasNUW=*/true);
1345 Builder.CreateBr(Header);
1346 IndVarPHI->addIncoming(Next, Latch);
1347
1348 Builder.SetInsertPoint(Exit);
1349 Builder.CreateBr(After);
1350
1351 // Remember and return the canonical control flow.
1352 LoopInfos.emplace_front();
1353 CanonicalLoopInfo *CL = &LoopInfos.front();
1354
1355 CL->Header = Header;
1356 CL->Cond = Cond;
1357 CL->Latch = Latch;
1358 CL->Exit = Exit;
1359
1360#ifndef NDEBUG
1361 CL->assertOK();
1362#endif
1363 return CL;
1364}
1365
1366CanonicalLoopInfo *
1367OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
1368 LoopBodyGenCallbackTy BodyGenCB,
1369 Value *TripCount, const Twine &Name) {
1370 BasicBlock *BB = Loc.IP.getBlock();
1371 BasicBlock *NextBB = BB->getNextNode();
1372
1373 CanonicalLoopInfo *CL = createLoopSkeleton(Loc.DL, TripCount, BB->getParent(),
1374 NextBB, NextBB, Name);
1375 BasicBlock *After = CL->getAfter();
1376
1377 // If location is not set, don't connect the loop.
1378 if (updateToLocation(Loc)) {
1379 // Split the loop at the insertion point: Branch to the preheader and move
1380 // every following instruction to after the loop (the After BB). Also, the
1381 // new successor is the loop's after block.
1382 Builder.CreateBr(CL->getPreheader());
1383 After->getInstList().splice(After->begin(), BB->getInstList(),
1384 Builder.GetInsertPoint(), BB->end());
1385 After->replaceSuccessorsPhiUsesWith(BB, After);
1386 }
1387
1388 // Emit the body content. We do it after connecting the loop to the CFG to
1389 // avoid that the callback encounters degenerate BBs.
1390 BodyGenCB(CL->getBodyIP(), CL->getIndVar());
1391
1392#ifndef NDEBUG
1393 CL->assertOK();
1394#endif
1395 return CL;
1396}
1397
1398CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop(
1399 const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
1400 Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
1401 InsertPointTy ComputeIP, const Twine &Name) {
1402
1403 // Consider the following difficulties (assuming 8-bit signed integers):
1404 // * Adding \p Step to the loop counter which passes \p Stop may overflow:
1405 // DO I = 1, 100, 50
1406 /// * A \p Step of INT_MIN cannot not be normalized to a positive direction:
1407 // DO I = 100, 0, -128
1408
1409 // Start, Stop and Step must be of the same integer type.
1410 auto *IndVarTy = cast<IntegerType>(Start->getType());
1411 assert(IndVarTy == Stop->getType() && "Stop type mismatch")(static_cast <bool> (IndVarTy == Stop->getType() &&
"Stop type mismatch") ? void (0) : __assert_fail ("IndVarTy == Stop->getType() && \"Stop type mismatch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1411, __extension__
__PRETTY_FUNCTION__))
;
1412 assert(IndVarTy == Step->getType() && "Step type mismatch")(static_cast <bool> (IndVarTy == Step->getType() &&
"Step type mismatch") ? void (0) : __assert_fail ("IndVarTy == Step->getType() && \"Step type mismatch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1412, __extension__
__PRETTY_FUNCTION__))
;
1413
1414 LocationDescription ComputeLoc =
1415 ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
1416 updateToLocation(ComputeLoc);
1417
1418 ConstantInt *Zero = ConstantInt::get(IndVarTy, 0);
1419 ConstantInt *One = ConstantInt::get(IndVarTy, 1);
1420
1421 // Like Step, but always positive.
1422 Value *Incr = Step;
1423
1424 // Distance between Start and Stop; always positive.
1425 Value *Span;
1426
1427 // Condition whether there are no iterations are executed at all, e.g. because
1428 // UB < LB.
1429 Value *ZeroCmp;
1430
1431 if (IsSigned) {
1432 // Ensure that increment is positive. If not, negate and invert LB and UB.
1433 Value *IsNeg = Builder.CreateICmpSLT(Step, Zero);
1434 Incr = Builder.CreateSelect(IsNeg, Builder.CreateNeg(Step), Step);
1435 Value *LB = Builder.CreateSelect(IsNeg, Stop, Start);
1436 Value *UB = Builder.CreateSelect(IsNeg, Start, Stop);
1437 Span = Builder.CreateSub(UB, LB, "", false, true);
1438 ZeroCmp = Builder.CreateICmp(
1439 InclusiveStop ? CmpInst::ICMP_SLT : CmpInst::ICMP_SLE, UB, LB);
1440 } else {
1441 Span = Builder.CreateSub(Stop, Start, "", true);
1442 ZeroCmp = Builder.CreateICmp(
1443 InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Stop, Start);
1444 }
1445
1446 Value *CountIfLooping;
1447 if (InclusiveStop) {
1448 CountIfLooping = Builder.CreateAdd(Builder.CreateUDiv(Span, Incr), One);
1449 } else {
1450 // Avoid incrementing past stop since it could overflow.
1451 Value *CountIfTwo = Builder.CreateAdd(
1452 Builder.CreateUDiv(Builder.CreateSub(Span, One), Incr), One);
1453 Value *OneCmp = Builder.CreateICmp(
1454 InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Span, Incr);
1455 CountIfLooping = Builder.CreateSelect(OneCmp, One, CountIfTwo);
1456 }
1457 Value *TripCount = Builder.CreateSelect(ZeroCmp, Zero, CountIfLooping,
1458 "omp_" + Name + ".tripcount");
1459
1460 auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
1461 Builder.restoreIP(CodeGenIP);
1462 Value *Span = Builder.CreateMul(IV, Step);
1463 Value *IndVar = Builder.CreateAdd(Span, Start);
1464 BodyGenCB(Builder.saveIP(), IndVar);
1465 };
1466 LocationDescription LoopLoc = ComputeIP.isSet() ? Loc.IP : Builder.saveIP();
1467 return createCanonicalLoop(LoopLoc, BodyGen, TripCount, Name);
1468}
1469
1470// Returns an LLVM function to call for initializing loop bounds using OpenMP
1471// static scheduling depending on `type`. Only i32 and i64 are supported by the
1472// runtime. Always interpret integers as unsigned similarly to
1473// CanonicalLoopInfo.
1474static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,
1475 OpenMPIRBuilder &OMPBuilder) {
1476 unsigned Bitwidth = Ty->getIntegerBitWidth();
1477 if (Bitwidth == 32)
1478 return OMPBuilder.getOrCreateRuntimeFunction(
1479 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
1480 if (Bitwidth == 64)
1481 return OMPBuilder.getOrCreateRuntimeFunction(
1482 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
1483 llvm_unreachable("unknown OpenMP loop iterator bitwidth")::llvm::llvm_unreachable_internal("unknown OpenMP loop iterator bitwidth"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1483)
;
1484}
1485
1486// Sets the number of loop iterations to the given value. This value must be
1487// valid in the condition block (i.e., defined in the preheader) and is
1488// interpreted as an unsigned integer.
1489void setCanonicalLoopTripCount(CanonicalLoopInfo *CLI, Value *TripCount) {
1490 Instruction *CmpI = &CLI->getCond()->front();
1491 assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount")(static_cast <bool> (isa<CmpInst>(CmpI) &&
"First inst must compare IV with TripCount") ? void (0) : __assert_fail
("isa<CmpInst>(CmpI) && \"First inst must compare IV with TripCount\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1491, __extension__
__PRETTY_FUNCTION__))
;
1492 CmpI->setOperand(1, TripCount);
1493 CLI->assertOK();
1494}
1495
1496OpenMPIRBuilder::InsertPointTy
1497OpenMPIRBuilder::applyStaticWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
1498 InsertPointTy AllocaIP,
1499 bool NeedsBarrier, Value *Chunk) {
1500 assert(CLI->isValid() && "Requires a valid canonical loop")(static_cast <bool> (CLI->isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("CLI->isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1500, __extension__
__PRETTY_FUNCTION__))
;
1501
1502 // Set up the source location value for OpenMP runtime.
1503 Builder.restoreIP(CLI->getPreheaderIP());
1504 Builder.SetCurrentDebugLocation(DL);
1505
1506 uint32_t SrcLocStrSize;
1507 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
1508 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1509
1510 // Declare useful OpenMP runtime functions.
1511 Value *IV = CLI->getIndVar();
1512 Type *IVTy = IV->getType();
1513 FunctionCallee StaticInit = getKmpcForStaticInitForType(IVTy, M, *this);
1514 FunctionCallee StaticFini =
1515 getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_for_static_fini);
1516
1517 // Allocate space for computed loop bounds as expected by the "init" function.
1518 Builder.restoreIP(AllocaIP);
1519 Type *I32Type = Type::getInt32Ty(M.getContext());
1520 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
1521 Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");
1522 Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");
1523 Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");
1524
1525 // At the end of the preheader, prepare for calling the "init" function by
1526 // storing the current loop bounds into the allocated space. A canonical loop
1527 // always iterates from 0 to trip-count with step 1. Note that "init" expects
1528 // and produces an inclusive upper bound.
1529 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
1530 Constant *Zero = ConstantInt::get(IVTy, 0);
1531 Constant *One = ConstantInt::get(IVTy, 1);
1532 Builder.CreateStore(Zero, PLowerBound);
1533 Value *UpperBound = Builder.CreateSub(CLI->getTripCount(), One);
1534 Builder.CreateStore(UpperBound, PUpperBound);
1535 Builder.CreateStore(One, PStride);
1536
1537 // FIXME: schedule(static) is NOT the same as schedule(static,1)
1538 if (!Chunk)
1539 Chunk = One;
1540
1541 Value *ThreadNum = getOrCreateThreadID(SrcLoc);
1542
1543 Constant *SchedulingType =
1544 ConstantInt::get(I32Type, static_cast<int>(OMPScheduleType::Static));
1545
1546 // Call the "init" function and update the trip count of the loop with the
1547 // value it produced.
1548 Builder.CreateCall(StaticInit,
1549 {SrcLoc, ThreadNum, SchedulingType, PLastIter, PLowerBound,
1550 PUpperBound, PStride, One, Chunk});
1551 Value *LowerBound = Builder.CreateLoad(IVTy, PLowerBound);
1552 Value *InclusiveUpperBound = Builder.CreateLoad(IVTy, PUpperBound);
1553 Value *TripCountMinusOne = Builder.CreateSub(InclusiveUpperBound, LowerBound);
1554 Value *TripCount = Builder.CreateAdd(TripCountMinusOne, One);
1555 setCanonicalLoopTripCount(CLI, TripCount);
1556
1557 // Update all uses of the induction variable except the one in the condition
1558 // block that compares it with the actual upper bound, and the increment in
1559 // the latch block.
1560 // TODO: this can eventually move to CanonicalLoopInfo or to a new
1561 // CanonicalLoopInfoUpdater interface.
1562 Builder.SetInsertPoint(CLI->getBody(), CLI->getBody()->getFirstInsertionPt());
1563 Value *UpdatedIV = Builder.CreateAdd(IV, LowerBound);
1564 IV->replaceUsesWithIf(UpdatedIV, [&](Use &U) {
1565 auto *Instr = dyn_cast<Instruction>(U.getUser());
1566 return !Instr ||
1567 (Instr->getParent() != CLI->getCond() &&
1568 Instr->getParent() != CLI->getLatch() && Instr != UpdatedIV);
1569 });
1570
1571 // In the "exit" block, call the "fini" function.
1572 Builder.SetInsertPoint(CLI->getExit(),
1573 CLI->getExit()->getTerminator()->getIterator());
1574 Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});
1575
1576 // Add the barrier if requested.
1577 if (NeedsBarrier)
1578 createBarrier(LocationDescription(Builder.saveIP(), DL),
1579 omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
1580 /* CheckCancelFlag */ false);
1581
1582 InsertPointTy AfterIP = CLI->getAfterIP();
1583 CLI->invalidate();
1584
1585 return AfterIP;
1586}
1587
1588OpenMPIRBuilder::InsertPointTy
1589OpenMPIRBuilder::applyWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
1590 InsertPointTy AllocaIP, bool NeedsBarrier) {
1591 // Currently only supports static schedules.
1592 return applyStaticWorkshareLoop(DL, CLI, AllocaIP, NeedsBarrier);
1593}
1594
1595/// Returns an LLVM function to call for initializing loop bounds using OpenMP
1596/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
1597/// the runtime. Always interpret integers as unsigned similarly to
1598/// CanonicalLoopInfo.
1599static FunctionCallee
1600getKmpcForDynamicInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
1601 unsigned Bitwidth = Ty->getIntegerBitWidth();
1602 if (Bitwidth == 32)
1603 return OMPBuilder.getOrCreateRuntimeFunction(
1604 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_4u);
1605 if (Bitwidth == 64)
1606 return OMPBuilder.getOrCreateRuntimeFunction(
1607 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_8u);
1608 llvm_unreachable("unknown OpenMP loop iterator bitwidth")::llvm::llvm_unreachable_internal("unknown OpenMP loop iterator bitwidth"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1608)
;
1609}
1610
1611/// Returns an LLVM function to call for updating the next loop using OpenMP
1612/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
1613/// the runtime. Always interpret integers as unsigned similarly to
1614/// CanonicalLoopInfo.
1615static FunctionCallee
1616getKmpcForDynamicNextForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
1617 unsigned Bitwidth = Ty->getIntegerBitWidth();
1618 if (Bitwidth == 32)
1619 return OMPBuilder.getOrCreateRuntimeFunction(
1620 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_4u);
1621 if (Bitwidth == 64)
1622 return OMPBuilder.getOrCreateRuntimeFunction(
1623 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_8u);
1624 llvm_unreachable("unknown OpenMP loop iterator bitwidth")::llvm::llvm_unreachable_internal("unknown OpenMP loop iterator bitwidth"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1624)
;
1625}
1626
1627OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyDynamicWorkshareLoop(
1628 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
1629 OMPScheduleType SchedType, bool NeedsBarrier, Value *Chunk) {
1630 assert(CLI->isValid() && "Requires a valid canonical loop")(static_cast <bool> (CLI->isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("CLI->isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1630, __extension__
__PRETTY_FUNCTION__))
;
1631
1632 // Set up the source location value for OpenMP runtime.
1633 Builder.SetCurrentDebugLocation(DL);
1634
1635 uint32_t SrcLocStrSize;
1636 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
1637 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1638
1639 // Declare useful OpenMP runtime functions.
1640 Value *IV = CLI->getIndVar();
1641 Type *IVTy = IV->getType();
1642 FunctionCallee DynamicInit = getKmpcForDynamicInitForType(IVTy, M, *this);
1643 FunctionCallee DynamicNext = getKmpcForDynamicNextForType(IVTy, M, *this);
1644
1645 // Allocate space for computed loop bounds as expected by the "init" function.
1646 Builder.restoreIP(AllocaIP);
1647 Type *I32Type = Type::getInt32Ty(M.getContext());
1648 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
1649 Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");
1650 Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");
1651 Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");
1652
1653 // At the end of the preheader, prepare for calling the "init" function by
1654 // storing the current loop bounds into the allocated space. A canonical loop
1655 // always iterates from 0 to trip-count with step 1. Note that "init" expects
1656 // and produces an inclusive upper bound.
1657 BasicBlock *PreHeader = CLI->getPreheader();
1658 Builder.SetInsertPoint(PreHeader->getTerminator());
1659 Constant *One = ConstantInt::get(IVTy, 1);
1660 Builder.CreateStore(One, PLowerBound);
1661 Value *UpperBound = CLI->getTripCount();
1662 Builder.CreateStore(UpperBound, PUpperBound);
1663 Builder.CreateStore(One, PStride);
1664
1665 BasicBlock *Header = CLI->getHeader();
1666 BasicBlock *Exit = CLI->getExit();
1667 BasicBlock *Cond = CLI->getCond();
1668 InsertPointTy AfterIP = CLI->getAfterIP();
1669
1670 // The CLI will be "broken" in the code below, as the loop is no longer
1671 // a valid canonical loop.
1672
1673 if (!Chunk)
1674 Chunk = One;
1675
1676 Value *ThreadNum = getOrCreateThreadID(SrcLoc);
1677
1678 Constant *SchedulingType =
1679 ConstantInt::get(I32Type, static_cast<int>(SchedType));
1680
1681 // Call the "init" function.
1682 Builder.CreateCall(DynamicInit,
1683 {SrcLoc, ThreadNum, SchedulingType, /* LowerBound */ One,
1684 UpperBound, /* step */ One, Chunk});
1685
1686 // An outer loop around the existing one.
1687 BasicBlock *OuterCond = BasicBlock::Create(
1688 PreHeader->getContext(), Twine(PreHeader->getName()) + ".outer.cond",
1689 PreHeader->getParent());
1690 // This needs to be 32-bit always, so can't use the IVTy Zero above.
1691 Builder.SetInsertPoint(OuterCond, OuterCond->getFirstInsertionPt());
1692 Value *Res =
1693 Builder.CreateCall(DynamicNext, {SrcLoc, ThreadNum, PLastIter,
1694 PLowerBound, PUpperBound, PStride});
1695 Constant *Zero32 = ConstantInt::get(I32Type, 0);
1696 Value *MoreWork = Builder.CreateCmp(CmpInst::ICMP_NE, Res, Zero32);
1697 Value *LowerBound =
1698 Builder.CreateSub(Builder.CreateLoad(IVTy, PLowerBound), One, "lb");
1699 Builder.CreateCondBr(MoreWork, Header, Exit);
1700
1701 // Change PHI-node in loop header to use outer cond rather than preheader,
1702 // and set IV to the LowerBound.
1703 Instruction *Phi = &Header->front();
1704 auto *PI = cast<PHINode>(Phi);
1705 PI->setIncomingBlock(0, OuterCond);
1706 PI->setIncomingValue(0, LowerBound);
1707
1708 // Then set the pre-header to jump to the OuterCond
1709 Instruction *Term = PreHeader->getTerminator();
1710 auto *Br = cast<BranchInst>(Term);
1711 Br->setSuccessor(0, OuterCond);
1712
1713 // Modify the inner condition:
1714 // * Use the UpperBound returned from the DynamicNext call.
1715 // * jump to the loop outer loop when done with one of the inner loops.
1716 Builder.SetInsertPoint(Cond, Cond->getFirstInsertionPt());
1717 UpperBound = Builder.CreateLoad(IVTy, PUpperBound, "ub");
1718 Instruction *Comp = &*Builder.GetInsertPoint();
1719 auto *CI = cast<CmpInst>(Comp);
1720 CI->setOperand(1, UpperBound);
1721 // Redirect the inner exit to branch to outer condition.
1722 Instruction *Branch = &Cond->back();
1723 auto *BI = cast<BranchInst>(Branch);
1724 assert(BI->getSuccessor(1) == Exit)(static_cast <bool> (BI->getSuccessor(1) == Exit) ? void
(0) : __assert_fail ("BI->getSuccessor(1) == Exit", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1724, __extension__ __PRETTY_FUNCTION__))
;
1725 BI->setSuccessor(1, OuterCond);
1726
1727 // Add the barrier if requested.
1728 if (NeedsBarrier) {
1729 Builder.SetInsertPoint(&Exit->back());
1730 createBarrier(LocationDescription(Builder.saveIP(), DL),
1731 omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
1732 /* CheckCancelFlag */ false);
1733 }
1734
1735 CLI->invalidate();
1736 return AfterIP;
1737}
1738
1739/// Make \p Source branch to \p Target.
1740///
1741/// Handles two situations:
1742/// * \p Source already has an unconditional branch.
1743/// * \p Source is a degenerate block (no terminator because the BB is
1744/// the current head of the IR construction).
1745static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL) {
1746 if (Instruction *Term = Source->getTerminator()) {
1747 auto *Br = cast<BranchInst>(Term);
1748 assert(!Br->isConditional() &&(static_cast <bool> (!Br->isConditional() &&
"BB's terminator must be an unconditional branch (or degenerate)"
) ? void (0) : __assert_fail ("!Br->isConditional() && \"BB's terminator must be an unconditional branch (or degenerate)\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1749, __extension__
__PRETTY_FUNCTION__))
1749 "BB's terminator must be an unconditional branch (or degenerate)")(static_cast <bool> (!Br->isConditional() &&
"BB's terminator must be an unconditional branch (or degenerate)"
) ? void (0) : __assert_fail ("!Br->isConditional() && \"BB's terminator must be an unconditional branch (or degenerate)\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1749, __extension__
__PRETTY_FUNCTION__))
;
1750 BasicBlock *Succ = Br->getSuccessor(0);
1751 Succ->removePredecessor(Source, /*KeepOneInputPHIs=*/true);
1752 Br->setSuccessor(0, Target);
1753 return;
1754 }
1755
1756 auto *NewBr = BranchInst::Create(Target, Source);
1757 NewBr->setDebugLoc(DL);
1758}
1759
1760/// Redirect all edges that branch to \p OldTarget to \p NewTarget. That is,
1761/// after this \p OldTarget will be orphaned.
1762static void redirectAllPredecessorsTo(BasicBlock *OldTarget,
1763 BasicBlock *NewTarget, DebugLoc DL) {
1764 for (BasicBlock *Pred : make_early_inc_range(predecessors(OldTarget)))
1765 redirectTo(Pred, NewTarget, DL);
1766}
1767
1768/// Determine which blocks in \p BBs are reachable from outside and remove the
1769/// ones that are not reachable from the function.
1770static void removeUnusedBlocksFromParent(ArrayRef<BasicBlock *> BBs) {
1771 SmallPtrSet<BasicBlock *, 6> BBsToErase{BBs.begin(), BBs.end()};
1772 auto HasRemainingUses = [&BBsToErase](BasicBlock *BB) {
1773 for (Use &U : BB->uses()) {
1774 auto *UseInst = dyn_cast<Instruction>(U.getUser());
1775 if (!UseInst)
1776 continue;
1777 if (BBsToErase.count(UseInst->getParent()))
1778 continue;
1779 return true;
1780 }
1781 return false;
1782 };
1783
1784 while (true) {
1785 bool Changed = false;
1786 for (BasicBlock *BB : make_early_inc_range(BBsToErase)) {
1787 if (HasRemainingUses(BB)) {
1788 BBsToErase.erase(BB);
1789 Changed = true;
1790 }
1791 }
1792 if (!Changed)
1793 break;
1794 }
1795
1796 SmallVector<BasicBlock *, 7> BBVec(BBsToErase.begin(), BBsToErase.end());
1797 DeleteDeadBlocks(BBVec);
1798}
1799
1800CanonicalLoopInfo *
1801OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
1802 InsertPointTy ComputeIP) {
1803 assert(Loops.size() >= 1 && "At least one loop required")(static_cast <bool> (Loops.size() >= 1 && "At least one loop required"
) ? void (0) : __assert_fail ("Loops.size() >= 1 && \"At least one loop required\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1803, __extension__
__PRETTY_FUNCTION__))
;
1804 size_t NumLoops = Loops.size();
1805
1806 // Nothing to do if there is already just one loop.
1807 if (NumLoops == 1)
1808 return Loops.front();
1809
1810 CanonicalLoopInfo *Outermost = Loops.front();
1811 CanonicalLoopInfo *Innermost = Loops.back();
1812 BasicBlock *OrigPreheader = Outermost->getPreheader();
1813 BasicBlock *OrigAfter = Outermost->getAfter();
1814 Function *F = OrigPreheader->getParent();
1815
1816 // Loop control blocks that may become orphaned later.
1817 SmallVector<BasicBlock *, 12> OldControlBBs;
1818 OldControlBBs.reserve(6 * Loops.size());
1819 for (CanonicalLoopInfo *Loop : Loops)
1820 Loop->collectControlBlocks(OldControlBBs);
1821
1822 // Setup the IRBuilder for inserting the trip count computation.
1823 Builder.SetCurrentDebugLocation(DL);
1824 if (ComputeIP.isSet())
1825 Builder.restoreIP(ComputeIP);
1826 else
1827 Builder.restoreIP(Outermost->getPreheaderIP());
1828
1829 // Derive the collapsed' loop trip count.
1830 // TODO: Find common/largest indvar type.
1831 Value *CollapsedTripCount = nullptr;
1832 for (CanonicalLoopInfo *L : Loops) {
1833 assert(L->isValid() &&(static_cast <bool> (L->isValid() && "All loops to collapse must be valid canonical loops"
) ? void (0) : __assert_fail ("L->isValid() && \"All loops to collapse must be valid canonical loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1834, __extension__
__PRETTY_FUNCTION__))
1834 "All loops to collapse must be valid canonical loops")(static_cast <bool> (L->isValid() && "All loops to collapse must be valid canonical loops"
) ? void (0) : __assert_fail ("L->isValid() && \"All loops to collapse must be valid canonical loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1834, __extension__
__PRETTY_FUNCTION__))
;
1835 Value *OrigTripCount = L->getTripCount();
1836 if (!CollapsedTripCount) {
1837 CollapsedTripCount = OrigTripCount;
1838 continue;
1839 }
1840
1841 // TODO: Enable UndefinedSanitizer to diagnose an overflow here.
1842 CollapsedTripCount = Builder.CreateMul(CollapsedTripCount, OrigTripCount,
1843 {}, /*HasNUW=*/true);
1844 }
1845
1846 // Create the collapsed loop control flow.
1847 CanonicalLoopInfo *Result =
1848 createLoopSkeleton(DL, CollapsedTripCount, F,
1849 OrigPreheader->getNextNode(), OrigAfter, "collapsed");
1850
1851 // Build the collapsed loop body code.
1852 // Start with deriving the input loop induction variables from the collapsed
1853 // one, using a divmod scheme. To preserve the original loops' order, the
1854 // innermost loop use the least significant bits.
1855 Builder.restoreIP(Result->getBodyIP());
1856
1857 Value *Leftover = Result->getIndVar();
1858 SmallVector<Value *> NewIndVars;
1859 NewIndVars.resize(NumLoops);
1860 for (int i = NumLoops - 1; i >= 1; --i) {
1861 Value *OrigTripCount = Loops[i]->getTripCount();
1862
1863 Value *NewIndVar = Builder.CreateURem(Leftover, OrigTripCount);
1864 NewIndVars[i] = NewIndVar;
1865
1866 Leftover = Builder.CreateUDiv(Leftover, OrigTripCount);
1867 }
1868 // Outermost loop gets all the remaining bits.
1869 NewIndVars[0] = Leftover;
1870
1871 // Construct the loop body control flow.
1872 // We progressively construct the branch structure following in direction of
1873 // the control flow, from the leading in-between code, the loop nest body, the
1874 // trailing in-between code, and rejoining the collapsed loop's latch.
1875 // ContinueBlock and ContinuePred keep track of the source(s) of next edge. If
1876 // the ContinueBlock is set, continue with that block. If ContinuePred, use
1877 // its predecessors as sources.
1878 BasicBlock *ContinueBlock = Result->getBody();
1879 BasicBlock *ContinuePred = nullptr;
1880 auto ContinueWith = [&ContinueBlock, &ContinuePred, DL](BasicBlock *Dest,
1881 BasicBlock *NextSrc) {
1882 if (ContinueBlock)
1883 redirectTo(ContinueBlock, Dest, DL);
1884 else
1885 redirectAllPredecessorsTo(ContinuePred, Dest, DL);
1886
1887 ContinueBlock = nullptr;
1888 ContinuePred = NextSrc;
1889 };
1890
1891 // The code before the nested loop of each level.
1892 // Because we are sinking it into the nest, it will be executed more often
1893 // that the original loop. More sophisticated schemes could keep track of what
1894 // the in-between code is and instantiate it only once per thread.
1895 for (size_t i = 0; i < NumLoops - 1; ++i)
1896 ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader());
1897
1898 // Connect the loop nest body.
1899 ContinueWith(Innermost->getBody(), Innermost->getLatch());
1900
1901 // The code after the nested loop at each level.
1902 for (size_t i = NumLoops - 1; i > 0; --i)
1903 ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch());
1904
1905 // Connect the finished loop to the collapsed loop latch.
1906 ContinueWith(Result->getLatch(), nullptr);
1907
1908 // Replace the input loops with the new collapsed loop.
1909 redirectTo(Outermost->getPreheader(), Result->getPreheader(), DL);
1910 redirectTo(Result->getAfter(), Outermost->getAfter(), DL);
1911
1912 // Replace the input loop indvars with the derived ones.
1913 for (size_t i = 0; i < NumLoops; ++i)
1914 Loops[i]->getIndVar()->replaceAllUsesWith(NewIndVars[i]);
1915
1916 // Remove unused parts of the input loops.
1917 removeUnusedBlocksFromParent(OldControlBBs);
1918
1919 for (CanonicalLoopInfo *L : Loops)
1920 L->invalidate();
1921
1922#ifndef NDEBUG
1923 Result->assertOK();
1924#endif
1925 return Result;
1926}
1927
1928std::vector<CanonicalLoopInfo *>
1929OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
1930 ArrayRef<Value *> TileSizes) {
1931 assert(TileSizes.size() == Loops.size() &&(static_cast <bool> (TileSizes.size() == Loops.size() &&
"Must pass as many tile sizes as there are loops") ? void (0
) : __assert_fail ("TileSizes.size() == Loops.size() && \"Must pass as many tile sizes as there are loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1932, __extension__
__PRETTY_FUNCTION__))
1932 "Must pass as many tile sizes as there are loops")(static_cast <bool> (TileSizes.size() == Loops.size() &&
"Must pass as many tile sizes as there are loops") ? void (0
) : __assert_fail ("TileSizes.size() == Loops.size() && \"Must pass as many tile sizes as there are loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1932, __extension__
__PRETTY_FUNCTION__))
;
1933 int NumLoops = Loops.size();
1934 assert(NumLoops >= 1 && "At least one loop to tile required")(static_cast <bool> (NumLoops >= 1 && "At least one loop to tile required"
) ? void (0) : __assert_fail ("NumLoops >= 1 && \"At least one loop to tile required\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1934, __extension__
__PRETTY_FUNCTION__))
;
1935
1936 CanonicalLoopInfo *OutermostLoop = Loops.front();
1937 CanonicalLoopInfo *InnermostLoop = Loops.back();
1938 Function *F = OutermostLoop->getBody()->getParent();
1939 BasicBlock *InnerEnter = InnermostLoop->getBody();
1940 BasicBlock *InnerLatch = InnermostLoop->getLatch();
1941
1942 // Loop control blocks that may become orphaned later.
1943 SmallVector<BasicBlock *, 12> OldControlBBs;
1944 OldControlBBs.reserve(6 * Loops.size());
1945 for (CanonicalLoopInfo *Loop : Loops)
1946 Loop->collectControlBlocks(OldControlBBs);
1947
1948 // Collect original trip counts and induction variable to be accessible by
1949 // index. Also, the structure of the original loops is not preserved during
1950 // the construction of the tiled loops, so do it before we scavenge the BBs of
1951 // any original CanonicalLoopInfo.
1952 SmallVector<Value *, 4> OrigTripCounts, OrigIndVars;
1953 for (CanonicalLoopInfo *L : Loops) {
1954 assert(L->isValid() && "All input loops must be valid canonical loops")(static_cast <bool> (L->isValid() && "All input loops must be valid canonical loops"
) ? void (0) : __assert_fail ("L->isValid() && \"All input loops must be valid canonical loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1954, __extension__
__PRETTY_FUNCTION__))
;
1955 OrigTripCounts.push_back(L->getTripCount());
1956 OrigIndVars.push_back(L->getIndVar());
1957 }
1958
1959 // Collect the code between loop headers. These may contain SSA definitions
1960 // that are used in the loop nest body. To be usable with in the innermost
1961 // body, these BasicBlocks will be sunk into the loop nest body. That is,
1962 // these instructions may be executed more often than before the tiling.
1963 // TODO: It would be sufficient to only sink them into body of the
1964 // corresponding tile loop.
1965 SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> InbetweenCode;
1966 for (int i = 0; i < NumLoops - 1; ++i) {
1967 CanonicalLoopInfo *Surrounding = Loops[i];
1968 CanonicalLoopInfo *Nested = Loops[i + 1];
1969
1970 BasicBlock *EnterBB = Surrounding->getBody();
1971 BasicBlock *ExitBB = Nested->getHeader();
1972 InbetweenCode.emplace_back(EnterBB, ExitBB);
1973 }
1974
1975 // Compute the trip counts of the floor loops.
1976 Builder.SetCurrentDebugLocation(DL);
1977 Builder.restoreIP(OutermostLoop->getPreheaderIP());
1978 SmallVector<Value *, 4> FloorCount, FloorRems;
1979 for (int i = 0; i < NumLoops; ++i) {
1980 Value *TileSize = TileSizes[i];
1981 Value *OrigTripCount = OrigTripCounts[i];
1982 Type *IVType = OrigTripCount->getType();
1983
1984 Value *FloorTripCount = Builder.CreateUDiv(OrigTripCount, TileSize);
1985 Value *FloorTripRem = Builder.CreateURem(OrigTripCount, TileSize);
1986
1987 // 0 if tripcount divides the tilesize, 1 otherwise.
1988 // 1 means we need an additional iteration for a partial tile.
1989 //
1990 // Unfortunately we cannot just use the roundup-formula
1991 // (tripcount + tilesize - 1)/tilesize
1992 // because the summation might overflow. We do not want introduce undefined
1993 // behavior when the untiled loop nest did not.
1994 Value *FloorTripOverflow =
1995 Builder.CreateICmpNE(FloorTripRem, ConstantInt::get(IVType, 0));
1996
1997 FloorTripOverflow = Builder.CreateZExt(FloorTripOverflow, IVType);
1998 FloorTripCount =
1999 Builder.CreateAdd(FloorTripCount, FloorTripOverflow,
2000 "omp_floor" + Twine(i) + ".tripcount", true);
2001
2002 // Remember some values for later use.
2003 FloorCount.push_back(FloorTripCount);
2004 FloorRems.push_back(FloorTripRem);
2005 }
2006
2007 // Generate the new loop nest, from the outermost to the innermost.
2008 std::vector<CanonicalLoopInfo *> Result;
2009 Result.reserve(NumLoops * 2);
2010
2011 // The basic block of the surrounding loop that enters the nest generated
2012 // loop.
2013 BasicBlock *Enter = OutermostLoop->getPreheader();
2014
2015 // The basic block of the surrounding loop where the inner code should
2016 // continue.
2017 BasicBlock *Continue = OutermostLoop->getAfter();
2018
2019 // Where the next loop basic block should be inserted.
2020 BasicBlock *OutroInsertBefore = InnermostLoop->getExit();
2021
2022 auto EmbeddNewLoop =
2023 [this, DL, F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](
2024 Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * {
2025 CanonicalLoopInfo *EmbeddedLoop = createLoopSkeleton(
2026 DL, TripCount, F, InnerEnter, OutroInsertBefore, Name);
2027 redirectTo(Enter, EmbeddedLoop->getPreheader(), DL);
2028 redirectTo(EmbeddedLoop->getAfter(), Continue, DL);
2029
2030 // Setup the position where the next embedded loop connects to this loop.
2031 Enter = EmbeddedLoop->getBody();
2032 Continue = EmbeddedLoop->getLatch();
2033 OutroInsertBefore = EmbeddedLoop->getLatch();
2034 return EmbeddedLoop;
2035 };
2036
2037 auto EmbeddNewLoops = [&Result, &EmbeddNewLoop](ArrayRef<Value *> TripCounts,
2038 const Twine &NameBase) {
2039 for (auto P : enumerate(TripCounts)) {
2040 CanonicalLoopInfo *EmbeddedLoop =
2041 EmbeddNewLoop(P.value(), NameBase + Twine(P.index()));
2042 Result.push_back(EmbeddedLoop);
2043 }
2044 };
2045
2046 EmbeddNewLoops(FloorCount, "floor");
2047
2048 // Within the innermost floor loop, emit the code that computes the tile
2049 // sizes.
2050 Builder.SetInsertPoint(Enter->getTerminator());
2051 SmallVector<Value *, 4> TileCounts;
2052 for (int i = 0; i < NumLoops; ++i) {
2053 CanonicalLoopInfo *FloorLoop = Result[i];
2054 Value *TileSize = TileSizes[i];
2055
2056 Value *FloorIsEpilogue =
2057 Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCount[i]);
2058 Value *TileTripCount =
2059 Builder.CreateSelect(FloorIsEpilogue, FloorRems[i], TileSize);
2060
2061 TileCounts.push_back(TileTripCount);
2062 }
2063
2064 // Create the tile loops.
2065 EmbeddNewLoops(TileCounts, "tile");
2066
2067 // Insert the inbetween code into the body.
2068 BasicBlock *BodyEnter = Enter;
2069 BasicBlock *BodyEntered = nullptr;
2070 for (std::pair<BasicBlock *, BasicBlock *> P : InbetweenCode) {
2071 BasicBlock *EnterBB = P.first;
2072 BasicBlock *ExitBB = P.second;
2073
2074 if (BodyEnter)
2075 redirectTo(BodyEnter, EnterBB, DL);
2076 else
2077 redirectAllPredecessorsTo(BodyEntered, EnterBB, DL);
2078
2079 BodyEnter = nullptr;
2080 BodyEntered = ExitBB;
2081 }
2082
2083 // Append the original loop nest body into the generated loop nest body.
2084 if (BodyEnter)
2085 redirectTo(BodyEnter, InnerEnter, DL);
2086 else
2087 redirectAllPredecessorsTo(BodyEntered, InnerEnter, DL);
2088 redirectAllPredecessorsTo(InnerLatch, Continue, DL);
2089
2090 // Replace the original induction variable with an induction variable computed
2091 // from the tile and floor induction variables.
2092 Builder.restoreIP(Result.back()->getBodyIP());
2093 for (int i = 0; i < NumLoops; ++i) {
2094 CanonicalLoopInfo *FloorLoop = Result[i];
2095 CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
2096 Value *OrigIndVar = OrigIndVars[i];
2097 Value *Size = TileSizes[i];
2098
2099 Value *Scale =
2100 Builder.CreateMul(Size, FloorLoop->getIndVar(), {}, /*HasNUW=*/true);
2101 Value *Shift =
2102 Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true);
2103 OrigIndVar->replaceAllUsesWith(Shift);
2104 }
2105
2106 // Remove unused parts of the original loops.
2107 removeUnusedBlocksFromParent(OldControlBBs);
2108
2109 for (CanonicalLoopInfo *L : Loops)
2110 L->invalidate();
2111
2112#ifndef NDEBUG
2113 for (CanonicalLoopInfo *GenL : Result)
2114 GenL->assertOK();
2115#endif
2116 return Result;
2117}
2118
2119/// Attach loop metadata \p Properties to the loop described by \p Loop. If the
2120/// loop already has metadata, the loop properties are appended.
2121static void addLoopMetadata(CanonicalLoopInfo *Loop,
2122 ArrayRef<Metadata *> Properties) {
2123 assert(Loop->isValid() && "Expecting a valid CanonicalLoopInfo")(static_cast <bool> (Loop->isValid() && "Expecting a valid CanonicalLoopInfo"
) ? void (0) : __assert_fail ("Loop->isValid() && \"Expecting a valid CanonicalLoopInfo\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2123, __extension__
__PRETTY_FUNCTION__))
;
2124
2125 // Nothing to do if no property to attach.
2126 if (Properties.empty())
2127 return;
2128
2129 LLVMContext &Ctx = Loop->getFunction()->getContext();
2130 SmallVector<Metadata *> NewLoopProperties;
2131 NewLoopProperties.push_back(nullptr);
2132
2133 // If the loop already has metadata, prepend it to the new metadata.
2134 BasicBlock *Latch = Loop->getLatch();
2135 assert(Latch && "A valid CanonicalLoopInfo must have a unique latch")(static_cast <bool> (Latch && "A valid CanonicalLoopInfo must have a unique latch"
) ? void (0) : __assert_fail ("Latch && \"A valid CanonicalLoopInfo must have a unique latch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2135, __extension__
__PRETTY_FUNCTION__))
;
2136 MDNode *Existing = Latch->getTerminator()->getMetadata(LLVMContext::MD_loop);
2137 if (Existing)
2138 append_range(NewLoopProperties, drop_begin(Existing->operands(), 1));
2139
2140 append_range(NewLoopProperties, Properties);
2141 MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
2142 LoopID->replaceOperandWith(0, LoopID);
2143
2144 Latch->getTerminator()->setMetadata(LLVMContext::MD_loop, LoopID);
2145}
2146
2147void OpenMPIRBuilder::unrollLoopFull(DebugLoc, CanonicalLoopInfo *Loop) {
2148 LLVMContext &Ctx = Builder.getContext();
2149 addLoopMetadata(
2150 Loop, {MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),
2151 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full"))});
2152}
2153
2154void OpenMPIRBuilder::unrollLoopHeuristic(DebugLoc, CanonicalLoopInfo *Loop) {
2155 LLVMContext &Ctx = Builder.getContext();
2156 addLoopMetadata(
2157 Loop, {
2158 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),
2159 });
2160}
2161
2162/// Create the TargetMachine object to query the backend for optimization
2163/// preferences.
2164///
2165/// Ideally, this would be passed from the front-end to the OpenMPBuilder, but
2166/// e.g. Clang does not pass it to its CodeGen layer and creates it only when
2167/// needed for the LLVM pass pipline. We use some default options to avoid
2168/// having to pass too many settings from the frontend that probably do not
2169/// matter.
2170///
2171/// Currently, TargetMachine is only used sometimes by the unrollLoopPartial
2172/// method. If we are going to use TargetMachine for more purposes, especially
2173/// those that are sensitive to TargetOptions, RelocModel and CodeModel, it
2174/// might become be worth requiring front-ends to pass on their TargetMachine,
2175/// or at least cache it between methods. Note that while fontends such as Clang
2176/// have just a single main TargetMachine per translation unit, "target-cpu" and
2177/// "target-features" that determine the TargetMachine are per-function and can
2178/// be overrided using __attribute__((target("OPTIONS"))).
2179static std::unique_ptr<TargetMachine>
2180createTargetMachine(Function *F, CodeGenOpt::Level OptLevel) {
2181 Module *M = F->getParent();
2182
2183 StringRef CPU = F->getFnAttribute("target-cpu").getValueAsString();
2184 StringRef Features = F->getFnAttribute("target-features").getValueAsString();
2185 const std::string &Triple = M->getTargetTriple();
2186
2187 std::string Error;
2188 const llvm::Target *TheTarget = TargetRegistry::lookupTarget(Triple, Error);
2189 if (!TheTarget)
2190 return {};
2191
2192 llvm::TargetOptions Options;
2193 return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(
2194 Triple, CPU, Features, Options, /*RelocModel=*/None, /*CodeModel=*/None,
2195 OptLevel));
2196}
2197
2198/// Heuristically determine the best-performant unroll factor for \p CLI. This
2199/// depends on the target processor. We are re-using the same heuristics as the
2200/// LoopUnrollPass.
2201static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI) {
2202 Function *F = CLI->getFunction();
2203
2204 // Assume the user requests the most aggressive unrolling, even if the rest of
2205 // the code is optimized using a lower setting.
2206 CodeGenOpt::Level OptLevel = CodeGenOpt::Aggressive;
2207 std::unique_ptr<TargetMachine> TM = createTargetMachine(F, OptLevel);
2208
2209 FunctionAnalysisManager FAM;
2210 FAM.registerPass([]() { return TargetLibraryAnalysis(); });
2211 FAM.registerPass([]() { return AssumptionAnalysis(); });
2212 FAM.registerPass([]() { return DominatorTreeAnalysis(); });
2213 FAM.registerPass([]() { return LoopAnalysis(); });
2214 FAM.registerPass([]() { return ScalarEvolutionAnalysis(); });
2215 FAM.registerPass([]() { return PassInstrumentationAnalysis(); });
2216 TargetIRAnalysis TIRA;
2217 if (TM)
2218 TIRA = TargetIRAnalysis(
2219 [&](const Function &F) { return TM->getTargetTransformInfo(F); });
2220 FAM.registerPass([&]() { return TIRA; });
2221
2222 TargetIRAnalysis::Result &&TTI = TIRA.run(*F, FAM);
2223 ScalarEvolutionAnalysis SEA;
2224 ScalarEvolution &&SE = SEA.run(*F, FAM);
2225 DominatorTreeAnalysis DTA;
2226 DominatorTree &&DT = DTA.run(*F, FAM);
2227 LoopAnalysis LIA;
2228 LoopInfo &&LI = LIA.run(*F, FAM);
2229 AssumptionAnalysis ACT;
2230 AssumptionCache &&AC = ACT.run(*F, FAM);
2231 OptimizationRemarkEmitter ORE{F};
2232
2233 Loop *L = LI.getLoopFor(CLI->getHeader());
2234 assert(L && "Expecting CanonicalLoopInfo to be recognized as a loop")(static_cast <bool> (L && "Expecting CanonicalLoopInfo to be recognized as a loop"
) ? void (0) : __assert_fail ("L && \"Expecting CanonicalLoopInfo to be recognized as a loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2234, __extension__
__PRETTY_FUNCTION__))
;
2235
2236 TargetTransformInfo::UnrollingPreferences UP =
2237 gatherUnrollingPreferences(L, SE, TTI,
2238 /*BlockFrequencyInfo=*/nullptr,
2239 /*ProfileSummaryInfo=*/nullptr, ORE, OptLevel,
2240 /*UserThreshold=*/None,
2241 /*UserCount=*/None,
2242 /*UserAllowPartial=*/true,
2243 /*UserAllowRuntime=*/true,
2244 /*UserUpperBound=*/None,
2245 /*UserFullUnrollMaxCount=*/None);
2246
2247 UP.Force = true;
2248
2249 // Account for additional optimizations taking place before the LoopUnrollPass
2250 // would unroll the loop.
2251 UP.Threshold *= UnrollThresholdFactor;
2252 UP.PartialThreshold *= UnrollThresholdFactor;
2253
2254 // Use normal unroll factors even if the rest of the code is optimized for
2255 // size.
2256 UP.OptSizeThreshold = UP.Threshold;
2257 UP.PartialOptSizeThreshold = UP.PartialThreshold;
2258
2259 LLVM_DEBUG(dbgs() << "Unroll heuristic thresholds:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
2260 << " Threshold=" << UP.Threshold << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
2261 << " PartialThreshold=" << UP.PartialThreshold << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
2262 << " OptSizeThreshold=" << UP.OptSizeThreshold << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
2263 << " PartialOptSizeThreshold="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
2264 << UP.PartialOptSizeThreshold << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
;
2265
2266 // Disable peeling.
2267 TargetTransformInfo::PeelingPreferences PP =
2268 gatherPeelingPreferences(L, SE, TTI,
2269 /*UserAllowPeeling=*/false,
2270 /*UserAllowProfileBasedPeeling=*/false,
2271 /*UnrollingSpecficValues=*/false);
2272
2273 SmallPtrSet<const Value *, 32> EphValues;
2274 CodeMetrics::collectEphemeralValues(L, &AC, EphValues);
2275
2276 // Assume that reads and writes to stack variables can be eliminated by
2277 // Mem2Reg, SROA or LICM. That is, don't count them towards the loop body's
2278 // size.
2279 for (BasicBlock *BB : L->blocks()) {
2280 for (Instruction &I : *BB) {
2281 Value *Ptr;
2282 if (auto *Load = dyn_cast<LoadInst>(&I)) {
2283 Ptr = Load->getPointerOperand();
2284 } else if (auto *Store = dyn_cast<StoreInst>(&I)) {
2285 Ptr = Store->getPointerOperand();
2286 } else
2287 continue;
2288
2289 Ptr = Ptr->stripPointerCasts();
2290
2291 if (auto *Alloca = dyn_cast<AllocaInst>(Ptr)) {
2292 if (Alloca->getParent() == &F->getEntryBlock())
2293 EphValues.insert(&I);
2294 }
2295 }
2296 }
2297
2298 unsigned NumInlineCandidates;
2299 bool NotDuplicatable;
2300 bool Convergent;
2301 unsigned LoopSize =
2302 ApproximateLoopSize(L, NumInlineCandidates, NotDuplicatable, Convergent,
2303 TTI, EphValues, UP.BEInsns);
2304 LLVM_DEBUG(dbgs() << "Estimated loop size is " << LoopSize << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Estimated loop size is "
<< LoopSize << "\n"; } } while (false)
;
2305
2306 // Loop is not unrollable if the loop contains certain instructions.
2307 if (NotDuplicatable || Convergent) {
2308 LLVM_DEBUG(dbgs() << "Loop not considered unrollable\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Loop not considered unrollable\n"
; } } while (false)
;
2309 return 1;
2310 }
2311
2312 // TODO: Determine trip count of \p CLI if constant, computeUnrollCount might
2313 // be able to use it.
2314 int TripCount = 0;
2315 int MaxTripCount = 0;
2316 bool MaxOrZero = false;
2317 unsigned TripMultiple = 0;
2318
2319 bool UseUpperBound = false;
2320 computeUnrollCount(L, TTI, DT, &LI, SE, EphValues, &ORE, TripCount,
2321 MaxTripCount, MaxOrZero, TripMultiple, LoopSize, UP, PP,
2322 UseUpperBound);
2323 unsigned Factor = UP.Count;
2324 LLVM_DEBUG(dbgs() << "Suggesting unroll factor of " << Factor << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Suggesting unroll factor of "
<< Factor << "\n"; } } while (false)
;
2325
2326 // This function returns 1 to signal to not unroll a loop.
2327 if (Factor == 0)
2328 return 1;
2329 return Factor;
2330}
2331
2332void OpenMPIRBuilder::unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop,
2333 int32_t Factor,
2334 CanonicalLoopInfo **UnrolledCLI) {
2335 assert(Factor >= 0 && "Unroll factor must not be negative")(static_cast <bool> (Factor >= 0 && "Unroll factor must not be negative"
) ? void (0) : __assert_fail ("Factor >= 0 && \"Unroll factor must not be negative\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2335, __extension__
__PRETTY_FUNCTION__))
;
2336
2337 Function *F = Loop->getFunction();
2338 LLVMContext &Ctx = F->getContext();
2339
2340 // If the unrolled loop is not used for another loop-associated directive, it
2341 // is sufficient to add metadata for the LoopUnrollPass.
2342 if (!UnrolledCLI) {
2343 SmallVector<Metadata *, 2> LoopMetadata;
2344 LoopMetadata.push_back(
2345 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")));
2346
2347 if (Factor >= 1) {
2348 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
2349 ConstantInt::get(Type::getInt32Ty(Ctx), APInt(32, Factor)));
2350 LoopMetadata.push_back(MDNode::get(
2351 Ctx, {MDString::get(Ctx, "llvm.loop.unroll.count"), FactorConst}));
2352 }
2353
2354 addLoopMetadata(Loop, LoopMetadata);
2355 return;
2356 }
2357
2358 // Heuristically determine the unroll factor.
2359 if (Factor == 0)
2360 Factor = computeHeuristicUnrollFactor(Loop);
2361
2362 // No change required with unroll factor 1.
2363 if (Factor == 1) {
2364 *UnrolledCLI = Loop;
2365 return;
2366 }
2367
2368 assert(Factor >= 2 &&(static_cast <bool> (Factor >= 2 && "unrolling only makes sense with a factor of 2 or larger"
) ? void (0) : __assert_fail ("Factor >= 2 && \"unrolling only makes sense with a factor of 2 or larger\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2369, __extension__
__PRETTY_FUNCTION__))
2369 "unrolling only makes sense with a factor of 2 or larger")(static_cast <bool> (Factor >= 2 && "unrolling only makes sense with a factor of 2 or larger"
) ? void (0) : __assert_fail ("Factor >= 2 && \"unrolling only makes sense with a factor of 2 or larger\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2369, __extension__
__PRETTY_FUNCTION__))
;
2370
2371 Type *IndVarTy = Loop->getIndVarType();
2372
2373 // Apply partial unrolling by tiling the loop by the unroll-factor, then fully
2374 // unroll the inner loop.
2375 Value *FactorVal =
2376 ConstantInt::get(IndVarTy, APInt(IndVarTy->getIntegerBitWidth(), Factor,
2377 /*isSigned=*/false));
2378 std::vector<CanonicalLoopInfo *> LoopNest =
2379 tileLoops(DL, {Loop}, {FactorVal});
2380 assert(LoopNest.size() == 2 && "Expect 2 loops after tiling")(static_cast <bool> (LoopNest.size() == 2 && "Expect 2 loops after tiling"
) ? void (0) : __assert_fail ("LoopNest.size() == 2 && \"Expect 2 loops after tiling\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2380, __extension__
__PRETTY_FUNCTION__))
;
2381 *UnrolledCLI = LoopNest[0];
2382 CanonicalLoopInfo *InnerLoop = LoopNest[1];
2383
2384 // LoopUnrollPass can only fully unroll loops with constant trip count.
2385 // Unroll by the unroll factor with a fallback epilog for the remainder
2386 // iterations if necessary.
2387 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
2388 ConstantInt::get(Type::getInt32Ty(Ctx), APInt(32, Factor)));
2389 addLoopMetadata(
2390 InnerLoop,
2391 {MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),
2392 MDNode::get(
2393 Ctx, {MDString::get(Ctx, "llvm.loop.unroll.count"), FactorConst})});
2394
2395#ifndef NDEBUG
2396 (*UnrolledCLI)->assertOK();
2397#endif
2398}
2399
2400OpenMPIRBuilder::InsertPointTy
2401OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc,
2402 llvm::Value *BufSize, llvm::Value *CpyBuf,
2403 llvm::Value *CpyFn, llvm::Value *DidIt) {
2404 if (!updateToLocation(Loc))
2405 return Loc.IP;
2406
2407 uint32_t SrcLocStrSize;
2408 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2409 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2410 Value *ThreadId = getOrCreateThreadID(Ident);
2411
2412 llvm::Value *DidItLD = Builder.CreateLoad(Builder.getInt32Ty(), DidIt);
2413
2414 Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};
2415
2416 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_copyprivate);
2417 Builder.CreateCall(Fn, Args);
2418
2419 return Builder.saveIP();
2420}
2421
2422OpenMPIRBuilder::InsertPointTy
2423OpenMPIRBuilder::createSingle(const LocationDescription &Loc,
2424 BodyGenCallbackTy BodyGenCB,
2425 FinalizeCallbackTy FiniCB, llvm::Value *DidIt) {
2426
2427 if (!updateToLocation(Loc))
2428 return Loc.IP;
2429
2430 // If needed (i.e. not null), initialize `DidIt` with 0
2431 if (DidIt) {
2432 Builder.CreateStore(Builder.getInt32(0), DidIt);
2433 }
2434
2435 Directive OMPD = Directive::OMPD_single;
2436 uint32_t SrcLocStrSize;
2437 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2438 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2439 Value *ThreadId = getOrCreateThreadID(Ident);
2440 Value *Args[] = {Ident, ThreadId};
2441
2442 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_single);
2443 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
2444
2445 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_single);
2446 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
2447
2448 // generates the following:
2449 // if (__kmpc_single()) {
2450 // .... single region ...
2451 // __kmpc_end_single
2452 // }
2453
2454 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
2455 /*Conditional*/ true, /*hasFinalize*/ true);
2456}
2457
2458OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCritical(
2459 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
2460 FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {
2461
2462 if (!updateToLocation(Loc))
2463 return Loc.IP;
2464
2465 Directive OMPD = Directive::OMPD_critical;
2466 uint32_t SrcLocStrSize;
2467 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2468 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2469 Value *ThreadId = getOrCreateThreadID(Ident);
2470 Value *LockVar = getOMPCriticalRegionLock(CriticalName);
2471 Value *Args[] = {Ident, ThreadId, LockVar};
2472
2473 SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args), std::end(Args));
2474 Function *RTFn = nullptr;
2475 if (HintInst) {
2476 // Add Hint to entry Args and create call
2477 EnterArgs.push_back(HintInst);
2478 RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical_with_hint);
2479 } else {
2480 RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical);
2481 }
2482 Instruction *EntryCall = Builder.CreateCall(RTFn, EnterArgs);
2483
2484 Function *ExitRTLFn =
2485 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_critical);
2486 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
2487
2488 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
2489 /*Conditional*/ false, /*hasFinalize*/ true);
2490}
2491
2492OpenMPIRBuilder::InsertPointTy
2493OpenMPIRBuilder::createOrderedDepend(const LocationDescription &Loc,
2494 InsertPointTy AllocaIP, unsigned NumLoops,
2495 ArrayRef<llvm::Value *> StoreValues,
2496 const Twine &Name, bool IsDependSource) {
2497 for (size_t I = 0; I < StoreValues.size(); I++)
2498 assert(StoreValues[I]->getType()->isIntegerTy(64) &&(static_cast <bool> (StoreValues[I]->getType()->isIntegerTy
(64) && "OpenMP runtime requires depend vec with i64 type"
) ? void (0) : __assert_fail ("StoreValues[I]->getType()->isIntegerTy(64) && \"OpenMP runtime requires depend vec with i64 type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2499, __extension__
__PRETTY_FUNCTION__))
2499 "OpenMP runtime requires depend vec with i64 type")(static_cast <bool> (StoreValues[I]->getType()->isIntegerTy
(64) && "OpenMP runtime requires depend vec with i64 type"
) ? void (0) : __assert_fail ("StoreValues[I]->getType()->isIntegerTy(64) && \"OpenMP runtime requires depend vec with i64 type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2499, __extension__
__PRETTY_FUNCTION__))
;
2500
2501 if (!updateToLocation(Loc))
2502 return Loc.IP;
2503
2504 // Allocate space for vector and generate alloc instruction.
2505 auto *ArrI64Ty = ArrayType::get(Int64, NumLoops);
2506 Builder.restoreIP(AllocaIP);
2507 AllocaInst *ArgsBase = Builder.CreateAlloca(ArrI64Ty, nullptr, Name);
2508 ArgsBase->setAlignment(Align(8));
2509 Builder.restoreIP(Loc.IP);
2510
2511 // Store the index value with offset in depend vector.
2512 for (unsigned I = 0; I < NumLoops; ++I) {
2513 Value *DependAddrGEPIter = Builder.CreateInBoundsGEP(
2514 ArrI64Ty, ArgsBase, {Builder.getInt64(0), Builder.getInt64(I)});
2515 StoreInst *STInst = Builder.CreateStore(StoreValues[I], DependAddrGEPIter);
2516 STInst->setAlignment(Align(8));
2517 }
2518
2519 Value *DependBaseAddrGEP = Builder.CreateInBoundsGEP(
2520 ArrI64Ty, ArgsBase, {Builder.getInt64(0), Builder.getInt64(0)});
2521
2522 uint32_t SrcLocStrSize;
2523 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2524 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2525 Value *ThreadId = getOrCreateThreadID(Ident);
2526 Value *Args[] = {Ident, ThreadId, DependBaseAddrGEP};
2527
2528 Function *RTLFn = nullptr;
2529 if (IsDependSource)
2530 RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_doacross_post);
2531 else
2532 RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_doacross_wait);
2533 Builder.CreateCall(RTLFn, Args);
2534
2535 return Builder.saveIP();
2536}
2537
2538OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createOrderedThreadsSimd(
2539 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
2540 FinalizeCallbackTy FiniCB, bool IsThreads) {
2541 if (!updateToLocation(Loc))
2542 return Loc.IP;
2543
2544 Directive OMPD = Directive::OMPD_ordered;
2545 Instruction *EntryCall = nullptr;
2546 Instruction *ExitCall = nullptr;
2547
2548 if (IsThreads) {
2549 uint32_t SrcLocStrSize;
2550 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2551 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2552 Value *ThreadId = getOrCreateThreadID(Ident);
2553 Value *Args[] = {Ident, ThreadId};
2554
2555 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_ordered);
2556 EntryCall = Builder.CreateCall(EntryRTLFn, Args);
2557
2558 Function *ExitRTLFn =
2559 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_ordered);
2560 ExitCall = Builder.CreateCall(ExitRTLFn, Args);
2561 }
2562
2563 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
2564 /*Conditional*/ false, /*hasFinalize*/ true);
2565}
2566
2567OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::EmitOMPInlinedRegion(
2568 Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,
2569 BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,
2570 bool HasFinalize, bool IsCancellable) {
2571
2572 if (HasFinalize)
2573 FinalizationStack.push_back({FiniCB, OMPD, IsCancellable});
2574
2575 // Create inlined region's entry and body blocks, in preparation
2576 // for conditional creation
2577 BasicBlock *EntryBB = Builder.GetInsertBlock();
2578 Instruction *SplitPos = EntryBB->getTerminator();
2579 if (!isa_and_nonnull<BranchInst>(SplitPos))
2580 SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);
2581 BasicBlock *ExitBB = EntryBB->splitBasicBlock(SplitPos, "omp_region.end");
2582 BasicBlock *FiniBB =
2583 EntryBB->splitBasicBlock(EntryBB->getTerminator(), "omp_region.finalize");
2584
2585 Builder.SetInsertPoint(EntryBB->getTerminator());
2586 emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
2587
2588 // generate body
2589 BodyGenCB(/* AllocaIP */ InsertPointTy(),
2590 /* CodeGenIP */ Builder.saveIP(), *FiniBB);
2591
2592 // If we didn't emit a branch to FiniBB during body generation, it means
2593 // FiniBB is unreachable (e.g. while(1);). stop generating all the
2594 // unreachable blocks, and remove anything we are not going to use.
2595 auto SkipEmittingRegion = FiniBB->hasNPredecessors(0);
2596 if (SkipEmittingRegion) {
2597 FiniBB->eraseFromParent();
2598 ExitCall->eraseFromParent();
2599 // Discard finalization if we have it.
2600 if (HasFinalize) {
2601 assert(!FinalizationStack.empty() &&(static_cast <bool> (!FinalizationStack.empty() &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2602, __extension__
__PRETTY_FUNCTION__))
2602 "Unexpected finalization stack state!")(static_cast <bool> (!FinalizationStack.empty() &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2602, __extension__
__PRETTY_FUNCTION__))
;
2603 FinalizationStack.pop_back();
2604 }
2605 } else {
2606 // emit exit call and do any needed finalization.
2607 auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt());
2608 assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&(static_cast <bool> (FiniBB->getTerminator()->getNumSuccessors
() == 1 && FiniBB->getTerminator()->getSuccessor
(0) == ExitBB && "Unexpected control flow graph state!!"
) ? void (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2610, __extension__
__PRETTY_FUNCTION__))
2609 FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&(static_cast <bool> (FiniBB->getTerminator()->getNumSuccessors
() == 1 && FiniBB->getTerminator()->getSuccessor
(0) == ExitBB && "Unexpected control flow graph state!!"
) ? void (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2610, __extension__
__PRETTY_FUNCTION__))
2610 "Unexpected control flow graph state!!")(static_cast <bool> (FiniBB->getTerminator()->getNumSuccessors
() == 1 && FiniBB->getTerminator()->getSuccessor
(0) == ExitBB && "Unexpected control flow graph state!!"
) ? void (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2610, __extension__
__PRETTY_FUNCTION__))
;
2611 emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
2612 assert(FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB &&(static_cast <bool> (FiniBB->getUniquePredecessor()->
getUniqueSuccessor() == FiniBB && "Unexpected Control Flow State!"
) ? void (0) : __assert_fail ("FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB && \"Unexpected Control Flow State!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2613, __extension__
__PRETTY_FUNCTION__))
2613 "Unexpected Control Flow State!")(static_cast <bool> (FiniBB->getUniquePredecessor()->
getUniqueSuccessor() == FiniBB && "Unexpected Control Flow State!"
) ? void (0) : __assert_fail ("FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB && \"Unexpected Control Flow State!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2613, __extension__
__PRETTY_FUNCTION__))
;
2614 MergeBlockIntoPredecessor(FiniBB);
2615 }
2616
2617 // If we are skipping the region of a non conditional, remove the exit
2618 // block, and clear the builder's insertion point.
2619 assert(SplitPos->getParent() == ExitBB &&(static_cast <bool> (SplitPos->getParent() == ExitBB
&& "Unexpected Insertion point location!") ? void (0
) : __assert_fail ("SplitPos->getParent() == ExitBB && \"Unexpected Insertion point location!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2620, __extension__
__PRETTY_FUNCTION__))
2620 "Unexpected Insertion point location!")(static_cast <bool> (SplitPos->getParent() == ExitBB
&& "Unexpected Insertion point location!") ? void (0
) : __assert_fail ("SplitPos->getParent() == ExitBB && \"Unexpected Insertion point location!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2620, __extension__
__PRETTY_FUNCTION__))
;
2621 if (!Conditional && SkipEmittingRegion) {
2622 ExitBB->eraseFromParent();
2623 Builder.ClearInsertionPoint();
2624 } else {
2625 auto merged = MergeBlockIntoPredecessor(ExitBB);
2626 BasicBlock *ExitPredBB = SplitPos->getParent();
2627 auto InsertBB = merged ? ExitPredBB : ExitBB;
2628 if (!isa_and_nonnull<BranchInst>(SplitPos))
2629 SplitPos->eraseFromParent();
2630 Builder.SetInsertPoint(InsertBB);
2631 }
2632
2633 return Builder.saveIP();
2634}
2635
2636OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(
2637 Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) {
2638 // if nothing to do, Return current insertion point.
2639 if (!Conditional || !EntryCall)
2640 return Builder.saveIP();
2641
2642 BasicBlock *EntryBB = Builder.GetInsertBlock();
2643 Value *CallBool = Builder.CreateIsNotNull(EntryCall);
2644 auto *ThenBB = BasicBlock::Create(M.getContext(), "omp_region.body");
2645 auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);
2646
2647 // Emit thenBB and set the Builder's insertion point there for
2648 // body generation next. Place the block after the current block.
2649 Function *CurFn = EntryBB->getParent();
2650 CurFn->getBasicBlockList().insertAfter(EntryBB->getIterator(), ThenBB);
2651
2652 // Move Entry branch to end of ThenBB, and replace with conditional
2653 // branch (If-stmt)
2654 Instruction *EntryBBTI = EntryBB->getTerminator();
2655 Builder.CreateCondBr(CallBool, ThenBB, ExitBB);
2656 EntryBBTI->removeFromParent();
2657 Builder.SetInsertPoint(UI);
2658 Builder.Insert(EntryBBTI);
2659 UI->eraseFromParent();
2660 Builder.SetInsertPoint(ThenBB->getTerminator());
2661
2662 // return an insertion point to ExitBB.
2663 return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt());
2664}
2665
2666OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit(
2667 omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,
2668 bool HasFinalize) {
2669
2670 Builder.restoreIP(FinIP);
2671
2672 // If there is finalization to do, emit it before the exit call
2673 if (HasFinalize) {
2674 assert(!FinalizationStack.empty() &&(static_cast <bool> (!FinalizationStack.empty() &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2675, __extension__
__PRETTY_FUNCTION__))
2675 "Unexpected finalization stack state!")(static_cast <bool> (!FinalizationStack.empty() &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2675, __extension__
__PRETTY_FUNCTION__))
;
2676
2677 FinalizationInfo Fi = FinalizationStack.pop_back_val();
2678 assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!")(static_cast <bool> (Fi.DK == OMPD && "Unexpected Directive for Finalization call!"
) ? void (0) : __assert_fail ("Fi.DK == OMPD && \"Unexpected Directive for Finalization call!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2678, __extension__
__PRETTY_FUNCTION__))
;
2679
2680 Fi.FiniCB(FinIP);
2681
2682 BasicBlock *FiniBB = FinIP.getBlock();
2683 Instruction *FiniBBTI = FiniBB->getTerminator();
2684
2685 // set Builder IP for call creation
2686 Builder.SetInsertPoint(FiniBBTI);
2687 }
2688
2689 if (!ExitCall)
2690 return Builder.saveIP();
2691
2692 // place the Exitcall as last instruction before Finalization block terminator
2693 ExitCall->removeFromParent();
2694 Builder.Insert(ExitCall);
2695
2696 return IRBuilder<>::InsertPoint(ExitCall->getParent(),
2697 ExitCall->getIterator());
2698}
2699
2700OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCopyinClauseBlocks(
2701 InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,
2702 llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {
2703 if (!IP.isSet())
2704 return IP;
2705
2706 IRBuilder<>::InsertPointGuard IPG(Builder);
2707
2708 // creates the following CFG structure
2709 // OMP_Entry : (MasterAddr != PrivateAddr)?
2710 // F T
2711 // | \
2712 // | copin.not.master
2713 // | /
2714 // v /
2715 // copyin.not.master.end
2716 // |
2717 // v
2718 // OMP.Entry.Next
2719
2720 BasicBlock *OMP_Entry = IP.getBlock();
2721 Function *CurFn = OMP_Entry->getParent();
2722 BasicBlock *CopyBegin =
2723 BasicBlock::Create(M.getContext(), "copyin.not.master", CurFn);
2724 BasicBlock *CopyEnd = nullptr;
2725
2726 // If entry block is terminated, split to preserve the branch to following
2727 // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.
2728 if (isa_and_nonnull<BranchInst>(OMP_Entry->getTerminator())) {
2729 CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(),
2730 "copyin.not.master.end");
2731 OMP_Entry->getTerminator()->eraseFromParent();
2732 } else {
2733 CopyEnd =
2734 BasicBlock::Create(M.getContext(), "copyin.not.master.end", CurFn);
2735 }
2736
2737 Builder.SetInsertPoint(OMP_Entry);
2738 Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy);
2739 Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy);
2740 Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr);
2741 Builder.CreateCondBr(cmp, CopyBegin, CopyEnd);
2742
2743 Builder.SetInsertPoint(CopyBegin);
2744 if (BranchtoEnd)
2745 Builder.SetInsertPoint(Builder.CreateBr(CopyEnd));
2746
2747 return Builder.saveIP();
2748}
2749
2750CallInst *OpenMPIRBuilder::createOMPAlloc(const LocationDescription &Loc,
2751 Value *Size, Value *Allocator,
2752 std::string Name) {
2753 IRBuilder<>::InsertPointGuard IPG(Builder);
2754 Builder.restoreIP(Loc.IP);
2755
2756 uint32_t SrcLocStrSize;
2757 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2758 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2759 Value *ThreadId = getOrCreateThreadID(Ident);
2760 Value *Args[] = {ThreadId, Size, Allocator};
2761
2762 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_alloc);
2763
2764 return Builder.CreateCall(Fn, Args, Name);
2765}
2766
2767CallInst *OpenMPIRBuilder::createOMPFree(const LocationDescription &Loc,
2768 Value *Addr, Value *Allocator,
2769 std::string Name) {
2770 IRBuilder<>::InsertPointGuard IPG(Builder);
2771 Builder.restoreIP(Loc.IP);
2772
2773 uint32_t SrcLocStrSize;
2774 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2775 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2776 Value *ThreadId = getOrCreateThreadID(Ident);
2777 Value *Args[] = {ThreadId, Addr, Allocator};
2778 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_free);
2779 return Builder.CreateCall(Fn, Args, Name);
2780}
2781
2782CallInst *OpenMPIRBuilder::createCachedThreadPrivate(
2783 const LocationDescription &Loc, llvm::Value *Pointer,
2784 llvm::ConstantInt *Size, const llvm::Twine &Name) {
2785 IRBuilder<>::InsertPointGuard IPG(Builder);
2786 Builder.restoreIP(Loc.IP);
2787
2788 uint32_t SrcLocStrSize;
2789 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2790 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2791 Value *ThreadId = getOrCreateThreadID(Ident);
2792 Constant *ThreadPrivateCache =
2793 getOrCreateOMPInternalVariable(Int8PtrPtr, Name);
2794 llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};
2795
2796 Function *Fn =
2797 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_threadprivate_cached);
2798
2799 return Builder.CreateCall(Fn, Args);
2800}
2801
2802OpenMPIRBuilder::InsertPointTy
2803OpenMPIRBuilder::createTargetInit(const LocationDescription &Loc, bool IsSPMD,
2804 bool RequiresFullRuntime) {
2805 if (!updateToLocation(Loc))
2806 return Loc.IP;
2807
2808 uint32_t SrcLocStrSize;
2809 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2810 Constant *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2811 ConstantInt *IsSPMDVal = ConstantInt::getSigned(
2812 IntegerType::getInt8Ty(Int8->getContext()),
2813 IsSPMD ? OMP_TGT_EXEC_MODE_SPMD : OMP_TGT_EXEC_MODE_GENERIC);
2814 ConstantInt *UseGenericStateMachine =
2815 ConstantInt::getBool(Int32->getContext(), !IsSPMD);
2816 ConstantInt *RequiresFullRuntimeVal =
2817 ConstantInt::getBool(Int32->getContext(), RequiresFullRuntime);
2818
2819 Function *Fn = getOrCreateRuntimeFunctionPtr(
2820 omp::RuntimeFunction::OMPRTL___kmpc_target_init);
2821
2822 CallInst *ThreadKind = Builder.CreateCall(
2823 Fn, {Ident, IsSPMDVal, UseGenericStateMachine, RequiresFullRuntimeVal});
2824
2825 Value *ExecUserCode = Builder.CreateICmpEQ(
2826 ThreadKind, ConstantInt::get(ThreadKind->getType(), -1),
2827 "exec_user_code");
2828
2829 // ThreadKind = __kmpc_target_init(...)
2830 // if (ThreadKind == -1)
2831 // user_code
2832 // else
2833 // return;
2834
2835 auto *UI = Builder.CreateUnreachable();
2836 BasicBlock *CheckBB = UI->getParent();
2837 BasicBlock *UserCodeEntryBB = CheckBB->splitBasicBlock(UI, "user_code.entry");
2838
2839 BasicBlock *WorkerExitBB = BasicBlock::Create(
2840 CheckBB->getContext(), "worker.exit", CheckBB->getParent());
2841 Builder.SetInsertPoint(WorkerExitBB);
2842 Builder.CreateRetVoid();
2843
2844 auto *CheckBBTI = CheckBB->getTerminator();
2845 Builder.SetInsertPoint(CheckBBTI);
2846 Builder.CreateCondBr(ExecUserCode, UI->getParent(), WorkerExitBB);
2847
2848 CheckBBTI->eraseFromParent();
2849 UI->eraseFromParent();
2850
2851 // Continue in the "user_code" block, see diagram above and in
2852 // openmp/libomptarget/deviceRTLs/common/include/target.h .
2853 return InsertPointTy(UserCodeEntryBB, UserCodeEntryBB->getFirstInsertionPt());
2854}
2855
2856void OpenMPIRBuilder::createTargetDeinit(const LocationDescription &Loc,
2857 bool IsSPMD,
2858 bool RequiresFullRuntime) {
2859 if (!updateToLocation(Loc))
2860 return;
2861
2862 uint32_t SrcLocStrSize;
2863 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2864 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2865 ConstantInt *IsSPMDVal = ConstantInt::getSigned(
2866 IntegerType::getInt8Ty(Int8->getContext()),
2867 IsSPMD ? OMP_TGT_EXEC_MODE_SPMD : OMP_TGT_EXEC_MODE_GENERIC);
2868 ConstantInt *RequiresFullRuntimeVal =
2869 ConstantInt::getBool(Int32->getContext(), RequiresFullRuntime);
2870
2871 Function *Fn = getOrCreateRuntimeFunctionPtr(
2872 omp::RuntimeFunction::OMPRTL___kmpc_target_deinit);
2873
2874 Builder.CreateCall(Fn, {Ident, IsSPMDVal, RequiresFullRuntimeVal});
2875}
2876
2877std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
2878 StringRef FirstSeparator,
2879 StringRef Separator) {
2880 SmallString<128> Buffer;
2881 llvm::raw_svector_ostream OS(Buffer);
2882 StringRef Sep = FirstSeparator;
2883 for (StringRef Part : Parts) {
2884 OS << Sep << Part;
2885 Sep = Separator;
2886 }
2887 return OS.str().str();
2888}
2889
2890Constant *OpenMPIRBuilder::getOrCreateOMPInternalVariable(
2891 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
2892 // TODO: Replace the twine arg with stringref to get rid of the conversion
2893 // logic. However This is taken from current implementation in clang as is.
2894 // Since this method is used in many places exclusively for OMP internal use
2895 // we will keep it as is for temporarily until we move all users to the
2896 // builder and then, if possible, fix it everywhere in one go.
2897 SmallString<256> Buffer;
2898 llvm::raw_svector_ostream Out(Buffer);
2899 Out << Name;
2900 StringRef RuntimeName = Out.str();
2901 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
2902 if (Elem.second) {
2903 assert(Elem.second->getType()->getPointerElementType() == Ty &&(static_cast <bool> (Elem.second->getType()->getPointerElementType
() == Ty && "OMP internal variable has different type than requested"
) ? void (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2904, __extension__
__PRETTY_FUNCTION__))
2904 "OMP internal variable has different type than requested")(static_cast <bool> (Elem.second->getType()->getPointerElementType
() == Ty && "OMP internal variable has different type than requested"
) ? void (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2904, __extension__
__PRETTY_FUNCTION__))
;
2905 } else {
2906 // TODO: investigate the appropriate linkage type used for the global
2907 // variable for possibly changing that to internal or private, or maybe
2908 // create different versions of the function for different OMP internal
2909 // variables.
2910 Elem.second = new llvm::GlobalVariable(
2911 M, Ty, /*IsConstant*/ false, llvm::GlobalValue::CommonLinkage,
2912 llvm::Constant::getNullValue(Ty), Elem.first(),
2913 /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
2914 AddressSpace);
2915 }
2916
2917 return Elem.second;
2918}
2919
2920Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {
2921 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
2922 std::string Name = getNameWithSeparators({Prefix, "var"}, ".", ".");
2923 return getOrCreateOMPInternalVariable(KmpCriticalNameTy, Name);
2924}
2925
2926GlobalVariable *
2927OpenMPIRBuilder::createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,
2928 std::string VarName) {
2929 llvm::Constant *MaptypesArrayInit =
2930 llvm::ConstantDataArray::get(M.getContext(), Mappings);
2931 auto *MaptypesArrayGlobal = new llvm::GlobalVariable(
2932 M, MaptypesArrayInit->getType(),
2933 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MaptypesArrayInit,
2934 VarName);
2935 MaptypesArrayGlobal->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
2936 return MaptypesArrayGlobal;
2937}
2938
2939void OpenMPIRBuilder::createMapperAllocas(const LocationDescription &Loc,
2940 InsertPointTy AllocaIP,
2941 unsigned NumOperands,
2942 struct MapperAllocas &MapperAllocas) {
2943 if (!updateToLocation(Loc))
2944 return;
2945
2946 auto *ArrI8PtrTy = ArrayType::get(Int8Ptr, NumOperands);
2947 auto *ArrI64Ty = ArrayType::get(Int64, NumOperands);
2948 Builder.restoreIP(AllocaIP);
2949 AllocaInst *ArgsBase = Builder.CreateAlloca(ArrI8PtrTy);
2950 AllocaInst *Args = Builder.CreateAlloca(ArrI8PtrTy);
2951 AllocaInst *ArgSizes = Builder.CreateAlloca(ArrI64Ty);
2952 Builder.restoreIP(Loc.IP);
2953 MapperAllocas.ArgsBase = ArgsBase;
2954 MapperAllocas.Args = Args;
2955 MapperAllocas.ArgSizes = ArgSizes;
2956}
2957
2958void OpenMPIRBuilder::emitMapperCall(const LocationDescription &Loc,
2959 Function *MapperFunc, Value *SrcLocInfo,
2960 Value *MaptypesArg, Value *MapnamesArg,
2961 struct MapperAllocas &MapperAllocas,
2962 int64_t DeviceID, unsigned NumOperands) {
2963 if (!updateToLocation(Loc))
2964 return;
2965
2966 auto *ArrI8PtrTy = ArrayType::get(Int8Ptr, NumOperands);
2967 auto *ArrI64Ty = ArrayType::get(Int64, NumOperands);
2968 Value *ArgsBaseGEP =
2969 Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.ArgsBase,
2970 {Builder.getInt32(0), Builder.getInt32(0)});
2971 Value *ArgsGEP =
2972 Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.Args,
2973 {Builder.getInt32(0), Builder.getInt32(0)});
2974 Value *ArgSizesGEP =
2975 Builder.CreateInBoundsGEP(ArrI64Ty, MapperAllocas.ArgSizes,
2976 {Builder.getInt32(0), Builder.getInt32(0)});
2977 Value *NullPtr = Constant::getNullValue(Int8Ptr->getPointerTo());
2978 Builder.CreateCall(MapperFunc,
2979 {SrcLocInfo, Builder.getInt64(DeviceID),
2980 Builder.getInt32(NumOperands), ArgsBaseGEP, ArgsGEP,
2981 ArgSizesGEP, MaptypesArg, MapnamesArg, NullPtr});
2982}
2983
2984bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
2985 const LocationDescription &Loc, llvm::AtomicOrdering AO, AtomicKind AK) {
2986 assert(!(AO == AtomicOrdering::NotAtomic ||(static_cast <bool> (!(AO == AtomicOrdering::NotAtomic ||
AO == llvm::AtomicOrdering::Unordered) && "Unexpected Atomic Ordering."
) ? void (0) : __assert_fail ("!(AO == AtomicOrdering::NotAtomic || AO == llvm::AtomicOrdering::Unordered) && \"Unexpected Atomic Ordering.\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2988, __extension__
__PRETTY_FUNCTION__))
2987 AO == llvm::AtomicOrdering::Unordered) &&(static_cast <bool> (!(AO == AtomicOrdering::NotAtomic ||
AO == llvm::AtomicOrdering::Unordered) && "Unexpected Atomic Ordering."
) ? void (0) : __assert_fail ("!(AO == AtomicOrdering::NotAtomic || AO == llvm::AtomicOrdering::Unordered) && \"Unexpected Atomic Ordering.\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2988, __extension__
__PRETTY_FUNCTION__))
2988 "Unexpected Atomic Ordering.")(static_cast <bool> (!(AO == AtomicOrdering::NotAtomic ||
AO == llvm::AtomicOrdering::Unordered) && "Unexpected Atomic Ordering."
) ? void (0) : __assert_fail ("!(AO == AtomicOrdering::NotAtomic || AO == llvm::AtomicOrdering::Unordered) && \"Unexpected Atomic Ordering.\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2988, __extension__
__PRETTY_FUNCTION__))
;
2989
2990 bool Flush = false;
2991 llvm::AtomicOrdering FlushAO = AtomicOrdering::Monotonic;
2992
2993 switch (AK) {
2994 case Read:
2995 if (AO == AtomicOrdering::Acquire || AO == AtomicOrdering::AcquireRelease ||
2996 AO == AtomicOrdering::SequentiallyConsistent) {
2997 FlushAO = AtomicOrdering::Acquire;
2998 Flush = true;
2999 }
3000 break;
3001 case Write:
3002 case Update:
3003 if (AO == AtomicOrdering::Release || AO == AtomicOrdering::AcquireRelease ||
3004 AO == AtomicOrdering::SequentiallyConsistent) {
3005 FlushAO = AtomicOrdering::Release;
3006 Flush = true;
3007 }
3008 break;
3009 case Capture:
3010 switch (AO) {
3011 case AtomicOrdering::Acquire:
3012 FlushAO = AtomicOrdering::Acquire;
3013 Flush = true;
3014 break;
3015 case AtomicOrdering::Release:
3016 FlushAO = AtomicOrdering::Release;
3017 Flush = true;
3018 break;
3019 case AtomicOrdering::AcquireRelease:
3020 case AtomicOrdering::SequentiallyConsistent:
3021 FlushAO = AtomicOrdering::AcquireRelease;
3022 Flush = true;
3023 break;
3024 default:
3025 // do nothing - leave silently.
3026 break;
3027 }
3028 }
3029
3030 if (Flush) {
3031 // Currently Flush RT call still doesn't take memory_ordering, so for when
3032 // that happens, this tries to do the resolution of which atomic ordering
3033 // to use with but issue the flush call
3034 // TODO: pass `FlushAO` after memory ordering support is added
3035 (void)FlushAO;
3036 emitFlush(Loc);
3037 }
3038
3039 // for AO == AtomicOrdering::Monotonic and all other case combinations
3040 // do nothing
3041 return Flush;
3042}
3043
3044OpenMPIRBuilder::InsertPointTy
3045OpenMPIRBuilder::createAtomicRead(const LocationDescription &Loc,
3046 AtomicOpValue &X, AtomicOpValue &V,
3047 AtomicOrdering AO) {
3048 if (!updateToLocation(Loc))
3049 return Loc.IP;
3050
3051 Type *XTy = X.Var->getType();
3052 assert(XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory")(static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3052, __extension__
__PRETTY_FUNCTION__))
;
3053 Type *XElemTy = XTy->getPointerElementType();
3054 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic read expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic read expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3056, __extension__
__PRETTY_FUNCTION__))
3055 XElemTy->isPointerTy()) &&(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic read expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic read expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3056, __extension__
__PRETTY_FUNCTION__))
3056 "OMP atomic read expected a scalar type")(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic read expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic read expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3056, __extension__
__PRETTY_FUNCTION__))
;
3057
3058 Value *XRead = nullptr;
3059
3060 if (XElemTy->isIntegerTy()) {
3061 LoadInst *XLD =
3062 Builder.CreateLoad(XElemTy, X.Var, X.IsVolatile, "omp.atomic.read");
3063 XLD->setAtomic(AO);
3064 XRead = cast<Value>(XLD);
3065 } else {
3066 // We need to bitcast and perform atomic op as integer
3067 unsigned Addrspace = cast<PointerType>(XTy)->getAddressSpace();
3068 IntegerType *IntCastTy =
3069 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
3070 Value *XBCast = Builder.CreateBitCast(
3071 X.Var, IntCastTy->getPointerTo(Addrspace), "atomic.src.int.cast");
3072 LoadInst *XLoad =
3073 Builder.CreateLoad(IntCastTy, XBCast, X.IsVolatile, "omp.atomic.load");
3074 XLoad->setAtomic(AO);
3075 if (XElemTy->isFloatingPointTy()) {
3076 XRead = Builder.CreateBitCast(XLoad, XElemTy, "atomic.flt.cast");
3077 } else {
3078 XRead = Builder.CreateIntToPtr(XLoad, XElemTy, "atomic.ptr.cast");
3079 }
3080 }
3081 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Read);
3082 Builder.CreateStore(XRead, V.Var, V.IsVolatile);
3083 return Builder.saveIP();
3084}
3085
3086OpenMPIRBuilder::InsertPointTy
3087OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
3088 AtomicOpValue &X, Value *Expr,
3089 AtomicOrdering AO) {
3090 if (!updateToLocation(Loc))
3091 return Loc.IP;
3092
3093 Type *XTy = X.Var->getType();
3094 assert(XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory")(static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3094, __extension__
__PRETTY_FUNCTION__))
;
3095 Type *XElemTy = XTy->getPointerElementType();
3096 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic write expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic write expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3098, __extension__
__PRETTY_FUNCTION__))
3097 XElemTy->isPointerTy()) &&(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic write expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic write expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3098, __extension__
__PRETTY_FUNCTION__))
3098 "OMP atomic write expected a scalar type")(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic write expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic write expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3098, __extension__
__PRETTY_FUNCTION__))
;
3099
3100 if (XElemTy->isIntegerTy()) {
3101 StoreInst *XSt = Builder.CreateStore(Expr, X.Var, X.IsVolatile);
3102 XSt->setAtomic(AO);
3103 } else {
3104 // We need to bitcast and perform atomic op as integers
3105 unsigned Addrspace = cast<PointerType>(XTy)->getAddressSpace();
3106 IntegerType *IntCastTy =
3107 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
3108 Value *XBCast = Builder.CreateBitCast(
3109 X.Var, IntCastTy->getPointerTo(Addrspace), "atomic.dst.int.cast");
3110 Value *ExprCast =
3111 Builder.CreateBitCast(Expr, IntCastTy, "atomic.src.int.cast");
3112 StoreInst *XSt = Builder.CreateStore(ExprCast, XBCast, X.IsVolatile);
3113 XSt->setAtomic(AO);
3114 }
3115
3116 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Write);
3117 return Builder.saveIP();
3118}
3119
3120OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicUpdate(
3121 const LocationDescription &Loc, Instruction *AllocIP, AtomicOpValue &X,
3122 Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
3123 AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr) {
3124 if (!updateToLocation(Loc))
3125 return Loc.IP;
3126
3127 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3130, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic update expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3134, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3137, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
1
Taking false branch
2
Assuming 'DebugFlag' is false
3
Loop condition is false. Exiting loop
3128 Type *XTy = X.Var->getType();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3130, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic update expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3134, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3137, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3129 assert(XTy->isPointerTy() &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3130, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic update expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3134, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3137, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3130 "OMP Atomic expects a pointer to target memory");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3130, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic update expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3134, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3137, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3131 Type *XElemTy = XTy->getPointerElementType();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3130, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic update expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3134, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3137, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3132 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3130, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic update expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3134, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3137, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3133 XElemTy->isPointerTy()) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3130, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic update expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3134, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3137, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3134 "OMP atomic update expected a scalar type");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3130, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic update expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3134, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3137, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3135 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3130, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic update expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3134, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3137, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3136 (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3130, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic update expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3134, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3137, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3137 "OpenMP atomic does not support LT or GT operations");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3130, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic update expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3134, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3137, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3138 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3130, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic update expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3134, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3137, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
;
3139
3140 emitAtomicUpdate(AllocIP, X.Var, Expr, AO, RMWOp, UpdateOp, X.IsVolatile,
4
Calling 'OpenMPIRBuilder::emitAtomicUpdate'
3141 IsXBinopExpr);
3142 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Update);
3143 return Builder.saveIP();
3144}
3145
3146Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
3147 AtomicRMWInst::BinOp RMWOp) {
3148 switch (RMWOp) {
3149 case AtomicRMWInst::Add:
3150 return Builder.CreateAdd(Src1, Src2);
3151 case AtomicRMWInst::Sub:
3152 return Builder.CreateSub(Src1, Src2);
3153 case AtomicRMWInst::And:
3154 return Builder.CreateAnd(Src1, Src2);
3155 case AtomicRMWInst::Nand:
3156 return Builder.CreateNeg(Builder.CreateAnd(Src1, Src2));
3157 case AtomicRMWInst::Or:
3158 return Builder.CreateOr(Src1, Src2);
3159 case AtomicRMWInst::Xor:
3160 return Builder.CreateXor(Src1, Src2);
3161 case AtomicRMWInst::Xchg:
3162 case AtomicRMWInst::FAdd:
3163 case AtomicRMWInst::FSub:
3164 case AtomicRMWInst::BAD_BINOP:
3165 case AtomicRMWInst::Max:
3166 case AtomicRMWInst::Min:
3167 case AtomicRMWInst::UMax:
3168 case AtomicRMWInst::UMin:
3169 llvm_unreachable("Unsupported atomic update operation")::llvm::llvm_unreachable_internal("Unsupported atomic update operation"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3169)
;
3170 }
3171 llvm_unreachable("Unsupported atomic update operation")::llvm::llvm_unreachable_internal("Unsupported atomic update operation"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3171)
;
3172}
3173
3174std::pair<Value *, Value *>
3175OpenMPIRBuilder::emitAtomicUpdate(Instruction *AllocIP, Value *X, Value *Expr,
3176 AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
3177 AtomicUpdateCallbackTy &UpdateOp,
3178 bool VolatileX, bool IsXBinopExpr) {
3179 Type *XElemTy = X->getType()->getPointerElementType();
3180
3181 bool DoCmpExch =
3182 ((RMWOp == AtomicRMWInst::BAD_BINOP) || (RMWOp == AtomicRMWInst::FAdd)) ||
5
Assuming 'RMWOp' is not equal to BAD_BINOP
6
Assuming 'RMWOp' is not equal to FAdd
3183 (RMWOp == AtomicRMWInst::FSub) ||
7
Assuming 'RMWOp' is not equal to FSub
3184 (RMWOp == AtomicRMWInst::Sub && !IsXBinopExpr);
8
Assuming 'RMWOp' is not equal to Sub
3185
3186 std::pair<Value *, Value *> Res;
3187 if (XElemTy->isIntegerTy() && !DoCmpExch) {
9
Calling 'Type::isIntegerTy'
12
Returning from 'Type::isIntegerTy'
3188 Res.first = Builder.CreateAtomicRMW(RMWOp, X, Expr, llvm::MaybeAlign(), AO);
3189 // not needed except in case of postfix captures. Generate anyway for
3190 // consistency with the else part. Will be removed with any DCE pass.
3191 Res.second = emitRMWOpAsInstruction(Res.first, Expr, RMWOp);
3192 } else {
3193 unsigned Addrspace = cast<PointerType>(X->getType())->getAddressSpace();
13
The object is a 'PointerType'
3194 IntegerType *IntCastTy =
3195 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
3196 Value *XBCast =
3197 Builder.CreateBitCast(X, IntCastTy->getPointerTo(Addrspace));
3198 LoadInst *OldVal =
3199 Builder.CreateLoad(IntCastTy, XBCast, X->getName() + ".atomic.load");
3200 OldVal->setAtomic(AO);
3201 // CurBB
3202 // | /---\
3203 // ContBB |
3204 // | \---/
3205 // ExitBB
3206 BasicBlock *CurBB = Builder.GetInsertBlock();
3207 Instruction *CurBBTI = CurBB->getTerminator();
3208 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
14
Assuming 'CurBBTI' is null
15
'?' condition is false
3209 BasicBlock *ExitBB =
3210 CurBB->splitBasicBlock(CurBBTI, X->getName() + ".atomic.exit");
3211 BasicBlock *ContBB = CurBB->splitBasicBlock(CurBB->getTerminator(),
3212 X->getName() + ".atomic.cont");
3213 ContBB->getTerminator()->eraseFromParent();
3214 Builder.SetInsertPoint(ContBB);
3215 llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(), 2);
3216 PHI->addIncoming(OldVal, CurBB);
3217 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(XElemTy);
3218 NewAtomicAddr->setName(X->getName() + "x.new.val");
3219 NewAtomicAddr->moveBefore(AllocIP);
3220 IntegerType *NewAtomicCastTy =
3221 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
3222 bool IsIntTy = XElemTy->isIntegerTy();
3223 Value *NewAtomicIntAddr =
3224 (IsIntTy
15.1
'IsIntTy' is false
15.1
'IsIntTy' is false
15.1
'IsIntTy' is false
)
16
'?' condition is false
3225 ? NewAtomicAddr
3226 : Builder.CreateBitCast(NewAtomicAddr,
3227 NewAtomicCastTy->getPointerTo(Addrspace));
3228 Value *OldExprVal = PHI;
3229 if (!IsIntTy
16.1
'IsIntTy' is false
16.1
'IsIntTy' is false
16.1
'IsIntTy' is false
) {
17
Taking true branch
3230 if (XElemTy->isFloatingPointTy()) {
18
Taking true branch
3231 OldExprVal = Builder.CreateBitCast(PHI, XElemTy,
3232 X->getName() + ".atomic.fltCast");
3233 } else {
3234 OldExprVal = Builder.CreateIntToPtr(PHI, XElemTy,
3235 X->getName() + ".atomic.ptrCast");
3236 }
3237 }
3238
3239 Value *Upd = UpdateOp(OldExprVal, Builder);
3240 Builder.CreateStore(Upd, NewAtomicAddr);
3241 LoadInst *DesiredVal = Builder.CreateLoad(XElemTy, NewAtomicIntAddr);
3242 Value *XAddr =
3243 (IsIntTy
18.1
'IsIntTy' is false
18.1
'IsIntTy' is false
18.1
'IsIntTy' is false
)
19
'?' condition is false
3244 ? X
3245 : Builder.CreateBitCast(X, IntCastTy->getPointerTo(Addrspace));
3246 AtomicOrdering Failure =
3247 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
3248 AtomicCmpXchgInst *Result = Builder.CreateAtomicCmpXchg(
3249 XAddr, OldExprVal, DesiredVal, llvm::MaybeAlign(), AO, Failure);
3250 Result->setVolatile(VolatileX);
3251 Value *PreviousVal = Builder.CreateExtractValue(Result, /*Idxs=*/0);
3252 Value *SuccessFailureVal = Builder.CreateExtractValue(Result, /*Idxs=*/1);
3253 PHI->addIncoming(PreviousVal, Builder.GetInsertBlock());
3254 Builder.CreateCondBr(SuccessFailureVal, ExitBB, ContBB);
3255
3256 Res.first = OldExprVal;
3257 Res.second = Upd;
3258
3259 // set Insertion point in exit block
3260 if (UnreachableInst *ExitTI
21.1
'ExitTI' is null
21.1
'ExitTI' is null
21.1
'ExitTI' is null
=
21
'ExitTI' initialized to a null pointer value
22
Taking false branch
3261 dyn_cast<UnreachableInst>(ExitBB->getTerminator())) {
20
Assuming the object is not a 'UnreachableInst'
3262 CurBBTI->eraseFromParent();
3263 Builder.SetInsertPoint(ExitBB);
3264 } else {
3265 Builder.SetInsertPoint(ExitTI);
23
Passing null pointer value via 1st parameter 'I'
24
Calling 'IRBuilderBase::SetInsertPoint'
3266 }
3267 }
3268
3269 return Res;
3270}
3271
3272OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCapture(
3273 const LocationDescription &Loc, Instruction *AllocIP, AtomicOpValue &X,
3274 AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
3275 AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
3276 bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr) {
3277 if (!updateToLocation(Loc))
3278 return Loc.IP;
3279
3280 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3283, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic capture expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3287, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3289, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3281 Type *XTy = X.Var->getType();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3283, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic capture expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3287, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3289, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3282 assert(XTy->isPointerTy() &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3283, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic capture expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3287, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3289, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3283 "OMP Atomic expects a pointer to target memory");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3283, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic capture expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3287, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3289, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3284 Type *XElemTy = XTy->getPointerElementType();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3283, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic capture expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3287, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3289, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3285 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3283, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic capture expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3287, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3289, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3286 XElemTy->isPointerTy()) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3283, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic capture expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3287, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3289, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3287 "OMP atomic capture expected a scalar type");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3283, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic capture expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3287, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3289, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3288 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3283, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic capture expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3287, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3289, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3289 "OpenMP atomic does not support LT or GT operations");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3283, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic capture expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3287, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3289, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
3290 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3283, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = XTy->getPointerElementType
(); (static_cast <bool> ((XElemTy->isFloatingPointTy
() || XElemTy->isIntegerTy() || XElemTy->isPointerTy())
&& "OMP atomic capture expected a scalar type") ? void
(0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3287, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3289, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
;
3291
3292 // If UpdateExpr is 'x' updated with some `expr` not based on 'x',
3293 // 'x' is simply atomically rewritten with 'expr'.
3294 AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
3295 std::pair<Value *, Value *> Result = emitAtomicUpdate(
3296 AllocIP, X.Var, Expr, AO, AtomicOp, UpdateOp, X.IsVolatile, IsXBinopExpr);
3297
3298 Value *CapturedVal = (IsPostfixUpdate ? Result.first : Result.second);
3299 Builder.CreateStore(CapturedVal, V.Var, V.IsVolatile);
3300
3301 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Capture);
3302 return Builder.saveIP();
3303}
3304
3305GlobalVariable *
3306OpenMPIRBuilder::createOffloadMapnames(SmallVectorImpl<llvm::Constant *> &Names,
3307 std::string VarName) {
3308 llvm::Constant *MapNamesArrayInit = llvm::ConstantArray::get(
3309 llvm::ArrayType::get(
3310 llvm::Type::getInt8Ty(M.getContext())->getPointerTo(), Names.size()),
3311 Names);
3312 auto *MapNamesArrayGlobal = new llvm::GlobalVariable(
3313 M, MapNamesArrayInit->getType(),
3314 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MapNamesArrayInit,
3315 VarName);
3316 return MapNamesArrayGlobal;
3317}
3318
3319// Create all simple and struct types exposed by the runtime and remember
3320// the llvm::PointerTypes of them for easy access later.
3321void OpenMPIRBuilder::initializeTypes(Module &M) {
3322 LLVMContext &Ctx = M.getContext();
3323 StructType *T;
3324#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
3325#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \
3326 VarName##Ty = ArrayType::get(ElemTy, ArraySize); \
3327 VarName##PtrTy = PointerType::getUnqual(VarName##Ty);
3328#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \
3329 VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \
3330 VarName##Ptr = PointerType::getUnqual(VarName);
3331#define OMP_STRUCT_TYPE(VarName, StructName, ...) \
3332 T = StructType::getTypeByName(Ctx, StructName); \
3333 if (!T) \
3334 T = StructType::create(Ctx, {__VA_ARGS__}, StructName); \
3335 VarName = T; \
3336 VarName##Ptr = PointerType::getUnqual(T);
3337#include "llvm/Frontend/OpenMP/OMPKinds.def"
3338}
3339
3340void OpenMPIRBuilder::OutlineInfo::collectBlocks(
3341 SmallPtrSetImpl<BasicBlock *> &BlockSet,
3342 SmallVectorImpl<BasicBlock *> &BlockVector) {
3343 SmallVector<BasicBlock *, 32> Worklist;
3344 BlockSet.insert(EntryBB);
3345 BlockSet.insert(ExitBB);
3346
3347 Worklist.push_back(EntryBB);
3348 while (!Worklist.empty()) {
3349 BasicBlock *BB = Worklist.pop_back_val();
3350 BlockVector.push_back(BB);
3351 for (BasicBlock *SuccBB : successors(BB))
3352 if (BlockSet.insert(SuccBB).second)
3353 Worklist.push_back(SuccBB);
3354 }
3355}
3356
3357void CanonicalLoopInfo::collectControlBlocks(
3358 SmallVectorImpl<BasicBlock *> &BBs) {
3359 // We only count those BBs as control block for which we do not need to
3360 // reverse the CFG, i.e. not the loop body which can contain arbitrary control
3361 // flow. For consistency, this also means we do not add the Body block, which
3362 // is just the entry to the body code.
3363 BBs.reserve(BBs.size() + 6);
3364 BBs.append({getPreheader(), Header, Cond, Latch, Exit, getAfter()});
3365}
3366
3367BasicBlock *CanonicalLoopInfo::getPreheader() const {
3368 assert(isValid() && "Requires a valid canonical loop")(static_cast <bool> (isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3368, __extension__
__PRETTY_FUNCTION__))
;
3369 for (BasicBlock *Pred : predecessors(Header)) {
3370 if (Pred != Latch)
3371 return Pred;
3372 }
3373 llvm_unreachable("Missing preheader")::llvm::llvm_unreachable_internal("Missing preheader", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 3373)
;
3374}
3375
3376void CanonicalLoopInfo::assertOK() const {
3377#ifndef NDEBUG
3378 // No constraints if this object currently does not describe a loop.
3379 if (!isValid())
3380 return;
3381
3382 BasicBlock *Preheader = getPreheader();
3383 BasicBlock *Body = getBody();
3384 BasicBlock *After = getAfter();
3385
3386 // Verify standard control-flow we use for OpenMP loops.
3387 assert(Preheader)(static_cast <bool> (Preheader) ? void (0) : __assert_fail
("Preheader", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3387
, __extension__ __PRETTY_FUNCTION__))
;
3388 assert(isa<BranchInst>(Preheader->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Preheader->
getTerminator()) && "Preheader must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Preheader->getTerminator()) && \"Preheader must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3389, __extension__
__PRETTY_FUNCTION__))
3389 "Preheader must terminate with unconditional branch")(static_cast <bool> (isa<BranchInst>(Preheader->
getTerminator()) && "Preheader must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Preheader->getTerminator()) && \"Preheader must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3389, __extension__
__PRETTY_FUNCTION__))
;
3390 assert(Preheader->getSingleSuccessor() == Header &&(static_cast <bool> (Preheader->getSingleSuccessor()
== Header && "Preheader must jump to header") ? void
(0) : __assert_fail ("Preheader->getSingleSuccessor() == Header && \"Preheader must jump to header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3391, __extension__
__PRETTY_FUNCTION__))
3391 "Preheader must jump to header")(static_cast <bool> (Preheader->getSingleSuccessor()
== Header && "Preheader must jump to header") ? void
(0) : __assert_fail ("Preheader->getSingleSuccessor() == Header && \"Preheader must jump to header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3391, __extension__
__PRETTY_FUNCTION__))
;
3392
3393 assert(Header)(static_cast <bool> (Header) ? void (0) : __assert_fail
("Header", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3393
, __extension__ __PRETTY_FUNCTION__))
;
3394 assert(isa<BranchInst>(Header->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Header->getTerminator
()) && "Header must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Header->getTerminator()) && \"Header must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3395, __extension__
__PRETTY_FUNCTION__))
3395 "Header must terminate with unconditional branch")(static_cast <bool> (isa<BranchInst>(Header->getTerminator
()) && "Header must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Header->getTerminator()) && \"Header must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3395, __extension__
__PRETTY_FUNCTION__))
;
3396 assert(Header->getSingleSuccessor() == Cond &&(static_cast <bool> (Header->getSingleSuccessor() ==
Cond && "Header must jump to exiting block") ? void (
0) : __assert_fail ("Header->getSingleSuccessor() == Cond && \"Header must jump to exiting block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3397, __extension__
__PRETTY_FUNCTION__))
3397 "Header must jump to exiting block")(static_cast <bool> (Header->getSingleSuccessor() ==
Cond && "Header must jump to exiting block") ? void (
0) : __assert_fail ("Header->getSingleSuccessor() == Cond && \"Header must jump to exiting block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3397, __extension__
__PRETTY_FUNCTION__))
;
3398
3399 assert(Cond)(static_cast <bool> (Cond) ? void (0) : __assert_fail (
"Cond", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3399, __extension__
__PRETTY_FUNCTION__))
;
3400 assert(Cond->getSinglePredecessor() == Header &&(static_cast <bool> (Cond->getSinglePredecessor() ==
Header && "Exiting block only reachable from header"
) ? void (0) : __assert_fail ("Cond->getSinglePredecessor() == Header && \"Exiting block only reachable from header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3401, __extension__
__PRETTY_FUNCTION__))
3401 "Exiting block only reachable from header")(static_cast <bool> (Cond->getSinglePredecessor() ==
Header && "Exiting block only reachable from header"
) ? void (0) : __assert_fail ("Cond->getSinglePredecessor() == Header && \"Exiting block only reachable from header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3401, __extension__
__PRETTY_FUNCTION__))
;
3402
3403 assert(isa<BranchInst>(Cond->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Cond->getTerminator
()) && "Exiting block must terminate with conditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Cond->getTerminator()) && \"Exiting block must terminate with conditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3404, __extension__
__PRETTY_FUNCTION__))
3404 "Exiting block must terminate with conditional branch")(static_cast <bool> (isa<BranchInst>(Cond->getTerminator
()) && "Exiting block must terminate with conditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Cond->getTerminator()) && \"Exiting block must terminate with conditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3404, __extension__
__PRETTY_FUNCTION__))
;
3405 assert(size(successors(Cond)) == 2 &&(static_cast <bool> (size(successors(Cond)) == 2 &&
"Exiting block must have two successors") ? void (0) : __assert_fail
("size(successors(Cond)) == 2 && \"Exiting block must have two successors\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3406, __extension__
__PRETTY_FUNCTION__))
3406 "Exiting block must have two successors")(static_cast <bool> (size(successors(Cond)) == 2 &&
"Exiting block must have two successors") ? void (0) : __assert_fail
("size(successors(Cond)) == 2 && \"Exiting block must have two successors\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3406, __extension__
__PRETTY_FUNCTION__))
;
3407 assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body &&(static_cast <bool> (cast<BranchInst>(Cond->getTerminator
())->getSuccessor(0) == Body && "Exiting block's first successor jump to the body"
) ? void (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body && \"Exiting block's first successor jump to the body\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3408, __extension__
__PRETTY_FUNCTION__))
3408 "Exiting block's first successor jump to the body")(static_cast <bool> (cast<BranchInst>(Cond->getTerminator
())->getSuccessor(0) == Body && "Exiting block's first successor jump to the body"
) ? void (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body && \"Exiting block's first successor jump to the body\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3408, __extension__
__PRETTY_FUNCTION__))
;
3409 assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit &&(static_cast <bool> (cast<BranchInst>(Cond->getTerminator
())->getSuccessor(1) == Exit && "Exiting block's second successor must exit the loop"
) ? void (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit && \"Exiting block's second successor must exit the loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3410, __extension__
__PRETTY_FUNCTION__))
3410 "Exiting block's second successor must exit the loop")(static_cast <bool> (cast<BranchInst>(Cond->getTerminator
())->getSuccessor(1) == Exit && "Exiting block's second successor must exit the loop"
) ? void (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit && \"Exiting block's second successor must exit the loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3410, __extension__
__PRETTY_FUNCTION__))
;
3411
3412 assert(Body)(static_cast <bool> (Body) ? void (0) : __assert_fail (
"Body", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3412, __extension__
__PRETTY_FUNCTION__))
;
3413 assert(Body->getSinglePredecessor() == Cond &&(static_cast <bool> (Body->getSinglePredecessor() ==
Cond && "Body only reachable from exiting block") ? void
(0) : __assert_fail ("Body->getSinglePredecessor() == Cond && \"Body only reachable from exiting block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3414, __extension__
__PRETTY_FUNCTION__))
3414 "Body only reachable from exiting block")(static_cast <bool> (Body->getSinglePredecessor() ==
Cond && "Body only reachable from exiting block") ? void
(0) : __assert_fail ("Body->getSinglePredecessor() == Cond && \"Body only reachable from exiting block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3414, __extension__
__PRETTY_FUNCTION__))
;
3415 assert(!isa<PHINode>(Body->front()))(static_cast <bool> (!isa<PHINode>(Body->front
())) ? void (0) : __assert_fail ("!isa<PHINode>(Body->front())"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3415, __extension__
__PRETTY_FUNCTION__))
;
3416
3417 assert(Latch)(static_cast <bool> (Latch) ? void (0) : __assert_fail (
"Latch", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3417, __extension__
__PRETTY_FUNCTION__))
;
3418 assert(isa<BranchInst>(Latch->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Latch->getTerminator
()) && "Latch must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Latch->getTerminator()) && \"Latch must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3419, __extension__
__PRETTY_FUNCTION__))
3419 "Latch must terminate with unconditional branch")(static_cast <bool> (isa<BranchInst>(Latch->getTerminator
()) && "Latch must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Latch->getTerminator()) && \"Latch must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3419, __extension__
__PRETTY_FUNCTION__))
;
3420 assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header")(static_cast <bool> (Latch->getSingleSuccessor() == Header
&& "Latch must jump to header") ? void (0) : __assert_fail
("Latch->getSingleSuccessor() == Header && \"Latch must jump to header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3420, __extension__
__PRETTY_FUNCTION__))
;
3421 // TODO: To support simple redirecting of the end of the body code that has
3422 // multiple; introduce another auxiliary basic block like preheader and after.
3423 assert(Latch->getSinglePredecessor() != nullptr)(static_cast <bool> (Latch->getSinglePredecessor() !=
nullptr) ? void (0) : __assert_fail ("Latch->getSinglePredecessor() != nullptr"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3423, __extension__
__PRETTY_FUNCTION__))
;
3424 assert(!isa<PHINode>(Latch->front()))(static_cast <bool> (!isa<PHINode>(Latch->front
())) ? void (0) : __assert_fail ("!isa<PHINode>(Latch->front())"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3424, __extension__
__PRETTY_FUNCTION__))
;
3425
3426 assert(Exit)(static_cast <bool> (Exit) ? void (0) : __assert_fail (
"Exit", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3426, __extension__
__PRETTY_FUNCTION__))
;
3427 assert(isa<BranchInst>(Exit->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Exit->getTerminator
()) && "Exit block must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Exit->getTerminator()) && \"Exit block must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3428, __extension__
__PRETTY_FUNCTION__))
3428 "Exit block must terminate with unconditional branch")(static_cast <bool> (isa<BranchInst>(Exit->getTerminator
()) && "Exit block must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Exit->getTerminator()) && \"Exit block must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3428, __extension__
__PRETTY_FUNCTION__))
;
3429 assert(Exit->getSingleSuccessor() == After &&(static_cast <bool> (Exit->getSingleSuccessor() == After
&& "Exit block must jump to after block") ? void (0)
: __assert_fail ("Exit->getSingleSuccessor() == After && \"Exit block must jump to after block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3430, __extension__
__PRETTY_FUNCTION__))
3430 "Exit block must jump to after block")(static_cast <bool> (Exit->getSingleSuccessor() == After
&& "Exit block must jump to after block") ? void (0)
: __assert_fail ("Exit->getSingleSuccessor() == After && \"Exit block must jump to after block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3430, __extension__
__PRETTY_FUNCTION__))
;
3431
3432 assert(After)(static_cast <bool> (After) ? void (0) : __assert_fail (
"After", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3432, __extension__
__PRETTY_FUNCTION__))
;
3433 assert(After->getSinglePredecessor() == Exit &&(static_cast <bool> (After->getSinglePredecessor() ==
Exit && "After block only reachable from exit block"
) ? void (0) : __assert_fail ("After->getSinglePredecessor() == Exit && \"After block only reachable from exit block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3434, __extension__
__PRETTY_FUNCTION__))
3434 "After block only reachable from exit block")(static_cast <bool> (After->getSinglePredecessor() ==
Exit && "After block only reachable from exit block"
) ? void (0) : __assert_fail ("After->getSinglePredecessor() == Exit && \"After block only reachable from exit block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3434, __extension__
__PRETTY_FUNCTION__))
;
3435 assert(After->empty() || !isa<PHINode>(After->front()))(static_cast <bool> (After->empty() || !isa<PHINode
>(After->front())) ? void (0) : __assert_fail ("After->empty() || !isa<PHINode>(After->front())"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3435, __extension__
__PRETTY_FUNCTION__))
;
3436
3437 Instruction *IndVar = getIndVar();
3438 assert(IndVar && "Canonical induction variable not found?")(static_cast <bool> (IndVar && "Canonical induction variable not found?"
) ? void (0) : __assert_fail ("IndVar && \"Canonical induction variable not found?\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3438, __extension__
__PRETTY_FUNCTION__))
;
3439 assert(isa<IntegerType>(IndVar->getType()) &&(static_cast <bool> (isa<IntegerType>(IndVar->
getType()) && "Induction variable must be an integer"
) ? void (0) : __assert_fail ("isa<IntegerType>(IndVar->getType()) && \"Induction variable must be an integer\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3440, __extension__
__PRETTY_FUNCTION__))
3440 "Induction variable must be an integer")(static_cast <bool> (isa<IntegerType>(IndVar->
getType()) && "Induction variable must be an integer"
) ? void (0) : __assert_fail ("isa<IntegerType>(IndVar->getType()) && \"Induction variable must be an integer\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3440, __extension__
__PRETTY_FUNCTION__))
;
3441 assert(cast<PHINode>(IndVar)->getParent() == Header &&(static_cast <bool> (cast<PHINode>(IndVar)->getParent
() == Header && "Induction variable must be a PHI in the loop header"
) ? void (0) : __assert_fail ("cast<PHINode>(IndVar)->getParent() == Header && \"Induction variable must be a PHI in the loop header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3442, __extension__
__PRETTY_FUNCTION__))
3442 "Induction variable must be a PHI in the loop header")(static_cast <bool> (cast<PHINode>(IndVar)->getParent
() == Header && "Induction variable must be a PHI in the loop header"
) ? void (0) : __assert_fail ("cast<PHINode>(IndVar)->getParent() == Header && \"Induction variable must be a PHI in the loop header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3442, __extension__
__PRETTY_FUNCTION__))
;
3443 assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader)(static_cast <bool> (cast<PHINode>(IndVar)->getIncomingBlock
(0) == Preheader) ? void (0) : __assert_fail ("cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3443, __extension__
__PRETTY_FUNCTION__))
;
3444 assert((static_cast <bool> (cast<ConstantInt>(cast<PHINode
>(IndVar)->getIncomingValue(0))->isZero()) ? void (0
) : __assert_fail ("cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero()"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3445, __extension__
__PRETTY_FUNCTION__))
3445 cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero())(static_cast <bool> (cast<ConstantInt>(cast<PHINode
>(IndVar)->getIncomingValue(0))->isZero()) ? void (0
) : __assert_fail ("cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero()"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3445, __extension__
__PRETTY_FUNCTION__))
;
3446 assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch)(static_cast <bool> (cast<PHINode>(IndVar)->getIncomingBlock
(1) == Latch) ? void (0) : __assert_fail ("cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3446, __extension__
__PRETTY_FUNCTION__))
;
3447
3448 auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
3449 assert(cast<Instruction>(NextIndVar)->getParent() == Latch)(static_cast <bool> (cast<Instruction>(NextIndVar
)->getParent() == Latch) ? void (0) : __assert_fail ("cast<Instruction>(NextIndVar)->getParent() == Latch"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3449, __extension__
__PRETTY_FUNCTION__))
;
3450 assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add)(static_cast <bool> (cast<BinaryOperator>(NextIndVar
)->getOpcode() == BinaryOperator::Add) ? void (0) : __assert_fail
("cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3450, __extension__
__PRETTY_FUNCTION__))
;
3451 assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar)(static_cast <bool> (cast<BinaryOperator>(NextIndVar
)->getOperand(0) == IndVar) ? void (0) : __assert_fail ("cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3451, __extension__
__PRETTY_FUNCTION__))
;
3452 assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))(static_cast <bool> (cast<ConstantInt>(cast<BinaryOperator
>(NextIndVar)->getOperand(1)) ->isOne()) ? void (0) :
__assert_fail ("cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1)) ->isOne()"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3453, __extension__
__PRETTY_FUNCTION__))
3453 ->isOne())(static_cast <bool> (cast<ConstantInt>(cast<BinaryOperator
>(NextIndVar)->getOperand(1)) ->isOne()) ? void (0) :
__assert_fail ("cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1)) ->isOne()"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3453, __extension__
__PRETTY_FUNCTION__))
;
3454
3455 Value *TripCount = getTripCount();
3456 assert(TripCount && "Loop trip count not found?")(static_cast <bool> (TripCount && "Loop trip count not found?"
) ? void (0) : __assert_fail ("TripCount && \"Loop trip count not found?\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3456, __extension__
__PRETTY_FUNCTION__))
;
3457 assert(IndVar->getType() == TripCount->getType() &&(static_cast <bool> (IndVar->getType() == TripCount->
getType() && "Trip count and induction variable must have the same type"
) ? void (0) : __assert_fail ("IndVar->getType() == TripCount->getType() && \"Trip count and induction variable must have the same type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3458, __extension__
__PRETTY_FUNCTION__))
3458 "Trip count and induction variable must have the same type")(static_cast <bool> (IndVar->getType() == TripCount->
getType() && "Trip count and induction variable must have the same type"
) ? void (0) : __assert_fail ("IndVar->getType() == TripCount->getType() && \"Trip count and induction variable must have the same type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3458, __extension__
__PRETTY_FUNCTION__))
;
3459
3460 auto *CmpI = cast<CmpInst>(&Cond->front());
3461 assert(CmpI->getPredicate() == CmpInst::ICMP_ULT &&(static_cast <bool> (CmpI->getPredicate() == CmpInst
::ICMP_ULT && "Exit condition must be a signed less-than comparison"
) ? void (0) : __assert_fail ("CmpI->getPredicate() == CmpInst::ICMP_ULT && \"Exit condition must be a signed less-than comparison\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3462, __extension__
__PRETTY_FUNCTION__))
3462 "Exit condition must be a signed less-than comparison")(static_cast <bool> (CmpI->getPredicate() == CmpInst
::ICMP_ULT && "Exit condition must be a signed less-than comparison"
) ? void (0) : __assert_fail ("CmpI->getPredicate() == CmpInst::ICMP_ULT && \"Exit condition must be a signed less-than comparison\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3462, __extension__
__PRETTY_FUNCTION__))
;
3463 assert(CmpI->getOperand(0) == IndVar &&(static_cast <bool> (CmpI->getOperand(0) == IndVar &&
"Exit condition must compare the induction variable") ? void
(0) : __assert_fail ("CmpI->getOperand(0) == IndVar && \"Exit condition must compare the induction variable\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3464, __extension__
__PRETTY_FUNCTION__))
3464 "Exit condition must compare the induction variable")(static_cast <bool> (CmpI->getOperand(0) == IndVar &&
"Exit condition must compare the induction variable") ? void
(0) : __assert_fail ("CmpI->getOperand(0) == IndVar && \"Exit condition must compare the induction variable\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3464, __extension__
__PRETTY_FUNCTION__))
;
3465 assert(CmpI->getOperand(1) == TripCount &&(static_cast <bool> (CmpI->getOperand(1) == TripCount
&& "Exit condition must compare with the trip count"
) ? void (0) : __assert_fail ("CmpI->getOperand(1) == TripCount && \"Exit condition must compare with the trip count\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3466, __extension__
__PRETTY_FUNCTION__))
3466 "Exit condition must compare with the trip count")(static_cast <bool> (CmpI->getOperand(1) == TripCount
&& "Exit condition must compare with the trip count"
) ? void (0) : __assert_fail ("CmpI->getOperand(1) == TripCount && \"Exit condition must compare with the trip count\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3466, __extension__
__PRETTY_FUNCTION__))
;
3467#endif
3468}
3469
3470void CanonicalLoopInfo::invalidate() {
3471 Header = nullptr;
3472 Cond = nullptr;
3473 Latch = nullptr;
3474 Exit = nullptr;
3475}

/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/include/llvm/IR/Type.h

1//===- llvm/Type.h - Classes for handling data types ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the declaration of the Type class. For more "Type"
10// stuff, look in DerivedTypes.h.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_IR_TYPE_H
15#define LLVM_IR_TYPE_H
16
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/Support/CBindingWrapping.h"
20#include "llvm/Support/Casting.h"
21#include "llvm/Support/Compiler.h"
22#include "llvm/Support/ErrorHandling.h"
23#include "llvm/Support/TypeSize.h"
24#include <cassert>
25#include <cstdint>
26#include <iterator>
27
28namespace llvm {
29
30class IntegerType;
31struct fltSemantics;
32class LLVMContext;
33class PointerType;
34class raw_ostream;
35class StringRef;
36
37/// The instances of the Type class are immutable: once they are created,
38/// they are never changed. Also note that only one instance of a particular
39/// type is ever created. Thus seeing if two types are equal is a matter of
40/// doing a trivial pointer comparison. To enforce that no two equal instances
41/// are created, Type instances can only be created via static factory methods
42/// in class Type and in derived classes. Once allocated, Types are never
43/// free'd.
44///
45class Type {
46public:
47 //===--------------------------------------------------------------------===//
48 /// Definitions of all of the base types for the Type system. Based on this
49 /// value, you can cast to a class defined in DerivedTypes.h.
50 /// Note: If you add an element to this, you need to add an element to the
51 /// Type::getPrimitiveType function, or else things will break!
52 /// Also update LLVMTypeKind and LLVMGetTypeKind () in the C binding.
53 ///
54 enum TypeID {
55 // PrimitiveTypes
56 HalfTyID = 0, ///< 16-bit floating point type
57 BFloatTyID, ///< 16-bit floating point type (7-bit significand)
58 FloatTyID, ///< 32-bit floating point type
59 DoubleTyID, ///< 64-bit floating point type
60 X86_FP80TyID, ///< 80-bit floating point type (X87)
61 FP128TyID, ///< 128-bit floating point type (112-bit significand)
62 PPC_FP128TyID, ///< 128-bit floating point type (two 64-bits, PowerPC)
63 VoidTyID, ///< type with no size
64 LabelTyID, ///< Labels
65 MetadataTyID, ///< Metadata
66 X86_MMXTyID, ///< MMX vectors (64 bits, X86 specific)
67 X86_AMXTyID, ///< AMX vectors (8192 bits, X86 specific)
68 TokenTyID, ///< Tokens
69
70 // Derived types... see DerivedTypes.h file.
71 IntegerTyID, ///< Arbitrary bit width integers
72 FunctionTyID, ///< Functions
73 PointerTyID, ///< Pointers
74 StructTyID, ///< Structures
75 ArrayTyID, ///< Arrays
76 FixedVectorTyID, ///< Fixed width SIMD vector type
77 ScalableVectorTyID ///< Scalable SIMD vector type
78 };
79
80private:
81 /// This refers to the LLVMContext in which this type was uniqued.
82 LLVMContext &Context;
83
84 TypeID ID : 8; // The current base type of this type.
85 unsigned SubclassData : 24; // Space for subclasses to store data.
86 // Note that this should be synchronized with
87 // MAX_INT_BITS value in IntegerType class.
88
89protected:
90 friend class LLVMContextImpl;
91
92 explicit Type(LLVMContext &C, TypeID tid)
93 : Context(C), ID(tid), SubclassData(0) {}
94 ~Type() = default;
95
96 unsigned getSubclassData() const { return SubclassData; }
97
98 void setSubclassData(unsigned val) {
99 SubclassData = val;
100 // Ensure we don't have any accidental truncation.
101 assert(getSubclassData() == val && "Subclass data too large for field")(static_cast <bool> (getSubclassData() == val &&
"Subclass data too large for field") ? void (0) : __assert_fail
("getSubclassData() == val && \"Subclass data too large for field\""
, "llvm/include/llvm/IR/Type.h", 101, __extension__ __PRETTY_FUNCTION__
))
;
102 }
103
104 /// Keeps track of how many Type*'s there are in the ContainedTys list.
105 unsigned NumContainedTys = 0;
106
107 /// A pointer to the array of Types contained by this Type. For example, this
108 /// includes the arguments of a function type, the elements of a structure,
109 /// the pointee of a pointer, the element type of an array, etc. This pointer
110 /// may be 0 for types that don't contain other types (Integer, Double,
111 /// Float).
112 Type * const *ContainedTys = nullptr;
113
114public:
115 /// Print the current type.
116 /// Omit the type details if \p NoDetails == true.
117 /// E.g., let %st = type { i32, i16 }
118 /// When \p NoDetails is true, we only print %st.
119 /// Put differently, \p NoDetails prints the type as if
120 /// inlined with the operands when printing an instruction.
121 void print(raw_ostream &O, bool IsForDebug = false,
122 bool NoDetails = false) const;
123
124 void dump() const;
125
126 /// Return the LLVMContext in which this type was uniqued.
127 LLVMContext &getContext() const { return Context; }
128
129 //===--------------------------------------------------------------------===//
130 // Accessors for working with types.
131 //
132
133 /// Return the type id for the type. This will return one of the TypeID enum
134 /// elements defined above.
135 TypeID getTypeID() const { return ID; }
136
137 /// Return true if this is 'void'.
138 bool isVoidTy() const { return getTypeID() == VoidTyID; }
139
140 /// Return true if this is 'half', a 16-bit IEEE fp type.
141 bool isHalfTy() const { return getTypeID() == HalfTyID; }
142
143 /// Return true if this is 'bfloat', a 16-bit bfloat type.
144 bool isBFloatTy() const { return getTypeID() == BFloatTyID; }
145
146 /// Return true if this is 'float', a 32-bit IEEE fp type.
147 bool isFloatTy() const { return getTypeID() == FloatTyID; }
148
149 /// Return true if this is 'double', a 64-bit IEEE fp type.
150 bool isDoubleTy() const { return getTypeID() == DoubleTyID; }
151
152 /// Return true if this is x86 long double.
153 bool isX86_FP80Ty() const { return getTypeID() == X86_FP80TyID; }
154
155 /// Return true if this is 'fp128'.
156 bool isFP128Ty() const { return getTypeID() == FP128TyID; }
157
158 /// Return true if this is powerpc long double.
159 bool isPPC_FP128Ty() const { return getTypeID() == PPC_FP128TyID; }
160
161 /// Return true if this is one of the six floating-point types
162 bool isFloatingPointTy() const {
163 return getTypeID() == HalfTyID || getTypeID() == BFloatTyID ||
164 getTypeID() == FloatTyID || getTypeID() == DoubleTyID ||
165 getTypeID() == X86_FP80TyID || getTypeID() == FP128TyID ||
166 getTypeID() == PPC_FP128TyID;
167 }
168
169 const fltSemantics &getFltSemantics() const;
170
171 /// Return true if this is X86 MMX.
172 bool isX86_MMXTy() const { return getTypeID() == X86_MMXTyID; }
173
174 /// Return true if this is X86 AMX.
175 bool isX86_AMXTy() const { return getTypeID() == X86_AMXTyID; }
176
177 /// Return true if this is a FP type or a vector of FP.
178 bool isFPOrFPVectorTy() const { return getScalarType()->isFloatingPointTy(); }
179
180 /// Return true if this is 'label'.
181 bool isLabelTy() const { return getTypeID() == LabelTyID; }
182
183 /// Return true if this is 'metadata'.
184 bool isMetadataTy() const { return getTypeID() == MetadataTyID; }
185
186 /// Return true if this is 'token'.
187 bool isTokenTy() const { return getTypeID() == TokenTyID; }
188
189 /// True if this is an instance of IntegerType.
190 bool isIntegerTy() const { return getTypeID() == IntegerTyID; }
10
Assuming the condition is false
11
Returning zero, which participates in a condition later
191
192 /// Return true if this is an IntegerType of the given width.
193 bool isIntegerTy(unsigned Bitwidth) const;
194
195 /// Return true if this is an integer type or a vector of integer types.
196 bool isIntOrIntVectorTy() const { return getScalarType()->isIntegerTy(); }
197
198 /// Return true if this is an integer type or a vector of integer types of
199 /// the given width.
200 bool isIntOrIntVectorTy(unsigned BitWidth) const {
201 return getScalarType()->isIntegerTy(BitWidth);
202 }
203
204 /// Return true if this is an integer type or a pointer type.
205 bool isIntOrPtrTy() const { return isIntegerTy() || isPointerTy(); }
206
207 /// True if this is an instance of FunctionType.
208 bool isFunctionTy() const { return getTypeID() == FunctionTyID; }
209
210 /// True if this is an instance of StructType.
211 bool isStructTy() const { return getTypeID() == StructTyID; }
212
213 /// True if this is an instance of ArrayType.
214 bool isArrayTy() const { return getTypeID() == ArrayTyID; }
215
216 /// True if this is an instance of PointerType.
217 bool isPointerTy() const { return getTypeID() == PointerTyID; }
218
219 /// True if this is an instance of an opaque PointerType.
220 bool isOpaquePointerTy() const;
221
222 /// Return true if this is a pointer type or a vector of pointer types.
223 bool isPtrOrPtrVectorTy() const { return getScalarType()->isPointerTy(); }
224
225 /// True if this is an instance of VectorType.
226 inline bool isVectorTy() const {
227 return getTypeID() == ScalableVectorTyID || getTypeID() == FixedVectorTyID;
228 }
229
230 /// Return true if this type could be converted with a lossless BitCast to
231 /// type 'Ty'. For example, i8* to i32*. BitCasts are valid for types of the
232 /// same size only where no re-interpretation of the bits is done.
233 /// Determine if this type could be losslessly bitcast to Ty
234 bool canLosslesslyBitCastTo(Type *Ty) const;
235
236 /// Return true if this type is empty, that is, it has no elements or all of
237 /// its elements are empty.
238 bool isEmptyTy() const;
239
240 /// Return true if the type is "first class", meaning it is a valid type for a
241 /// Value.
242 bool isFirstClassType() const {
243 return getTypeID() != FunctionTyID && getTypeID() != VoidTyID;
244 }
245
246 /// Return true if the type is a valid type for a register in codegen. This
247 /// includes all first-class types except struct and array types.
248 bool isSingleValueType() const {
249 return isFloatingPointTy() || isX86_MMXTy() || isIntegerTy() ||
250 isPointerTy() || isVectorTy() || isX86_AMXTy();
251 }
252
253 /// Return true if the type is an aggregate type. This means it is valid as
254 /// the first operand of an insertvalue or extractvalue instruction. This
255 /// includes struct and array types, but does not include vector types.
256 bool isAggregateType() const {
257 return getTypeID() == StructTyID || getTypeID() == ArrayTyID;
258 }
259
260 /// Return true if it makes sense to take the size of this type. To get the
261 /// actual size for a particular target, it is reasonable to use the
262 /// DataLayout subsystem to do this.
263 bool isSized(SmallPtrSetImpl<Type*> *Visited = nullptr) const {
264 // If it's a primitive, it is always sized.
265 if (getTypeID() == IntegerTyID || isFloatingPointTy() ||
266 getTypeID() == PointerTyID || getTypeID() == X86_MMXTyID ||
267 getTypeID() == X86_AMXTyID)
268 return true;
269 // If it is not something that can have a size (e.g. a function or label),
270 // it doesn't have a size.
271 if (getTypeID() != StructTyID && getTypeID() != ArrayTyID && !isVectorTy())
272 return false;
273 // Otherwise we have to try harder to decide.
274 return isSizedDerivedType(Visited);
275 }
276
277 /// Return the basic size of this type if it is a primitive type. These are
278 /// fixed by LLVM and are not target-dependent.
279 /// This will return zero if the type does not have a size or is not a
280 /// primitive type.
281 ///
282 /// If this is a scalable vector type, the scalable property will be set and
283 /// the runtime size will be a positive integer multiple of the base size.
284 ///
285 /// Note that this may not reflect the size of memory allocated for an
286 /// instance of the type or the number of bytes that are written when an
287 /// instance of the type is stored to memory. The DataLayout class provides
288 /// additional query functions to provide this information.
289 ///
290 TypeSize getPrimitiveSizeInBits() const LLVM_READONLY__attribute__((__pure__));
291
292 /// If this is a vector type, return the getPrimitiveSizeInBits value for the
293 /// element type. Otherwise return the getPrimitiveSizeInBits value for this
294 /// type.
295 unsigned getScalarSizeInBits() const LLVM_READONLY__attribute__((__pure__));
296
297 /// Return the width of the mantissa of this type. This is only valid on
298 /// floating-point types. If the FP type does not have a stable mantissa (e.g.
299 /// ppc long double), this method returns -1.
300 int getFPMantissaWidth() const;
301
302 /// Return whether the type is IEEE compatible, as defined by the eponymous
303 /// method in APFloat.
304 bool isIEEE() const;
305
306 /// If this is a vector type, return the element type, otherwise return
307 /// 'this'.
308 inline Type *getScalarType() const {
309 if (isVectorTy())
310 return getContainedType(0);
311 return const_cast<Type *>(this);
312 }
313
314 //===--------------------------------------------------------------------===//
315 // Type Iteration support.
316 //
317 using subtype_iterator = Type * const *;
318
319 subtype_iterator subtype_begin() const { return ContainedTys; }
320 subtype_iterator subtype_end() const { return &ContainedTys[NumContainedTys];}
321 ArrayRef<Type*> subtypes() const {
322 return makeArrayRef(subtype_begin(), subtype_end());
323 }
324
325 using subtype_reverse_iterator = std::reverse_iterator<subtype_iterator>;
326
327 subtype_reverse_iterator subtype_rbegin() const {
328 return subtype_reverse_iterator(subtype_end());
329 }
330 subtype_reverse_iterator subtype_rend() const {
331 return subtype_reverse_iterator(subtype_begin());
332 }
333
334 /// This method is used to implement the type iterator (defined at the end of
335 /// the file). For derived types, this returns the types 'contained' in the
336 /// derived type.
337 Type *getContainedType(unsigned i) const {
338 assert(i < NumContainedTys && "Index out of range!")(static_cast <bool> (i < NumContainedTys && "Index out of range!"
) ? void (0) : __assert_fail ("i < NumContainedTys && \"Index out of range!\""
, "llvm/include/llvm/IR/Type.h", 338, __extension__ __PRETTY_FUNCTION__
))
;
339 return ContainedTys[i];
340 }
341
342 /// Return the number of types in the derived type.
343 unsigned getNumContainedTypes() const { return NumContainedTys; }
344
345 //===--------------------------------------------------------------------===//
346 // Helper methods corresponding to subclass methods. This forces a cast to
347 // the specified subclass and calls its accessor. "getArrayNumElements" (for
348 // example) is shorthand for cast<ArrayType>(Ty)->getNumElements(). This is
349 // only intended to cover the core methods that are frequently used, helper
350 // methods should not be added here.
351
352 inline unsigned getIntegerBitWidth() const;
353
354 inline Type *getFunctionParamType(unsigned i) const;
355 inline unsigned getFunctionNumParams() const;
356 inline bool isFunctionVarArg() const;
357
358 inline StringRef getStructName() const;
359 inline unsigned getStructNumElements() const;
360 inline Type *getStructElementType(unsigned N) const;
361
362 inline uint64_t getArrayNumElements() const;
363
364 Type *getArrayElementType() const {
365 assert(getTypeID() == ArrayTyID)(static_cast <bool> (getTypeID() == ArrayTyID) ? void (
0) : __assert_fail ("getTypeID() == ArrayTyID", "llvm/include/llvm/IR/Type.h"
, 365, __extension__ __PRETTY_FUNCTION__))
;
366 return ContainedTys[0];
367 }
368
369 Type *getPointerElementType() const {
370 assert(getTypeID() == PointerTyID)(static_cast <bool> (getTypeID() == PointerTyID) ? void
(0) : __assert_fail ("getTypeID() == PointerTyID", "llvm/include/llvm/IR/Type.h"
, 370, __extension__ __PRETTY_FUNCTION__))
;
371 assert(NumContainedTys &&(static_cast <bool> (NumContainedTys && "Attempting to get element type of opaque pointer"
) ? void (0) : __assert_fail ("NumContainedTys && \"Attempting to get element type of opaque pointer\""
, "llvm/include/llvm/IR/Type.h", 372, __extension__ __PRETTY_FUNCTION__
))
372 "Attempting to get element type of opaque pointer")(static_cast <bool> (NumContainedTys && "Attempting to get element type of opaque pointer"
) ? void (0) : __assert_fail ("NumContainedTys && \"Attempting to get element type of opaque pointer\""
, "llvm/include/llvm/IR/Type.h", 372, __extension__ __PRETTY_FUNCTION__
))
;
373 return ContainedTys[0];
374 }
375
376 /// Given vector type, change the element type,
377 /// whilst keeping the old number of elements.
378 /// For non-vectors simply returns \p EltTy.
379 inline Type *getWithNewType(Type *EltTy) const;
380
381 /// Given an integer or vector type, change the lane bitwidth to NewBitwidth,
382 /// whilst keeping the old number of lanes.
383 inline Type *getWithNewBitWidth(unsigned NewBitWidth) const;
384
385 /// Given scalar/vector integer type, returns a type with elements twice as
386 /// wide as in the original type. For vectors, preserves element count.
387 inline Type *getExtendedType() const;
388
389 /// Get the address space of this pointer or pointer vector type.
390 inline unsigned getPointerAddressSpace() const;
391
392 //===--------------------------------------------------------------------===//
393 // Static members exported by the Type class itself. Useful for getting
394 // instances of Type.
395 //
396
397 /// Return a type based on an identifier.
398 static Type *getPrimitiveType(LLVMContext &C, TypeID IDNumber);
399
400 //===--------------------------------------------------------------------===//
401 // These are the builtin types that are always available.
402 //
403 static Type *getVoidTy(LLVMContext &C);
404 static Type *getLabelTy(LLVMContext &C);
405 static Type *getHalfTy(LLVMContext &C);
406 static Type *getBFloatTy(LLVMContext &C);
407 static Type *getFloatTy(LLVMContext &C);
408 static Type *getDoubleTy(LLVMContext &C);
409 static Type *getMetadataTy(LLVMContext &C);
410 static Type *getX86_FP80Ty(LLVMContext &C);
411 static Type *getFP128Ty(LLVMContext &C);
412 static Type *getPPC_FP128Ty(LLVMContext &C);
413 static Type *getX86_MMXTy(LLVMContext &C);
414 static Type *getX86_AMXTy(LLVMContext &C);
415 static Type *getTokenTy(LLVMContext &C);
416 static IntegerType *getIntNTy(LLVMContext &C, unsigned N);
417 static IntegerType *getInt1Ty(LLVMContext &C);
418 static IntegerType *getInt8Ty(LLVMContext &C);
419 static IntegerType *getInt16Ty(LLVMContext &C);
420 static IntegerType *getInt32Ty(LLVMContext &C);
421 static IntegerType *getInt64Ty(LLVMContext &C);
422 static IntegerType *getInt128Ty(LLVMContext &C);
423 template <typename ScalarTy> static Type *getScalarTy(LLVMContext &C) {
424 int noOfBits = sizeof(ScalarTy) * CHAR_BIT8;
425 if (std::is_integral<ScalarTy>::value) {
426 return (Type*) Type::getIntNTy(C, noOfBits);
427 } else if (std::is_floating_point<ScalarTy>::value) {
428 switch (noOfBits) {
429 case 32:
430 return Type::getFloatTy(C);
431 case 64:
432 return Type::getDoubleTy(C);
433 }
434 }
435 llvm_unreachable("Unsupported type in Type::getScalarTy")::llvm::llvm_unreachable_internal("Unsupported type in Type::getScalarTy"
, "llvm/include/llvm/IR/Type.h", 435)
;
436 }
437 static Type *getFloatingPointTy(LLVMContext &C, const fltSemantics &S);
438
439 //===--------------------------------------------------------------------===//
440 // Convenience methods for getting pointer types with one of the above builtin
441 // types as pointee.
442 //
443 static PointerType *getHalfPtrTy(LLVMContext &C, unsigned AS = 0);
444 static PointerType *getBFloatPtrTy(LLVMContext &C, unsigned AS = 0);
445 static PointerType *getFloatPtrTy(LLVMContext &C, unsigned AS = 0);
446 static PointerType *getDoublePtrTy(LLVMContext &C, unsigned AS = 0);
447 static PointerType *getX86_FP80PtrTy(LLVMContext &C, unsigned AS = 0);
448 static PointerType *getFP128PtrTy(LLVMContext &C, unsigned AS = 0);
449 static PointerType *getPPC_FP128PtrTy(LLVMContext &C, unsigned AS = 0);
450 static PointerType *getX86_MMXPtrTy(LLVMContext &C, unsigned AS = 0);
451 static PointerType *getX86_AMXPtrTy(LLVMContext &C, unsigned AS = 0);
452 static PointerType *getIntNPtrTy(LLVMContext &C, unsigned N, unsigned AS = 0);
453 static PointerType *getInt1PtrTy(LLVMContext &C, unsigned AS = 0);
454 static PointerType *getInt8PtrTy(LLVMContext &C, unsigned AS = 0);
455 static PointerType *getInt16PtrTy(LLVMContext &C, unsigned AS = 0);
456 static PointerType *getInt32PtrTy(LLVMContext &C, unsigned AS = 0);
457 static PointerType *getInt64PtrTy(LLVMContext &C, unsigned AS = 0);
458
459 /// Return a pointer to the current type. This is equivalent to
460 /// PointerType::get(Foo, AddrSpace).
461 /// TODO: Remove this after opaque pointer transition is complete.
462 PointerType *getPointerTo(unsigned AddrSpace = 0) const;
463
464private:
465 /// Derived types like structures and arrays are sized iff all of the members
466 /// of the type are sized as well. Since asking for their size is relatively
467 /// uncommon, move this operation out-of-line.
468 bool isSizedDerivedType(SmallPtrSetImpl<Type*> *Visited = nullptr) const;
469};
470
471// Printing of types.
472inline raw_ostream &operator<<(raw_ostream &OS, const Type &T) {
473 T.print(OS);
474 return OS;
475}
476
477// allow isa<PointerType>(x) to work without DerivedTypes.h included.
478template <> struct isa_impl<PointerType, Type> {
479 static inline bool doit(const Type &Ty) {
480 return Ty.getTypeID() == Type::PointerTyID;
481 }
482};
483
484// Create wrappers for C Binding types (see CBindingWrapping.h).
485DEFINE_ISA_CONVERSION_FUNCTIONS(Type, LLVMTypeRef)inline Type *unwrap(LLVMTypeRef P) { return reinterpret_cast<
Type*>(P); } inline LLVMTypeRef wrap(const Type *P) { return
reinterpret_cast<LLVMTypeRef>(const_cast<Type*>(
P)); } template<typename T> inline T *unwrap(LLVMTypeRef
P) { return cast<T>(unwrap(P)); }
486
487/* Specialized opaque type conversions.
488 */
489inline Type **unwrap(LLVMTypeRef* Tys) {
490 return reinterpret_cast<Type**>(Tys);
491}
492
493inline LLVMTypeRef *wrap(Type **Tys) {
494 return reinterpret_cast<LLVMTypeRef*>(const_cast<Type**>(Tys));
495}
496
497} // end namespace llvm
498
499#endif // LLVM_IR_TYPE_H

/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/include/llvm/IR/IRBuilder.h

1//===- llvm/IRBuilder.h - Builder for LLVM Instructions ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_IR_IRBUILDER_H
15#define LLVM_IR_IRBUILDER_H
16
17#include "llvm-c/Types.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/None.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/IR/BasicBlock.h"
24#include "llvm/IR/Constant.h"
25#include "llvm/IR/ConstantFolder.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/IR/DebugInfoMetadata.h"
29#include "llvm/IR/DebugLoc.h"
30#include "llvm/IR/DerivedTypes.h"
31#include "llvm/IR/Function.h"
32#include "llvm/IR/GlobalVariable.h"
33#include "llvm/IR/InstrTypes.h"
34#include "llvm/IR/Instruction.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/IR/IntrinsicInst.h"
37#include "llvm/IR/LLVMContext.h"
38#include "llvm/IR/Module.h"
39#include "llvm/IR/Operator.h"
40#include "llvm/IR/Type.h"
41#include "llvm/IR/Value.h"
42#include "llvm/IR/ValueHandle.h"
43#include "llvm/Support/AtomicOrdering.h"
44#include "llvm/Support/CBindingWrapping.h"
45#include "llvm/Support/Casting.h"
46#include <cassert>
47#include <cstddef>
48#include <cstdint>
49#include <functional>
50#include <utility>
51
52namespace llvm {
53
54class APInt;
55class MDNode;
56class Use;
57
58/// This provides the default implementation of the IRBuilder
59/// 'InsertHelper' method that is called whenever an instruction is created by
60/// IRBuilder and needs to be inserted.
61///
62/// By default, this inserts the instruction at the insertion point.
63class IRBuilderDefaultInserter {
64public:
65 virtual ~IRBuilderDefaultInserter();
66
67 virtual void InsertHelper(Instruction *I, const Twine &Name,
68 BasicBlock *BB,
69 BasicBlock::iterator InsertPt) const {
70 if (BB) BB->getInstList().insert(InsertPt, I);
71 I->setName(Name);
72 }
73};
74
75/// Provides an 'InsertHelper' that calls a user-provided callback after
76/// performing the default insertion.
77class IRBuilderCallbackInserter : public IRBuilderDefaultInserter {
78 std::function<void(Instruction *)> Callback;
79
80public:
81 virtual ~IRBuilderCallbackInserter();
82
83 IRBuilderCallbackInserter(std::function<void(Instruction *)> Callback)
84 : Callback(std::move(Callback)) {}
85
86 void InsertHelper(Instruction *I, const Twine &Name,
87 BasicBlock *BB,
88 BasicBlock::iterator InsertPt) const override {
89 IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
90 Callback(I);
91 }
92};
93
94/// Common base class shared among various IRBuilders.
95class IRBuilderBase {
96 /// Pairs of (metadata kind, MDNode *) that should be added to all newly
97 /// created instructions, like !dbg metadata.
98 SmallVector<std::pair<unsigned, MDNode *>, 2> MetadataToCopy;
99
100 /// Add or update the an entry (Kind, MD) to MetadataToCopy, if \p MD is not
101 /// null. If \p MD is null, remove the entry with \p Kind.
102 void AddOrRemoveMetadataToCopy(unsigned Kind, MDNode *MD) {
103 if (!MD) {
104 erase_if(MetadataToCopy, [Kind](const std::pair<unsigned, MDNode *> &KV) {
105 return KV.first == Kind;
106 });
107 return;
108 }
109
110 for (auto &KV : MetadataToCopy)
111 if (KV.first == Kind) {
112 KV.second = MD;
113 return;
114 }
115
116 MetadataToCopy.emplace_back(Kind, MD);
117 }
118
119protected:
120 BasicBlock *BB;
121 BasicBlock::iterator InsertPt;
122 LLVMContext &Context;
123 const IRBuilderFolder &Folder;
124 const IRBuilderDefaultInserter &Inserter;
125
126 MDNode *DefaultFPMathTag;
127 FastMathFlags FMF;
128
129 bool IsFPConstrained;
130 fp::ExceptionBehavior DefaultConstrainedExcept;
131 RoundingMode DefaultConstrainedRounding;
132
133 ArrayRef<OperandBundleDef> DefaultOperandBundles;
134
135public:
136 IRBuilderBase(LLVMContext &context, const IRBuilderFolder &Folder,
137 const IRBuilderDefaultInserter &Inserter,
138 MDNode *FPMathTag, ArrayRef<OperandBundleDef> OpBundles)
139 : Context(context), Folder(Folder), Inserter(Inserter),
140 DefaultFPMathTag(FPMathTag), IsFPConstrained(false),
141 DefaultConstrainedExcept(fp::ebStrict),
142 DefaultConstrainedRounding(RoundingMode::Dynamic),
143 DefaultOperandBundles(OpBundles) {
144 ClearInsertionPoint();
145 }
146
147 /// Insert and return the specified instruction.
148 template<typename InstTy>
149 InstTy *Insert(InstTy *I, const Twine &Name = "") const {
150 Inserter.InsertHelper(I, Name, BB, InsertPt);
151 AddMetadataToInst(I);
152 return I;
153 }
154
155 /// No-op overload to handle constants.
156 Constant *Insert(Constant *C, const Twine& = "") const {
157 return C;
158 }
159
160 Value *Insert(Value *V, const Twine &Name = "") const {
161 if (Instruction *I = dyn_cast<Instruction>(V))
162 return Insert(I, Name);
163 assert(isa<Constant>(V))(static_cast <bool> (isa<Constant>(V)) ? void (0)
: __assert_fail ("isa<Constant>(V)", "llvm/include/llvm/IR/IRBuilder.h"
, 163, __extension__ __PRETTY_FUNCTION__))
;
164 return V;
165 }
166
167 //===--------------------------------------------------------------------===//
168 // Builder configuration methods
169 //===--------------------------------------------------------------------===//
170
171 /// Clear the insertion point: created instructions will not be
172 /// inserted into a block.
173 void ClearInsertionPoint() {
174 BB = nullptr;
175 InsertPt = BasicBlock::iterator();
176 }
177
178 BasicBlock *GetInsertBlock() const { return BB; }
179 BasicBlock::iterator GetInsertPoint() const { return InsertPt; }
180 LLVMContext &getContext() const { return Context; }
181
182 /// This specifies that created instructions should be appended to the
183 /// end of the specified block.
184 void SetInsertPoint(BasicBlock *TheBB) {
185 BB = TheBB;
186 InsertPt = BB->end();
187 }
188
189 /// This specifies that created instructions should be inserted before
190 /// the specified instruction.
191 void SetInsertPoint(Instruction *I) {
192 BB = I->getParent();
25
Called C++ object pointer is null
193 InsertPt = I->getIterator();
194 assert(InsertPt != BB->end() && "Can't read debug loc from end()")(static_cast <bool> (InsertPt != BB->end() &&
"Can't read debug loc from end()") ? void (0) : __assert_fail
("InsertPt != BB->end() && \"Can't read debug loc from end()\""
, "llvm/include/llvm/IR/IRBuilder.h", 194, __extension__ __PRETTY_FUNCTION__
))
;
195 SetCurrentDebugLocation(I->getDebugLoc());
196 }
197
198 /// This specifies that created instructions should be inserted at the
199 /// specified point.
200 void SetInsertPoint(BasicBlock *TheBB, BasicBlock::iterator IP) {
201 BB = TheBB;
202 InsertPt = IP;
203 if (IP != TheBB->end())
204 SetCurrentDebugLocation(IP->getDebugLoc());
205 }
206
207 /// Set location information used by debugging information.
208 void SetCurrentDebugLocation(DebugLoc L) {
209 AddOrRemoveMetadataToCopy(LLVMContext::MD_dbg, L.getAsMDNode());
210 }
211
212 /// Collect metadata with IDs \p MetadataKinds from \p Src which should be
213 /// added to all created instructions. Entries present in MedataDataToCopy but
214 /// not on \p Src will be dropped from MetadataToCopy.
215 void CollectMetadataToCopy(Instruction *Src,
216 ArrayRef<unsigned> MetadataKinds) {
217 for (unsigned K : MetadataKinds)
218 AddOrRemoveMetadataToCopy(K, Src->getMetadata(K));
219 }
220
221 /// Get location information used by debugging information.
222 DebugLoc getCurrentDebugLocation() const {
223 for (auto &KV : MetadataToCopy)
224 if (KV.first == LLVMContext::MD_dbg)
225 return {cast<DILocation>(KV.second)};
226
227 return {};
228 }
229
230 /// If this builder has a current debug location, set it on the
231 /// specified instruction.
232 void SetInstDebugLocation(Instruction *I) const {
233 for (const auto &KV : MetadataToCopy)
234 if (KV.first == LLVMContext::MD_dbg) {
235 I->setDebugLoc(DebugLoc(KV.second));
236 return;
237 }
238 }
239
240 /// Add all entries in MetadataToCopy to \p I.
241 void AddMetadataToInst(Instruction *I) const {
242 for (auto &KV : MetadataToCopy)
243 I->setMetadata(KV.first, KV.second);
244 }
245
246 /// Get the return type of the current function that we're emitting
247 /// into.
248 Type *getCurrentFunctionReturnType() const;
249
250 /// InsertPoint - A saved insertion point.
251 class InsertPoint {
252 BasicBlock *Block = nullptr;
253 BasicBlock::iterator Point;
254
255 public:
256 /// Creates a new insertion point which doesn't point to anything.
257 InsertPoint() = default;
258
259 /// Creates a new insertion point at the given location.
260 InsertPoint(BasicBlock *InsertBlock, BasicBlock::iterator InsertPoint)
261 : Block(InsertBlock), Point(InsertPoint) {}
262
263 /// Returns true if this insert point is set.
264 bool isSet() const { return (Block != nullptr); }
265
266 BasicBlock *getBlock() const { return Block; }
267 BasicBlock::iterator getPoint() const { return Point; }
268 };
269
270 /// Returns the current insert point.
271 InsertPoint saveIP() const {
272 return InsertPoint(GetInsertBlock(), GetInsertPoint());
273 }
274
275 /// Returns the current insert point, clearing it in the process.
276 InsertPoint saveAndClearIP() {
277 InsertPoint IP(GetInsertBlock(), GetInsertPoint());
278 ClearInsertionPoint();
279 return IP;
280 }
281
282 /// Sets the current insert point to a previously-saved location.
283 void restoreIP(InsertPoint IP) {
284 if (IP.isSet())
285 SetInsertPoint(IP.getBlock(), IP.getPoint());
286 else
287 ClearInsertionPoint();
288 }
289
290 /// Get the floating point math metadata being used.
291 MDNode *getDefaultFPMathTag() const { return DefaultFPMathTag; }
292
293 /// Get the flags to be applied to created floating point ops
294 FastMathFlags getFastMathFlags() const { return FMF; }
295
296 FastMathFlags &getFastMathFlags() { return FMF; }
297
298 /// Clear the fast-math flags.
299 void clearFastMathFlags() { FMF.clear(); }
300
301 /// Set the floating point math metadata to be used.
302 void setDefaultFPMathTag(MDNode *FPMathTag) { DefaultFPMathTag = FPMathTag; }
303
304 /// Set the fast-math flags to be used with generated fp-math operators
305 void setFastMathFlags(FastMathFlags NewFMF) { FMF = NewFMF; }
306
307 /// Enable/Disable use of constrained floating point math. When
308 /// enabled the CreateF<op>() calls instead create constrained
309 /// floating point intrinsic calls. Fast math flags are unaffected
310 /// by this setting.
311 void setIsFPConstrained(bool IsCon) { IsFPConstrained = IsCon; }
312
313 /// Query for the use of constrained floating point math
314 bool getIsFPConstrained() { return IsFPConstrained; }
315
316 /// Set the exception handling to be used with constrained floating point
317 void setDefaultConstrainedExcept(fp::ExceptionBehavior NewExcept) {
318#ifndef NDEBUG
319 Optional<StringRef> ExceptStr = convertExceptionBehaviorToStr(NewExcept);
320 assert(ExceptStr.hasValue() && "Garbage strict exception behavior!")(static_cast <bool> (ExceptStr.hasValue() && "Garbage strict exception behavior!"
) ? void (0) : __assert_fail ("ExceptStr.hasValue() && \"Garbage strict exception behavior!\""
, "llvm/include/llvm/IR/IRBuilder.h", 320, __extension__ __PRETTY_FUNCTION__
))
;
321#endif
322 DefaultConstrainedExcept = NewExcept;
323 }
324
325 /// Set the rounding mode handling to be used with constrained floating point
326 void setDefaultConstrainedRounding(RoundingMode NewRounding) {
327#ifndef NDEBUG
328 Optional<StringRef> RoundingStr = convertRoundingModeToStr(NewRounding);
329 assert(RoundingStr.hasValue() && "Garbage strict rounding mode!")(static_cast <bool> (RoundingStr.hasValue() && "Garbage strict rounding mode!"
) ? void (0) : __assert_fail ("RoundingStr.hasValue() && \"Garbage strict rounding mode!\""
, "llvm/include/llvm/IR/IRBuilder.h", 329, __extension__ __PRETTY_FUNCTION__
))
;
330#endif
331 DefaultConstrainedRounding = NewRounding;
332 }
333
334 /// Get the exception handling used with constrained floating point
335 fp::ExceptionBehavior getDefaultConstrainedExcept() {
336 return DefaultConstrainedExcept;
337 }
338
339 /// Get the rounding mode handling used with constrained floating point
340 RoundingMode getDefaultConstrainedRounding() {
341 return DefaultConstrainedRounding;
342 }
343
344 void setConstrainedFPFunctionAttr() {
345 assert(BB && "Must have a basic block to set any function attributes!")(static_cast <bool> (BB && "Must have a basic block to set any function attributes!"
) ? void (0) : __assert_fail ("BB && \"Must have a basic block to set any function attributes!\""
, "llvm/include/llvm/IR/IRBuilder.h", 345, __extension__ __PRETTY_FUNCTION__
))
;
346
347 Function *F = BB->getParent();
348 if (!F->hasFnAttribute(Attribute::StrictFP)) {
349 F->addFnAttr(Attribute::StrictFP);
350 }
351 }
352
353 void setConstrainedFPCallAttr(CallBase *I) {
354 I->addFnAttr(Attribute::StrictFP);
355 }
356
357 void setDefaultOperandBundles(ArrayRef<OperandBundleDef> OpBundles) {
358 DefaultOperandBundles = OpBundles;
359 }
360
361 //===--------------------------------------------------------------------===//
362 // RAII helpers.
363 //===--------------------------------------------------------------------===//
364
365 // RAII object that stores the current insertion point and restores it
366 // when the object is destroyed. This includes the debug location.
367 class InsertPointGuard {
368 IRBuilderBase &Builder;
369 AssertingVH<BasicBlock> Block;
370 BasicBlock::iterator Point;
371 DebugLoc DbgLoc;
372
373 public:
374 InsertPointGuard(IRBuilderBase &B)
375 : Builder(B), Block(B.GetInsertBlock()), Point(B.GetInsertPoint()),
376 DbgLoc(B.getCurrentDebugLocation()) {}
377
378 InsertPointGuard(const InsertPointGuard &) = delete;
379 InsertPointGuard &operator=(const InsertPointGuard &) = delete;
380
381 ~InsertPointGuard() {
382 Builder.restoreIP(InsertPoint(Block, Point));
383 Builder.SetCurrentDebugLocation(DbgLoc);
384 }
385 };
386
387 // RAII object that stores the current fast math settings and restores
388 // them when the object is destroyed.
389 class FastMathFlagGuard {
390 IRBuilderBase &Builder;
391 FastMathFlags FMF;
392 MDNode *FPMathTag;
393 bool IsFPConstrained;
394 fp::ExceptionBehavior DefaultConstrainedExcept;
395 RoundingMode DefaultConstrainedRounding;
396
397 public:
398 FastMathFlagGuard(IRBuilderBase &B)
399 : Builder(B), FMF(B.FMF), FPMathTag(B.DefaultFPMathTag),
400 IsFPConstrained(B.IsFPConstrained),
401 DefaultConstrainedExcept(B.DefaultConstrainedExcept),
402 DefaultConstrainedRounding(B.DefaultConstrainedRounding) {}
403
404 FastMathFlagGuard(const FastMathFlagGuard &) = delete;
405 FastMathFlagGuard &operator=(const FastMathFlagGuard &) = delete;
406
407 ~FastMathFlagGuard() {
408 Builder.FMF = FMF;
409 Builder.DefaultFPMathTag = FPMathTag;
410 Builder.IsFPConstrained = IsFPConstrained;
411 Builder.DefaultConstrainedExcept = DefaultConstrainedExcept;
412 Builder.DefaultConstrainedRounding = DefaultConstrainedRounding;
413 }
414 };
415
416 // RAII object that stores the current default operand bundles and restores
417 // them when the object is destroyed.
418 class OperandBundlesGuard {
419 IRBuilderBase &Builder;
420 ArrayRef<OperandBundleDef> DefaultOperandBundles;
421
422 public:
423 OperandBundlesGuard(IRBuilderBase &B)
424 : Builder(B), DefaultOperandBundles(B.DefaultOperandBundles) {}
425
426 OperandBundlesGuard(const OperandBundlesGuard &) = delete;
427 OperandBundlesGuard &operator=(const OperandBundlesGuard &) = delete;
428
429 ~OperandBundlesGuard() {
430 Builder.DefaultOperandBundles = DefaultOperandBundles;
431 }
432 };
433
434
435 //===--------------------------------------------------------------------===//
436 // Miscellaneous creation methods.
437 //===--------------------------------------------------------------------===//
438
439 /// Make a new global variable with initializer type i8*
440 ///
441 /// Make a new global variable with an initializer that has array of i8 type
442 /// filled in with the null terminated string value specified. The new global
443 /// variable will be marked mergable with any others of the same contents. If
444 /// Name is specified, it is the name of the global variable created.
445 ///
446 /// If no module is given via \p M, it is take from the insertion point basic
447 /// block.
448 GlobalVariable *CreateGlobalString(StringRef Str, const Twine &Name = "",
449 unsigned AddressSpace = 0,
450 Module *M = nullptr);
451
452 /// Get a constant value representing either true or false.
453 ConstantInt *getInt1(bool V) {
454 return ConstantInt::get(getInt1Ty(), V);
455 }
456
457 /// Get the constant value for i1 true.
458 ConstantInt *getTrue() {
459 return ConstantInt::getTrue(Context);
460 }
461
462 /// Get the constant value for i1 false.
463 ConstantInt *getFalse() {
464 return ConstantInt::getFalse(Context);
465 }
466
467 /// Get a constant 8-bit value.
468 ConstantInt *getInt8(uint8_t C) {
469 return ConstantInt::get(getInt8Ty(), C);
470 }
471
472 /// Get a constant 16-bit value.
473 ConstantInt *getInt16(uint16_t C) {
474 return ConstantInt::get(getInt16Ty(), C);
475 }
476
477 /// Get a constant 32-bit value.
478 ConstantInt *getInt32(uint32_t C) {
479 return ConstantInt::get(getInt32Ty(), C);
480 }
481
482 /// Get a constant 64-bit value.
483 ConstantInt *getInt64(uint64_t C) {
484 return ConstantInt::get(getInt64Ty(), C);
485 }
486
487 /// Get a constant N-bit value, zero extended or truncated from
488 /// a 64-bit value.
489 ConstantInt *getIntN(unsigned N, uint64_t C) {
490 return ConstantInt::get(getIntNTy(N), C);
491 }
492
493 /// Get a constant integer value.
494 ConstantInt *getInt(const APInt &AI) {
495 return ConstantInt::get(Context, AI);
496 }
497
498 //===--------------------------------------------------------------------===//
499 // Type creation methods
500 //===--------------------------------------------------------------------===//
501
502 /// Fetch the type representing a single bit
503 IntegerType *getInt1Ty() {
504 return Type::getInt1Ty(Context);
505 }
506
507 /// Fetch the type representing an 8-bit integer.
508 IntegerType *getInt8Ty() {
509 return Type::getInt8Ty(Context);
510 }
511
512 /// Fetch the type representing a 16-bit integer.
513 IntegerType *getInt16Ty() {
514 return Type::getInt16Ty(Context);
515 }
516
517 /// Fetch the type representing a 32-bit integer.
518 IntegerType *getInt32Ty() {
519 return Type::getInt32Ty(Context);
520 }
521
522 /// Fetch the type representing a 64-bit integer.
523 IntegerType *getInt64Ty() {
524 return Type::getInt64Ty(Context);
525 }
526
527 /// Fetch the type representing a 128-bit integer.
528 IntegerType *getInt128Ty() { return Type::getInt128Ty(Context); }
529
530 /// Fetch the type representing an N-bit integer.
531 IntegerType *getIntNTy(unsigned N) {
532 return Type::getIntNTy(Context, N);
533 }
534
535 /// Fetch the type representing a 16-bit floating point value.
536 Type *getHalfTy() {
537 return Type::getHalfTy(Context);
538 }
539
540 /// Fetch the type representing a 16-bit brain floating point value.
541 Type *getBFloatTy() {
542 return Type::getBFloatTy(Context);
543 }
544
545 /// Fetch the type representing a 32-bit floating point value.
546 Type *getFloatTy() {
547 return Type::getFloatTy(Context);
548 }
549
550 /// Fetch the type representing a 64-bit floating point value.
551 Type *getDoubleTy() {
552 return Type::getDoubleTy(Context);
553 }
554
555 /// Fetch the type representing void.
556 Type *getVoidTy() {
557 return Type::getVoidTy(Context);
558 }
559
560 /// Fetch the type representing a pointer to an 8-bit integer value.
561 PointerType *getInt8PtrTy(unsigned AddrSpace = 0) {
562 return Type::getInt8PtrTy(Context, AddrSpace);
563 }
564
565 /// Fetch the type representing a pointer to an integer value.
566 IntegerType *getIntPtrTy(const DataLayout &DL, unsigned AddrSpace = 0) {
567 return DL.getIntPtrType(Context, AddrSpace);
568 }
569
570 //===--------------------------------------------------------------------===//
571 // Intrinsic creation methods
572 //===--------------------------------------------------------------------===//
573
574 /// Create and insert a memset to the specified pointer and the
575 /// specified value.
576 ///
577 /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
578 /// specified, it will be added to the instruction. Likewise with alias.scope
579 /// and noalias tags.
580 CallInst *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size,
581 MaybeAlign Align, bool isVolatile = false,
582 MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr,
583 MDNode *NoAliasTag = nullptr) {
584 return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile,
585 TBAATag, ScopeTag, NoAliasTag);
586 }
587
588 CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size, MaybeAlign Align,
589 bool isVolatile = false, MDNode *TBAATag = nullptr,
590 MDNode *ScopeTag = nullptr,
591 MDNode *NoAliasTag = nullptr);
592
593 /// Create and insert an element unordered-atomic memset of the region of
594 /// memory starting at the given pointer to the given value.
595 ///
596 /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
597 /// specified, it will be added to the instruction. Likewise with alias.scope
598 /// and noalias tags.
599 CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
600 uint64_t Size, Align Alignment,
601 uint32_t ElementSize,
602 MDNode *TBAATag = nullptr,
603 MDNode *ScopeTag = nullptr,
604 MDNode *NoAliasTag = nullptr) {
605 return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size),
606 Align(Alignment), ElementSize,
607 TBAATag, ScopeTag, NoAliasTag);
608 }
609
610 CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
611 Value *Size, Align Alignment,
612 uint32_t ElementSize,
613 MDNode *TBAATag = nullptr,
614 MDNode *ScopeTag = nullptr,
615 MDNode *NoAliasTag = nullptr);
616
617 /// Create and insert a memcpy between the specified pointers.
618 ///
619 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
620 /// specified, it will be added to the instruction. Likewise with alias.scope
621 /// and noalias tags.
622 CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
623 MaybeAlign SrcAlign, uint64_t Size,
624 bool isVolatile = false, MDNode *TBAATag = nullptr,
625 MDNode *TBAAStructTag = nullptr,
626 MDNode *ScopeTag = nullptr,
627 MDNode *NoAliasTag = nullptr) {
628 return CreateMemCpy(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
629 isVolatile, TBAATag, TBAAStructTag, ScopeTag,
630 NoAliasTag);
631 }
632
633 CallInst *CreateMemTransferInst(
634 Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
635 MaybeAlign SrcAlign, Value *Size, bool isVolatile = false,
636 MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
637 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr);
638
639 CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
640 MaybeAlign SrcAlign, Value *Size,
641 bool isVolatile = false, MDNode *TBAATag = nullptr,
642 MDNode *TBAAStructTag = nullptr,
643 MDNode *ScopeTag = nullptr,
644 MDNode *NoAliasTag = nullptr) {
645 return CreateMemTransferInst(Intrinsic::memcpy, Dst, DstAlign, Src,
646 SrcAlign, Size, isVolatile, TBAATag,
647 TBAAStructTag, ScopeTag, NoAliasTag);
648 }
649
650 CallInst *
651 CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign, Value *Src,
652 MaybeAlign SrcAlign, Value *Size, bool IsVolatile = false,
653 MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
654 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr);
655
656 /// Create and insert an element unordered-atomic memcpy between the
657 /// specified pointers.
658 ///
659 /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers, respectively.
660 ///
661 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
662 /// specified, it will be added to the instruction. Likewise with alias.scope
663 /// and noalias tags.
664 CallInst *CreateElementUnorderedAtomicMemCpy(
665 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
666 uint32_t ElementSize, MDNode *TBAATag = nullptr,
667 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
668 MDNode *NoAliasTag = nullptr);
669
670 CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
671 MaybeAlign SrcAlign, uint64_t Size,
672 bool isVolatile = false, MDNode *TBAATag = nullptr,
673 MDNode *ScopeTag = nullptr,
674 MDNode *NoAliasTag = nullptr) {
675 return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
676 isVolatile, TBAATag, ScopeTag, NoAliasTag);
677 }
678
679 CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
680 MaybeAlign SrcAlign, Value *Size,
681 bool isVolatile = false, MDNode *TBAATag = nullptr,
682 MDNode *ScopeTag = nullptr,
683 MDNode *NoAliasTag = nullptr);
684
685 /// \brief Create and insert an element unordered-atomic memmove between the
686 /// specified pointers.
687 ///
688 /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers,
689 /// respectively.
690 ///
691 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
692 /// specified, it will be added to the instruction. Likewise with alias.scope
693 /// and noalias tags.
694 CallInst *CreateElementUnorderedAtomicMemMove(
695 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
696 uint32_t ElementSize, MDNode *TBAATag = nullptr,
697 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
698 MDNode *NoAliasTag = nullptr);
699
700 /// Create a sequential vector fadd reduction intrinsic of the source vector.
701 /// The first parameter is a scalar accumulator value. An unordered reduction
702 /// can be created by adding the reassoc fast-math flag to the resulting
703 /// sequential reduction.
704 CallInst *CreateFAddReduce(Value *Acc, Value *Src);
705
706 /// Create a sequential vector fmul reduction intrinsic of the source vector.
707 /// The first parameter is a scalar accumulator value. An unordered reduction
708 /// can be created by adding the reassoc fast-math flag to the resulting
709 /// sequential reduction.
710 CallInst *CreateFMulReduce(Value *Acc, Value *Src);
711
712 /// Create a vector int add reduction intrinsic of the source vector.
713 CallInst *CreateAddReduce(Value *Src);
714
715 /// Create a vector int mul reduction intrinsic of the source vector.
716 CallInst *CreateMulReduce(Value *Src);
717
718 /// Create a vector int AND reduction intrinsic of the source vector.
719 CallInst *CreateAndReduce(Value *Src);
720
721 /// Create a vector int OR reduction intrinsic of the source vector.
722 CallInst *CreateOrReduce(Value *Src);
723
724 /// Create a vector int XOR reduction intrinsic of the source vector.
725 CallInst *CreateXorReduce(Value *Src);
726
727 /// Create a vector integer max reduction intrinsic of the source
728 /// vector.
729 CallInst *CreateIntMaxReduce(Value *Src, bool IsSigned = false);
730
731 /// Create a vector integer min reduction intrinsic of the source
732 /// vector.
733 CallInst *CreateIntMinReduce(Value *Src, bool IsSigned = false);
734
735 /// Create a vector float max reduction intrinsic of the source
736 /// vector.
737 CallInst *CreateFPMaxReduce(Value *Src);
738
739 /// Create a vector float min reduction intrinsic of the source
740 /// vector.
741 CallInst *CreateFPMinReduce(Value *Src);
742
743 /// Create a lifetime.start intrinsic.
744 ///
745 /// If the pointer isn't i8* it will be converted.
746 CallInst *CreateLifetimeStart(Value *Ptr, ConstantInt *Size = nullptr);
747
748 /// Create a lifetime.end intrinsic.
749 ///
750 /// If the pointer isn't i8* it will be converted.
751 CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr);
752
753 /// Create a call to invariant.start intrinsic.
754 ///
755 /// If the pointer isn't i8* it will be converted.
756 CallInst *CreateInvariantStart(Value *Ptr, ConstantInt *Size = nullptr);
757
758 /// Create a call to Masked Load intrinsic
759 CallInst *CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask,
760 Value *PassThru = nullptr, const Twine &Name = "");
761
762 /// Create a call to Masked Store intrinsic
763 CallInst *CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
764 Value *Mask);
765
766 /// Create a call to Masked Gather intrinsic
767 CallInst *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
768 Value *Mask = nullptr, Value *PassThru = nullptr,
769 const Twine &Name = "");
770
771 /// Create a call to Masked Scatter intrinsic
772 CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment,
773 Value *Mask = nullptr);
774
775 /// Create an assume intrinsic call that allows the optimizer to
776 /// assume that the provided condition will be true.
777 ///
778 /// The optional argument \p OpBundles specifies operand bundles that are
779 /// added to the call instruction.
780 CallInst *CreateAssumption(Value *Cond,
781 ArrayRef<OperandBundleDef> OpBundles = llvm::None);
782
783 /// Create a llvm.experimental.noalias.scope.decl intrinsic call.
784 Instruction *CreateNoAliasScopeDeclaration(Value *Scope);
785 Instruction *CreateNoAliasScopeDeclaration(MDNode *ScopeTag) {
786 return CreateNoAliasScopeDeclaration(
787 MetadataAsValue::get(Context, ScopeTag));
788 }
789
790 /// Create a call to the experimental.gc.statepoint intrinsic to
791 /// start a new statepoint sequence.
792 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
793 Value *ActualCallee,
794 ArrayRef<Value *> CallArgs,
795 Optional<ArrayRef<Value *>> DeoptArgs,
796 ArrayRef<Value *> GCArgs,
797 const Twine &Name = "");
798
799 /// Create a call to the experimental.gc.statepoint intrinsic to
800 /// start a new statepoint sequence.
801 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
802 Value *ActualCallee, uint32_t Flags,
803 ArrayRef<Value *> CallArgs,
804 Optional<ArrayRef<Use>> TransitionArgs,
805 Optional<ArrayRef<Use>> DeoptArgs,
806 ArrayRef<Value *> GCArgs,
807 const Twine &Name = "");
808
809 /// Conveninence function for the common case when CallArgs are filled
810 /// in using makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be
811 /// .get()'ed to get the Value pointer.
812 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
813 Value *ActualCallee, ArrayRef<Use> CallArgs,
814 Optional<ArrayRef<Value *>> DeoptArgs,
815 ArrayRef<Value *> GCArgs,
816 const Twine &Name = "");
817
818 /// Create an invoke to the experimental.gc.statepoint intrinsic to
819 /// start a new statepoint sequence.
820 InvokeInst *
821 CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
822 Value *ActualInvokee, BasicBlock *NormalDest,
823 BasicBlock *UnwindDest, ArrayRef<Value *> InvokeArgs,
824 Optional<ArrayRef<Value *>> DeoptArgs,
825 ArrayRef<Value *> GCArgs, const Twine &Name = "");
826
827 /// Create an invoke to the experimental.gc.statepoint intrinsic to
828 /// start a new statepoint sequence.
829 InvokeInst *CreateGCStatepointInvoke(
830 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
831 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
832 ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs,
833 Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
834 const Twine &Name = "");
835
836 // Convenience function for the common case when CallArgs are filled in using
837 // makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be .get()'ed to
838 // get the Value *.
839 InvokeInst *
840 CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
841 Value *ActualInvokee, BasicBlock *NormalDest,
842 BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
843 Optional<ArrayRef<Value *>> DeoptArgs,
844 ArrayRef<Value *> GCArgs, const Twine &Name = "");
845
846 /// Create a call to the experimental.gc.result intrinsic to extract
847 /// the result from a call wrapped in a statepoint.
848 CallInst *CreateGCResult(Instruction *Statepoint,
849 Type *ResultType,
850 const Twine &Name = "");
851
852 /// Create a call to the experimental.gc.relocate intrinsics to
853 /// project the relocated value of one pointer from the statepoint.
854 CallInst *CreateGCRelocate(Instruction *Statepoint,
855 int BaseOffset,
856 int DerivedOffset,
857 Type *ResultType,
858 const Twine &Name = "");
859
860 /// Create a call to the experimental.gc.pointer.base intrinsic to get the
861 /// base pointer for the specified derived pointer.
862 CallInst *CreateGCGetPointerBase(Value *DerivedPtr, const Twine &Name = "");
863
864 /// Create a call to the experimental.gc.get.pointer.offset intrinsic to get
865 /// the offset of the specified derived pointer from its base.
866 CallInst *CreateGCGetPointerOffset(Value *DerivedPtr, const Twine &Name = "");
867
868 /// Create a call to llvm.vscale, multiplied by \p Scaling. The type of VScale
869 /// will be the same type as that of \p Scaling.
870 Value *CreateVScale(Constant *Scaling, const Twine &Name = "");
871
872 /// Creates a vector of type \p DstType with the linear sequence <0, 1, ...>
873 Value *CreateStepVector(Type *DstType, const Twine &Name = "");
874
875 /// Create a call to intrinsic \p ID with 1 operand which is mangled on its
876 /// type.
877 CallInst *CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
878 Instruction *FMFSource = nullptr,
879 const Twine &Name = "");
880
881 /// Create a call to intrinsic \p ID with 2 operands which is mangled on the
882 /// first type.
883 CallInst *CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS,
884 Instruction *FMFSource = nullptr,
885 const Twine &Name = "");
886
887 /// Create a call to intrinsic \p ID with \p args, mangled using \p Types. If
888 /// \p FMFSource is provided, copy fast-math-flags from that instruction to
889 /// the intrinsic.
890 CallInst *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Type *> Types,
891 ArrayRef<Value *> Args,
892 Instruction *FMFSource = nullptr,
893 const Twine &Name = "");
894
895 /// Create call to the minnum intrinsic.
896 CallInst *CreateMinNum(Value *LHS, Value *RHS, const Twine &Name = "") {
897 return CreateBinaryIntrinsic(Intrinsic::minnum, LHS, RHS, nullptr, Name);
898 }
899
900 /// Create call to the maxnum intrinsic.
901 CallInst *CreateMaxNum(Value *LHS, Value *RHS, const Twine &Name = "") {
902 return CreateBinaryIntrinsic(Intrinsic::maxnum, LHS, RHS, nullptr, Name);
903 }
904
905 /// Create call to the minimum intrinsic.
906 CallInst *CreateMinimum(Value *LHS, Value *RHS, const Twine &Name = "") {
907 return CreateBinaryIntrinsic(Intrinsic::minimum, LHS, RHS, nullptr, Name);
908 }
909
910 /// Create call to the maximum intrinsic.
911 CallInst *CreateMaximum(Value *LHS, Value *RHS, const Twine &Name = "") {
912 return CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS, nullptr, Name);
913 }
914
915 /// Create a call to the arithmetic_fence intrinsic.
916 CallInst *CreateArithmeticFence(Value *Val, Type *DstType,
917 const Twine &Name = "") {
918 return CreateIntrinsic(Intrinsic::arithmetic_fence, DstType, Val, nullptr,
919 Name);
920 }
921
922 /// Create a call to the experimental.vector.extract intrinsic.
923 CallInst *CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx,
924 const Twine &Name = "") {
925 return CreateIntrinsic(Intrinsic::experimental_vector_extract,
926 {DstType, SrcVec->getType()}, {SrcVec, Idx}, nullptr,
927 Name);
928 }
929
930 /// Create a call to the experimental.vector.insert intrinsic.
931 CallInst *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
932 Value *Idx, const Twine &Name = "") {
933 return CreateIntrinsic(Intrinsic::experimental_vector_insert,
934 {DstType, SubVec->getType()}, {SrcVec, SubVec, Idx},
935 nullptr, Name);
936 }
937
938private:
939 /// Create a call to a masked intrinsic with given Id.
940 CallInst *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops,
941 ArrayRef<Type *> OverloadedTypes,
942 const Twine &Name = "");
943
944 Value *getCastedInt8PtrValue(Value *Ptr);
945
946 //===--------------------------------------------------------------------===//
947 // Instruction creation methods: Terminators
948 //===--------------------------------------------------------------------===//
949
950private:
951 /// Helper to add branch weight and unpredictable metadata onto an
952 /// instruction.
953 /// \returns The annotated instruction.
954 template <typename InstTy>
955 InstTy *addBranchMetadata(InstTy *I, MDNode *Weights, MDNode *Unpredictable) {
956 if (Weights)
957 I->setMetadata(LLVMContext::MD_prof, Weights);
958 if (Unpredictable)
959 I->setMetadata(LLVMContext::MD_unpredictable, Unpredictable);
960 return I;
961 }
962
963public:
964 /// Create a 'ret void' instruction.
965 ReturnInst *CreateRetVoid() {
966 return Insert(ReturnInst::Create(Context));
967 }
968
969 /// Create a 'ret <val>' instruction.
970 ReturnInst *CreateRet(Value *V) {
971 return Insert(ReturnInst::Create(Context, V));
972 }
973
974 /// Create a sequence of N insertvalue instructions,
975 /// with one Value from the retVals array each, that build a aggregate
976 /// return value one value at a time, and a ret instruction to return
977 /// the resulting aggregate value.
978 ///
979 /// This is a convenience function for code that uses aggregate return values
980 /// as a vehicle for having multiple return values.
981 ReturnInst *CreateAggregateRet(Value *const *retVals, unsigned N) {
982 Value *V = UndefValue::get(getCurrentFunctionReturnType());
983 for (unsigned i = 0; i != N; ++i)
984 V = CreateInsertValue(V, retVals[i], i, "mrv");
985 return Insert(ReturnInst::Create(Context, V));
986 }
987
988 /// Create an unconditional 'br label X' instruction.
989 BranchInst *CreateBr(BasicBlock *Dest) {
990 return Insert(BranchInst::Create(Dest));
991 }
992
993 /// Create a conditional 'br Cond, TrueDest, FalseDest'
994 /// instruction.
995 BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False,
996 MDNode *BranchWeights = nullptr,
997 MDNode *Unpredictable = nullptr) {
998 return Insert(addBranchMetadata(BranchInst::Create(True, False, Cond),
999 BranchWeights, Unpredictable));
1000 }
1001
1002 /// Create a conditional 'br Cond, TrueDest, FalseDest'
1003 /// instruction. Copy branch meta data if available.
1004 BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False,
1005 Instruction *MDSrc) {
1006 BranchInst *Br = BranchInst::Create(True, False, Cond);
1007 if (MDSrc) {
1008 unsigned WL[4] = {LLVMContext::MD_prof, LLVMContext::MD_unpredictable,
1009 LLVMContext::MD_make_implicit, LLVMContext::MD_dbg};
1010 Br->copyMetadata(*MDSrc, makeArrayRef(&WL[0], 4));
1011 }
1012 return Insert(Br);
1013 }
1014
1015 /// Create a switch instruction with the specified value, default dest,
1016 /// and with a hint for the number of cases that will be added (for efficient
1017 /// allocation).
1018 SwitchInst *CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases = 10,
1019 MDNode *BranchWeights = nullptr,
1020 MDNode *Unpredictable = nullptr) {
1021 return Insert(addBranchMetadata(SwitchInst::Create(V, Dest, NumCases),
1022 BranchWeights, Unpredictable));
1023 }
1024
1025 /// Create an indirect branch instruction with the specified address
1026 /// operand, with an optional hint for the number of destinations that will be
1027 /// added (for efficient allocation).
1028 IndirectBrInst *CreateIndirectBr(Value *Addr, unsigned NumDests = 10) {
1029 return Insert(IndirectBrInst::Create(Addr, NumDests));
1030 }
1031
1032 /// Create an invoke instruction.
1033 InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee,
1034 BasicBlock *NormalDest, BasicBlock *UnwindDest,
1035 ArrayRef<Value *> Args,
1036 ArrayRef<OperandBundleDef> OpBundles,
1037 const Twine &Name = "") {
1038 InvokeInst *II =
1039 InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args, OpBundles);
1040 if (IsFPConstrained)
1041 setConstrainedFPCallAttr(II);
1042 return Insert(II, Name);
1043 }
1044 InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee,
1045 BasicBlock *NormalDest, BasicBlock *UnwindDest,
1046 ArrayRef<Value *> Args = None,
1047 const Twine &Name = "") {
1048 InvokeInst *II =
1049 InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args);
1050 if (IsFPConstrained)
1051 setConstrainedFPCallAttr(II);
1052 return Insert(II, Name);
1053 }
1054
1055 InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest,
1056 BasicBlock *UnwindDest, ArrayRef<Value *> Args,
1057 ArrayRef<OperandBundleDef> OpBundles,
1058 const Twine &Name = "") {
1059 return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(),
1060 NormalDest, UnwindDest, Args, OpBundles, Name);
1061 }
1062
1063 InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest,
1064 BasicBlock *UnwindDest,
1065 ArrayRef<Value *> Args = None,
1066 const Twine &Name = "") {
1067 return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(),
1068 NormalDest, UnwindDest, Args, Name);
1069 }
1070
1071 /// \brief Create a callbr instruction.
1072 CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee,
1073 BasicBlock *DefaultDest,
1074 ArrayRef<BasicBlock *> IndirectDests,
1075 ArrayRef<Value *> Args = None,
1076 const Twine &Name = "") {
1077 return Insert(CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests,
1078 Args), Name);
1079 }
1080 CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee,
1081 BasicBlock *DefaultDest,
1082 ArrayRef<BasicBlock *> IndirectDests,
1083 ArrayRef<Value *> Args,
1084 ArrayRef<OperandBundleDef> OpBundles,
1085 const Twine &Name = "") {
1086 return Insert(
1087 CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args,
1088 OpBundles), Name);
1089 }
1090
1091 CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest,
1092 ArrayRef<BasicBlock *> IndirectDests,
1093 ArrayRef<Value *> Args = None,
1094 const Twine &Name = "") {
1095 return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(),
1096 DefaultDest, IndirectDests, Args, Name);
1097 }
1098 CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest,
1099 ArrayRef<BasicBlock *> IndirectDests,
1100 ArrayRef<Value *> Args,
1101 ArrayRef<OperandBundleDef> OpBundles,
1102 const Twine &Name = "") {
1103 return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(),
1104 DefaultDest, IndirectDests, Args, Name);
1105 }
1106
1107 ResumeInst *CreateResume(Value *Exn) {
1108 return Insert(ResumeInst::Create(Exn));
1109 }
1110
1111 CleanupReturnInst *CreateCleanupRet(CleanupPadInst *CleanupPad,
1112 BasicBlock *UnwindBB = nullptr) {
1113 return Insert(CleanupReturnInst::Create(CleanupPad, UnwindBB));
1114 }
1115
1116 CatchSwitchInst *CreateCatchSwitch(Value *ParentPad, BasicBlock *UnwindBB,
1117 unsigned NumHandlers,
1118 const Twine &Name = "") {
1119 return Insert(CatchSwitchInst::Create(ParentPad, UnwindBB, NumHandlers),
1120 Name);
1121 }
1122
1123 CatchPadInst *CreateCatchPad(Value *ParentPad, ArrayRef<Value *> Args,
1124 const Twine &Name = "") {
1125 return Insert(CatchPadInst::Create(ParentPad, Args), Name);
1126 }
1127
1128 CleanupPadInst *CreateCleanupPad(Value *ParentPad,
1129 ArrayRef<Value *> Args = None,
1130 const Twine &Name = "") {
1131 return Insert(CleanupPadInst::Create(ParentPad, Args), Name);
1132 }
1133
1134 CatchReturnInst *CreateCatchRet(CatchPadInst *CatchPad, BasicBlock *BB) {
1135 return Insert(CatchReturnInst::Create(CatchPad, BB));
1136 }
1137
1138 UnreachableInst *CreateUnreachable() {
1139 return Insert(new UnreachableInst(Context));
1140 }
1141
1142 //===--------------------------------------------------------------------===//
1143 // Instruction creation methods: Binary Operators
1144 //===--------------------------------------------------------------------===//
1145private:
1146 BinaryOperator *CreateInsertNUWNSWBinOp(BinaryOperator::BinaryOps Opc,
1147 Value *LHS, Value *RHS,
1148 const Twine &Name,
1149 bool HasNUW, bool HasNSW) {
1150 BinaryOperator *BO = Insert(BinaryOperator::Create(Opc, LHS, RHS), Name);
1151 if (HasNUW) BO->setHasNoUnsignedWrap();
1152 if (HasNSW) BO->setHasNoSignedWrap();
1153 return BO;
1154 }
1155
1156 Instruction *setFPAttrs(Instruction *I, MDNode *FPMD,
1157 FastMathFlags FMF) const {
1158 if (!FPMD)
1159 FPMD = DefaultFPMathTag;
1160 if (FPMD)
1161 I->setMetadata(LLVMContext::MD_fpmath, FPMD);
1162 I->setFastMathFlags(FMF);
1163 return I;
1164 }
1165
1166 Value *foldConstant(Instruction::BinaryOps Opc, Value *L,
1167 Value *R, const Twine &Name) const {
1168 auto *LC = dyn_cast<Constant>(L);
1169 auto *RC = dyn_cast<Constant>(R);
1170 return (LC && RC) ? Insert(Folder.CreateBinOp(Opc, LC, RC), Name) : nullptr;
1171 }
1172
1173 Value *getConstrainedFPRounding(Optional<RoundingMode> Rounding) {
1174 RoundingMode UseRounding = DefaultConstrainedRounding;
1175
1176 if (Rounding.hasValue())
1177 UseRounding = Rounding.getValue();
1178
1179 Optional<StringRef> RoundingStr = convertRoundingModeToStr(UseRounding);
1180 assert(RoundingStr.hasValue() && "Garbage strict rounding mode!")(static_cast <bool> (RoundingStr.hasValue() && "Garbage strict rounding mode!"
) ? void (0) : __assert_fail ("RoundingStr.hasValue() && \"Garbage strict rounding mode!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1180, __extension__ __PRETTY_FUNCTION__
))
;
1181 auto *RoundingMDS = MDString::get(Context, RoundingStr.getValue());
1182
1183 return MetadataAsValue::get(Context, RoundingMDS);
1184 }
1185
1186 Value *getConstrainedFPExcept(Optional<fp::ExceptionBehavior> Except) {
1187 fp::ExceptionBehavior UseExcept = DefaultConstrainedExcept;
1188
1189 if (Except.hasValue())
1190 UseExcept = Except.getValue();
1191
1192 Optional<StringRef> ExceptStr = convertExceptionBehaviorToStr(UseExcept);
1193 assert(ExceptStr.hasValue() && "Garbage strict exception behavior!")(static_cast <bool> (ExceptStr.hasValue() && "Garbage strict exception behavior!"
) ? void (0) : __assert_fail ("ExceptStr.hasValue() && \"Garbage strict exception behavior!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1193, __extension__ __PRETTY_FUNCTION__
))
;
1194 auto *ExceptMDS = MDString::get(Context, ExceptStr.getValue());
1195
1196 return MetadataAsValue::get(Context, ExceptMDS);
1197 }
1198
1199 Value *getConstrainedFPPredicate(CmpInst::Predicate Predicate) {
1200 assert(CmpInst::isFPPredicate(Predicate) &&(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1203, __extension__ __PRETTY_FUNCTION__
))
1201 Predicate != CmpInst::FCMP_FALSE &&(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1203, __extension__ __PRETTY_FUNCTION__
))
1202 Predicate != CmpInst::FCMP_TRUE &&(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1203, __extension__ __PRETTY_FUNCTION__
))
1203 "Invalid constrained FP comparison predicate!")(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1203, __extension__ __PRETTY_FUNCTION__
))
;
1204
1205 StringRef PredicateStr = CmpInst::getPredicateName(Predicate);
1206 auto *PredicateMDS = MDString::get(Context, PredicateStr);
1207
1208 return MetadataAsValue::get(Context, PredicateMDS);
1209 }
1210
1211public:
1212 Value *CreateAdd(Value *LHS, Value *RHS, const Twine &Name = "",
1213 bool HasNUW = false, bool HasNSW = false) {
1214 if (auto *V = Folder.FoldAdd(LHS, RHS, HasNUW, HasNSW))
1215 return V;
1216 return CreateInsertNUWNSWBinOp(Instruction::Add, LHS, RHS, Name,
1217 HasNUW, HasNSW);
1218 }
1219
1220 Value *CreateNSWAdd(Value *LHS, Value *RHS, const Twine &Name = "") {
1221 return CreateAdd(LHS, RHS, Name, false, true);
1222 }
1223
1224 Value *CreateNUWAdd(Value *LHS, Value *RHS, const Twine &Name = "") {
1225 return CreateAdd(LHS, RHS, Name, true, false);
1226 }
1227
1228 Value *CreateSub(Value *LHS, Value *RHS, const Twine &Name = "",
1229 bool HasNUW = false, bool HasNSW = false) {
1230 if (auto *LC = dyn_cast<Constant>(LHS))
1231 if (auto *RC = dyn_cast<Constant>(RHS))
1232 return Insert(Folder.CreateSub(LC, RC, HasNUW, HasNSW), Name);
1233 return CreateInsertNUWNSWBinOp(Instruction::Sub, LHS, RHS, Name,
1234 HasNUW, HasNSW);
1235 }
1236
1237 Value *CreateNSWSub(Value *LHS, Value *RHS, const Twine &Name = "") {
1238 return CreateSub(LHS, RHS, Name, false, true);
1239 }
1240
1241 Value *CreateNUWSub(Value *LHS, Value *RHS, const Twine &Name = "") {
1242 return CreateSub(LHS, RHS, Name, true, false);
1243 }
1244
1245 Value *CreateMul(Value *LHS, Value *RHS, const Twine &Name = "",
1246 bool HasNUW = false, bool HasNSW = false) {
1247 if (auto *LC = dyn_cast<Constant>(LHS))
1248 if (auto *RC = dyn_cast<Constant>(RHS))
1249 return Insert(Folder.CreateMul(LC, RC, HasNUW, HasNSW), Name);
1250 return CreateInsertNUWNSWBinOp(Instruction::Mul, LHS, RHS, Name,
1251 HasNUW, HasNSW);
1252 }
1253
1254 Value *CreateNSWMul(Value *LHS, Value *RHS, const Twine &Name = "") {
1255 return CreateMul(LHS, RHS, Name, false, true);
1256 }
1257
1258 Value *CreateNUWMul(Value *LHS, Value *RHS, const Twine &Name = "") {
1259 return CreateMul(LHS, RHS, Name, true, false);
1260 }
1261
1262 Value *CreateUDiv(Value *LHS, Value *RHS, const Twine &Name = "",
1263 bool isExact = false) {
1264 if (auto *LC = dyn_cast<Constant>(LHS))
1265 if (auto *RC = dyn_cast<Constant>(RHS))
1266 return Insert(Folder.CreateUDiv(LC, RC, isExact), Name);
1267 if (!isExact)
1268 return Insert(BinaryOperator::CreateUDiv(LHS, RHS), Name);
1269 return Insert(BinaryOperator::CreateExactUDiv(LHS, RHS), Name);
1270 }
1271
1272 Value *CreateExactUDiv(Value *LHS, Value *RHS, const Twine &Name = "") {
1273 return CreateUDiv(LHS, RHS, Name, true);
1274 }
1275
1276 Value *CreateSDiv(Value *LHS, Value *RHS, const Twine &Name = "",
1277 bool isExact = false) {
1278 if (auto *LC = dyn_cast<Constant>(LHS))
1279 if (auto *RC = dyn_cast<Constant>(RHS))
1280 return Insert(Folder.CreateSDiv(LC, RC, isExact), Name);
1281 if (!isExact)
1282 return Insert(BinaryOperator::CreateSDiv(LHS, RHS), Name);
1283 return Insert(BinaryOperator::CreateExactSDiv(LHS, RHS), Name);
1284 }
1285
1286 Value *CreateExactSDiv(Value *LHS, Value *RHS, const Twine &Name = "") {
1287 return CreateSDiv(LHS, RHS, Name, true);
1288 }
1289
1290 Value *CreateURem(Value *LHS, Value *RHS, const Twine &Name = "") {
1291 if (Value *V = foldConstant(Instruction::URem, LHS, RHS, Name)) return V;
1292 return Insert(BinaryOperator::CreateURem(LHS, RHS), Name);
1293 }
1294
1295 Value *CreateSRem(Value *LHS, Value *RHS, const Twine &Name = "") {
1296 if (Value *V = foldConstant(Instruction::SRem, LHS, RHS, Name)) return V;
1297 return Insert(BinaryOperator::CreateSRem(LHS, RHS), Name);
1298 }
1299
1300 Value *CreateShl(Value *LHS, Value *RHS, const Twine &Name = "",
1301 bool HasNUW = false, bool HasNSW = false) {
1302 if (auto *LC = dyn_cast<Constant>(LHS))
1303 if (auto *RC = dyn_cast<Constant>(RHS))
1304 return Insert(Folder.CreateShl(LC, RC, HasNUW, HasNSW), Name);
1305 return CreateInsertNUWNSWBinOp(Instruction::Shl, LHS, RHS, Name,
1306 HasNUW, HasNSW);
1307 }
1308
1309 Value *CreateShl(Value *LHS, const APInt &RHS, const Twine &Name = "",
1310 bool HasNUW = false, bool HasNSW = false) {
1311 return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name,
1312 HasNUW, HasNSW);
1313 }
1314
1315 Value *CreateShl(Value *LHS, uint64_t RHS, const Twine &Name = "",
1316 bool HasNUW = false, bool HasNSW = false) {
1317 return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name,
1318 HasNUW, HasNSW);
1319 }
1320
1321 Value *CreateLShr(Value *LHS, Value *RHS, const Twine &Name = "",
1322 bool isExact = false) {
1323 if (auto *LC = dyn_cast<Constant>(LHS))
1324 if (auto *RC = dyn_cast<Constant>(RHS))
1325 return Insert(Folder.CreateLShr(LC, RC, isExact), Name);
1326 if (!isExact)
1327 return Insert(BinaryOperator::CreateLShr(LHS, RHS), Name);
1328 return Insert(BinaryOperator::CreateExactLShr(LHS, RHS), Name);
1329 }
1330
1331 Value *CreateLShr(Value *LHS, const APInt &RHS, const Twine &Name = "",
1332 bool isExact = false) {
1333 return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1334 }
1335
1336 Value *CreateLShr(Value *LHS, uint64_t RHS, const Twine &Name = "",
1337 bool isExact = false) {
1338 return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1339 }
1340
1341 Value *CreateAShr(Value *LHS, Value *RHS, const Twine &Name = "",
1342 bool isExact = false) {
1343 if (auto *LC = dyn_cast<Constant>(LHS))
1344 if (auto *RC = dyn_cast<Constant>(RHS))
1345 return Insert(Folder.CreateAShr(LC, RC, isExact), Name);
1346 if (!isExact)
1347 return Insert(BinaryOperator::CreateAShr(LHS, RHS), Name);
1348 return Insert(BinaryOperator::CreateExactAShr(LHS, RHS), Name);
1349 }
1350
1351 Value *CreateAShr(Value *LHS, const APInt &RHS, const Twine &Name = "",
1352 bool isExact =